From d2d81c0af026b3bbf4c27b152df9102ee6c33224 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 18 Sep 2024 11:45:17 +0200 Subject: [PATCH 01/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Upgrade=20models-lib?= =?UTF-8?q?rary=20(pydantic=20v2)=20(#6333)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu <5694077+GitHK@users.noreply.github.com> --- packages/models-library/requirements/_base.in | 2 + .../models-library/requirements/_base.txt | 18 +- .../models-library/requirements/_test.txt | 10 +- .../models-library/requirements/_tools.txt | 10 - .../scripts/validate-pg-projects.py | 12 +- .../src/models_library/access_rights.py | 5 +- .../src/models_library/aiodocker_api.py | 11 +- .../api_schemas__common/meta.py | 9 +- .../api_schemas_api_server/api_keys.py | 5 +- .../api_schemas_catalog/services.py | 54 +- .../api_schemas_catalog/services_ports.py | 12 +- .../services_specifications.py | 3 - .../api_schemas_directorv2/clusters.py | 64 +- .../api_schemas_directorv2/comp_tasks.py | 7 +- .../dynamic_services.py | 27 +- .../dynamic_services_service.py | 30 +- .../api_schemas_directorv2/health.py | 10 +- .../api_schemas_directorv2/services.py | 30 +- .../dynamic_services.py | 24 +- .../api_schemas_dynamic_sidecar/containers.py | 10 +- .../api_schemas_dynamic_sidecar/telemetry.py | 4 +- .../api_schemas_invitations/invitations.py | 20 +- .../api_schemas_long_running_tasks/base.py | 13 +- .../api_schemas_long_running_tasks/tasks.py | 4 +- .../credit_transactions.py | 4 +- .../pricing_plans.py | 30 +- .../src/models_library/api_schemas_storage.py | 74 +- .../api_schemas_webserver/_base.py | 38 +- .../api_schemas_webserver/auth.py | 25 +- .../api_schemas_webserver/catalog.py | 62 +- .../api_schemas_webserver/clusters.py | 25 +- .../api_schemas_webserver/folders.py | 12 +- .../api_schemas_webserver/folders_v2.py | 22 +- .../api_schemas_webserver/groups.py | 50 +- .../api_schemas_webserver/product.py | 18 +- .../api_schemas_webserver/projects.py | 46 +- .../api_schemas_webserver/projects_nodes.py | 35 +- .../api_schemas_webserver/projects_ports.py | 9 +- .../api_schemas_webserver/resource_usage.py | 37 +- .../api_schemas_webserver/wallets.py | 21 +- .../api_schemas_webserver/workspaces.py | 8 +- .../src/models_library/basic_regex.py | 2 +- .../src/models_library/basic_types.py | 108 +- .../src/models_library/boot_options.py | 15 +- .../src/models_library/callbacks_mapping.py | 69 +- .../src/models_library/clusters.py | 62 +- .../src/models_library/docker.py | 38 +- .../src/models_library/emails.py | 8 +- .../src/models_library/errors_classes.py | 15 +- .../src/models_library/folders.py | 5 +- .../function_services_catalog/_settings.py | 2 +- .../generated_models/docker_rest_api.py | 1297 ++++++++++------- .../src/models_library/generics.py | 40 +- .../src/models_library/groups.py | 16 +- .../src/models_library/invitations.py | 6 +- .../osparc_variable_identifier.py | 24 +- .../src/models_library/payments.py | 36 +- .../src/models_library/products.py | 17 +- .../src/models_library/progress_bar.py | 22 +- .../src/models_library/projects.py | 58 +- .../src/models_library/projects_access.py | 28 +- .../src/models_library/projects_comments.py | 10 +- .../src/models_library/projects_networks.py | 26 +- .../src/models_library/projects_nodes.py | 62 +- .../src/models_library/projects_nodes_io.py | 85 +- .../src/models_library/projects_nodes_ui.py | 14 +- .../src/models_library/projects_pipeline.py | 18 +- .../src/models_library/projects_state.py | 43 +- .../src/models_library/projects_ui.py | 26 +- .../models_library/rabbitmq_basic_types.py | 10 +- .../src/models_library/rabbitmq_messages.py | 40 +- .../src/models_library/resource_tracker.py | 63 +- .../src/models_library/rest_ordering.py | 5 +- .../src/models_library/rest_pagination.py | 89 +- .../models_library/rest_pagination_utils.py | 3 +- .../src/models_library/rpc_pagination.py | 14 +- .../models_library/service_settings_labels.py | 167 ++- .../service_settings_nat_rule.py | 40 +- .../src/models_library/services_access.py | 9 +- .../src/models_library/services_authoring.py | 16 +- .../src/models_library/services_base.py | 13 +- .../src/models_library/services_creation.py | 9 +- .../src/models_library/services_history.py | 19 +- .../src/models_library/services_io.py | 51 +- .../services_metadata_editable.py | 11 +- .../services_metadata_published.py | 35 +- .../src/models_library/services_resources.py | 61 +- .../src/models_library/services_types.py | 49 +- .../src/models_library/services_ui.py | 16 +- .../src/models_library/user_preferences.py | 33 +- .../src/models_library/users.py | 15 +- .../utils/_original_fastapi_encoders.py | 153 +- .../utils/json_serialization.py | 100 +- .../utils/pydantic_tools_extension.py | 5 +- .../utils/specs_substitution.py | 4 +- .../src/models_library/wallets.py | 11 +- .../src/models_library/workspaces.py | 8 +- .../tests/test__pydantic_models.py | 64 +- .../tests/test__pydantic_models_and_enums.py | 4 +- .../models-library/tests/test_basic_types.py | 40 +- packages/models-library/tests/test_docker.py | 2 +- packages/models-library/tests/test_emails.py | 15 +- packages/models-library/tests/test_errors.py | 18 +- .../tests/test_errors_classes.py | 11 +- .../tests/test_function_services_catalog.py | 2 +- .../models-library/tests/test_generics.py | 15 +- .../tests/test_project_nodes_io.py | 12 +- .../tests/test_rabbit_messages.py | 17 +- .../tests/test_rest_pagination.py | 6 +- .../tests/test_rpc_pagination.py | 6 +- .../tests/test_service_resources.py | 36 +- .../tests/test_service_settings_labels.py | 55 +- .../models-library/tests/test_services.py | 9 +- .../tests/test_utils_common_validators.py | 20 +- .../models-library/tests/test_utils_nodes.py | 5 +- .../test_utils_pydantic_tools_extension.py | 11 +- .../tests/test_utils_service_io.py | 10 +- .../src/pytest_simcore/pydantic_models.py | 8 +- requirements/constraints.txt | 5 - .../modules/s3.py | 5 +- .../services/resource_tracker_service_runs.py | 4 +- .../handlers_health.py | 5 +- .../storage/tests/unit/test_dsm_soft_links.py | 4 +- services/storage/tests/unit/test_models.py | 16 +- services/storage/tests/unit/test_utils.py | 4 +- .../02/test_projects_states_handlers.py | 5 +- 126 files changed, 2444 insertions(+), 2081 deletions(-) mode change 100755 => 100644 packages/models-library/scripts/validate-pg-projects.py diff --git a/packages/models-library/requirements/_base.in b/packages/models-library/requirements/_base.in index 01da93156ec..e2f8eae097a 100644 --- a/packages/models-library/requirements/_base.in +++ b/packages/models-library/requirements/_base.in @@ -7,3 +7,5 @@ arrow jsonschema orjson pydantic[email] +pydantic-settings +pydantic-extra-types diff --git a/packages/models-library/requirements/_base.txt b/packages/models-library/requirements/_base.txt index f900e7715f0..03e1b39d677 100644 --- a/packages/models-library/requirements/_base.txt +++ b/packages/models-library/requirements/_base.txt @@ -1,3 +1,5 @@ +annotated-types==0.7.0 + # via pydantic arrow==1.3.0 # via -r requirements/_base.in attrs==24.2.0 @@ -18,12 +20,22 @@ orjson==3.10.7 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in -pydantic==1.10.17 +pydantic==2.9.1 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.3 + # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/_base.in +pydantic-settings==2.4.0 + # via -r requirements/_base.in python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings referencing==0.35.1 # via # jsonschema @@ -37,4 +49,6 @@ six==1.16.0 types-python-dateutil==2.9.0.20240821 # via arrow typing-extensions==4.12.2 - # via pydantic + # via + # pydantic + # pydantic-core diff --git a/packages/models-library/requirements/_test.txt b/packages/models-library/requirements/_test.txt index 9457e941448..b44ab4c5a3f 100644 --- a/packages/models-library/requirements/_test.txt +++ b/packages/models-library/requirements/_test.txt @@ -8,8 +8,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in flexcache==0.3 @@ -68,7 +66,9 @@ python-dateutil==2.9.0.post0 # -c requirements/_base.txt # faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt @@ -87,10 +87,6 @@ six==1.16.0 # python-dateutil termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # pytest types-jsonschema==4.23.0.20240813 # via -r requirements/_test.in types-pyyaml==6.0.12.20240808 diff --git a/packages/models-library/requirements/_tools.txt b/packages/models-library/requirements/_tools.txt index 0ea376ae45a..0efdb113971 100644 --- a/packages/models-library/requirements/_tools.txt +++ b/packages/models-library/requirements/_tools.txt @@ -78,14 +78,6 @@ setuptools==73.0.1 # via pip-tools shellingham==1.5.4 # via typer -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typer==0.12.4 @@ -94,8 +86,6 @@ typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy # typer virtualenv==20.26.3 diff --git a/packages/models-library/scripts/validate-pg-projects.py b/packages/models-library/scripts/validate-pg-projects.py old mode 100755 new mode 100644 index e74b438698d..978e32cfc6f --- a/packages/models-library/scripts/validate-pg-projects.py +++ b/packages/models-library/scripts/validate-pg-projects.py @@ -4,14 +4,10 @@ import typer from models_library.projects import ProjectAtDB -from pydantic import Json, ValidationError, validator -from pydantic.main import Extra +from pydantic import ConfigDict, Json, ValidationError, field_validator class ProjectFromCsv(ProjectAtDB): - class Config(ProjectAtDB.Config): - extra = Extra.forbid - # TODO: missing in ProjectAtDB access_rights: Json @@ -22,9 +18,11 @@ class Config(ProjectAtDB.Config): hidden: bool + model_config = ConfigDict(extra="forbid") + # NOTE: validators introduced to parse CSV - @validator("published", "hidden", pre=True, check_fields=False) + @field_validator("published", "hidden", mode="before", check_fields=False) @classmethod def empty_str_as_false(cls, v): # See booleans for >v1.0 https://pydantic-docs.helpmanual.io/usage/types/#booleans @@ -32,7 +30,7 @@ def empty_str_as_false(cls, v): return False return v - @validator("workbench", pre=True, check_fields=False) + @field_validator("workbench", mode="before", check_fields=False) @classmethod def jsonstr_to_dict(cls, v): if isinstance(v, str): diff --git a/packages/models-library/src/models_library/access_rights.py b/packages/models-library/src/models_library/access_rights.py index b1218b858a1..a6cea15a946 100644 --- a/packages/models-library/src/models_library/access_rights.py +++ b/packages/models-library/src/models_library/access_rights.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field class AccessRights(BaseModel): @@ -6,5 +6,4 @@ class AccessRights(BaseModel): write: bool = Field(..., description="has write access") delete: bool = Field(..., description="has deletion rights") - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/aiodocker_api.py b/packages/models-library/src/models_library/aiodocker_api.py index 757036f31e8..c5c3b3333a0 100644 --- a/packages/models-library/src/models_library/aiodocker_api.py +++ b/packages/models-library/src/models_library/aiodocker_api.py @@ -1,4 +1,4 @@ -from pydantic import Field, validator +from pydantic import ConfigDict, Field, field_validator from .generated_models.docker_rest_api import ( ContainerSpec, @@ -16,7 +16,7 @@ class AioDockerContainerSpec(ContainerSpec): description="aiodocker expects here a dictionary and re-convert it back internally`.\n", ) - @validator("Env", pre=True) + @field_validator("Env", mode="before") @classmethod def convert_list_to_dict(cls, v): if v is not None and isinstance(v, list): @@ -37,8 +37,7 @@ class AioDockerResources1(Resources1): None, description="Define resources reservation.", alias="Reservations" ) - class Config(Resources1.Config): # type: ignore - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class AioDockerTaskSpec(TaskSpec): @@ -55,6 +54,4 @@ class AioDockerTaskSpec(TaskSpec): class AioDockerServiceSpec(ServiceSpec): TaskTemplate: AioDockerTaskSpec | None = None - class Config(ServiceSpec.Config): # type: ignore - alias_generator = camel_to_snake - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True, alias_generator=camel_to_snake) diff --git a/packages/models-library/src/models_library/api_schemas__common/meta.py b/packages/models-library/src/models_library/api_schemas__common/meta.py index 8cd2db53ae6..514abdc7d6d 100644 --- a/packages/models-library/src/models_library/api_schemas__common/meta.py +++ b/packages/models-library/src/models_library/api_schemas__common/meta.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from ..basic_types import VersionStr @@ -12,11 +10,12 @@ class BaseMeta(BaseModel): default=None, description="Maps every route's path tag with a released version" ) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "simcore_service_foo", "version": "2.4.45", "released": {"v1": "1.3.4", "v2": "2.4.45"}, } } + ) diff --git a/packages/models-library/src/models_library/api_schemas_api_server/api_keys.py b/packages/models-library/src/models_library/api_schemas_api_server/api_keys.py index d828fc6507d..999cb2f192c 100644 --- a/packages/models-library/src/models_library/api_schemas_api_server/api_keys.py +++ b/packages/models-library/src/models_library/api_schemas_api_server/api_keys.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field, SecretStr +from pydantic import BaseModel, ConfigDict, Field, SecretStr class ApiKey(BaseModel): @@ -15,5 +15,4 @@ class ApiKeyInDB(BaseModel): user_id: int product_name: str - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index ab0c98c4dc5..db386a8714f 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -1,8 +1,8 @@ from datetime import datetime -from typing import Any, ClassVar, TypeAlias +from typing import Any, TypeAlias from models_library.rpc_pagination import PageRpc -from pydantic import BaseModel, Extra, Field, HttpUrl, NonNegativeInt +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, NonNegativeInt from ..boot_options import BootOptions from ..emails import LowerCaseEmailStr @@ -23,23 +23,23 @@ class ServiceUpdate(ServiceMetaDataEditable, ServiceAccessRights): - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { # ServiceAccessRights "accessRights": { 1: { "execute_access": False, "write_access": False, - }, + }, # type: ignore[dict-item] 2: { "execute_access": True, "write_access": True, - }, + }, # type: ignore[dict-item] 44: { "execute_access": False, "write_access": False, - }, + }, # type: ignore[dict-item] }, # ServiceMetaData = ServiceCommonData + "name": "My Human Readable Service Name", @@ -72,6 +72,7 @@ class Config: }, } } + ) _EXAMPLE_FILEPICKER: dict[str, Any] = { @@ -206,12 +207,11 @@ class ServiceGet( ): # pylint: disable=too-many-ancestors owner: LowerCaseEmailStr | None - class Config: - allow_population_by_field_name = True - extra = Extra.ignore - schema_extra: ClassVar[dict[str, Any]] = { - "examples": [_EXAMPLE_FILEPICKER, _EXAMPLE_SLEEPER] - } + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + json_schema_extra={"examples": [_EXAMPLE_FILEPICKER, _EXAMPLE_SLEEPER]}, + ) class ServiceGetV2(BaseModel): @@ -229,7 +229,7 @@ class ServiceGetV2(BaseModel): service_type: ServiceType = Field(default=..., alias="type") contact: LowerCaseEmailStr | None - authors: list[Author] = Field(..., min_items=1) + authors: list[Author] = Field(..., min_length=1) owner: LowerCaseEmailStr | None inputs: ServiceInputsDict @@ -249,11 +249,11 @@ class ServiceGetV2(BaseModel): " It includes current release.", ) - class Config: - extra = Extra.forbid - alias_generator = snake_to_camel - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + alias_generator=snake_to_camel, + json_schema_extra={ "examples": [ { **_EXAMPLE_SLEEPER, # v2.2.1 (latest) @@ -304,7 +304,8 @@ class Config: ], }, ] - } + }, + ) PageRpcServicesGetV2: TypeAlias = PageRpc[ @@ -330,12 +331,13 @@ class ServiceUpdateV2(BaseModel): access_rights: dict[GroupID, ServiceGroupAccessRightsV2] | None = None - class Config: - extra = Extra.forbid - alias_generator = snake_to_camel - allow_population_by_field_name = True + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + alias_generator=snake_to_camel, + ) -assert set(ServiceUpdateV2.__fields__.keys()) - set( # nosec - ServiceGetV2.__fields__.keys() +assert set(ServiceUpdateV2.model_fields.keys()) - set( # nosec + ServiceGetV2.model_fields.keys() ) == {"deprecated"} diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services_ports.py b/packages/models-library/src/models_library/api_schemas_catalog/services_ports.py index ada65d69e28..8393594b0c8 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services_ports.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services_ports.py @@ -1,6 +1,6 @@ -from typing import Any, ClassVar, Literal +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from ..basic_regex import PUBLIC_VARIABLE_NAME_RE from ..services import ServiceInput, ServiceOutput @@ -17,7 +17,7 @@ class ServicePortGet(BaseModel): key: str = Field( ..., description="port identifier name", - regex=PUBLIC_VARIABLE_NAME_RE, + pattern=PUBLIC_VARIABLE_NAME_RE, title="Key name", ) kind: PortKindStr @@ -26,9 +26,8 @@ class ServicePortGet(BaseModel): None, description="jsonschema for the port's value. SEE https://json-schema.org/understanding-json-schema/", ) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "key": "input_1", "kind": "input", @@ -41,6 +40,7 @@ class Config: }, } } + ) @classmethod def from_service_io( diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py b/packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py index aaa2b0489ae..331ef23f83e 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py @@ -13,9 +13,6 @@ class ServiceSpecifications(BaseModel): description="schedule-time specifications specifications for the service (follows Docker Service creation API (specifically only the Resources part), see https://docs.docker.com/engine/api/v1.41/#tag/Service/operation/ServiceCreate", ) - class Config: - pass - class ServiceSpecificationsGet(ServiceSpecifications): ... diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py index ca06a39b129..1c9892a7201 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py @@ -1,13 +1,14 @@ -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias from pydantic import ( AnyHttpUrl, BaseModel, + ConfigDict, Field, HttpUrl, NonNegativeFloat, - root_validator, - validator, + field_validator, + model_validator, ) from pydantic.networks import AnyUrl from pydantic.types import ByteSize, PositiveFloat @@ -44,7 +45,7 @@ class WorkerMetrics(BaseModel): class UsedResources(DictModel[str, NonNegativeFloat]): - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def ensure_negative_value_is_zero(cls, values): # dasks adds/remove resource values and sometimes @@ -72,7 +73,7 @@ class Scheduler(BaseModel): status: str = Field(..., description="The running status of the scheduler") workers: WorkersDict | None = Field(default_factory=dict) - @validator("workers", pre=True, always=True) + @field_validator("workers", mode="before") @classmethod def ensure_workers_is_empty_dict(cls, v): if v is None: @@ -95,10 +96,9 @@ class ClusterGet(Cluster): alias="accessRights", default_factory=dict ) - class Config(Cluster.Config): - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def ensure_access_rights_converted(cls, values): if "access_rights" in values: @@ -118,21 +118,8 @@ class ClusterCreate(BaseCluster): alias="accessRights", default_factory=dict ) - @validator("thumbnail", always=True, pre=True) - @classmethod - def set_default_thumbnail_if_empty(cls, v, values): - if v is None: - cluster_type = values["type"] - default_thumbnails = { - ClusterTypeInModel.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", - ClusterTypeInModel.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", - ClusterTypeInModel.ON_DEMAND.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", - } - return default_thumbnails[cluster_type] - return v - - class Config(BaseCluster.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "name": "My awesome cluster", @@ -156,13 +143,27 @@ class Config(BaseCluster.Config): "password": "somepassword", }, "accessRights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, + 154: CLUSTER_ADMIN_RIGHTS, # type: ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS, # type: ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS, # type: ignore[dict-item] }, }, ] } + ) + + @field_validator("thumbnail", mode="before") + @classmethod + def set_default_thumbnail_if_empty(cls, v, values): + if v is None: + cluster_type = values["type"] + default_thumbnails = { + ClusterTypeInModel.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", + ClusterTypeInModel.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", + ClusterTypeInModel.ON_DEMAND.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", + } + return default_thumbnails[cluster_type] + return v class ClusterPatch(BaseCluster): @@ -177,8 +178,8 @@ class ClusterPatch(BaseCluster): alias="accessRights" ) - class Config(BaseCluster.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "name": "Changing the name of my cluster", @@ -188,13 +189,14 @@ class Config(BaseCluster.Config): }, { "accessRights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, + 154: CLUSTER_ADMIN_RIGHTS, # type: ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS, # type: ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS, # type: ignore[dict-item] }, }, ] } + ) class ClusterPing(BaseModel): diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py index cb8dab74d65..e383d45f20e 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py @@ -1,7 +1,7 @@ from typing import Any, TypeAlias from models_library.basic_types import IDStr -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, validator +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, field_validator from ..clusters import ClusterID from ..projects import ProjectID @@ -43,13 +43,14 @@ class ComputationCreate(BaseModel): use_on_demand_clusters: bool = Field( default=False, description="if True, a cluster will be created as necessary (wallet_id cannot be None, and cluster_id must be None)", + validate_default=True, ) wallet_info: WalletInfo | None = Field( default=None, description="contains information about the wallet used to bill the running service", ) - @validator("product_name", always=True) + @field_validator("product_name") @classmethod def ensure_product_name_defined_if_computation_starts(cls, v, values): if "start_pipeline" in values and values["start_pipeline"] and v is None: @@ -57,7 +58,7 @@ def ensure_product_name_defined_if_computation_starts(cls, v, values): raise ValueError(msg) return v - @validator("use_on_demand_clusters", always=True) + @field_validator("use_on_demand_clusters") @classmethod def ensure_expected_options(cls, v, values): if v is True and ("cluster_id" in values and values["cluster_id"] is not None): diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py index 3515c38a5d7..151611271a4 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py @@ -1,6 +1,6 @@ -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, ByteSize, Field +from pydantic import BaseModel, ByteSize, ConfigDict, Field from ..resource_tracker import HardwareInfo, PricingInfo from ..services import ServicePortKey @@ -30,10 +30,9 @@ def from_transferred_bytes( ) -> "RetrieveDataOutEnveloped": return cls(data=RetrieveDataOut(size_bytes=ByteSize(transferred_bytes))) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { - "examples": [{"data": {"size_bytes": 42}}] - } + model_config = ConfigDict( + json_schema_extra={"examples": [{"data": {"size_bytes": 42}}]} + ) class DynamicServiceCreate(ServiceDetails): @@ -55,9 +54,8 @@ class DynamicServiceCreate(ServiceDetails): default=None, description="contains harware information (ex. aws_ec2_instances)", ) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "key": "simcore/services/dynamic/3dviewer", "version": "2.4.5", @@ -67,14 +65,13 @@ class Config: "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", "product_name": "osparc", "can_save": True, - "service_resources": ServiceResourcesDictHelpers.Config.schema_extra[ - "examples" - ][0], - "wallet_info": WalletInfo.Config.schema_extra["examples"][0], - "pricing_info": PricingInfo.Config.schema_extra["examples"][0], - "hardware_info": HardwareInfo.Config.schema_extra["examples"][0], + "service_resources": ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "wallet_info": WalletInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "pricing_info": PricingInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "hardware_info": HardwareInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] } } + ) DynamicServiceGet: TypeAlias = RunningDynamicServiceDetails diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py index 281589614ab..f0958695e15 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py @@ -1,8 +1,7 @@ from functools import cached_property from pathlib import Path -from typing import Any, ClassVar -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from ..basic_types import PortInt from ..projects import ProjectID @@ -39,10 +38,9 @@ class ServiceDetails(CommonServiceDetails): description="predefined path where the dynamic service should be served. If empty, the service shall use the root endpoint.", alias="service_basepath", ) - - class Config: - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + populate_by_name=True, + json_schema_extra={ "example": { "key": "simcore/services/dynamic/3dviewer", "version": "2.4.5", @@ -51,7 +49,8 @@ class Config: "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa", } - } + }, + ) class RunningDynamicServiceDetails(ServiceDetails): @@ -89,13 +88,9 @@ class RunningDynamicServiceDetails(ServiceDetails): alias="service_message", ) - @cached_property - def legacy_service_url(self) -> str: - return f"http://{self.host}:{self.internal_port}{self.basepath}" # NOSONAR - - class Config(ServiceDetails.Config): - keep_untouched = (cached_property,) - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + ignored_types=(cached_property,), + json_schema_extra={ "examples": [ { "boot_type": "V0", @@ -125,4 +120,9 @@ class Config(ServiceDetails.Config): "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa", }, ] - } + }, + ) + + @cached_property + def legacy_service_url(self) -> str: + return f"http://{self.host}:{self.internal_port}{self.basepath}" # NOSONAR diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/health.py b/packages/models-library/src/models_library/api_schemas_directorv2/health.py index 1f578888b18..827ec533418 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/health.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/health.py @@ -1,14 +1,12 @@ -from typing import Any, ClassVar - -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class HealthCheckGet(BaseModel): timestamp: str - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "timestamp": "simcore_service_directorv2.api.routes.health@2023-07-03T12:59:12.024551+00:00" } } + ) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/services.py b/packages/models-library/src/models_library/api_schemas_directorv2/services.py index 6e429be4d50..3905680b374 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/services.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/services.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic.types import ByteSize, NonNegativeInt from ..service_settings_labels import ContainerSpec @@ -23,6 +21,7 @@ class NodeRequirements(BaseModel): None, description="defines the required (maximum) GPU for running the services", alias="GPU", + validate_default=True ) ram: ByteSize = Field( ..., @@ -33,17 +32,18 @@ class NodeRequirements(BaseModel): default=None, description="defines the required (maximum) amount of VRAM for running the services", alias="VRAM", + validate_default=True ) - @validator("vram", "gpu", always=True, pre=True) + @field_validator("vram", "gpu", mode="before") @classmethod def check_0_is_none(cls, v): if v == 0: v = None return v - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"CPU": 1.0, "RAM": 4194304}, {"CPU": 1.0, "GPU": 1, "RAM": 4194304}, @@ -53,6 +53,7 @@ class Config: }, ] } + ) class ServiceExtras(BaseModel): @@ -60,11 +61,13 @@ class ServiceExtras(BaseModel): service_build_details: ServiceBuildDetails | None = None container_spec: ContainerSpec | None = None - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"node_requirements": node_example} - for node_example in NodeRequirements.Config.schema_extra["examples"] + for node_example in NodeRequirements.model_config["json_schema_extra"][ + "examples" + ] # type: ignore[index,union-attr] ] + [ { @@ -75,7 +78,9 @@ class Config: "vcs_url": "git@github.com:ITISFoundation/osparc-simcore.git", }, } - for node_example in NodeRequirements.Config.schema_extra["examples"] + for node_example in NodeRequirements.model_config["json_schema_extra"][ + "examples" + ] # type: ignore[index,dict-item, union-attr] ] + [ { @@ -87,6 +92,9 @@ class Config: }, "container_spec": {"Command": ["run", "subcommand"]}, } - for node_example in NodeRequirements.Config.schema_extra["examples"] + for node_example in NodeRequirements.model_config["json_schema_extra"][ + "examples" + ] # type: ignore[index,union-attr] ] } + ) diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py index 48ef3c48445..47c4fc69a18 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py @@ -1,5 +1,3 @@ -from typing import Any, ClassVar - from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID @@ -7,7 +5,7 @@ from models_library.services_resources import ServiceResourcesDictHelpers from models_library.users import UserID from models_library.wallets import WalletInfo -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class DynamicServiceStart(DynamicServiceCreate): @@ -15,8 +13,8 @@ class DynamicServiceStart(DynamicServiceCreate): request_scheme: str simcore_user_agent: str - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "product_name": "osparc", "can_save": True, @@ -28,14 +26,13 @@ class Config: "request_dns": "some.local", "request_scheme": "http", "simcore_user_agent": "", - "service_resources": ServiceResourcesDictHelpers.Config.schema_extra[ - "examples" - ][0], - "wallet_info": WalletInfo.Config.schema_extra["examples"][0], - "pricing_info": PricingInfo.Config.schema_extra["examples"][0], - "hardware_info": HardwareInfo.Config.schema_extra["examples"][0], + "service_resources": ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "wallet_info": WalletInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "pricing_info": PricingInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "hardware_info": HardwareInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] } } + ) class DynamicServiceStop(BaseModel): @@ -45,8 +42,8 @@ class DynamicServiceStop(BaseModel): simcore_user_agent: str save_state: bool - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "user_id": 234, "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", @@ -55,3 +52,4 @@ class Config: "save_state": True, } } + ) diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py index 26fddb502e3..2e14ed62c16 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py @@ -1,18 +1,18 @@ -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, NonNegativeFloat +from pydantic import BaseModel, ConfigDict, NonNegativeFloat class ActivityInfo(BaseModel): seconds_inactive: NonNegativeFloat - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"seconds_inactive": 0}, {"seconds_inactive": 100}, ] } + ) ActivityInfoOrNone: TypeAlias = ActivityInfo | None diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py index 22d151221d8..b6fd731e9a7 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py @@ -34,8 +34,8 @@ class DiskUsage(BaseModel): total: ByteSize = Field(description="total space = free + used") used_percent: float = Field( - gte=0.00, - lte=100.00, + ge=0.00, + le=100.00, description="Percent of used space relative to the total space", ) diff --git a/packages/models-library/src/models_library/api_schemas_invitations/invitations.py b/packages/models-library/src/models_library/api_schemas_invitations/invitations.py index e25aee7c804..8c5fd85d2e8 100644 --- a/packages/models-library/src/models_library/api_schemas_invitations/invitations.py +++ b/packages/models-library/src/models_library/api_schemas_invitations/invitations.py @@ -1,6 +1,6 @@ -from typing import Any, ClassVar +from typing import Any -from pydantic import BaseModel, Field, HttpUrl +from pydantic import BaseModel, ConfigDict, Field, HttpUrl from ..invitations import InvitationContent, InvitationInputs from ..products import ProductName @@ -13,8 +13,7 @@ class ApiInvitationInputs(InvitationInputs): - class Config: - schema_extra: ClassVar[dict[str, Any]] = {"example": _INPUTS_EXAMPLE} + model_config = ConfigDict(json_schema_extra={"example": _INPUTS_EXAMPLE}) class ApiInvitationContent(InvitationContent): @@ -23,26 +22,27 @@ class ApiInvitationContent(InvitationContent): ..., description="This invitations can only be used for this product." ) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { **_INPUTS_EXAMPLE, "product": "osparc", "created": "2023-01-11 13:11:47.293595", } } + ) class ApiInvitationContentAndLink(ApiInvitationContent): invitation_url: HttpUrl = Field(..., description="Invitation link") - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { - **ApiInvitationContent.Config.schema_extra["example"], + **ApiInvitationContent.model_config["json_schema_extra"]["example"], # type: ignore[index,dict-item] "invitation_url": "https://foo.com/#/registration?invitation=1234", } } + ) class ApiEncryptedInvitation(BaseModel): diff --git a/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py b/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py index 1995d8c3849..a3bb93813dc 100644 --- a/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py +++ b/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py @@ -1,7 +1,7 @@ import logging -from typing import TypeAlias +from typing import Annotated, TypeAlias -from pydantic import BaseModel, ConstrainedFloat, Field, validate_arguments, validator +from pydantic import BaseModel, Field, field_validator, validate_call _logger = logging.getLogger(__name__) @@ -9,10 +9,7 @@ ProgressMessage: TypeAlias = str - -class ProgressPercent(ConstrainedFloat): - ge = 0.0 - le = 1.0 +ProgressPercent: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] class TaskProgress(BaseModel): @@ -25,7 +22,7 @@ class TaskProgress(BaseModel): message: ProgressMessage = Field(default="") percent: ProgressPercent = Field(default=0.0) - @validate_arguments + @validate_call def update( self, *, @@ -47,7 +44,7 @@ def update( def create(cls, task_id: TaskId | None = None) -> "TaskProgress": return cls(task_id=task_id) - @validator("percent") + @field_validator("percent") @classmethod def round_value_to_3_digit(cls, v): return round(v, 3) diff --git a/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py b/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py index dd0e0c0a72b..b5a8d8443b9 100644 --- a/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py +++ b/packages/models-library/src/models_library/api_schemas_long_running_tasks/tasks.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Any -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from .base import TaskId, TaskProgress @@ -25,7 +25,7 @@ class TaskGet(BaseModel): result_href: str abort_href: str - @validator("task_name") + @field_validator("task_name") @classmethod def unquote_str(cls, v) -> str: return urllib.parse.unquote(v) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index edb308ff39a..c24291ebbdb 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -1,7 +1,7 @@ from datetime import datetime from decimal import Decimal -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from ..products import ProductName from ..resource_tracker import CreditTransactionId @@ -13,7 +13,7 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID available_osparc_credits: Decimal - @validator("available_osparc_credits", always=True) + @field_validator("available_osparc_credits") @classmethod def ensure_rounded(cls, v): return round(v, 2) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py index 9c3ca4ba2b1..bbb5d52f906 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py @@ -1,8 +1,7 @@ from datetime import datetime from decimal import Decimal -from typing import Any, ClassVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from ..resource_tracker import ( HardwareInfo, @@ -24,21 +23,24 @@ class PricingUnitGet(BaseModel): default: bool specific_info: HardwareInfo - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_unit_id": 1, "unit_name": "SMALL", - "unit_extra_info": UnitExtraInfo.Config.schema_extra["examples"][0], + "unit_extra_info": UnitExtraInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "current_cost_per_unit": 5.7, "current_cost_per_unit_id": 1, "default": True, "specific_info": hw_config_example, } - for hw_config_example in HardwareInfo.Config.schema_extra["examples"] + for hw_config_example in HardwareInfo.model_config["json_schema_extra"][ + "examples" + ] # type: ignore[index,union-attr] ] } + ) class PricingPlanGet(BaseModel): @@ -51,8 +53,8 @@ class PricingPlanGet(BaseModel): pricing_units: list[PricingUnitGet] | None is_active: bool - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_plan_id": 1, @@ -64,11 +66,14 @@ class Config: "pricing_units": [pricing_unit_get_example], "is_active": True, } - for pricing_unit_get_example in PricingUnitGet.Config.schema_extra[ + for pricing_unit_get_example in PricingUnitGet.model_config[ + "json_schema_extra" + ][ "examples" - ] + ] # type: ignore[index,union-attr] ] } + ) class PricingPlanToServiceGet(BaseModel): @@ -77,8 +82,8 @@ class PricingPlanToServiceGet(BaseModel): service_version: ServiceVersion created: datetime - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_plan_id": 1, @@ -88,3 +93,4 @@ class Config: } ] } + ) diff --git a/packages/models-library/src/models_library/api_schemas_storage.py b/packages/models-library/src/models_library/api_schemas_storage.py index 29e341456bc..fac13d42d40 100644 --- a/packages/models-library/src/models_library/api_schemas_storage.py +++ b/packages/models-library/src/models_library/api_schemas_storage.py @@ -6,22 +6,21 @@ IMPORTANT: DO NOT COUPLE these schemas until storage is refactored """ -import re from datetime import datetime from enum import Enum -from re import Pattern -from typing import Any, ClassVar, TypeAlias +from typing import Annotated, Any, TypeAlias from uuid import UUID from pydantic import ( BaseModel, ByteSize, - ConstrainedStr, - Extra, + ConfigDict, Field, PositiveInt, - root_validator, - validator, + RootModel, + StringConstraints, + field_validator, + model_validator, ) from pydantic.networks import AnyUrl @@ -39,12 +38,9 @@ ETag: TypeAlias = str -class S3BucketName(ConstrainedStr): - regex: Pattern[str] | None = re.compile(S3_BUCKET_NAME_RE) +S3BucketName: TypeAlias = Annotated[str, StringConstraints(pattern=S3_BUCKET_NAME_RE)] - -class DatCoreDatasetName(ConstrainedStr): - regex: Pattern[str] | None = re.compile(DATCORE_DATASET_NAME_RE) +DatCoreDatasetName: TypeAlias = Annotated[str, StringConstraints(pattern=DATCORE_DATASET_NAME_RE)] # / @@ -60,14 +56,15 @@ class FileLocation(BaseModel): name: LocationName id: LocationID - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ {"name": "simcore.s3", "id": 0}, {"name": "datcore", "id": 1}, ] - } + }, + ) FileLocationArray: TypeAlias = ListModel[FileLocation] @@ -77,11 +74,10 @@ class Config: class DatasetMetaDataGet(BaseModel): dataset_id: UUID | DatCoreDatasetName display_name: str - - class Config: - extra = Extra.forbid - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + from_attributes=True, + json_schema_extra={ "examples": [ # simcore dataset { @@ -106,7 +102,8 @@ class Config: "display_name": "YetAnotherTest", }, ] - } + }, + ) # /locations/{location_id}/files/metadata: @@ -150,17 +147,10 @@ class FileMetaDataGet(BaseModel): description="SHA256 message digest of the file content. Main purpose: cheap lookup.", ) - @validator("location_id", pre=True) - @classmethod - def ensure_location_is_integer(cls, v): - if v is not None: - return int(v) - return v - - class Config: - extra = Extra.forbid - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + from_attributes=True, + json_schema_extra={ "examples": [ # typical S3 entry { @@ -234,11 +224,19 @@ class Config: "project_name": None, }, ] - } + }, + ) + + @field_validator("location_id", mode="before") + @classmethod + def ensure_location_is_integer(cls, v): + if v is not None: + return int(v) + return v -class FileMetaDataArray(BaseModel): - __root__: list[FileMetaDataGet] = [] +class FileMetaDataArray(RootModel[list[FileMetaDataGet]]): + root: list[FileMetaDataGet] = Field(default_factory=list) # /locations/{location_id}/files/{file_id} @@ -279,7 +277,7 @@ class UploadedPart(BaseModel): class FileUploadCompletionBody(BaseModel): parts: list[UploadedPart] - @validator("parts") + @field_validator("parts") @classmethod def ensure_sorted(cls, value: list[UploadedPart]) -> list[UploadedPart]: return sorted(value, key=lambda uploaded_part: uploaded_part.number) @@ -312,7 +310,7 @@ class FoldersBody(BaseModel): destination: dict[str, Any] = Field(default_factory=dict) nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict) - @root_validator() + @model_validator(mode="after") @classmethod def ensure_consistent_entries(cls, values): source_node_keys = (NodeID(n) for n in values["source"].get("workbench", {})) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/_base.py b/packages/models-library/src/models_library/api_schemas_webserver/_base.py index 9856a4743e9..718984116c7 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/_base.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/_base.py @@ -5,38 +5,36 @@ from typing import Any -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict from ..utils.change_case import snake_to_camel class EmptyModel(BaseModel): - # Used to represent body={} - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class InputSchemaWithoutCamelCase(BaseModel): - # Added to tmp keep backwards compatibility - # until all bodies are updated - # - class Config: - allow_population_by_field_name = False - extra = Extra.ignore # Non-strict inputs policy: Used to prune extra field - allow_mutations = False + model_config = ConfigDict( + populate_by_name=False, + extra="ignore", # Non-strict inputs policy: Used to prune extra field + frozen=True, + ) class InputSchema(BaseModel): - class Config(InputSchemaWithoutCamelCase.Config): - alias_generator = snake_to_camel + model_config = ConfigDict( + **InputSchemaWithoutCamelCase.model_config, alias_generator=snake_to_camel + ) class OutputSchema(BaseModel): - class Config: - allow_population_by_field_name = True - extra = Extra.ignore # Used to prune extra fields from internal data - allow_mutations = False - alias_generator = snake_to_camel + model_config = ConfigDict( + alias_generator=snake_to_camel, + populate_by_name=True, + extra="ignore", # Used to prune extra fields from internal data + frozen=True, + ) def data( self, @@ -47,7 +45,7 @@ def data( **kwargs ) -> dict[str, Any]: """Helper function to get envelope's data as a dict""" - return self.dict( + return self.model_dump( by_alias=True, exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, @@ -64,7 +62,7 @@ def data_json( **kwargs ) -> str: """Helper function to get envelope's data as a json str""" - return self.json( + return self.model_dump_json( by_alias=True, exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/auth.py b/packages/models-library/src/models_library/api_schemas_webserver/auth.py index d9f2754171d..b0b11661cb3 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/auth.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/auth.py @@ -1,7 +1,7 @@ from datetime import timedelta -from typing import Any, ClassVar +from typing import Any -from pydantic import BaseModel, Field, SecretStr +from pydantic import BaseModel, ConfigDict, Field, SecretStr from ..emails import LowerCaseEmailStr from ._base import InputSchema @@ -11,11 +11,13 @@ class AccountRequestInfo(InputSchema): form: dict[str, Any] captcha: str - class Config: + model_config = ConfigDict( + str_strip_whitespace=True, + str_max_length=200, # NOTE: this is just informative. The format of the form is defined # currently in the front-end and it might change # SEE image in https://github.com/ITISFoundation/osparc-simcore/pull/5378 - schema_extra: ClassVar[dict[str, Any]] = { + json_schema_extra={ "example": { "form": { "firstName": "James", @@ -35,9 +37,8 @@ class Config: }, "captcha": "A12B34", } - } - anystr_strip_whitespace = True - max_anystr_length = 200 + }, + ) class UnregisterCheck(InputSchema): @@ -57,8 +58,8 @@ class ApiKeyCreate(BaseModel): description="Time delta from creation time to expiration. If None, then it does not expire.", ) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "display_name": "test-api-forever", @@ -73,6 +74,7 @@ class Config: }, ] } + ) class ApiKeyGet(BaseModel): @@ -80,9 +82,10 @@ class ApiKeyGet(BaseModel): api_key: str api_secret: str - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"display_name": "myapi", "api_key": "key", "api_secret": "secret"}, ] } + ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/catalog.py b/packages/models-library/src/models_library/api_schemas_webserver/catalog.py index 172575a8f92..09bfa36499a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/catalog.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/catalog.py @@ -1,13 +1,12 @@ -from typing import Any, ClassVar, TypeAlias +from typing import Any, TypeAlias -from pydantic import Extra, Field +from pydantic import ConfigDict, Field from pydantic.main import BaseModel from ..api_schemas_catalog import services as api_schemas_catalog_services from ..services_io import ServiceInput, ServiceOutput from ..services_types import ServicePortKey from ..utils.change_case import snake_to_camel -from ..utils.json_serialization import json_dumps, json_loads from ._base import InputSchema, OutputSchema ServiceInputKey: TypeAlias = ServicePortKey @@ -24,12 +23,9 @@ class _BaseCommonApiExtension(BaseModel): description="Short name for the unit for display (html-compatible), if available", ) - class Config: - alias_generator = snake_to_camel - allow_population_by_field_name = True - extra = Extra.forbid - json_dumps = json_dumps - json_loads = json_loads + model_config = ConfigDict( + alias_generator=snake_to_camel, populate_by_name=True, extra="forbid" + ) class ServiceInputGet(ServiceInput, _BaseCommonApiExtension): @@ -39,8 +35,8 @@ class ServiceInputGet(ServiceInput, _BaseCommonApiExtension): ..., description="Unique name identifier for this input" ) - class Config(_BaseCommonApiExtension.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "displayOrder": 2, "label": "Sleep Time", @@ -70,6 +66,7 @@ class Config(_BaseCommonApiExtension.Config): } ], } + ) class ServiceOutputGet(ServiceOutput, _BaseCommonApiExtension): @@ -79,8 +76,8 @@ class ServiceOutputGet(ServiceOutput, _BaseCommonApiExtension): ..., description="Unique name identifier for this input" ) - class Config(_BaseCommonApiExtension.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "displayOrder": 2, "label": "Time Slept", @@ -92,6 +89,7 @@ class Config(_BaseCommonApiExtension.Config): "keyId": "output_2", } } + ) ServiceInputsGetDict: TypeAlias = dict[ServicePortKey, ServiceInputGet] @@ -99,7 +97,7 @@ class Config(_BaseCommonApiExtension.Config): _EXAMPLE_FILEPICKER: dict[str, Any] = { - **api_schemas_catalog_services.ServiceGet.Config.schema_extra["examples"][1], + **api_schemas_catalog_services.ServiceGet.model_config["json_schema_extra"]["examples"][1], # type: ignore [index,dict-item] "inputs": {}, "outputs": { "outFile": { @@ -114,7 +112,7 @@ class Config(_BaseCommonApiExtension.Config): } _EXAMPLE_SLEEPER: dict[str, Any] = { - **api_schemas_catalog_services.ServiceGet.Config.schema_extra["examples"][0], + **api_schemas_catalog_services.ServiceGet.model_config["json_schema_extra"]["examples"][0], # type: ignore[index,dict-item] "inputs": { "input_1": { "displayOrder": 1, @@ -224,15 +222,14 @@ class ServiceGet(api_schemas_catalog_services.ServiceGet): ..., description="outputs with extended information" ) - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { - "examples": [_EXAMPLE_FILEPICKER, _EXAMPLE_SLEEPER] - } + model_config = ConfigDict( + **OutputSchema.model_config, + json_schema_extra={"examples": [_EXAMPLE_FILEPICKER, _EXAMPLE_SLEEPER]}, + ) class ServiceResourcesGet(api_schemas_catalog_services.ServiceResourcesGet): - class Config(OutputSchema.Config): - ... + model_config = OutputSchema.model_config class CatalogServiceGet(api_schemas_catalog_services.ServiceGetV2): @@ -246,23 +243,26 @@ class CatalogServiceGet(api_schemas_catalog_services.ServiceGetV2): ..., description="outputs with extended information" ) - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + **OutputSchema.model_config, + json_schema_extra={ "example": { - **api_schemas_catalog_services.ServiceGetV2.Config.schema_extra[ - "examples" - ][0], + **api_schemas_catalog_services.ServiceGetV2.model_config["json_schema_extra"]["examples"][0], # type: ignore [index,dict-item] "inputs": { f"input{i}": example for i, example in enumerate( - ServiceInputGet.Config.schema_extra["examples"] + ServiceInputGet.model_config["json_schema_extra"]["examples"] # type: ignore[index,arg-type] ) }, - "outputs": {"outFile": ServiceOutputGet.Config.schema_extra["example"]}, + "outputs": { + "outFile": ServiceOutputGet.model_config["json_schema_extra"][ + "example" + ] # type: ignore[index] + }, } - } + }, + ) class CatalogServiceUpdate(api_schemas_catalog_services.ServiceUpdateV2): - class Config(InputSchema.Config): - ... + model_config = InputSchema.model_config diff --git a/packages/models-library/src/models_library/api_schemas_webserver/clusters.py b/packages/models-library/src/models_library/api_schemas_webserver/clusters.py index b1b89746238..109e0618b98 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/clusters.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict from ..api_schemas_directorv2 import clusters as directorv2_clusters from ..clusters import ClusterID @@ -7,32 +7,27 @@ class ClusterPathParams(BaseModel): cluster_id: ClusterID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict( + populate_by_name=True, + extra="forbid", + ) class ClusterGet(directorv2_clusters.ClusterGet): - class Config(OutputSchema.Config): - ... + model_config = OutputSchema.model_config class ClusterCreate(directorv2_clusters.ClusterCreate): - class Config(InputSchema.Config): - ... + model_config = InputSchema.model_config class ClusterPatch(directorv2_clusters.ClusterPatch): - class Config(InputSchema.Config): - ... + model_config = InputSchema.model_config class ClusterPing(directorv2_clusters.ClusterPing): - class Config(InputSchema.Config): - ... + model_config = InputSchema.model_config class ClusterDetails(directorv2_clusters.ClusterDetails): - class Config(OutputSchema.Config): - ... + model_config = OutputSchema.model_config diff --git a/packages/models-library/src/models_library/api_schemas_webserver/folders.py b/packages/models-library/src/models_library/api_schemas_webserver/folders.py index e971b1f8c73..48a2ae605e4 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/folders.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/folders.py @@ -6,7 +6,7 @@ from models_library.projects_access import AccessRights from models_library.users import GroupID from models_library.utils.common_validators import null_or_none_str_to_none_validator -from pydantic import Extra, PositiveInt, validator +from pydantic import ConfigDict, PositiveInt, field_validator from ._base import InputSchema, OutputSchema @@ -33,11 +33,10 @@ class CreateFolderBodyParams(InputSchema): description: str parent_folder_id: FolderID | None = None - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") - _null_or_none_str_to_none_validator = validator( - "parent_folder_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator = field_validator( + "parent_folder_id", mode="before" )(null_or_none_str_to_none_validator) @@ -45,5 +44,4 @@ class PutFolderBodyParams(InputSchema): name: IDStr description: str - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py b/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py index e2d4918c435..29fed6baced 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py @@ -7,7 +7,7 @@ from models_library.users import GroupID from models_library.utils.common_validators import null_or_none_str_to_none_validator from models_library.workspaces import WorkspaceID -from pydantic import Extra, PositiveInt, validator +from pydantic import ConfigDict, PositiveInt, field_validator from ._base import InputSchema, OutputSchema @@ -32,26 +32,22 @@ class CreateFolderBodyParams(InputSchema): name: IDStr parent_folder_id: FolderID | None = None workspace_id: WorkspaceID | None = None + model_config = ConfigDict(extra="forbid") - class Config: - extra = Extra.forbid - - _null_or_none_str_to_none_validator = validator( - "parent_folder_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator = field_validator( + "parent_folder_id", mode="before" )(null_or_none_str_to_none_validator) - _null_or_none_str_to_none_validator2 = validator( - "workspace_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator2 = field_validator( + "workspace_id", mode="before" )(null_or_none_str_to_none_validator) class PutFolderBodyParams(InputSchema): name: IDStr parent_folder_id: FolderID | None + model_config = ConfigDict(extra="forbid") - class Config: - extra = Extra.forbid - - _null_or_none_str_to_none_validator = validator( - "parent_folder_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator = field_validator( + "parent_folder_id", mode="before" )(null_or_none_str_to_none_validator) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/groups.py b/packages/models-library/src/models_library/api_schemas_webserver/groups.py index e0b6d3fbb37..46e9da3dc52 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/groups.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/groups.py @@ -1,7 +1,14 @@ from contextlib import suppress -from typing import Any, ClassVar -from pydantic import AnyUrl, BaseModel, Field, ValidationError, parse_obj_as, validator +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + TypeAdapter, + ValidationError, + field_validator, +) from ..emails import LowerCaseEmailStr @@ -18,15 +25,15 @@ class GroupAccessRights(BaseModel): read: bool write: bool delete: bool - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"read": True, "write": False, "delete": False}, {"read": True, "write": True, "delete": False}, {"read": True, "write": True, "delete": True}, ] } + ) class UsersGroup(BaseModel): @@ -43,17 +50,8 @@ class UsersGroup(BaseModel): alias="inclusionRules", ) - @validator("thumbnail", pre=True) - @classmethod - def sanitize_legacy_data(cls, v): - if v: - # Enforces null if thumbnail is not valid URL or empty - with suppress(ValidationError): - return parse_obj_as(AnyUrl, v) - return None - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "gid": "27", @@ -84,6 +82,16 @@ class Config: }, ] } + ) + + @field_validator("thumbnail", mode="before") + @classmethod + def _sanitize_legacy_data(cls, v): + if v: + # Enforces null if thumbnail is not valid URL or empty + with suppress(ValidationError): + return TypeAdapter(AnyUrl).validate_python(v) + return None class AllUsersGroups(BaseModel): @@ -92,8 +100,8 @@ class AllUsersGroups(BaseModel): all: UsersGroup | None = None product: UsersGroup | None = None - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "me": { "gid": "27", @@ -131,6 +139,7 @@ class Config: }, } } + ) class GroupUserGet(BaseModel): @@ -142,8 +151,8 @@ class GroupUserGet(BaseModel): gid: str | None = Field(None, description="the user primary gid") access_rights: GroupAccessRights = Field(..., alias="accessRights") - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "id": "1", "login": "mr.smith@matrix.com", @@ -158,3 +167,4 @@ class Config: }, } } + ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/product.py b/packages/models-library/src/models_library/api_schemas_webserver/product.py index da0db603202..f967e15d548 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/product.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/product.py @@ -1,7 +1,7 @@ from datetime import datetime -from typing import Any, ClassVar +from typing import Annotated, TypeAlias -from pydantic import ConstrainedInt, Field, HttpUrl, NonNegativeInt, PositiveInt +from pydantic import ConfigDict, Field, HttpUrl, NonNegativeInt, PositiveInt from ..basic_types import IDStr, NonNegativeDecimal from ..emails import LowerCaseEmailStr @@ -22,8 +22,8 @@ class GetCreditPrice(OutputSchema): "Can be None if this product's price is UNDEFINED", ) - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "productName": "osparc", @@ -37,6 +37,7 @@ class Config(OutputSchema.Config): }, ] } + ) class GetProductTemplate(OutputSchema): @@ -75,9 +76,7 @@ class GetProduct(OutputSchema): ) -class ExtraCreditsUsdRangeInt(ConstrainedInt): - ge = 0 - lt = 500 +ExtraCreditsUsdRangeInt: TypeAlias = Annotated[int, Field(ge=0, lt=500)] class GenerateInvitation(InputSchema): @@ -95,8 +94,8 @@ class InvitationGenerated(OutputSchema): created: datetime invitation_link: HttpUrl - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "productName": "osparc", @@ -117,3 +116,4 @@ class Config(OutputSchema.Config): }, ] } + ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index b644ac52d0b..dd1625301fa 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -9,14 +9,10 @@ from models_library.folders import FolderID from models_library.workspaces import WorkspaceID -from pydantic import Field, validator +from pydantic import Field, HttpUrl, field_validator from ..api_schemas_long_running_tasks.tasks import TaskGet -from ..basic_types import ( - HttpUrlWithCustomMinLength, - LongTruncatedStr, - ShortTruncatedStr, -) +from ..basic_types import LongTruncatedStr, ShortTruncatedStr from ..emails import LowerCaseEmailStr from ..projects import ClassifierID, DateTimeStr, NodesDict, ProjectID from ..projects_access import AccessRights, GroupIDStr @@ -36,7 +32,7 @@ class ProjectCreateNew(InputSchema): uuid: ProjectID | None = None # NOTE: suggested uuid! but could be different! name: str description: str | None - thumbnail: HttpUrlWithCustomMinLength | None + thumbnail: HttpUrl | None workbench: NodesDict access_rights: dict[GroupIDStr, AccessRights] tags: list[int] = Field(default_factory=list) @@ -45,23 +41,23 @@ class ProjectCreateNew(InputSchema): workspace_id: WorkspaceID | None = None folder_id: FolderID | None = None - _empty_is_none = validator( - "uuid", "thumbnail", "description", allow_reuse=True, pre=True - )(empty_str_to_none_pre_validator) + _empty_is_none = field_validator("uuid", "thumbnail", "description", mode="before")( + empty_str_to_none_pre_validator + ) - _null_or_none_to_none = validator( - "workspace_id", "folder_id", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) + _null_or_none_to_none = field_validator("workspace_id", "folder_id", mode="before")( + null_or_none_str_to_none_validator + ) # NOTE: based on OVERRIDABLE_DOCUMENT_KEYS class ProjectCopyOverride(InputSchema): name: str description: str | None - thumbnail: HttpUrlWithCustomMinLength | None + thumbnail: HttpUrl | None prj_owner: LowerCaseEmailStr - _empty_is_none = validator("thumbnail", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) @@ -70,7 +66,7 @@ class ProjectGet(OutputSchema): uuid: ProjectID name: str description: str - thumbnail: HttpUrlWithCustomMinLength | Literal[""] + thumbnail: HttpUrl | Literal[""] creation_date: DateTimeStr last_change_date: DateTimeStr workbench: NodesDict @@ -78,14 +74,14 @@ class ProjectGet(OutputSchema): access_rights: dict[GroupIDStr, AccessRights] tags: list[int] classifiers: list[ClassifierID] = [] - state: ProjectState | None - ui: EmptyModel | StudyUI | None + state: ProjectState | None = None + ui: EmptyModel | StudyUI | None = None quality: dict[str, Any] = {} - dev: dict | None + dev: dict | None = None permalink: ProjectPermalink = FieldNotRequired() - workspace_id: WorkspaceID | None + workspace_id: WorkspaceID | None = None - _empty_description = validator("description", allow_reuse=True, pre=True)( + _empty_description = field_validator("description", mode="before")( none_to_empty_str_pre_validator ) @@ -101,7 +97,7 @@ class ProjectReplace(InputSchema): uuid: ProjectID name: ShortTruncatedStr description: LongTruncatedStr - thumbnail: HttpUrlWithCustomMinLength | None + thumbnail: HttpUrl | None creation_date: DateTimeStr last_change_date: DateTimeStr workbench: NodesDict @@ -115,7 +111,7 @@ class ProjectReplace(InputSchema): default_factory=dict, ) - _empty_is_none = validator("thumbnail", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) @@ -123,7 +119,7 @@ class ProjectReplace(InputSchema): class ProjectUpdate(InputSchema): name: ShortTruncatedStr = FieldNotRequired() description: LongTruncatedStr = FieldNotRequired() - thumbnail: HttpUrlWithCustomMinLength = FieldNotRequired() + thumbnail: HttpUrl = FieldNotRequired() workbench: NodesDict = FieldNotRequired() access_rights: dict[GroupIDStr, AccessRights] = FieldNotRequired() tags: list[int] = FieldNotRequired() @@ -135,7 +131,7 @@ class ProjectUpdate(InputSchema): class ProjectPatch(InputSchema): name: ShortTruncatedStr = FieldNotRequired() description: LongTruncatedStr = FieldNotRequired() - thumbnail: HttpUrlWithCustomMinLength = FieldNotRequired() + thumbnail: HttpUrl = FieldNotRequired() access_rights: dict[GroupIDStr, AccessRights] = FieldNotRequired() classifiers: list[ClassifierID] = FieldNotRequired() dev: dict | None = FieldNotRequired() diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 25a6f5fb0dd..cda166e0d13 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -1,7 +1,7 @@ # mypy: disable-error-code=truthy-function -from typing import Any, ClassVar, Literal, TypeAlias +from typing import Any, Literal, TypeAlias -from pydantic import Field +from pydantic import ConfigDict, Field from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt @@ -62,13 +62,13 @@ class NodeGet(OutputSchema): service_key: ServiceKey = Field( ..., description="distinctive name for the node based on the docker registry path", - example=[ + examples=[ "simcore/services/comp/itis/sleeper", "simcore/services/dynamic/3dviewer", ], ) service_version: ServiceVersion = Field( - ..., description="semantic version number", example=["1.0.0", "0.0.1"] + ..., description="semantic version number", examples=["1.0.0", "0.0.1"] ) service_host: str = Field( ..., @@ -90,9 +90,8 @@ class NodeGet(OutputSchema): description="the service message", ) user_id: str = Field(..., description="the user that started the service") - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "published_port": 30000, "entrypoint": "/the/entry/point/is/here", @@ -107,6 +106,7 @@ class Config: "user_id": 123, } } + ) class NodeGetIdle(OutputSchema): @@ -117,30 +117,32 @@ class NodeGetIdle(OutputSchema): def from_node_id(cls, node_id: NodeID) -> "NodeGetIdle": return cls(service_state="idle", service_uuid=node_id) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "service_uuid": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "service_state": "idle", } } + ) class NodeGetUnknown(OutputSchema): service_state: Literal["unknown"] service_uuid: NodeID - @classmethod - def from_node_id(cls, node_id: NodeID) -> "NodeGetUnknown": - return cls(service_state="unknown", service_uuid=node_id) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "service_uuid": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "service_state": "unknown", } } + ) + + @classmethod + def from_node_id(cls, node_id: NodeID) -> "NodeGetUnknown": + return cls(service_state="unknown", service_uuid=node_id) class NodeOutputs(InputSchemaWithoutCamelCase): @@ -152,5 +154,4 @@ class NodeRetrieve(InputSchemaWithoutCamelCase): class NodeRetrieved(RetrieveDataOut): - class Config(OutputSchema.Config): - ... + model_config = OutputSchema.model_config diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_ports.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_ports.py index df38c862900..6582542525b 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_ports.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_ports.py @@ -15,19 +15,16 @@ class _ProjectIOBase(BaseModel): class ProjectInputUpdate(_ProjectIOBase): - class Config(InputSchemaWithoutCamelCase): - ... + model_config = InputSchemaWithoutCamelCase.model_config class ProjectInputGet(OutputSchema, _ProjectIOBase): label: str - class Config(InputSchemaWithoutCamelCase): - ... + model_config = InputSchemaWithoutCamelCase.model_config class ProjectOutputGet(_ProjectIOBase): label: str - class Config(OutputSchema): - ... + model_config = OutputSchema.model_config diff --git a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py index fa150f9ffc6..8242105f55a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py @@ -1,7 +1,7 @@ from datetime import datetime from decimal import Decimal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from ..projects import ProjectID from ..projects_nodes_io import NodeID @@ -95,9 +95,10 @@ class CreatePricingPlanBodyParams(InputSchema): classification: PricingPlanClassification pricing_plan_key: str - class Config: - anystr_strip_whitespace = True - max_anystr_length = 200 + model_config = ConfigDict( + str_strip_whitespace=True, + str_max_length=200, + ) class UpdatePricingPlanBodyParams(InputSchema): @@ -105,9 +106,10 @@ class UpdatePricingPlanBodyParams(InputSchema): description: str is_active: bool - class Config: - anystr_strip_whitespace = True - max_anystr_length = 200 + model_config = ConfigDict( + str_strip_whitespace=True, + str_max_length=200, + ) class CreatePricingUnitBodyParams(InputSchema): @@ -118,9 +120,10 @@ class CreatePricingUnitBodyParams(InputSchema): cost_per_unit: Decimal comment: str - class Config: - anystr_strip_whitespace = True - max_anystr_length = 200 + model_config = ConfigDict( + str_strip_whitespace=True, + str_max_length=200, + ) class UpdatePricingUnitBodyParams(InputSchema): @@ -130,15 +133,17 @@ class UpdatePricingUnitBodyParams(InputSchema): specific_info: SpecificInfo pricing_unit_cost_update: PricingUnitCostUpdate | None - class Config: - anystr_strip_whitespace = True - max_anystr_length = 200 + model_config = ConfigDict( + str_strip_whitespace=True, + str_max_length=200, + ) class ConnectServiceToPricingPlanBodyParams(InputSchema): service_key: ServiceKey service_version: ServiceVersion - class Config: - anystr_strip_whitespace = True - max_anystr_length = 200 + model_config = ConfigDict( + str_strip_whitespace=True, + str_max_length=200, + ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index af0aa61ac80..f9ebbd9fb2d 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -1,8 +1,8 @@ from datetime import datetime from decimal import Decimal -from typing import Any, ClassVar, Literal, TypeAlias +from typing import Literal, TypeAlias -from pydantic import Field, HttpUrl, validator +from pydantic import ConfigDict, Field, HttpUrl, field_validator from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID @@ -91,8 +91,8 @@ class PaymentMethodInitiated(OutputSchema): ..., description="Link to external site that holds the payment submission form" ) - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "wallet_id": 1, @@ -101,6 +101,7 @@ class Config(OutputSchema.Config): } ] } + ) class PaymentMethodTransaction(OutputSchema): @@ -109,8 +110,8 @@ class PaymentMethodTransaction(OutputSchema): payment_method_id: PaymentMethodID state: Literal["PENDING", "SUCCESS", "FAILED", "CANCELED"] - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "walletId": 1, @@ -119,6 +120,7 @@ class Config(OutputSchema.Config): } ] } + ) class PaymentMethodGet(OutputSchema): @@ -135,8 +137,8 @@ class PaymentMethodGet(OutputSchema): description="If true, this payment-method is used for auto-recharge", ) - class Config(OutputSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "idr": "pm_1234567890", @@ -157,6 +159,7 @@ class Config(OutputSchema.Config): }, ], } + ) # @@ -194,7 +197,7 @@ class ReplaceWalletAutoRecharge(InputSchema): top_up_amount_in_usd: NonNegativeDecimal monthly_limit_in_usd: NonNegativeDecimal | None - @validator("monthly_limit_in_usd") + @field_validator("monthly_limit_in_usd") @classmethod def _monthly_limit_greater_than_top_up(cls, v, values): top_up = values["top_up_amount_in_usd"] diff --git a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py index 0ba98ab4ec3..32f17200ee4 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py @@ -4,7 +4,7 @@ from models_library.basic_types import IDStr from models_library.users import GroupID from models_library.workspaces import WorkspaceID -from pydantic import Extra, PositiveInt +from pydantic import ConfigDict, PositiveInt from ..access_rights import AccessRights from ._base import InputSchema, OutputSchema @@ -31,8 +31,7 @@ class CreateWorkspaceBodyParams(InputSchema): description: str | None = None thumbnail: str | None = None - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class PutWorkspaceBodyParams(InputSchema): @@ -40,5 +39,4 @@ class PutWorkspaceBodyParams(InputSchema): description: str | None = None thumbnail: str | None = None - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/basic_regex.py b/packages/models-library/src/models_library/basic_regex.py index eb202fa188b..51441fe39e6 100644 --- a/packages/models-library/src/models_library/basic_regex.py +++ b/packages/models-library/src/models_library/basic_regex.py @@ -46,7 +46,7 @@ # Storage basic file ID SIMCORE_S3_FILE_ID_RE = rf"^(api|({UUID_RE_BASE}))\/({UUID_RE_BASE})\/(.+)$" -SIMCORE_S3_DIRECTORY_ID_RE = rf"^({UUID_RE_BASE})\/({UUID_RE_BASE})\/(.+)/$" +SIMCORE_S3_DIRECTORY_ID_RE = rf"^({UUID_RE_BASE})\/({UUID_RE_BASE})\/(.+)\/$" # S3 - AWS bucket names [https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html] S3_BUCKET_NAME_RE = r"(?!(^xn--|-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$" diff --git a/packages/models-library/src/models_library/basic_types.py b/packages/models-library/src/models_library/basic_types.py index 18788e188a8..d61eff22617 100644 --- a/packages/models-library/src/models_library/basic_types.py +++ b/packages/models-library/src/models_library/basic_types.py @@ -1,15 +1,11 @@ -import re +from decimal import Decimal from enum import StrEnum -from typing import Final, TypeAlias +from re import Pattern +from typing import Annotated, Final, TypeAlias - -from pydantic import ( - ConstrainedDecimal, - ConstrainedInt, - ConstrainedStr, - HttpUrl, - PositiveInt, -) +import pydantic +from pydantic import Field, PositiveInt, StringConstraints +from pydantic_core import core_schema from .basic_regex import ( PROPERTY_KEY_RE, @@ -18,68 +14,44 @@ UUID_RE, ) +NonNegativeDecimal: TypeAlias = Annotated[Decimal, Field(ge=0)] -class NonNegativeDecimal(ConstrainedDecimal): - ge = 0 - - -class PositiveDecimal(ConstrainedDecimal): - gt = 0 - - -class AmountDecimal(ConstrainedDecimal): - # Used for amounts like credits or dollars - # NOTE: upper limit to avoid https://github.com/ITISFoundation/appmotion-exchange/issues/2 - # NOTE: do not contraint in decimal places. Too strong validation error rather Decimal.quantize - # before passing the value - gt = 0 - lt = 1e6 +PositiveDecimal: TypeAlias = Annotated[Decimal, Field(gt=0)] +# Used for amounts like credits or dollars +# NOTE: upper limit to avoid https://github.com/ITISFoundation/appmotion-exchange/issues/2 +# NOTE: do not contraint in decimal places. Too strong validation error rather Decimal.quantize +# before passing the value +AmountDecimal: TypeAlias = Annotated[Decimal, Field(gt=0, lt=1e6)] # port number range -class PortInt(ConstrainedInt): - gt = 0 - lt = 65535 - +PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)] # e.g. 'v5' -class VersionTag(ConstrainedStr): - regex = re.compile(r"^v\d$") - - -class VersionStr(ConstrainedStr): - regex = re.compile(SIMPLE_VERSION_RE) +VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")] +VersionStr: TypeAlias = Annotated[str, StringConstraints(pattern=SIMPLE_VERSION_RE)] # e.g. '1.23.11' or '2.1.0-rc2' or not 0.1.0-alpha (see test_SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS) -class SemanticVersionStr(ConstrainedStr): - regex = re.compile(SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS) - +SemanticVersionStr: TypeAlias = Annotated[ + str, StringConstraints(pattern=SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS) +] # checksums # sha1sum path/to/file -class SHA1Str(ConstrainedStr): - regex = re.compile(r"^[a-fA-F0-9]{40}$") - +SHA1Str: TypeAlias = Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{40}$")] # sha256sum path/to/file -class SHA256Str(ConstrainedStr): - regex = re.compile(r"^[a-fA-F0-9]{64}$") - +SHA256Str: TypeAlias = Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{64}$")] # md5sum path/to/file -class MD5Str(ConstrainedStr): - regex = re.compile(r"^[a-fA-F0-9]{32}$") - +MD5Str: TypeAlias = Annotated[str, StringConstraints(pattern=r"^[a-fA-F0-9]{32}$")] # env var -class EnvVarKey(ConstrainedStr): - regex = re.compile(r"[a-zA-Z]\w*") - +EnvVarKey: TypeAlias = Annotated[str, StringConstraints(pattern=r"^[a-zA-Z]\w*")] # e.g. '5c833a78-1af3-43a7-9ed7-6a63b188f4d8' -class UUIDStr(ConstrainedStr): - regex = re.compile(UUID_RE) +UUIDStr: TypeAlias = Annotated[str, StringConstraints(pattern=UUID_RE)] # non-empty bounded string used as identifier @@ -87,6 +59,32 @@ class UUIDStr(ConstrainedStr): _ELLIPSIS_CHAR: Final[str] = "..." +class ConstrainedStr(str): + pattern: str | Pattern[str] | None = None + min_length: int | None = None + max_length: int | None = None + strip_whitespace: bool = False + curtail_length: int | None = None + + @classmethod + def _validate(cls, __input_value: str) -> str: + if cls.curtail_length and len(__input_value) > cls.curtail_length: + __input_value = __input_value[: cls.curtail_length] + return cls(__input_value) + + @classmethod + def __get_pydantic_core_schema__(cls, _source_type, _handler): + return core_schema.no_info_after_validator_function( + cls._validate, + core_schema.str_schema( + pattern=cls.pattern, + min_length=cls.min_length, + max_length=cls.max_length, + strip_whitespace=cls.strip_whitespace, + ), + ) + + class IDStr(ConstrainedStr): strip_whitespace = True min_length = 1 @@ -132,6 +130,9 @@ class LongTruncatedStr(ConstrainedStr): IdInt: TypeAlias = PositiveInt PrimaryKeyInt: TypeAlias = PositiveInt +AnyHttpUrl = Annotated[str, pydantic.AnyHttpUrl] + +HttpUrl = Annotated[str, pydantic.HttpUrl] # https e.g. https://techterms.com/definition/https class HttpSecureUrl(HttpUrl): @@ -180,5 +181,4 @@ class BuildTargetEnum(StrEnum): DEVELOPMENT = "development" -class KeyIDStr(ConstrainedStr): - regex = re.compile(PROPERTY_KEY_RE) +KeyIDStr = Annotated[str, StringConstraints(pattern=PROPERTY_KEY_RE)] diff --git a/packages/models-library/src/models_library/boot_options.py b/packages/models-library/src/models_library/boot_options.py index ec1aabd546b..52756bf1097 100644 --- a/packages/models-library/src/models_library/boot_options.py +++ b/packages/models-library/src/models_library/boot_options.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator from typing_extensions import TypedDict from .basic_types import EnvVarKey @@ -17,17 +15,17 @@ class BootOption(BaseModel): default: str items: dict[str, BootChoice] - @validator("items") + @field_validator("items") @classmethod - def ensure_default_included(cls, v, values): - default = values["default"] + def ensure_default_included(cls, v, info: ValidationInfo): + default = info.data["default"] if default not in v: msg = f"Expected default={default} to be present a key of items={v}" raise ValueError(msg) return v - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "label": "Boot mode", @@ -61,6 +59,7 @@ class Config: }, ] } + ) BootOptions = dict[EnvVarKey, BootOption] diff --git a/packages/models-library/src/models_library/callbacks_mapping.py b/packages/models-library/src/models_library/callbacks_mapping.py index 9e4e88214ce..498766ed750 100644 --- a/packages/models-library/src/models_library/callbacks_mapping.py +++ b/packages/models-library/src/models_library/callbacks_mapping.py @@ -1,7 +1,7 @@ from collections.abc import Sequence -from typing import Any, ClassVar, Final +from typing import Final -from pydantic import BaseModel, Extra, Field, NonNegativeFloat, validator +from pydantic import BaseModel, ConfigDict, Field, NonNegativeFloat, field_validator INACTIVITY_TIMEOUT_CAP: Final[NonNegativeFloat] = 5 TIMEOUT_MIN: Final[NonNegativeFloat] = 1 @@ -15,15 +15,15 @@ class UserServiceCommand(BaseModel): timeout: NonNegativeFloat = Field( ..., description="after this interval the command will be timed-out" ) - - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ {"service": "rt-web", "command": "ls", "timeout": 1}, {"service": "s4l-core", "command": ["ls", "-lah"], "timeout": 1}, ] - } + }, + ) class CallbacksMapping(BaseModel): @@ -47,24 +47,9 @@ class CallbacksMapping(BaseModel): ), ) - @validator("inactivity") - @classmethod - def ensure_inactivity_timeout_is_capped( - cls, v: UserServiceCommand - ) -> UserServiceCommand: - if v is not None and ( - v.timeout < TIMEOUT_MIN or v.timeout > INACTIVITY_TIMEOUT_CAP - ): - msg = ( - f"Constraint not respected for inactivity timeout={v.timeout}: " - f"interval=({TIMEOUT_MIN}, {INACTIVITY_TIMEOUT_CAP})" - ) - raise ValueError(msg) - return v - - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { # empty validates @@ -73,21 +58,37 @@ class Config: "metrics": None, "before_shutdown": [], }, - {"metrics": UserServiceCommand.Config.schema_extra["examples"][0]}, + {"metrics": UserServiceCommand.model_config["json_schema_extra"]["examples"][0]}, # type: ignore [index] { - "metrics": UserServiceCommand.Config.schema_extra["examples"][0], + "metrics": UserServiceCommand.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "before_shutdown": [ - UserServiceCommand.Config.schema_extra["examples"][0], - UserServiceCommand.Config.schema_extra["examples"][1], + UserServiceCommand.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + UserServiceCommand.model_config["json_schema_extra"]["examples"][1], # type: ignore [index] ], }, { - "metrics": UserServiceCommand.Config.schema_extra["examples"][0], + "metrics": UserServiceCommand.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "before_shutdown": [ - UserServiceCommand.Config.schema_extra["examples"][0], - UserServiceCommand.Config.schema_extra["examples"][1], + UserServiceCommand.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + UserServiceCommand.model_config["json_schema_extra"]["examples"][1], # type: ignore [index] ], - "inactivity": UserServiceCommand.Config.schema_extra["examples"][0], + "inactivity": UserServiceCommand.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] }, ] - } + }, + ) + + @field_validator("inactivity") + @classmethod + def ensure_inactivity_timeout_is_capped( + cls, v: UserServiceCommand + ) -> UserServiceCommand: + if v is not None and ( + v.timeout < TIMEOUT_MIN or v.timeout > INACTIVITY_TIMEOUT_CAP + ): + msg = ( + f"Constraint not respected for inactivity timeout={v.timeout}: " + f"interval=({TIMEOUT_MIN}, {INACTIVITY_TIMEOUT_CAP})" + ) + raise ValueError(msg) + return v diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index c51598b06ee..1dbcff0bc70 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -1,16 +1,16 @@ from enum import auto from pathlib import Path -from typing import Any, ClassVar, Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias from pydantic import ( AnyUrl, BaseModel, - Extra, + ConfigDict, Field, HttpUrl, SecretStr, - root_validator, - validator, + field_validator, + model_validator, ) from pydantic.types import NonNegativeInt @@ -32,8 +32,7 @@ class ClusterAccessRights(BaseModel): write: bool = Field(..., description="allows to modify the cluster") delete: bool = Field(..., description="allows to delete a cluster") - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") CLUSTER_ADMIN_RIGHTS = ClusterAccessRights(read=True, write=True, delete=True) @@ -45,9 +44,7 @@ class Config: class BaseAuthentication(BaseModel): type: str - class Config: - frozen = True - extra = Extra.forbid + model_config = ConfigDict(frozen=True, extra="forbid") class SimpleAuthentication(BaseAuthentication): @@ -55,8 +52,8 @@ class SimpleAuthentication(BaseAuthentication): username: str password: SecretStr - class Config(BaseAuthentication.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "type": "simple", @@ -65,32 +62,34 @@ class Config(BaseAuthentication.Config): }, ] } + ) class KerberosAuthentication(BaseAuthentication): type: Literal["kerberos"] = "kerberos" - # NOTE: the entries here still need to be defined - class Config(BaseAuthentication.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "type": "kerberos", }, ] } + ) class JupyterHubTokenAuthentication(BaseAuthentication): type: Literal["jupyterhub"] = "jupyterhub" api_token: str - class Config(BaseAuthentication.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"type": "jupyterhub", "api_token": "some_jupyterhub_token"}, ] } + ) class NoAuthentication(BaseAuthentication): @@ -103,8 +102,8 @@ class TLSAuthentication(BaseAuthentication): tls_client_cert: Path tls_client_key: Path - class Config(BaseAuthentication.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "type": "tls", @@ -114,6 +113,7 @@ class Config(BaseAuthentication.Config): }, ] } + ) InternalClusterAuthentication: TypeAlias = NoAuthentication | TLSAuthentication @@ -134,6 +134,7 @@ class BaseCluster(BaseModel): default=None, description="url to the image describing this cluster", examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"], + validate_default=True ) endpoint: AnyUrl authentication: ClusterAuthentication = Field( @@ -141,13 +142,11 @@ class BaseCluster(BaseModel): ) access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict) - _from_equivalent_enums = validator("type", allow_reuse=True, pre=True)( + _from_equivalent_enums = field_validator("type", mode="before")( create_enums_pre_validator(ClusterTypeInModel) ) - class Config: - extra = Extra.forbid - use_enum_values = True + model_config = ConfigDict(extra="forbid", use_enum_values=True) ClusterID: TypeAlias = NonNegativeInt @@ -157,8 +156,8 @@ class Config: class Cluster(BaseCluster): id: ClusterID = Field(..., description="The cluster ID") - class Config(BaseCluster.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "id": DEFAULT_CLUSTER_ID, @@ -193,9 +192,9 @@ class Config(BaseCluster.Config): "endpoint": "https://registry.osparc-development.fake.dev", "authentication": {"type": "kerberos"}, "access_rights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, + 154: CLUSTER_ADMIN_RIGHTS, # type: ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS, # type: ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS, # type: ignore[dict-item] }, }, { @@ -210,15 +209,16 @@ class Config(BaseCluster.Config): "api_token": "some_fake_token", }, "access_rights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, + 154: CLUSTER_ADMIN_RIGHTS, # type: ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS, # type: ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS, # type: ignore[dict-item] }, }, ] } + ) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def check_owner_has_access_rights(cls, values): is_default_cluster = bool(values["id"] == DEFAULT_CLUSTER_ID) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index 732dfc08197..75c003fff89 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -1,18 +1,20 @@ import contextlib import re -from typing import Any, ClassVar, Final +from typing import Annotated, Any, Final, TypeAlias from pydantic import ( BaseModel, ByteSize, - ConstrainedStr, + ConfigDict, Field, + StringConstraints, + TypeAdapter, ValidationError, - parse_obj_as, - root_validator, + model_validator, ) from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX +from .basic_types import ConstrainedStr from .generated_models.docker_rest_api import Task from .products import ProductName from .projects import ProjectID @@ -23,16 +25,15 @@ class DockerLabelKey(ConstrainedStr): # NOTE: https://docs.docker.com/config/labels-custom-metadata/#key-format-recommendations # good practice: use reverse DNS notation - regex: re.Pattern[str] | None = DOCKER_LABEL_KEY_REGEX + pattern = DOCKER_LABEL_KEY_REGEX @classmethod def from_key(cls, key: str) -> "DockerLabelKey": return cls(key.lower().replace("_", "-")) -class DockerGenericTag(ConstrainedStr): - # NOTE: https://docs.docker.com/engine/reference/commandline/tag/#description - regex: re.Pattern[str] | None = DOCKER_GENERIC_TAG_KEY_RE +# NOTE: https://docs.docker.com/engine/reference/commandline/tag/#description +DockerGenericTag: TypeAlias = Annotated[str, StringConstraints(pattern=DOCKER_GENERIC_TAG_KEY_RE)] class DockerPlacementConstraint(ConstrainedStr): @@ -60,7 +61,7 @@ class DockerPlacementConstraint(ConstrainedStr): DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY: Final[ DockerLabelKey -] = parse_obj_as(DockerLabelKey, "ec2-instance-type") +] = TypeAdapter(DockerLabelKey).validate_python("ec2-instance-type") def to_simcore_runtime_docker_label_key(key: str) -> DockerLabelKey: @@ -99,7 +100,7 @@ class StandardSimcoreDockerLabels(BaseModel): ..., alias=f"{_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX}cpu-limit" ) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def _backwards_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]: # NOTE: this is necessary for dy-sidecar and legacy service until they are adjusted @@ -122,7 +123,7 @@ def _backwards_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]: def _convert_nano_cpus_to_cpus(nano_cpu: str) -> str: with contextlib.suppress(ValidationError): - return f"{parse_obj_as(float, nano_cpu) / (1.0*10**9):.2f}" + return f"{TypeAdapter(float).validate_python(nano_cpu) / (1.0*10**9):.2f}" return _UNDEFINED_LABEL_VALUE_INT mapped_values.setdefault( @@ -144,13 +145,13 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: @classmethod def from_docker_task(cls, docker_task: Task) -> "StandardSimcoreDockerLabels": assert docker_task.Spec # nosec - assert docker_task.Spec.ContainerSpec # nosec - task_labels = docker_task.Spec.ContainerSpec.Labels or {} - return cls.parse_obj(task_labels) + assert docker_task.Spec.ContainerSpec_ # nosec + task_labels = docker_task.Spec.ContainerSpec_.Labels or {} + return cls.model_validate(task_labels) - class Config: - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + populate_by_name=True, + json_schema_extra={ "examples": [ # legacy service labels { @@ -219,4 +220,5 @@ class Config: "io.simcore.runtime.user-id": "5", }, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/emails.py b/packages/models-library/src/models_library/emails.py index 80996eed76f..72835f4c754 100644 --- a/packages/models-library/src/models_library/emails.py +++ b/packages/models-library/src/models_library/emails.py @@ -1,7 +1,5 @@ -from pydantic import EmailStr +from typing import Annotated, TypeAlias +from pydantic import AfterValidator, EmailStr -class LowerCaseEmailStr(EmailStr): - @classmethod - def validate(cls, value: str) -> str: - return super().validate(value).lower() +LowerCaseEmailStr: TypeAlias = Annotated[str, EmailStr, AfterValidator(str.lower)] diff --git a/packages/models-library/src/models_library/errors_classes.py b/packages/models-library/src/models_library/errors_classes.py index dab24fb4009..6f6d9e0c266 100644 --- a/packages/models-library/src/models_library/errors_classes.py +++ b/packages/models-library/src/models_library/errors_classes.py @@ -7,12 +7,21 @@ def __missing__(self, key): class OsparcErrorMixin(PydanticErrorMixin): - def __new__(cls, *args, **kwargs): + msg_template: str + + def __new__(cls, *_args, **_kwargs): if not hasattr(cls, "code"): - cls.code = cls._get_full_class_name() - return super().__new__(cls, *args, **kwargs) + cls.code = cls._get_full_class_name() # type: ignore[assignment] + return super().__new__(cls) + + def __init__(self, *_args, **kwargs) -> None: + self.__dict__ = kwargs + super().__init__(message=self._build_message(), code=self.code) def __str__(self) -> str: + return self._build_message() + + def _build_message(self) -> str: # NOTE: safe. Does not raise KeyError return self.msg_template.format_map(_DefaultDict(**self.__dict__)) diff --git a/packages/models-library/src/models_library/folders.py b/packages/models-library/src/models_library/folders.py index 73262e1e647..829e3865c51 100644 --- a/packages/models-library/src/models_library/folders.py +++ b/packages/models-library/src/models_library/folders.py @@ -3,7 +3,7 @@ from models_library.users import GroupID, UserID from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt FolderID: TypeAlias = PositiveInt @@ -32,5 +32,4 @@ class FolderDB(BaseModel): user_id: UserID | None workspace_id: WorkspaceID | None - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/packages/models-library/src/models_library/function_services_catalog/_settings.py b/packages/models-library/src/models_library/function_services_catalog/_settings.py index 3ca4260d8ce..05812b81879 100644 --- a/packages/models-library/src/models_library/function_services_catalog/_settings.py +++ b/packages/models-library/src/models_library/function_services_catalog/_settings.py @@ -1,7 +1,7 @@ import json import os -from pydantic import BaseSettings +from pydantic_settings import BaseSettings # Expects env var: FUNCTION_SERVICES_AUTHORS='{"OM":{"name": ...}, "EN":{...} }' try: diff --git a/packages/models-library/src/models_library/generated_models/docker_rest_api.py b/packages/models-library/src/models_library/generated_models/docker_rest_api.py index 835141ea037..0170672cb26 100644 --- a/packages/models-library/src/models_library/generated_models/docker_rest_api.py +++ b/packages/models-library/src/models_library/generated_models/docker_rest_api.py @@ -8,11 +8,11 @@ from enum import Enum from typing import Any -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field, RootModel -class Model(BaseModel): - __root__: Any +class Model(RootModel[Any]): + ... class Type(str, Enum): @@ -61,42 +61,42 @@ class MountPoint(BaseModel): Type: Type1 | None = Field( None, description="The mount type:\n\n- `bind` a mount of a file or directory from the host into the container.\n- `volume` a docker volume with the given `Name`.\n- `tmpfs` a `tmpfs`.\n- `npipe` a named pipe from the host into the container.\n", - example="volume", + examples=["volume"], ) Name: str | None = Field( None, description="Name is the name reference to the underlying data defined by `Source`\ne.g., the volume name.\n", - example="myvolume", + examples=["myvolume"], ) Source: str | None = Field( None, description="Source location of the mount.\n\nFor volumes, this contains the storage location of the volume (within\n`/var/lib/docker/volumes/`). For bind-mounts, and `npipe`, this contains\nthe source (host) part of the bind-mount. For `tmpfs` mount points, this\nfield is empty.\n", - example="/var/lib/docker/volumes/myvolume/_data", + examples=["/var/lib/docker/volumes/myvolume/_data"], ) Destination: str | None = Field( None, description="Destination is the path relative to the container root (`/`) where\nthe `Source` is mounted inside the container.\n", - example="/usr/share/nginx/html/", + examples=["/usr/share/nginx/html/"], ) Driver: str | None = Field( None, description="Driver is the volume driver used to create the volume (if it is a volume).\n", - example="local", + examples=["local"], ) Mode: str | None = Field( None, description='Mode is a comma separated list of options supplied by the user when\ncreating the bind/volume mount.\n\nThe default is platform-specific (`"z"` on Linux, empty on Windows).\n', - example="z", + examples=["z"], ) RW: bool | None = Field( None, description="Whether the mount is mounted writable (read-write).\n", - example=True, + examples=[True], ) Propagation: str | None = Field( None, description="Propagation describes how mounts are propagated from the host into the\nmount point, and vice-versa. Refer to the [Linux kernel documentation](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt)\nfor details. This field is not used on Windows.\n", - example="", + examples=[""], ) @@ -115,15 +115,15 @@ class DeviceRequest(BaseModel): A request for devices to be sent to device drivers """ - Driver: str | None = Field(None, example="nvidia") - Count: int | None = Field(None, example=-1) + Driver: str | None = Field(None, examples=["nvidia"]) + Count: int | None = Field(None, examples=[-1]) DeviceIDs: list[str] | None = Field( - None, example=["0", "1", "GPU-fef8089b-4820-abfc-e83e-94318197576e"] + None, examples=[["0", "1", "GPU-fef8089b-4820-abfc-e83e-94318197576e"]] ) Capabilities: list[list[str]] | None = Field( None, description="A list of capabilities; an OR list of AND lists of capabilities.\n", - example=[["gpu", "nvidia", "compute"]], + examples=[[["gpu", "nvidia", "compute"]]], ) Options: dict[str, str] | None = Field( None, @@ -154,8 +154,9 @@ class BindOptions(BaseModel): Optional configuration for the `bind` type. """ - Propagation: Propagation | None = Field( + Propagation_: Propagation | None = Field( None, + alias="Propagation", description="A propagation mode with the value `[r]private`, `[r]shared`, or `[r]slave`.", ) NonRecursive: bool | None = Field( @@ -187,8 +188,8 @@ class VolumeOptions(BaseModel): Labels: dict[str, str] | None = Field( None, description="User-defined key/value metadata." ) - DriverConfig: DriverConfig | None = Field( - None, description="Map of driver specific options" + DriverConfig_: DriverConfig | None = Field( + None, alias="DriverConfig", description="Map of driver specific options" ) @@ -221,14 +222,20 @@ class Mount(BaseModel): None, description="The consistency requirement for the mount: `default`, `consistent`, `cached`, or `delegated`.", ) - BindOptions: BindOptions | None = Field( - None, description="Optional configuration for the `bind` type." + BindOptions_: BindOptions | None = Field( + None, + alias="BindOptions", + description="Optional configuration for the `bind` type.", ) - VolumeOptions: VolumeOptions | None = Field( - None, description="Optional configuration for the `volume` type." + VolumeOptions_: VolumeOptions | None = Field( + None, + alias="VolumeOptions", + description="Optional configuration for the `volume` type.", ) - TmpfsOptions: TmpfsOptions | None = Field( - None, description="Optional configuration for the `tmpfs` type." + TmpfsOptions_: TmpfsOptions | None = Field( + None, + alias="TmpfsOptions", + description="Optional configuration for the `tmpfs` type.", ) @@ -259,8 +266,9 @@ class RestartPolicy(BaseModel): """ - Name: Name | None = Field( + Name_: Name | None = Field( None, + alias="Name", description="- Empty string means not to restart\n- `no` Do not automatically restart\n- `always` Always restart\n- `unless-stopped` Restart always except when the user has manually stopped the container\n- `on-failure` Restart only when the container exit code is non-zero\n", ) MaximumRetryCount: int | None = Field( @@ -335,7 +343,7 @@ class Resources(BaseModel): CpusetCpus: str | None = Field( None, description="CPUs in which to allow execution (e.g., `0-3`, `0,1`).\n", - example="0-3", + examples=["0-3"], ) CpusetMems: str | None = Field( None, @@ -354,7 +362,7 @@ class Resources(BaseModel): KernelMemory: int | None = Field( None, description="Kernel memory limit in bytes.\n\n


\n\n> **Deprecated**: This field is deprecated as the kernel 5.4 deprecated\n> `kmem.limit_in_bytes`.\n", - example=209715200, + examples=[209715200], ) KernelMemoryTCP: int | None = Field( None, description="Hard limit for kernel TCP buffer memory (in bytes)." @@ -413,12 +421,12 @@ class Limit(BaseModel): """ - NanoCPUs: int | None = Field(None, example=4000000000) - MemoryBytes: int | None = Field(None, example=8272408576) + NanoCPUs: int | None = Field(None, examples=[4000000000]) + MemoryBytes: int | None = Field(None, examples=[8272408576]) Pids: int | None = Field( 0, description="Limits the maximum number of PIDs in the container. Set `0` for unlimited.\n", - example=100, + examples=[100], ) @@ -433,24 +441,30 @@ class DiscreteResourceSpec(BaseModel): class GenericResource(BaseModel): - NamedResourceSpec: NamedResourceSpec | None = None - DiscreteResourceSpec: DiscreteResourceSpec | None = None + NamedResourceSpec_: NamedResourceSpec | None = Field( + None, alias="NamedResourceSpec" + ) + DiscreteResourceSpec_: DiscreteResourceSpec | None = Field( + None, alias="DiscreteResourceSpec" + ) -class GenericResources(BaseModel): +class GenericResources(RootModel): """ User-defined resources can be either Integer resources (e.g, `SSD=3`) or String resources (e.g, `GPU=UUID1`). """ - __root__: list[GenericResource] = Field( + root: list[GenericResource] = Field( ..., description="User-defined resources can be either Integer resources (e.g, `SSD=3`) or\nString resources (e.g, `GPU=UUID1`).\n", - example=[ - {"DiscreteResourceSpec": {"Kind": "SSD", "Value": 3}}, - {"NamedResourceSpec": {"Kind": "GPU", "Value": "UUID1"}}, - {"NamedResourceSpec": {"Kind": "GPU", "Value": "UUID2"}}, + examples=[ + [ + {"DiscreteResourceSpec": {"Kind": "SSD", "Value": 3}}, + {"NamedResourceSpec": {"Kind": "GPU", "Value": "UUID1"}}, + {"NamedResourceSpec": {"Kind": "GPU", "Value": "UUID2"}}, + ] ], ) @@ -508,17 +522,17 @@ class HealthcheckResult(BaseModel): Start: datetime | None = Field( None, description="Date and time at which this check started in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2020-01-04T10:44:24.496525531Z", + examples=["2020-01-04T10:44:24.496525531Z"], ) End: str | None = Field( None, description="Date and time at which this check ended in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2020-01-04T10:45:21.364524523Z", + examples=["2020-01-04T10:45:21.364524523Z"], ) ExitCode: int | None = Field( None, description="ExitCode meanings:\n\n- `0` healthy\n- `1` unhealthy\n- `2` reserved (considered unhealthy)\n- other values: error running probe\n", - example=0, + examples=[0], ) Output: str | None = Field(None, description="Output from last check") @@ -560,8 +574,8 @@ class CgroupnsMode(str, Enum): host = "host" -class ConsoleSizeItem(BaseModel): - __root__: int = Field(..., ge=0) +class ConsoleSizeItem(RootModel): + root: int = Field(..., ge=0) class Isolation(str, Enum): @@ -591,7 +605,7 @@ class ContainerConfig(BaseModel): Hostname: str | None = Field( None, description="The hostname to use for the container, as a valid RFC 1123 hostname.\n", - example="439f4e91bd1d", + examples=["439f4e91bd1d"], ) Domainname: str | None = Field( None, description="The domain name to use for the container.\n" @@ -609,7 +623,7 @@ class ContainerConfig(BaseModel): ExposedPorts: dict[str, dict[str, Any]] | None = Field( None, description='An object mapping ports to an empty object in the form:\n\n`{"/": {}}`\n', - example={"80/tcp": {}, "443/tcp": {}}, + examples=[{"80/tcp": {}, "443/tcp": {}}], ) Tty: bool | None = Field( False, @@ -622,21 +636,23 @@ class ContainerConfig(BaseModel): Env: list[str] | None = Field( None, description='A list of environment variables to set inside the container in the\nform `["VAR=value", ...]`. A variable without `=` is removed from the\nenvironment, rather than to have an empty value.\n', - example=["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"], + examples=[ + ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] + ], ) Cmd: list[str] | None = Field( None, description="Command to run specified as a string or an array of strings.\n", - example=["/bin/sh"], + examples=[["/bin/sh"]], ) Healthcheck: HealthConfig | None = None ArgsEscaped: bool | None = Field( - False, description="Command is already escaped (Windows only)", example=False + False, description="Command is already escaped (Windows only)", examples=[False] ) Image: str | None = Field( None, description="The name (or reference) of the image to use when creating the container,\nor which was used when the container was created.\n", - example="example-image:1.0", + examples=["example-image:1.0"], ) Volumes: dict[str, dict[str, Any]] | None = Field( None, @@ -645,12 +661,12 @@ class ContainerConfig(BaseModel): WorkingDir: str | None = Field( None, description="The working directory for commands to run in.", - example="/public/", + examples=["/public/"], ) Entrypoint: list[str] | None = Field( None, description='The entry point for the container as a string or an array of strings.\n\nIf the array consists of exactly one empty string (`[""]`) then the\nentry point is reset to system default (i.e., the entry point used by\ndocker when there is no `ENTRYPOINT` instruction in the `Dockerfile`).\n', - example=[], + examples=[[]], ) NetworkDisabled: bool | None = Field( None, description="Disable networking for the container." @@ -659,20 +675,22 @@ class ContainerConfig(BaseModel): OnBuild: list[str] | None = Field( None, description="`ONBUILD` metadata that were defined in the image's `Dockerfile`.\n", - example=[], + examples=[[]], ) Labels: dict[str, str] | None = Field( None, description="User-defined key/value metadata.", - example={ - "com.example.some-label": "some-value", - "com.example.some-other-label": "some-other-value", - }, + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], ) StopSignal: str | None = Field( None, description="Signal to stop a container as a string or unsigned integer.\n", - example="SIGTERM", + examples=["SIGTERM"], ) StopTimeout: int | None = Field( 10, description="Timeout to stop a container in seconds." @@ -680,7 +698,7 @@ class ContainerConfig(BaseModel): Shell: list[str] | None = Field( None, description="Shell for when `RUN`, `CMD`, and `ENTRYPOINT` uses a shell.\n", - example=["/bin/sh", "-c"], + examples=[["/bin/sh", "-c"]], ) @@ -704,8 +722,7 @@ class PortMap(BaseModel): """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") class PortBinding(BaseModel): @@ -718,12 +735,12 @@ class PortBinding(BaseModel): HostIp: str | None = Field( None, description="Host IP address that the container's port is mapped to.", - example="127.0.0.1", + examples=["127.0.0.1"], ) HostPort: str | None = Field( None, description="Host port number that the container's port is mapped to.", - example="4443", + examples=["4443"], ) @@ -735,16 +752,18 @@ class GraphDriverData(BaseModel): """ Name: str = Field( - ..., description="Name of the storage driver.", example="overlay2" + ..., description="Name of the storage driver.", examples=["overlay2"] ) Data: dict[str, str] = Field( ..., description="Low-level storage metadata, provided as key/value pairs.\n\nThis information is driver-specific, and depends on the storage-driver\nin use, and should be used for informational purposes only.\n", - example={ - "MergedDir": "/var/lib/docker/overlay2/ef749362d13333e65fc95c572eb525abbe0052e16e086cb64bc3b98ae9aa6d74/merged", - "UpperDir": "/var/lib/docker/overlay2/ef749362d13333e65fc95c572eb525abbe0052e16e086cb64bc3b98ae9aa6d74/diff", - "WorkDir": "/var/lib/docker/overlay2/ef749362d13333e65fc95c572eb525abbe0052e16e086cb64bc3b98ae9aa6d74/work", - }, + examples=[ + { + "MergedDir": "/var/lib/docker/overlay2/ef749362d13333e65fc95c572eb525abbe0052e16e086cb64bc3b98ae9aa6d74/merged", + "UpperDir": "/var/lib/docker/overlay2/ef749362d13333e65fc95c572eb525abbe0052e16e086cb64bc3b98ae9aa6d74/diff", + "WorkDir": "/var/lib/docker/overlay2/ef749362d13333e65fc95c572eb525abbe0052e16e086cb64bc3b98ae9aa6d74/work", + } + ], ) @@ -754,12 +773,14 @@ class RootFS(BaseModel): """ - Type: str = Field(..., example="layers") + Type: str = Field(..., examples=["layers"]) Layers: list[str] | None = Field( None, - example=[ - "sha256:1834950e52ce4d5a88a1bbd131c537f4d0e56d10ff0dd69e66be3b7dfa9df7e6", - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef", + examples=[ + [ + "sha256:1834950e52ce4d5a88a1bbd131c537f4d0e56d10ff0dd69e66be3b7dfa9df7e6", + "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef", + ] ], ) @@ -774,7 +795,7 @@ class Metadata(BaseModel): LastTagTime: str | None = Field( None, description="Date and time at which the image was last tagged in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n\nThis information is only available if the image was tagged locally,\nand omitted otherwise.\n", - example="2022-02-28T14:40:02.623929178Z", + examples=["2022-02-28T14:40:02.623929178Z"], ) @@ -787,95 +808,103 @@ class ImageInspect(BaseModel): Id: str | None = Field( None, description="ID is the content-addressable ID of an image.\n\nThis identifier is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n", - example="sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710", + examples=[ + "sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710" + ], ) RepoTags: list[str] | None = Field( None, description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same imagem and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n', - example=[ - "example:1.0", - "example:latest", - "example:stable", - "internal.registry.example.com:5000/example:1.0", + examples=[ + [ + "example:1.0", + "example:latest", + "example:stable", + "internal.registry.example.com:5000/example:1.0", + ] ], ) RepoDigests: list[str] | None = Field( None, description="List of content-addressable digests of locally available image manifests\nthat the image is referenced from. Multiple manifests can refer to the\nsame image.\n\nThese digests are usually only available if the image was either pulled\nfrom a registry, or if the image was pushed to a registry, which is when\nthe manifest is generated and its digest calculated.\n", - example=[ - "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", - "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578", + examples=[ + [ + "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", + "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578", + ] ], ) Parent: str | None = Field( None, description="ID of the parent image.\n\nDepending on how the image was created, this field may be empty and\nis only set for images that were built/created locally. This field\nis empty if the image was pulled from an image registry.\n", - example="", + examples=[""], ) Comment: str | None = Field( None, description="Optional message that was set when committing or importing the image.\n", - example="", + examples=[""], ) Created: str | None = Field( None, description="Date and time at which the image was created, formatted in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2022-02-04T21:20:12.497794809Z", + examples=["2022-02-04T21:20:12.497794809Z"], ) Container: str | None = Field( None, description="The ID of the container that was used to create the image.\n\nDepending on how the image was created, this field may be empty.\n", - example="65974bc86f1770ae4bff79f651ebdbce166ae9aada632ee3fa9af3a264911735", + examples=["65974bc86f1770ae4bff79f651ebdbce166ae9aada632ee3fa9af3a264911735"], ) - ContainerConfig: ContainerConfig | None = None + ContainerConfig_: ContainerConfig | None = Field(None, alias="ContainerConfig") DockerVersion: str | None = Field( None, description="The version of Docker that was used to build the image.\n\nDepending on how the image was created, this field may be empty.\n", - example="20.10.7", + examples=["20.10.7"], ) Author: str | None = Field( None, description="Name of the author that was specified when committing the image, or as\nspecified through MAINTAINER (deprecated) in the Dockerfile.\n", - example="", + examples=[""], ) - Config_: ContainerConfig | None = Field(None, alias="Config") # type: ignore + Config_: ContainerConfig | None = Field(None, alias="Config") Architecture: str | None = Field( None, description="Hardware CPU architecture that the image runs on.\n", - example="arm", + examples=["arm"], ) Variant: str | None = Field( None, description="CPU architecture variant (presently ARM-only).\n", - example="v7", + examples=["v7"], ) Os: str | None = Field( None, description="Operating System the image is built to run on.\n", - example="linux", + examples=["linux"], ) OsVersion: str | None = Field( None, description="Operating System version the image is built to run on (especially\nfor Windows).\n", - example="", + examples=[""], ) Size: int | None = Field( None, description="Total size of the image including all layers it is composed of.\n", - example=1239828, + examples=[1239828], ) VirtualSize: int | None = Field( None, description="Total size of the image including all layers it is composed of.\n\nIn versions of Docker before v1.10, this field was calculated from\nthe image itself and all of its parent images. Docker v1.10 and up\nstore images self-contained, and no longer use a parent-chain, making\nthis field an equivalent of the Size field.\n\nThis field is kept for backward compatibility, but may be removed in\na future version of the API.\n", - example=1239828, + examples=[1239828], ) GraphDriver: GraphDriverData | None = None - RootFS: RootFS | None = Field( + RootFS_: RootFS | None = Field( None, + alias="RootFS", description="Information about the image's RootFS, including the layer IDs.\n", ) - Metadata: Metadata | None = Field( + Metadata_: Metadata | None = Field( None, + alias="Metadata", description="Additional metadata of the image in the local cache. This information\nis local to the daemon, and not part of the image itself.\n", ) @@ -884,63 +913,71 @@ class ImageSummary(BaseModel): Id: str = Field( ..., description="ID is the content-addressable ID of an image.\n\nThis identifier is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n", - example="sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710", + examples=[ + "sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710" + ], ) ParentId: str = Field( ..., description="ID of the parent image.\n\nDepending on how the image was created, this field may be empty and\nis only set for images that were built/created locally. This field\nis empty if the image was pulled from an image registry.\n", - example="", + examples=[""], ) RepoTags: list[str] = Field( ..., description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same imagem and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n', - example=[ - "example:1.0", - "example:latest", - "example:stable", - "internal.registry.example.com:5000/example:1.0", + examples=[ + [ + "example:1.0", + "example:latest", + "example:stable", + "internal.registry.example.com:5000/example:1.0", + ] ], ) RepoDigests: list[str] = Field( ..., description="List of content-addressable digests of locally available image manifests\nthat the image is referenced from. Multiple manifests can refer to the\nsame image.\n\nThese digests are usually only available if the image was either pulled\nfrom a registry, or if the image was pushed to a registry, which is when\nthe manifest is generated and its digest calculated.\n", - example=[ - "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", - "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578", + examples=[ + [ + "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", + "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578", + ] ], ) Created: int = Field( ..., description="Date and time at which the image was created as a Unix timestamp\n(number of seconds sinds EPOCH).\n", - example="1644009612", + examples=["1644009612"], ) Size: int = Field( ..., description="Total size of the image including all layers it is composed of.\n", - example=172064416, + examples=[172064416], ) SharedSize: int = Field( ..., description="Total size of image layers that are shared between this image and other\nimages.\n\nThis size is not calculated by default. `-1` indicates that the value\nhas not been set / calculated.\n", - example=1239828, + examples=[1239828], ) VirtualSize: int = Field( ..., description="Total size of the image including all layers it is composed of.\n\nIn versions of Docker before v1.10, this field was calculated from\nthe image itself and all of its parent images. Docker v1.10 and up\nstore images self-contained, and no longer use a parent-chain, making\nthis field an equivalent of the Size field.\n\nThis field is kept for backward compatibility, but may be removed in\na future version of the API.\n", - example=172064416, + examples=[172064416], ) Labels: dict[str, str] = Field( ..., description="User-defined key/value metadata.", - example={ - "com.example.some-label": "some-value", - "com.example.some-other-label": "some-other-value", - }, + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], ) Containers: int = Field( ..., description="Number of containers using this image. Includes both stopped and running\ncontainers.\n\nThis size is not calculated by default, and depends on which API endpoint\nis used. `-1` indicates that the value has not been set / calculated.\n", - example=2, + examples=[2], ) @@ -988,47 +1025,51 @@ class UsageData(BaseModel): class Volume(BaseModel): - Name: str = Field(..., description="Name of the volume.", example="tardis") + Name: str = Field(..., description="Name of the volume.", examples=["tardis"]) Driver: str = Field( ..., description="Name of the volume driver used by the volume.", - example="custom", + examples=["custom"], ) Mountpoint: str = Field( ..., description="Mount path of the volume on the host.", - example="/var/lib/docker/volumes/tardis", + examples=["/var/lib/docker/volumes/tardis"], ) CreatedAt: str | None = Field( None, description="Date/Time the volume was created.", - example="2016-06-07T20:31:11.853781916Z", + examples=["2016-06-07T20:31:11.853781916Z"], ) Status: dict[str, dict[str, Any]] | None = Field( None, description='Low-level details about the volume, provided by the volume driver.\nDetails are returned as a map with key/value pairs:\n`{"key":"value","key2":"value2"}`.\n\nThe `Status` field is optional, and is omitted if the volume driver\ndoes not support this feature.\n', - example={"hello": "world"}, + examples=[{"hello": "world"}], ) Labels: dict[str, str] = Field( ..., description="User-defined key/value metadata.", - example={ - "com.example.some-label": "some-value", - "com.example.some-other-label": "some-other-value", - }, + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], ) - Scope: Scope = Field( + Scope_: Scope = Field( ..., + alias="Scope", description="The level at which the volume exists. Either `global` for cluster-wide,\nor `local` for machine level.\n", - example="local", + examples=["local"], ) Options: dict[str, str] = Field( ..., description="The driver specific options used when creating the volume.\n", - example={"device": "tmpfs", "o": "size=100m,uid=1000", "type": "tmpfs"}, + examples=[{"device": "tmpfs", "o": "size=100m,uid=1000", "type": "tmpfs"}], ) - UsageData: UsageData | None = Field( + UsageData_: UsageData | None = Field( None, + alias="UsageData", description="Usage details about the volume. This information is used by the\n`GET /system/df` endpoint, and omitted in other endpoints.\n", ) @@ -1041,23 +1082,25 @@ class VolumeConfig(BaseModel): Name: str | None = Field( None, description="The new volume's name. If not specified, Docker generates a name.\n", - example="tardis", + examples=["tardis"], ) Driver: str | None = Field( - "local", description="Name of the volume driver to use.", example="custom" + "local", description="Name of the volume driver to use.", examples=["custom"] ) DriverOpts: dict[str, str] | None = Field( None, description="A mapping of driver options and values. These options are\npassed directly to the driver and are driver specific.\n", - example={"device": "tmpfs", "o": "size=100m,uid=1000", "type": "tmpfs"}, + examples=[{"device": "tmpfs", "o": "size=100m,uid=1000", "type": "tmpfs"}], ) Labels: dict[str, str] | None = Field( None, description="User-defined key/value metadata.", - example={ - "com.example.some-label": "some-value", - "com.example.some-other-label": "some-other-value", - }, + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], ) @@ -1099,43 +1142,45 @@ class BuildCache(BaseModel): ID: str | None = Field( None, description="Unique ID of the build cache record.\n", - example="ndlpt0hhvkqcdfkputsk4cq9c", + examples=["ndlpt0hhvkqcdfkputsk4cq9c"], ) Parent: str | None = Field( None, description="ID of the parent build cache record.\n", - example="hw53o5aio51xtltp5xjp8v7fx", + examples=["hw53o5aio51xtltp5xjp8v7fx"], ) Type: Type4 | None = Field( - None, description="Cache record type.\n", example="regular" + None, description="Cache record type.\n", examples=["regular"] ) Description: str | None = Field( None, description="Description of the build-step that produced the build cache.\n", - example="mount / from exec /bin/sh -c echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/keep-cache", + examples=[ + "mount / from exec /bin/sh -c echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/keep-cache" + ], ) InUse: bool | None = Field( - None, description="Indicates if the build cache is in use.\n", example=False + None, description="Indicates if the build cache is in use.\n", examples=[False] ) Shared: bool | None = Field( - None, description="Indicates if the build cache is shared.\n", example=True + None, description="Indicates if the build cache is shared.\n", examples=[True] ) Size: int | None = Field( None, description="Amount of disk space used by the build cache (in bytes).\n", - example=51, + examples=[51], ) CreatedAt: str | None = Field( None, description="Date and time at which the build cache was created in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2016-08-18T10:44:24.496525531Z", + examples=["2016-08-18T10:44:24.496525531Z"], ) LastUsedAt: str | None = Field( None, description="Date and time at which the build cache was last used in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2017-08-09T07:09:37.632105588Z", + examples=["2017-08-09T07:09:37.632105588Z"], ) - UsageCount: int | None = Field(None, example=26) + UsageCount: int | None = Field(None, examples=[26]) class ImageID(BaseModel): @@ -1178,28 +1223,30 @@ class EndpointIPAMConfig(BaseModel): """ - IPv4Address: str | None = Field(None, example="172.20.30.33") - IPv6Address: str | None = Field(None, example="2001:db8:abcd::3033") + IPv4Address: str | None = Field(None, examples=["172.20.30.33"]) + IPv6Address: str | None = Field(None, examples=["2001:db8:abcd::3033"]) LinkLocalIPs: list[str] | None = Field( - None, example=["169.254.34.68", "fe80::3468"] + None, examples=[["169.254.34.68", "fe80::3468"]] ) class PluginMount(BaseModel): - Name: str = Field(..., example="some-mount") - Description: str = Field(..., example="This is a mount that's used by the plugin.") + Name: str = Field(..., examples=["some-mount"]) + Description: str = Field( + ..., examples=["This is a mount that's used by the plugin."] + ) Settable: list[str] - Source: str = Field(..., example="/var/lib/docker/plugins/") - Destination: str = Field(..., example="/mnt/state") - Type: str = Field(..., example="bind") - Options: list[str] = Field(..., example=["rbind", "rw"]) + Source: str = Field(..., examples=["/var/lib/docker/plugins/"]) + Destination: str = Field(..., examples=["/mnt/state"]) + Type: str = Field(..., examples=["bind"]) + Options: list[str] = Field(..., examples=[["rbind", "rw"]]) class PluginDevice(BaseModel): Name: str Description: str Settable: list[str] - Path: str = Field(..., example="/dev/fuse") + Path: str = Field(..., examples=["/dev/fuse"]) class PluginEnv(BaseModel): @@ -1222,9 +1269,9 @@ class PluginPrivilege(BaseModel): """ - Name: str | None = Field(None, example="network") + Name: str | None = Field(None, examples=["network"]) Description: str | None = None - Value: list[str] | None = Field(None, example=["host"]) + Value: list[str] | None = Field(None, examples=[["host"]]) class Settings(BaseModel): @@ -1233,7 +1280,7 @@ class Settings(BaseModel): """ Mounts: list[PluginMount] - Env: list[str] = Field(..., example=["DEBUG=0"]) + Env: list[str] = Field(..., examples=[["DEBUG=0"]]) Args: list[str] Devices: list[PluginDevice] @@ -1252,44 +1299,49 @@ class Interface(BaseModel): The interface between Docker and the plugin """ - Types: list[PluginInterfaceType] = Field(..., example=["docker.volumedriver/1.0"]) - Socket: str = Field(..., example="plugins.sock") - ProtocolScheme: ProtocolScheme | None = Field( + Types: list[PluginInterfaceType] = Field( + ..., examples=[["docker.volumedriver/1.0"]] + ) + Socket: str = Field(..., examples=["plugins.sock"]) + ProtocolScheme_: ProtocolScheme | None = Field( None, + alias="ProtocolScheme", description="Protocol to use for clients connecting to the plugin.", - example="some.protocol/v1.0", + examples=["some.protocol/v1.0"], ) class User(BaseModel): - UID: int | None = Field(None, example=1000) - GID: int | None = Field(None, example=1000) + UID: int | None = Field(None, examples=[1000]) + GID: int | None = Field(None, examples=[1000]) class Network1(BaseModel): - Type: str = Field(..., example="host") + Type: str = Field(..., examples=["host"]) class Linux(BaseModel): - Capabilities: list[str] = Field(..., example=["CAP_SYS_ADMIN", "CAP_SYSLOG"]) - AllowAllDevices: bool = Field(..., example=False) + Capabilities: list[str] = Field(..., examples=[["CAP_SYS_ADMIN", "CAP_SYSLOG"]]) + AllowAllDevices: bool = Field(..., examples=[False]) Devices: list[PluginDevice] class Args(BaseModel): - Name: str = Field(..., example="args") - Description: str = Field(..., example="command line arguments") + Name: str = Field(..., examples=["args"]) + Description: str = Field(..., examples=["command line arguments"]) Settable: list[str] Value: list[str] class Rootfs(BaseModel): - type: str | None = Field(None, example="layers") + type: str | None = Field(None, examples=["layers"]) diff_ids: list[str] | None = Field( None, - example=[ - "sha256:675532206fbf3030b8458f88d6e26d4eb1577688a25efec97154c94e8b6b4887", - "sha256:e216a057b1cb1efc11f8a268f37ef62083e70b1b38323ba252e25ac88904a7e8", + examples=[ + [ + "sha256:675532206fbf3030b8458f88d6e26d4eb1577688a25efec97154c94e8b6b4887", + "sha256:e216a057b1cb1efc11f8a268f37ef62083e70b1b38323ba252e25ac88904a7e8", + ] ], ) @@ -1302,33 +1354,37 @@ class Config(BaseModel): DockerVersion: str | None = Field( None, description="Docker Version used to create the plugin", - example="17.06.0-ce", + examples=["17.06.0-ce"], ) - Description: str = Field(..., example="A sample volume plugin for Docker") - Documentation: str = Field(..., example="/engine/extend/plugins/") - Interface: Interface = Field( - ..., description="The interface between Docker and the plugin" + Description: str = Field(..., examples=["A sample volume plugin for Docker"]) + Documentation: str = Field(..., examples=["/engine/extend/plugins/"]) + Interface_: Interface = Field( + ..., + alias="Interface", + description="The interface between Docker and the plugin", ) Entrypoint: list[str] = Field( - ..., example=["/usr/bin/sample-volume-plugin", "/data"] + ..., examples=[["/usr/bin/sample-volume-plugin", "/data"]] ) - WorkDir: str = Field(..., example="/bin/") - User: User | None = None + WorkDir: str = Field(..., examples=["/bin/"]) + User_: User | None = Field(None, alias="User") Network: Network1 Linux: Linux - PropagatedMount: str = Field(..., example="/mnt/volumes") - IpcHost: bool = Field(..., example=False) - PidHost: bool = Field(..., example=False) + PropagatedMount: str = Field(..., examples=["/mnt/volumes"]) + IpcHost: bool = Field(..., examples=[False]) + PidHost: bool = Field(..., examples=[False]) Mounts: list[PluginMount] Env: list[PluginEnv] = Field( ..., - example=[ - { - "Name": "DEBUG", - "Description": "If set, prints debug messages", - "Settable": None, - "Value": "0", - } + examples=[ + [ + { + "Name": "DEBUG", + "Description": "If set, prints debug messages", + "Settable": None, + "Value": "0", + } + ] ], ) Args: Args @@ -1341,21 +1397,22 @@ class Plugin(BaseModel): """ Id: str | None = Field( - None, example="5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078" + None, + examples=["5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078"], ) - Name: str = Field(..., example="tiborvass/sample-volume-plugin") + Name: str = Field(..., examples=["tiborvass/sample-volume-plugin"]) Enabled: bool = Field( ..., description="True if the plugin is running. False if the plugin is not running, only installed.", - example=True, + examples=[True], ) - Settings: Settings = Field( - ..., description="Settings that can be modified by users." + Settings_: Settings = Field( + ..., alias="Settings", description="Settings that can be modified by users." ) PluginReference: str | None = Field( None, description="plugin remote reference used to push/pull the plugin", - example="localhost:5000/tiborvass/sample-volume-plugin:latest", + examples=["localhost:5000/tiborvass/sample-volume-plugin:latest"], ) Config_: Config = Field(..., alias="Config", description="The config of a plugin.") @@ -1375,7 +1432,7 @@ class ObjectVersion(BaseModel): """ - Index: int | None = Field(None, example=373531) + Index: int | None = Field(None, examples=[373531]) class Role(str, Enum): @@ -1398,13 +1455,20 @@ class Availability(str, Enum): class NodeSpec(BaseModel): - Name: str | None = Field(None, description="Name for the node.", example="my-node") + Name: str | None = Field( + None, description="Name for the node.", examples=["my-node"] + ) Labels: dict[str, str] | None = Field( None, description="User-defined key/value metadata." ) - Role: Role | None = Field(None, description="Role of the node.", example="manager") - Availability: Availability | None = Field( - None, description="Availability of the node.", example="active" + Role_: Role | None = Field( + None, alias="Role", description="Role of the node.", examples=["manager"] + ) + Availability_: Availability | None = Field( + None, + alias="Availability", + description="Availability of the node.", + examples=["active"], ) @@ -1417,12 +1481,12 @@ class Platform(BaseModel): Architecture: str | None = Field( None, description="Architecture represents the hardware architecture (for example,\n`x86_64`).\n", - example="x86_64", + examples=["x86_64"], ) OS: str | None = Field( None, description="OS represents the Operating System (for example, `linux` or `windows`).\n", - example="linux", + examples=["linux"], ) @@ -1436,29 +1500,31 @@ class EngineDescription(BaseModel): EngineDescription provides information about an engine. """ - EngineVersion: str | None = Field(None, example="17.06.0") - Labels: dict[str, str] | None = Field(None, example={"foo": "bar"}) + EngineVersion: str | None = Field(None, examples=["17.06.0"]) + Labels: dict[str, str] | None = Field(None, examples=[{"foo": "bar"}]) Plugins: list[Plugin1] | None = Field( None, - example=[ - {"Type": "Log", "Name": "awslogs"}, - {"Type": "Log", "Name": "fluentd"}, - {"Type": "Log", "Name": "gcplogs"}, - {"Type": "Log", "Name": "gelf"}, - {"Type": "Log", "Name": "journald"}, - {"Type": "Log", "Name": "json-file"}, - {"Type": "Log", "Name": "logentries"}, - {"Type": "Log", "Name": "splunk"}, - {"Type": "Log", "Name": "syslog"}, - {"Type": "Network", "Name": "bridge"}, - {"Type": "Network", "Name": "host"}, - {"Type": "Network", "Name": "ipvlan"}, - {"Type": "Network", "Name": "macvlan"}, - {"Type": "Network", "Name": "null"}, - {"Type": "Network", "Name": "overlay"}, - {"Type": "Volume", "Name": "local"}, - {"Type": "Volume", "Name": "localhost:5000/vieux/sshfs:latest"}, - {"Type": "Volume", "Name": "vieux/sshfs:latest"}, + examples=[ + [ + {"Type": "Log", "Name": "awslogs"}, + {"Type": "Log", "Name": "fluentd"}, + {"Type": "Log", "Name": "gcplogs"}, + {"Type": "Log", "Name": "gelf"}, + {"Type": "Log", "Name": "journald"}, + {"Type": "Log", "Name": "json-file"}, + {"Type": "Log", "Name": "logentries"}, + {"Type": "Log", "Name": "splunk"}, + {"Type": "Log", "Name": "syslog"}, + {"Type": "Network", "Name": "bridge"}, + {"Type": "Network", "Name": "host"}, + {"Type": "Network", "Name": "ipvlan"}, + {"Type": "Network", "Name": "macvlan"}, + {"Type": "Network", "Name": "null"}, + {"Type": "Network", "Name": "overlay"}, + {"Type": "Volume", "Name": "local"}, + {"Type": "Volume", "Name": "localhost:5000/vieux/sshfs:latest"}, + {"Type": "Volume", "Name": "vieux/sshfs:latest"}, + ] ], ) @@ -1512,7 +1578,7 @@ class Orchestration(BaseModel): TaskHistoryRetentionLimit: int | None = Field( None, description="The number of historic tasks to keep per instance or node. If\nnegative, never remove completed or failed tasks.\n", - example=10, + examples=[10], ) @@ -1522,7 +1588,9 @@ class Raft(BaseModel): """ SnapshotInterval: int | None = Field( - None, description="The number of log entries between snapshots.", example=10000 + None, + description="The number of log entries between snapshots.", + examples=[10000], ) KeepOldSnapshots: int | None = Field( None, @@ -1531,17 +1599,17 @@ class Raft(BaseModel): LogEntriesForSlowFollowers: int | None = Field( None, description="The number of log entries to keep around to sync up slow followers\nafter a snapshot is created.\n", - example=500, + examples=[500], ) ElectionTick: int | None = Field( None, description="The number of ticks that a follower will wait for a message from\nthe leader before becoming a candidate and starting an election.\n`ElectionTick` must be greater than `HeartbeatTick`.\n\nA tick currently defaults to one second, so these translate\ndirectly to seconds currently, but this is NOT guaranteed.\n", - example=3, + examples=[3], ) HeartbeatTick: int | None = Field( None, description="The number of ticks between heartbeats. Every HeartbeatTick ticks,\nthe leader will send a heartbeat to the followers.\n\nA tick currently defaults to one second, so these translate\ndirectly to seconds currently, but this is NOT guaranteed.\n", - example=1, + examples=[1], ) @@ -1553,7 +1621,7 @@ class Dispatcher(BaseModel): HeartbeatPeriod: int | None = Field( None, description="The delay for an agent to send a heartbeat to the dispatcher.\n", - example=5000000000, + examples=[5000000000], ) @@ -1568,8 +1636,9 @@ class Protocol(str, Enum): class ExternalCA(BaseModel): - Protocol: Protocol | None = Field( + Protocol_: Protocol | None = Field( Protocol.cfssl, + alias="Protocol", description="Protocol for communication with the external CA (currently\nonly `cfssl` is supported).\n", ) URL: str | None = Field( @@ -1593,7 +1662,7 @@ class CAConfig(BaseModel): NodeCertExpiry: int | None = Field( None, description="The duration node certificates are issued for.", - example=7776000000000000, + examples=[7776000000000000], ) ExternalCAs: list[ExternalCA] | None = Field( None, @@ -1621,7 +1690,7 @@ class EncryptionConfig(BaseModel): AutoLockManagers: bool | None = Field( None, description="If set, generate a key and use it to lock data stored on the\nmanagers.\n", - example=False, + examples=[False], ) @@ -1638,12 +1707,12 @@ class LogDriver(BaseModel): Name: str | None = Field( None, description="The log driver to use as a default for new tasks.\n", - example="json-file", + examples=["json-file"], ) Options: dict[str, str] | None = Field( None, description="Driver-specific options for the selectd log driver, specified\nas key/value pairs.\n", - example={"max-file": "10", "max-size": "100m"}, + examples=[{"max-file": "10", "max-size": "100m"}], ) @@ -1652,8 +1721,9 @@ class TaskDefaults(BaseModel): Defaults for creating tasks in this cluster. """ - LogDriver: LogDriver | None = Field( + LogDriver_: LogDriver | None = Field( None, + alias="LogDriver", description="The log driver to use for tasks created in the orchestrator if\nunspecified by a service.\n\nUpdating this value only affects new tasks. Existing tasks continue\nto use their previously configured log driver until recreated.\n", ) @@ -1663,26 +1733,38 @@ class SwarmSpec(BaseModel): User modifiable swarm configuration. """ - Name: str | None = Field(None, description="Name of the swarm.", example="default") + Name: str | None = Field( + None, description="Name of the swarm.", examples=["default"] + ) Labels: dict[str, str] | None = Field( None, description="User-defined key/value metadata.", - example={ - "com.example.corp.type": "production", - "com.example.corp.department": "engineering", - }, + examples=[ + { + "com.example.corp.type": "production", + "com.example.corp.department": "engineering", + } + ], + ) + Orchestration_: Orchestration | None = Field( + None, alias="Orchestration", description="Orchestration configuration." ) - Orchestration: Orchestration | None = Field( - None, description="Orchestration configuration." + Raft_: Raft | None = Field(None, alias="Raft", description="Raft configuration.") + Dispatcher_: Dispatcher | None = Field( + None, alias="Dispatcher", description="Dispatcher configuration." ) - Raft: Raft | None = Field(None, description="Raft configuration.") - Dispatcher: Dispatcher | None = Field(None, description="Dispatcher configuration.") - CAConfig: CAConfig | None = Field(None, description="CA configuration.") - EncryptionConfig: EncryptionConfig | None = Field( - None, description="Parameters related to encryption-at-rest." + CAConfig_: CAConfig | None = Field( + None, alias="CAConfig", description="CA configuration." ) - TaskDefaults: TaskDefaults | None = Field( - None, description="Defaults for creating tasks in this cluster." + EncryptionConfig_: EncryptionConfig | None = Field( + None, + alias="EncryptionConfig", + description="Parameters related to encryption-at-rest.", + ) + TaskDefaults_: TaskDefaults | None = Field( + None, + alias="TaskDefaults", + description="Defaults for creating tasks in this cluster.", ) @@ -1694,30 +1776,30 @@ class ClusterInfo(BaseModel): """ ID: str | None = Field( - None, description="The ID of the swarm.", example="abajmipo7b4xz5ip2nrla6b11" + None, description="The ID of the swarm.", examples=["abajmipo7b4xz5ip2nrla6b11"] ) Version: ObjectVersion | None = None CreatedAt: str | None = Field( None, description="Date and time at which the swarm was initialised in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2016-08-18T10:44:24.496525531Z", + examples=["2016-08-18T10:44:24.496525531Z"], ) UpdatedAt: str | None = Field( None, description="Date and time at which the swarm was last updated in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2017-08-09T07:09:37.632105588Z", + examples=["2017-08-09T07:09:37.632105588Z"], ) Spec: SwarmSpec | None = None - TLSInfo: TLSInfo | None = None + TLSInfo_: TLSInfo | None = Field(None, alias="TLSInfo") RootRotationInProgress: bool | None = Field( None, description="Whether there is currently a root CA rotation in progress for the swarm\n", - example=False, + examples=[False], ) DataPathPort: int | None = Field( 4789, description="DataPathPort specifies the data path port number for data traffic.\nAcceptable port range is 1024 to 49151.\nIf no port is set or is set to 0, the default port (4789) is used.\n", - example=4789, + examples=[4789], ) DefaultAddrPool: list[str] | None = Field( None, @@ -1726,7 +1808,7 @@ class ClusterInfo(BaseModel): SubnetSize: int | None = Field( 24, description="SubnetSize specifies the subnet size of the networks created from the\ndefault subnet pool.\n", - example=24, + examples=[24], le=29, ) @@ -1740,17 +1822,21 @@ class JoinTokens(BaseModel): Worker: str | None = Field( None, description="The token workers can use to join the swarm.\n", - example="SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx", + examples=[ + "SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx" + ], ) Manager: str | None = Field( None, description="The token managers can use to join the swarm.\n", - example="SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-7p73s1dx5in4tatdymyhg9hu2", + examples=[ + "SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-7p73s1dx5in4tatdymyhg9hu2" + ], ) class Swarm(ClusterInfo): - JoinTokens: JoinTokens | None = None + JoinTokens_: JoinTokens | None = Field(None, alias="JoinTokens") class PluginSpec(BaseModel): @@ -1785,12 +1871,12 @@ class CredentialSpec(BaseModel): None, alias="Config", description="Load credential spec from a Swarm Config with the given ID.\nThe specified config must also be present in the Configs\nfield with the Runtime property set.\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", - example="0bt9dmxjvjiqermk6xrop3ekq", + examples=["0bt9dmxjvjiqermk6xrop3ekq"], ) File: str | None = Field( None, description="Load credential spec from this file. The file is read by\nthe daemon, and must be present in the `CredentialSpecs`\nsubdirectory in the docker data directory, which defaults\nto `C:\\ProgramData\\Docker\\` on Windows.\n\nFor example, specifying `spec.json` loads\n`C:\\ProgramData\\Docker\\CredentialSpecs\\spec.json`.\n\n


\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", - example="spec.json", + examples=["spec.json"], ) Registry: str | None = Field( None, @@ -1815,11 +1901,13 @@ class Privileges(BaseModel): Security options for the container """ - CredentialSpec: CredentialSpec | None = Field( - None, description="CredentialSpec for managed service account (Windows only)" + CredentialSpec_: CredentialSpec | None = Field( + None, + alias="CredentialSpec", + description="CredentialSpec for managed service account (Windows only)", ) - SELinuxContext: SELinuxContext | None = Field( - None, description="SELinux labels of the container" + SELinuxContext_: SELinuxContext | None = Field( + None, alias="SELinuxContext", description="SELinux labels of the container" ) @@ -1859,8 +1947,9 @@ class File(BaseModel): class Secret(BaseModel): - File: File | None = Field( + File_: File | None = Field( None, + alias="File", description="File represents a specific target that is backed by a file.\n", ) SecretID: str | None = Field( @@ -1945,8 +2034,8 @@ class ContainerSpec(BaseModel): None, description="A list of additional groups that the container process will run as.\n", ) - Privileges: Privileges | None = Field( - None, description="Security options for the container" + Privileges_: Privileges | None = Field( + None, alias="Privileges", description="Security options for the container" ) TTY: bool | None = Field( None, description="Whether a pseudo-TTY should be allocated." @@ -1969,8 +2058,9 @@ class ContainerSpec(BaseModel): None, description="A list of hostname/IP mappings to add to the container's `hosts`\nfile. The format of extra hosts is specified in the\n[hosts(5)](http://man7.org/linux/man-pages/man5/hosts.5.html)\nman page:\n\n IP_address canonical_hostname [aliases...]\n", ) - DNSConfig: DNSConfig | None = Field( + DNSConfig_: DNSConfig | None = Field( None, + alias="DNSConfig", description="Specification for DNS related configurations in resolver configuration\nfile (`resolv.conf`).\n", ) Secrets: list[Secret] | None = Field( @@ -1981,8 +2071,9 @@ class ContainerSpec(BaseModel): None, description="Configs contains references to zero or more configs that will be\nexposed to the service.\n", ) - Isolation: Isolation | None = Field( + Isolation_: Isolation | None = Field( None, + alias="Isolation", description="Isolation technology of the containers running the service.\n(Windows only)\n", ) Init: bool | None = Field( @@ -1996,12 +2087,12 @@ class ContainerSpec(BaseModel): CapabilityAdd: list[str] | None = Field( None, description="A list of kernel capabilities to add to the default set\nfor the container.\n", - example=["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_CHROOT", "CAP_SYSLOG"], + examples=[["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_CHROOT", "CAP_SYSLOG"]], ) CapabilityDrop: list[str] | None = Field( None, description="A list of kernel capabilities to drop from the default set\nfor the container.\n", - example=["CAP_NET_RAW"], + examples=[["CAP_NET_RAW"]], ) Ulimits: list[Ulimit1] | None = Field( None, @@ -2045,7 +2136,9 @@ class RestartPolicy1(BaseModel): """ - Condition: Condition | None = Field(None, description="Condition for restart.") + Condition_: Condition | None = Field( + None, alias="Condition", description="Condition for restart." + ) Delay: int | None = Field(None, description="Delay between restart attempts.") MaxAttempts: int | None = Field( 0, @@ -2064,27 +2157,31 @@ class Spread(BaseModel): class Preference(BaseModel): - Spread: Spread | None = None + Spread_: Spread | None = Field(None, alias="Spread") class Placement(BaseModel): Constraints: list[str] | None = Field( None, description="An array of constraint expressions to limit the set of nodes where\na task can be scheduled. Constraint expressions can either use a\n_match_ (`==`) or _exclude_ (`!=`) rule. Multiple constraints find\nnodes that satisfy every expression (AND match). Constraints can\nmatch node or Docker Engine labels as follows:\n\nnode attribute | matches | example\n---------------------|--------------------------------|-----------------------------------------------\n`node.id` | Node ID | `node.id==2ivku8v2gvtg4`\n`node.hostname` | Node hostname | `node.hostname!=node-2`\n`node.role` | Node role (`manager`/`worker`) | `node.role==manager`\n`node.platform.os` | Node operating system | `node.platform.os==windows`\n`node.platform.arch` | Node architecture | `node.platform.arch==x86_64`\n`node.labels` | User-defined node labels | `node.labels.security==high`\n`engine.labels` | Docker Engine's labels | `engine.labels.operatingsystem==ubuntu-14.04`\n\n`engine.labels` apply to Docker Engine labels like operating system,\ndrivers, etc. Swarm administrators add `node.labels` for operational\npurposes by using the [`node update endpoint`](#operation/NodeUpdate).\n", - example=[ - "node.hostname!=node3.corp.example.com", - "node.role!=manager", - "node.labels.type==production", - "node.platform.os==linux", - "node.platform.arch==x86_64", + examples=[ + [ + "node.hostname!=node3.corp.example.com", + "node.role!=manager", + "node.labels.type==production", + "node.platform.os==linux", + "node.platform.arch==x86_64", + ] ], ) Preferences: list[Preference] | None = Field( None, description="Preferences provide a way to make the scheduler aware of factors\nsuch as topology. They are provided in order from highest to\nlowest precedence.\n", - example=[ - {"Spread": {"SpreadDescriptor": "node.labels.datacenter"}}, - {"Spread": {"SpreadDescriptor": "node.labels.rack"}}, + examples=[ + [ + {"Spread": {"SpreadDescriptor": "node.labels.datacenter"}}, + {"Spread": {"SpreadDescriptor": "node.labels.rack"}}, + ] ], ) MaxReplicas: int | None = Field( @@ -2138,7 +2235,7 @@ class Status1(BaseModel): State: TaskState | None = None Message: str | None = None Err: str | None = None - ContainerStatus: ContainerStatus | None = None + ContainerStatus_: ContainerStatus | None = Field(None, alias="ContainerStatus") class Replicated(BaseModel): @@ -2166,10 +2263,11 @@ class Mode(BaseModel): Scheduling mode for the service. """ - Replicated: Replicated | None = None + Replicated_: Replicated | None = Field(None, alias="Replicated") Global: dict[str, Any] | None = None - ReplicatedJob: ReplicatedJob | None = Field( + ReplicatedJob_: ReplicatedJob | None = Field( None, + alias="ReplicatedJob", description="The mode used for services with a finite number of tasks that run\nto a completed state.\n", ) GlobalJob: dict[str, Any] | None = Field( @@ -2214,8 +2312,9 @@ class UpdateConfig(BaseModel): Delay: int | None = Field( None, description="Amount of time between updates, in nanoseconds." ) - FailureAction: FailureAction | None = Field( + FailureAction_: FailureAction | None = Field( None, + alias="FailureAction", description="Action to take if an updated task fails to run, or stops running\nduring the update.\n", ) Monitor: int | None = Field( @@ -2226,8 +2325,9 @@ class UpdateConfig(BaseModel): 0, description="The fraction of tasks that may fail during an update before the\nfailure action is invoked, specified as a floating point number\nbetween 0 and 1.\n", ) - Order: Order | None = Field( + Order_: Order | None = Field( None, + alias="Order", description="The order of operations when rolling out an updated task. Either\nthe old task is shut down before the new task is started, or the\nnew task is started before the old task is shut down.\n", ) @@ -2268,8 +2368,9 @@ class RollbackConfig(BaseModel): 0, description="The fraction of tasks that may fail during a rollback before the\nfailure action is invoked, specified as a floating point number\nbetween 0 and 1.\n", ) - Order: Order | None = Field( + Order_: Order | None = Field( None, + alias="Order", description="The order of operations when rolling back a task. Either the old\ntask is shut down before the new task is started, or the new task\nis started before the old task is shut down.\n", ) @@ -2297,10 +2398,11 @@ class EndpointPortConfig(BaseModel): Protocol: Type | None = None TargetPort: int | None = Field(None, description="The port inside the container.") PublishedPort: int | None = Field(None, description="The port on the swarm hosts.") - PublishMode: PublishMode | None = Field( + PublishMode_: PublishMode | None = Field( PublishMode.ingress, + alias="PublishMode", description='The mode in which port is published.\n\n


\n\n- "ingress" makes the target port accessible on every node,\n regardless of whether there is a task for the service running on\n that node or not.\n- "host" bypasses the routing mesh and publish the port directly on\n the swarm node where that service is running.\n', - example="ingress", + examples=["ingress"], ) @@ -2351,7 +2453,7 @@ class UpdateStatus(BaseModel): The status of a service update. """ - State: State | None = None + State_: State | None = Field(None, alias="State") StartedAt: str | None = None CompletedAt: str | None = None Message: str | None = None @@ -2367,12 +2469,12 @@ class ServiceStatus(BaseModel): RunningTasks: int | None = Field( None, description="The number of tasks for the service currently in the Running state.\n", - example=7, + examples=[7], ) DesiredTasks: int | None = Field( None, description="The number of tasks for the service desired to be running.\nFor replicated services, this is the replica count from the\nservice spec. For global services, this is computed by taking\ncount of all tasks for the service with a Desired State other\nthan Shutdown.\n", - example=10, + examples=[10], ) CompletedTasks: int | None = Field( None, @@ -2425,10 +2527,12 @@ class Driver(BaseModel): options: dict[str, str] | None = Field( None, description="Key/value map of driver-specific options.", - example={ - "OptionA": "value for driver-specific option A", - "OptionB": "value for driver-specific option B", - }, + examples=[ + { + "OptionA": "value for driver-specific option A", + "OptionB": "value for driver-specific option B", + } + ], ) @@ -2437,15 +2541,17 @@ class SecretSpec(BaseModel): labels: dict[str, str] | None = Field( None, description="User-defined key/value metadata.", - example={ - "com.example.some-label": "some-value", - "com.example.some-other-label": "some-other-value", - }, + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], ) data: str | None = Field( None, description="Base64-url-safe-encoded ([RFC 4648](https://tools.ietf.org/html/rfc4648#section-5))\ndata to store as secret.\n\nThis field is only used to _create_ a secret, and is not returned by\nother endpoints.\n", - example="", + examples=[""], ) driver: Driver | None = Field( None, @@ -2458,10 +2564,10 @@ class SecretSpec(BaseModel): class Secret1(BaseModel): - ID: str | None = Field(None, example="blt1owaxmitz71s9v5zh81zun") + ID: str | None = Field(None, examples=["blt1owaxmitz71s9v5zh81zun"]) Version: ObjectVersion | None = None - CreatedAt: str | None = Field(None, example="2017-07-20T13:55:28.678958722Z") - UpdatedAt: str | None = Field(None, example="2017-07-20T13:55:28.678958722Z") + CreatedAt: str | None = Field(None, examples=["2017-07-20T13:55:28.678958722Z"]) + UpdatedAt: str | None = Field(None, examples=["2017-07-20T13:55:28.678958722Z"]) Spec: SecretSpec | None = None @@ -2517,9 +2623,9 @@ class Platform1(BaseModel): class Component(BaseModel): - Name: str = Field(..., description="Name of the component\n", example="Engine") + Name: str = Field(..., description="Name of the component\n", examples=["Engine"]) Version: str = Field( - ..., description="Version of the component\n", example="19.03.12" + ..., description="Version of the component\n", examples=["19.03.12"] ) Details: dict[str, Any] | None = Field( None, @@ -2538,52 +2644,52 @@ class SystemVersion(BaseModel): None, description="Information about system components\n" ) Version: str | None = Field( - None, description="The version of the daemon", example="19.03.12" + None, description="The version of the daemon", examples=["19.03.12"] ) ApiVersion: str | None = Field( None, description="The default (and highest) API version that is supported by the daemon\n", - example="1.40", + examples=["1.40"], ) MinAPIVersion: str | None = Field( None, description="The minimum API version that is supported by the daemon\n", - example="1.12", + examples=["1.12"], ) GitCommit: str | None = Field( None, description="The Git commit of the source code that was used to build the daemon\n", - example="48a66213fe", + examples=["48a66213fe"], ) GoVersion: str | None = Field( None, description="The version Go used to compile the daemon, and the version of the Go\nruntime in use.\n", - example="go1.13.14", + examples=["go1.13.14"], ) Os: str | None = Field( None, description='The operating system that the daemon is running on ("linux" or "windows")\n', - example="linux", + examples=["linux"], ) Arch: str | None = Field( None, description="The architecture that the daemon is running on\n", - example="amd64", + examples=["amd64"], ) KernelVersion: str | None = Field( None, description="The kernel version (`uname -r`) that the daemon is running on.\n\nThis field is omitted when empty.\n", - example="4.19.76-linuxkit", + examples=["4.19.76-linuxkit"], ) Experimental: bool | None = Field( None, description="Indicates if the daemon is started with experimental features enabled.\n\nThis field is omitted when empty / false.\n", - example=True, + examples=[True], ) BuildTime: str | None = Field( None, description="The date and time that the daemon was compiled.\n", - example="2020-06-22T15:49:27.000000000+00:00", + examples=["2020-06-22T15:49:27.000000000+00:00"], ) @@ -2627,9 +2733,11 @@ class Isolation2(str, Enum): class DefaultAddressPool(BaseModel): Base: str | None = Field( - None, description="The network address in CIDR format", example="10.10.0.0/16" + None, + description="The network address in CIDR format", + examples=["10.10.0.0/16"], ) - Size: int | None = Field(None, description="The network pool size", example="24") + Size: int | None = Field(None, description="The network pool size", examples=["24"]) class PluginsInfo(BaseModel): @@ -2647,31 +2755,33 @@ class PluginsInfo(BaseModel): Volume: list[str] | None = Field( None, description="Names of available volume-drivers, and network-driver plugins.", - example=["local"], + examples=[["local"]], ) Network: list[str] | None = Field( None, description="Names of available network-drivers, and network-driver plugins.", - example=["bridge", "host", "ipvlan", "macvlan", "null", "overlay"], + examples=[["bridge", "host", "ipvlan", "macvlan", "null", "overlay"]], ) Authorization: list[str] | None = Field( None, description="Names of available authorization plugins.", - example=["img-authz-plugin", "hbm"], + examples=[["img-authz-plugin", "hbm"]], ) Log: list[str] | None = Field( None, description="Names of available logging-drivers, and logging-driver plugins.", - example=[ - "awslogs", - "fluentd", - "gcplogs", - "gelf", - "journald", - "json-file", - "logentries", - "splunk", - "syslog", + examples=[ + [ + "awslogs", + "fluentd", + "gcplogs", + "gelf", + "journald", + "json-file", + "logentries", + "splunk", + "syslog", + ] ], ) @@ -2684,26 +2794,28 @@ class IndexInfo(BaseModel): Name: str | None = Field( None, description='Name of the registry, such as "docker.io".\n', - example="docker.io", + examples=["docker.io"], ) Mirrors: list[str] | None = Field( None, description="List of mirrors, expressed as URIs.\n", - example=[ - "https://hub-mirror.corp.example.com:5000/", - "https://registry-2.docker.io/", - "https://registry-3.docker.io/", + examples=[ + [ + "https://hub-mirror.corp.example.com:5000/", + "https://registry-2.docker.io/", + "https://registry-3.docker.io/", + ] ], ) Secure: bool | None = Field( None, description="Indicates if the registry is part of the list of insecure\nregistries.\n\nIf `false`, the registry is insecure. Insecure registries accept\nun-encrypted (HTTP) and/or untrusted (HTTPS with certificates from\nunknown CAs) communication.\n\n> **Warning**: Insecure registries can be useful when running a local\n> registry. However, because its use creates security vulnerabilities\n> it should ONLY be enabled for testing purposes. For increased\n> security, users should add their CA to their system's list of\n> trusted CAs instead of enabling this option.\n", - example=True, + examples=[True], ) Official: bool | None = Field( None, description="Indicates whether this is an official registry (i.e., Docker Hub / docker.io)\n", - example=True, + examples=[True], ) @@ -2721,12 +2833,12 @@ class Runtime(BaseModel): path: str | None = Field( None, description="Name and, optional, path, of the OCI executable binary.\n\nIf the path is omitted, the daemon searches the host's `$PATH` for the\nbinary and uses the first result.\n", - example="/usr/local/bin/my-oci-runtime", + examples=["/usr/local/bin/my-oci-runtime"], ) runtimeArgs: list[str] | None = Field( None, description="List of command-line arguments to pass to the runtime when invoked.\n", - example=["--debug", "--systemd-cgroup=false"], + examples=[["--debug", "--systemd-cgroup=false"]], ) @@ -2741,12 +2853,12 @@ class Commit(BaseModel): ID: str | None = Field( None, description="Actual commit ID of external tool.", - example="cfb82a876ecc11b5ca0977d1733adbe58599088a", + examples=["cfb82a876ecc11b5ca0977d1733adbe58599088a"], ) Expected: str | None = Field( None, description="Commit ID of external tool expected by dockerd as set at build time.\n", - example="2d41c047c83e09a6d61d464906feb2a2f3c52aa4", + examples=["2d41c047c83e09a6d61d464906feb2a2f3c52aa4"], ) @@ -2805,16 +2917,18 @@ class EventActor(BaseModel): ID: str | None = Field( None, description="The ID of the object emitting the event", - example="ede54ee1afda366ab42f824e8a5ffd195155d853ceaec74a927f249ea270c743", + examples=["ede54ee1afda366ab42f824e8a5ffd195155d853ceaec74a927f249ea270c743"], ) Attributes: dict[str, str] | None = Field( None, description="Various key/value attributes of the object, depending on its type.\n", - example={ - "com.example.some-label": "some-label-value", - "image": "alpine:latest", - "name": "my-container", - }, + examples=[ + { + "com.example.some-label": "some-label-value", + "image": "alpine:latest", + "name": "my-container", + } + ], ) @@ -2854,19 +2968,25 @@ class SystemEventsResponse(BaseModel): """ Type: Type5 | None = Field( - None, description="The type of object emitting the event", example="container" + None, + description="The type of object emitting the event", + examples=["container"], + ) + Action: str | None = Field( + None, description="The type of event", examples=["create"] ) - Action: str | None = Field(None, description="The type of event", example="create") Actor: EventActor | None = None scope: Scope1 | None = Field( None, description="Scope of the event. Engine events are `local` scope. Cluster (Swarm)\nevents are `swarm` scope.\n", ) - time: int | None = Field(None, description="Timestamp of event", example=1629574695) + time: int | None = Field( + None, description="Timestamp of event", examples=[1629574695] + ) timeNano: int | None = Field( None, description="Timestamp of event, with nanosecond accuracy", - example=1629574695515050031, + examples=[1629574695515050031], ) @@ -2880,15 +3000,17 @@ class OCIDescriptor(BaseModel): mediaType: str | None = Field( None, description="The media type of the object this schema refers to.\n", - example="application/vnd.docker.distribution.manifest.v2+json", + examples=["application/vnd.docker.distribution.manifest.v2+json"], ) digest: str | None = Field( None, description="The digest of the targeted content.\n", - example="sha256:c0537ff6a5218ef531ece93d4984efc99bbf3f7497c0a7726c88e2bb7584dc96", + examples=[ + "sha256:c0537ff6a5218ef531ece93d4984efc99bbf3f7497c0a7726c88e2bb7584dc96" + ], ) size: int | None = Field( - None, description="The size in bytes of the blob.\n", example=3987495 + None, description="The size in bytes of the blob.\n", examples=[3987495] ) @@ -2902,29 +3024,29 @@ class OCIPlatform(BaseModel): architecture: str | None = Field( None, description="The CPU architecture, for example `amd64` or `ppc64`.\n", - example="arm", + examples=["arm"], ) os: str | None = Field( None, description="The operating system, for example `linux` or `windows`.\n", - example="windows", + examples=["windows"], ) os_version: str | None = Field( None, alias="os.version", description="Optional field specifying the operating system version, for example on\nWindows `10.0.19041.1165`.\n", - example="10.0.19041.1165", + examples=["10.0.19041.1165"], ) os_features: list[str] | None = Field( None, alias="os.features", description="Optional field specifying an array of strings, each listing a required\nOS feature (for example on Windows `win32k`).\n", - example=["win32k"], + examples=[["win32k"]], ) variant: str | None = Field( None, description="Optional field specifying a variant of the CPU, for example `v7` to\nspecify ARMv7 when architecture is `arm`.\n", - example="v7", + examples=["v7"], ) @@ -2948,9 +3070,9 @@ class ResourceObject(BaseModel): """ - NanoCPUs: int | None = Field(None, example=4000000000) - MemoryBytes: int | None = Field(None, example=8272408576) - GenericResources: GenericResources | None = None + NanoCPUs: int | None = Field(None, examples=[4000000000]) + MemoryBytes: int | None = Field(None, examples=[8272408576]) + GenericResources_: GenericResources | None = Field(None, alias="GenericResources") class Health(BaseModel): @@ -2959,15 +3081,16 @@ class Health(BaseModel): """ - Status: Status | None = Field( + Status_: Status | None = Field( None, + alias="Status", description='Status is one of `none`, `starting`, `healthy` or `unhealthy`\n\n- "none" Indicates there is no healthcheck\n- "starting" Starting indicates that the container is not yet ready\n- "healthy" Healthy indicates that the container is running correctly\n- "unhealthy" Unhealthy indicates that the container has a problem\n', - example="healthy", + examples=["healthy"], ) FailingStreak: int | None = Field( None, description="FailingStreak is the number of consecutive failures", - example=0, + examples=[0], ) Log: list[HealthcheckResult] | None = Field( None, description="Log contains the last few results (oldest first)\n" @@ -2986,15 +3109,17 @@ class HostConfig(Resources): ContainerIDFile: str | None = Field( None, description="Path to a file where the container ID is written" ) - LogConfig: LogConfig | None = Field( - None, description="The logging configuration for this container" + LogConfig_: LogConfig | None = Field( + None, + alias="LogConfig", + description="The logging configuration for this container", ) NetworkMode: str | None = Field( None, description="Network mode to use for this container. Supported standard values\nare: `bridge`, `host`, `none`, and `container:`. Any\nother value is taken as a custom network's name to which this\ncontainer should connect to.\n", ) PortBindings: PortMap | None = None - RestartPolicy: RestartPolicy | None = None + RestartPolicy_: RestartPolicy | None = Field(None, alias="RestartPolicy") AutoRemove: bool | None = Field( None, description="Automatically remove the container when the container's process\nexits. This has no effect if `RestartPolicy` is set.\n", @@ -3017,8 +3142,9 @@ class HostConfig(Resources): None, description="A list of kernel capabilities to drop from the container. Conflicts\nwith option 'Capabilities'.\n", ) - CgroupnsMode: CgroupnsMode | None = Field( + CgroupnsMode_: CgroupnsMode | None = Field( None, + alias="CgroupnsMode", description='cgroup namespace mode for the container. Possible values are:\n\n- `"private"`: the container runs in its own private cgroup namespace\n- `"host"`: use the host system\'s cgroup namespace\n\nIf not specified, the daemon default is used, which can either be `"private"`\nor `"host"`, depending on daemon version, kernel support and configuration.\n', ) Dns: list[str] | None = Field( @@ -3048,7 +3174,7 @@ class HostConfig(Resources): OomScoreAdj: int | None = Field( None, description="An integer value containing the score given to the container in\norder to tune OOM killer preferences.\n", - example=500, + examples=[500], ) PidMode: str | None = Field( None, @@ -3096,11 +3222,13 @@ class HostConfig(Resources): ConsoleSize: list[ConsoleSizeItem] | None = Field( None, description="Initial console size, as an `[height, width]` array. (Windows only)\n", - max_items=2, - min_items=2, + max_length=2, + min_length=2, ) - Isolation: Isolation | None = Field( - None, description="Isolation technology of the container. (Windows only)\n" + Isolation_: Isolation | None = Field( + None, + alias="Isolation", + description="Isolation technology of the container. (Windows only)\n", ) MaskedPaths: list[str] | None = Field( None, @@ -3156,48 +3284,50 @@ class EndpointSettings(BaseModel): """ IPAMConfig: EndpointIPAMConfig | None = None - Links: list[str] | None = Field(None, example=["container_1", "container_2"]) - Aliases: list[str] | None = Field(None, example=["server_x", "server_y"]) + Links: list[str] | None = Field(None, examples=[["container_1", "container_2"]]) + Aliases: list[str] | None = Field(None, examples=[["server_x", "server_y"]]) NetworkID: str | None = Field( None, description="Unique ID of the network.\n", - example="08754567f1f40222263eab4102e1c733ae697e8e354aa9cd6e18d7402835292a", + examples=["08754567f1f40222263eab4102e1c733ae697e8e354aa9cd6e18d7402835292a"], ) EndpointID: str | None = Field( None, description="Unique ID for the service endpoint in a Sandbox.\n", - example="b88f5b905aabf2893f3cbc4ee42d1ea7980bbc0a92e2c8922b1e1795298afb0b", + examples=["b88f5b905aabf2893f3cbc4ee42d1ea7980bbc0a92e2c8922b1e1795298afb0b"], ) Gateway: str | None = Field( - None, description="Gateway address for this network.\n", example="172.17.0.1" + None, description="Gateway address for this network.\n", examples=["172.17.0.1"] ) IPAddress: str | None = Field( - None, description="IPv4 address.\n", example="172.17.0.4" + None, description="IPv4 address.\n", examples=["172.17.0.4"] ) IPPrefixLen: int | None = Field( - None, description="Mask length of the IPv4 address.\n", example=16 + None, description="Mask length of the IPv4 address.\n", examples=[16] ) IPv6Gateway: str | None = Field( - None, description="IPv6 gateway address.\n", example="2001:db8:2::100" + None, description="IPv6 gateway address.\n", examples=["2001:db8:2::100"] ) GlobalIPv6Address: str | None = Field( - None, description="Global IPv6 address.\n", example="2001:db8::5689" + None, description="Global IPv6 address.\n", examples=["2001:db8::5689"] ) GlobalIPv6PrefixLen: int | None = Field( - None, description="Mask length of the global IPv6 address.\n", example=64 + None, description="Mask length of the global IPv6 address.\n", examples=[64] ) MacAddress: str | None = Field( None, description="MAC address for the endpoint on this network.\n", - example="02:42:ac:11:00:04", + examples=["02:42:ac:11:00:04"], ) DriverOpts: dict[str, str] | None = Field( None, description="DriverOpts is a mapping of driver options and values. These options\nare passed directly to the driver and are driver specific.\n", - example={ - "com.example.some-label": "some-value", - "com.example.some-other-label": "some-other-value", - }, + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], ) @@ -3208,11 +3338,11 @@ class NodeDescription(BaseModel): """ - Hostname: str | None = Field(None, example="bf3067039e47") - Platform: Platform | None = None + Hostname: str | None = Field(None, examples=["bf3067039e47"]) + Platform_: Platform | None = Field(None, alias="Platform") Resources: ResourceObject | None = None Engine: EngineDescription | None = None - TLSInfo: TLSInfo | None = None + TLSInfo_: TLSInfo | None = Field(None, alias="TLSInfo") class NodeStatus(BaseModel): @@ -3224,9 +3354,9 @@ class NodeStatus(BaseModel): """ State: NodeState | None = None - Message: str | None = Field(None, example="") + Message: str | None = Field(None, examples=[""]) Addr: str | None = Field( - None, description="IP address of the node.", example="172.17.0.2" + None, description="IP address of the node.", examples=["172.17.0.2"] ) @@ -3239,12 +3369,12 @@ class ManagerStatus(BaseModel): """ - Leader: bool | None = Field(False, example=True) - Reachability: Reachability | None = None + Leader: bool | None = Field(False, examples=[True]) + Reachability_: Reachability | None = Field(None, alias="Reachability") Addr: str | None = Field( None, description="The IP address and port at which the manager is reachable.\n", - example="10.0.0.46:2377", + examples=["10.0.0.46:2377"], ) @@ -3266,16 +3396,19 @@ class TaskSpec(BaseModel): User modifiable task configuration. """ - PluginSpec: PluginSpec | None = Field( + PluginSpec_: PluginSpec | None = Field( None, + alias="PluginSpec", description="Plugin spec for the service. *(Experimental release only.)*\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n", ) - ContainerSpec: ContainerSpec | None = Field( + ContainerSpec_: ContainerSpec | None = Field( None, + alias="ContainerSpec", description="Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n", ) - NetworkAttachmentSpec: NetworkAttachmentSpec | None = Field( + NetworkAttachmentSpec_: NetworkAttachmentSpec | None = Field( None, + alias="NetworkAttachmentSpec", description="Read-only spec type for non-swarm containers attached to swarm overlay\nnetworks.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n", ) Resources: Resources1 | None = Field( @@ -3286,7 +3419,7 @@ class TaskSpec(BaseModel): None, description="Specification for the restart policy which applies to containers\ncreated as part of this service.\n", ) - Placement: Placement | None = None + Placement_: Placement | None = Field(None, alias="Placement") ForceUpdate: int | None = Field( None, description="A counter that triggers an update even if no relevant parameters have\nbeen changed.\n", @@ -3340,17 +3473,23 @@ class ServiceSpec(BaseModel): None, description="User-defined key/value metadata." ) TaskTemplate: TaskSpec | None = None - Mode: Mode | None = Field(None, description="Scheduling mode for the service.") - UpdateConfig: UpdateConfig | None = Field( - None, description="Specification for the update strategy of the service." + Mode_: Mode | None = Field( + None, alias="Mode", description="Scheduling mode for the service." ) - RollbackConfig: RollbackConfig | None = Field( - None, description="Specification for the rollback strategy of the service." + UpdateConfig_: UpdateConfig | None = Field( + None, + alias="UpdateConfig", + description="Specification for the update strategy of the service.", + ) + RollbackConfig_: RollbackConfig | None = Field( + None, + alias="RollbackConfig", + description="Specification for the rollback strategy of the service.", ) Networks: list[NetworkAttachmentConfig] | None = Field( None, description="Specifies which networks the service should attach to." ) - EndpointSpec: EndpointSpec | None = None + EndpointSpec_: EndpointSpec | None = Field(None, alias="EndpointSpec") class Service(BaseModel): @@ -3359,16 +3498,18 @@ class Service(BaseModel): CreatedAt: str | None = None UpdatedAt: str | None = None Spec: ServiceSpec | None = None - Endpoint: Endpoint | None = None - UpdateStatus: UpdateStatus | None = Field( - None, description="The status of a service update." + Endpoint_: Endpoint | None = Field(None, alias="Endpoint") + UpdateStatus_: UpdateStatus | None = Field( + None, alias="UpdateStatus", description="The status of a service update." ) - ServiceStatus: ServiceStatus | None = Field( + ServiceStatus_: ServiceStatus | None = Field( None, + alias="ServiceStatus", description="The status of the service's tasks. Provided only when requested as\npart of a ServiceList operation.\n", ) - JobStatus: JobStatus | None = Field( + JobStatus_: JobStatus | None = Field( None, + alias="JobStatus", description="The status of the service when it is in one of ReplicatedJob or\nGlobalJob modes. Absent on Replicated and Global mode services. The\nJobIteration is an ObjectVersion, but unlike the Service's version,\ndoes not need to be sent with an update request.\n", ) @@ -3433,43 +3574,43 @@ class ContainerState(BaseModel): Status: Status2 | None = Field( None, description='String representation of the container state. Can be one of "created",\n"running", "paused", "restarting", "removing", "exited", or "dead".\n', - example="running", + examples=["running"], ) Running: bool | None = Field( None, description='Whether this container is running.\n\nNote that a running container can be _paused_. The `Running` and `Paused`\nbooleans are not mutually exclusive:\n\nWhen pausing a container (on Linux), the freezer cgroup is used to suspend\nall processes in the container. Freezing the process requires the process to\nbe running. As a result, paused containers are both `Running` _and_ `Paused`.\n\nUse the `Status` field instead to determine if a container\'s state is "running".\n', - example=True, + examples=[True], ) Paused: bool | None = Field( - None, description="Whether this container is paused.", example=False + None, description="Whether this container is paused.", examples=[False] ) Restarting: bool | None = Field( - None, description="Whether this container is restarting.", example=False + None, description="Whether this container is restarting.", examples=[False] ) OOMKilled: bool | None = Field( None, description="Whether this container has been killed because it ran out of memory.\n", - example=False, + examples=[False], ) - Dead: bool | None = Field(None, example=False) + Dead: bool | None = Field(None, examples=[False]) Pid: int | None = Field( - None, description="The process ID of this container", example=1234 + None, description="The process ID of this container", examples=[1234] ) ExitCode: int | None = Field( - None, description="The last exit code of this container", example=0 + None, description="The last exit code of this container", examples=[0] ) Error: str | None = None StartedAt: str | None = Field( None, description="The time when this container was last started.", - example="2020-01-06T09:06:59.461876391Z", + examples=["2020-01-06T09:06:59.461876391Z"], ) FinishedAt: str | None = Field( None, description="The time when this container last exited.", - example="2020-01-06T09:07:59.461876391Z", + examples=["2020-01-06T09:07:59.461876391Z"], ) - Health: Health | None = None + Health_: Health | None = Field(None, alias="Health") class ContainerWaitResponse(BaseModel): @@ -3490,56 +3631,62 @@ class RegistryServiceConfig(BaseModel): AllowNondistributableArtifactsCIDRs: list[str] | None = Field( None, description="List of IP ranges to which nondistributable artifacts can be pushed,\nusing the CIDR syntax [RFC 4632](https://tools.ietf.org/html/4632).\n\nSome images (for example, Windows base images) contain artifacts\nwhose distribution is restricted by license. When these images are\npushed to a registry, restricted artifacts are not included.\n\nThis configuration override this behavior, and enables the daemon to\npush nondistributable artifacts to all registries whose resolved IP\naddress is within the subnet described by the CIDR syntax.\n\nThis option is useful when pushing images containing\nnondistributable artifacts to a registry on an air-gapped network so\nhosts on that network can pull the images without connecting to\nanother server.\n\n> **Warning**: Nondistributable artifacts typically have restrictions\n> on how and where they can be distributed and shared. Only use this\n> feature to push artifacts to private registries and ensure that you\n> are in compliance with any terms that cover redistributing\n> nondistributable artifacts.\n", - example=["::1/128", "127.0.0.0/8"], + examples=[["::1/128", "127.0.0.0/8"]], ) AllowNondistributableArtifactsHostnames: list[str] | None = Field( None, description="List of registry hostnames to which nondistributable artifacts can be\npushed, using the format `[:]` or `[:]`.\n\nSome images (for example, Windows base images) contain artifacts\nwhose distribution is restricted by license. When these images are\npushed to a registry, restricted artifacts are not included.\n\nThis configuration override this behavior for the specified\nregistries.\n\nThis option is useful when pushing images containing\nnondistributable artifacts to a registry on an air-gapped network so\nhosts on that network can pull the images without connecting to\nanother server.\n\n> **Warning**: Nondistributable artifacts typically have restrictions\n> on how and where they can be distributed and shared. Only use this\n> feature to push artifacts to private registries and ensure that you\n> are in compliance with any terms that cover redistributing\n> nondistributable artifacts.\n", - example=[ - "registry.internal.corp.example.com:3000", - "[2001:db8:a0b:12f0::1]:443", + examples=[ + [ + "registry.internal.corp.example.com:3000", + "[2001:db8:a0b:12f0::1]:443", + ] ], ) InsecureRegistryCIDRs: list[str] | None = Field( None, description="List of IP ranges of insecure registries, using the CIDR syntax\n([RFC 4632](https://tools.ietf.org/html/4632)). Insecure registries\naccept un-encrypted (HTTP) and/or untrusted (HTTPS with certificates\nfrom unknown CAs) communication.\n\nBy default, local registries (`127.0.0.0/8`) are configured as\ninsecure. All other registries are secure. Communicating with an\ninsecure registry is not possible if the daemon assumes that registry\nis secure.\n\nThis configuration override this behavior, insecure communication with\nregistries whose resolved IP address is within the subnet described by\nthe CIDR syntax.\n\nRegistries can also be marked insecure by hostname. Those registries\nare listed under `IndexConfigs` and have their `Secure` field set to\n`false`.\n\n> **Warning**: Using this option can be useful when running a local\n> registry, but introduces security vulnerabilities. This option\n> should therefore ONLY be used for testing purposes. For increased\n> security, users should add their CA to their system's list of trusted\n> CAs instead of enabling this option.\n", - example=["::1/128", "127.0.0.0/8"], + examples=[["::1/128", "127.0.0.0/8"]], ) IndexConfigs: dict[str, IndexInfo] | None = Field( None, - example={ - "127.0.0.1:5000": { - "Name": "127.0.0.1:5000", - "Mirrors": [], - "Secure": False, - "Official": False, - }, - "[2001:db8:a0b:12f0::1]:80": { - "Name": "[2001:db8:a0b:12f0::1]:80", - "Mirrors": [], - "Secure": False, - "Official": False, - }, - "docker.io": { - "Name": "docker.io", - "Mirrors": ["https://hub-mirror.corp.example.com:5000/"], - "Secure": True, - "Official": True, - }, - "registry.internal.corp.example.com:3000": { - "Name": "registry.internal.corp.example.com:3000", - "Mirrors": [], - "Secure": False, - "Official": False, - }, - }, + examples=[ + { + "127.0.0.1:5000": { + "Name": "127.0.0.1:5000", + "Mirrors": [], + "Secure": False, + "Official": False, + }, + "[2001:db8:a0b:12f0::1]:80": { + "Name": "[2001:db8:a0b:12f0::1]:80", + "Mirrors": [], + "Secure": False, + "Official": False, + }, + "docker.io": { + "Name": "docker.io", + "Mirrors": ["https://hub-mirror.corp.example.com:5000/"], + "Secure": True, + "Official": True, + }, + "registry.internal.corp.example.com:3000": { + "Name": "registry.internal.corp.example.com:3000", + "Mirrors": [], + "Secure": False, + "Official": False, + }, + } + ], ) Mirrors: list[str] | None = Field( None, description="List of registry URLs that act as a mirror for the official\n(`docker.io`) registry.\n", - example=[ - "https://hub-mirror.corp.example.com:5000/", - "https://[2001:db8:a0b:12f0::1]/", + examples=[ + [ + "https://hub-mirror.corp.example.com:5000/", + "https://[2001:db8:a0b:12f0::1]/", + ] ], ) @@ -3553,30 +3700,32 @@ class SwarmInfo(BaseModel): NodeID: str | None = Field( "", description="Unique identifier of for this node in the swarm.", - example="k67qz4598weg5unwwffg6z1m1", + examples=["k67qz4598weg5unwwffg6z1m1"], ) NodeAddr: str | None = Field( "", description="IP address at which this node can be reached by other nodes in the\nswarm.\n", - example="10.0.0.46", + examples=["10.0.0.46"], ) - LocalNodeState: LocalNodeState | None = None - ControlAvailable: bool | None = Field(False, example=True) + LocalNodeState_: LocalNodeState | None = Field(None, alias="LocalNodeState") + ControlAvailable: bool | None = Field(False, examples=[True]) Error: str | None = "" RemoteManagers: list[PeerNode] | None = Field( None, description="List of ID's and addresses of other managers in the swarm.\n", - example=[ - {"NodeID": "71izy0goik036k48jg985xnds", "Addr": "10.0.0.158:2377"}, - {"NodeID": "79y6h1o4gv8n120drcprv5nmc", "Addr": "10.0.0.159:2377"}, - {"NodeID": "k67qz4598weg5unwwffg6z1m1", "Addr": "10.0.0.46:2377"}, + examples=[ + [ + {"NodeID": "71izy0goik036k48jg985xnds", "Addr": "10.0.0.158:2377"}, + {"NodeID": "79y6h1o4gv8n120drcprv5nmc", "Addr": "10.0.0.159:2377"}, + {"NodeID": "k67qz4598weg5unwwffg6z1m1", "Addr": "10.0.0.46:2377"}, + ] ], ) Nodes: int | None = Field( - None, description="Total number of nodes in the swarm.", example=4 + None, description="Total number of nodes in the swarm.", examples=[4] ) Managers: int | None = Field( - None, description="Total number of managers in the swarm.", example=3 + None, description="Total number of managers in the swarm.", examples=[3] ) Cluster: ClusterInfo | None = None @@ -3604,73 +3753,73 @@ class NetworkSettings(BaseModel): Bridge: str | None = Field( None, description="Name of the network'a bridge (for example, `docker0`).", - example="docker0", + examples=["docker0"], ) SandboxID: str | None = Field( None, description="SandboxID uniquely represents a container's network stack.", - example="9d12daf2c33f5959c8bf90aa513e4f65b561738661003029ec84830cd503a0c3", + examples=["9d12daf2c33f5959c8bf90aa513e4f65b561738661003029ec84830cd503a0c3"], ) HairpinMode: bool | None = Field( None, description="Indicates if hairpin NAT should be enabled on the virtual interface.\n", - example=False, + examples=[False], ) LinkLocalIPv6Address: str | None = Field( None, description="IPv6 unicast address using the link-local prefix.", - example="fe80::42:acff:fe11:1", + examples=["fe80::42:acff:fe11:1"], ) LinkLocalIPv6PrefixLen: int | None = Field( - None, description="Prefix length of the IPv6 unicast address.", example="64" + None, description="Prefix length of the IPv6 unicast address.", examples=["64"] ) Ports: PortMap | None = None SandboxKey: str | None = Field( None, description="SandboxKey identifies the sandbox", - example="/var/run/docker/netns/8ab54b426c38", + examples=["/var/run/docker/netns/8ab54b426c38"], ) SecondaryIPAddresses: list[Address] | None = Field(None, description="") SecondaryIPv6Addresses: list[Address] | None = Field(None, description="") EndpointID: str | None = Field( None, description='EndpointID uniquely represents a service endpoint in a Sandbox.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example="b88f5b905aabf2893f3cbc4ee42d1ea7980bbc0a92e2c8922b1e1795298afb0b", + examples=["b88f5b905aabf2893f3cbc4ee42d1ea7980bbc0a92e2c8922b1e1795298afb0b"], ) Gateway: str | None = Field( None, description='Gateway address for the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example="172.17.0.1", + examples=["172.17.0.1"], ) GlobalIPv6Address: str | None = Field( None, description='Global IPv6 address for the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example="2001:db8::5689", + examples=["2001:db8::5689"], ) GlobalIPv6PrefixLen: int | None = Field( None, description='Mask length of the global IPv6 address.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example=64, + examples=[64], ) IPAddress: str | None = Field( None, description='IPv4 address for the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example="172.17.0.4", + examples=["172.17.0.4"], ) IPPrefixLen: int | None = Field( None, description='Mask length of the IPv4 address.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example=16, + examples=[16], ) IPv6Gateway: str | None = Field( None, description='IPv6 gateway address for this network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example="2001:db8:2::100", + examples=["2001:db8:2::100"], ) MacAddress: str | None = Field( None, description='MAC address for the container on the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', - example="02:42:ac:11:00:04", + examples=["02:42:ac:11:00:04"], ) Networks: dict[str, EndpointSettings] | None = Field( None, @@ -3685,7 +3834,7 @@ class Network(BaseModel): Scope: str | None = None Driver: str | None = None EnableIPv6: bool | None = None - IPAM: IPAM | None = None + IPAM_: IPAM | None = Field(None, alias="IPAM") Internal: bool | None = None Attachable: bool | None = None Ingress: bool | None = None @@ -3695,267 +3844,279 @@ class Network(BaseModel): class Node(BaseModel): - ID: str | None = Field(None, example="24ifsmvkjbyhk") + ID: str | None = Field(None, examples=["24ifsmvkjbyhk"]) Version: ObjectVersion | None = None CreatedAt: str | None = Field( None, description="Date and time at which the node was added to the swarm in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2016-08-18T10:44:24.496525531Z", + examples=["2016-08-18T10:44:24.496525531Z"], ) UpdatedAt: str | None = Field( None, description="Date and time at which the node was last updated in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", - example="2017-08-09T07:09:37.632105588Z", + examples=["2017-08-09T07:09:37.632105588Z"], ) Spec: NodeSpec | None = None Description: NodeDescription | None = None Status: NodeStatus | None = None - ManagerStatus: ManagerStatus | None = None + ManagerStatus_: ManagerStatus | None = Field(None, alias="ManagerStatus") class SystemInfo(BaseModel): ID: str | None = Field( None, description="Unique identifier of the daemon.\n\n


\n\n> **Note**: The format of the ID itself is not part of the API, and\n> should not be considered stable.\n", - example="7TRN:IPZB:QYBB:VPBQ:UMPP:KARE:6ZNR:XE6T:7EWV:PKF4:ZOJD:TPYS", + examples=["7TRN:IPZB:QYBB:VPBQ:UMPP:KARE:6ZNR:XE6T:7EWV:PKF4:ZOJD:TPYS"], ) Containers: int | None = Field( - None, description="Total number of containers on the host.", example=14 + None, description="Total number of containers on the host.", examples=[14] ) ContainersRunning: int | None = Field( - None, description='Number of containers with status `"running"`.\n', example=3 + None, + description='Number of containers with status `"running"`.\n', + examples=[3], ) ContainersPaused: int | None = Field( - None, description='Number of containers with status `"paused"`.\n', example=1 + None, description='Number of containers with status `"paused"`.\n', examples=[1] ) ContainersStopped: int | None = Field( - None, description='Number of containers with status `"stopped"`.\n', example=10 + None, + description='Number of containers with status `"stopped"`.\n', + examples=[10], ) Images: int | None = Field( None, description="Total number of images on the host.\n\nBoth _tagged_ and _untagged_ (dangling) images are counted.\n", - example=508, + examples=[508], ) Driver: str | None = Field( - None, description="Name of the storage driver in use.", example="overlay2" + None, description="Name of the storage driver in use.", examples=["overlay2"] ) DriverStatus: list[list[str]] | None = Field( None, description='Information specific to the storage driver, provided as\n"label" / "value" pairs.\n\nThis information is provided by the storage driver, and formatted\nin a way consistent with the output of `docker info` on the command\nline.\n\n


\n\n> **Note**: The information returned in this field, including the\n> formatting of values and labels, should not be considered stable,\n> and may change without notice.\n', - example=[ - ["Backing Filesystem", "extfs"], - ["Supports d_type", "true"], - ["Native Overlay Diff", "true"], + examples=[ + [ + ["Backing Filesystem", "extfs"], + ["Supports d_type", "true"], + ["Native Overlay Diff", "true"], + ] ], ) DockerRootDir: str | None = Field( None, description="Root directory of persistent Docker state.\n\nDefaults to `/var/lib/docker` on Linux, and `C:\\ProgramData\\docker`\non Windows.\n", - example="/var/lib/docker", + examples=["/var/lib/docker"], ) Plugins: PluginsInfo | None = None MemoryLimit: bool | None = Field( None, description="Indicates if the host has memory limit support enabled.", - example=True, + examples=[True], ) SwapLimit: bool | None = Field( None, description="Indicates if the host has memory swap limit support enabled.", - example=True, + examples=[True], ) KernelMemory: bool | None = Field( None, description="Indicates if the host has kernel memory limit support enabled.\n\n


\n\n> **Deprecated**: This field is deprecated as the kernel 5.4 deprecated\n> `kmem.limit_in_bytes`.\n", - example=True, + examples=[True], ) KernelMemoryTCP: bool | None = Field( None, description="Indicates if the host has kernel memory TCP limit support enabled.\n\nKernel memory TCP limits are not supported when using cgroups v2, which\ndoes not support the corresponding `memory.kmem.tcp.limit_in_bytes` cgroup.\n", - example=True, + examples=[True], ) CpuCfsPeriod: bool | None = Field( None, description="Indicates if CPU CFS(Completely Fair Scheduler) period is supported by\nthe host.\n", - example=True, + examples=[True], ) CpuCfsQuota: bool | None = Field( None, description="Indicates if CPU CFS(Completely Fair Scheduler) quota is supported by\nthe host.\n", - example=True, + examples=[True], ) CPUShares: bool | None = Field( None, description="Indicates if CPU Shares limiting is supported by the host.\n", - example=True, + examples=[True], ) CPUSet: bool | None = Field( None, description="Indicates if CPUsets (cpuset.cpus, cpuset.mems) are supported by the host.\n\nSee [cpuset(7)](https://www.kernel.org/doc/Documentation/cgroup-v1/cpusets.txt)\n", - example=True, + examples=[True], ) PidsLimit: bool | None = Field( None, description="Indicates if the host kernel has PID limit support enabled.", - example=True, + examples=[True], ) OomKillDisable: bool | None = Field( None, description="Indicates if OOM killer disable is supported on the host." ) IPv4Forwarding: bool | None = Field( - None, description="Indicates IPv4 forwarding is enabled.", example=True + None, description="Indicates IPv4 forwarding is enabled.", examples=[True] ) BridgeNfIptables: bool | None = Field( None, description="Indicates if `bridge-nf-call-iptables` is available on the host.", - example=True, + examples=[True], ) BridgeNfIp6tables: bool | None = Field( None, description="Indicates if `bridge-nf-call-ip6tables` is available on the host.", - example=True, + examples=[True], ) Debug: bool | None = Field( None, description="Indicates if the daemon is running in debug-mode / with debug-level\nlogging enabled.\n", - example=True, + examples=[True], ) NFd: int | None = Field( None, description="The total number of file Descriptors in use by the daemon process.\n\nThis information is only returned if debug-mode is enabled.\n", - example=64, + examples=[64], ) NGoroutines: int | None = Field( None, description="The number of goroutines that currently exist.\n\nThis information is only returned if debug-mode is enabled.\n", - example=174, + examples=[174], ) SystemTime: str | None = Field( None, description="Current system-time in [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt)\nformat with nano-seconds.\n", - example="2017-08-08T20:28:29.06202363Z", + examples=["2017-08-08T20:28:29.06202363Z"], ) LoggingDriver: str | None = Field( None, description="The logging driver to use as a default for new containers.\n" ) - CgroupDriver: CgroupDriver | None = Field( + CgroupDriver_: CgroupDriver | None = Field( CgroupDriver.cgroupfs, + alias="CgroupDriver", description="The driver to use for managing cgroups.\n", - example="cgroupfs", + examples=["cgroupfs"], ) - CgroupVersion: CgroupVersion | None = Field( - CgroupVersion.field_1, description="The version of the cgroup.\n", example="1" + CgroupVersion_: CgroupVersion | None = Field( + CgroupVersion.field_1, + alias="CgroupVersion", + description="The version of the cgroup.\n", + examples=["1"], ) NEventsListener: int | None = Field( - None, description="Number of event listeners subscribed.", example=30 + None, description="Number of event listeners subscribed.", examples=[30] ) KernelVersion: str | None = Field( None, description='Kernel version of the host.\n\nOn Linux, this information obtained from `uname`. On Windows this\ninformation is queried from the HKEY_LOCAL_MACHINE\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\\nregistry value, for example _"10.0 14393 (14393.1198.amd64fre.rs1_release_sec.170427-1353)"_.\n', - example="4.9.38-moby", + examples=["4.9.38-moby"], ) OperatingSystem: str | None = Field( None, description='Name of the host\'s operating system, for example: "Ubuntu 16.04.2 LTS"\nor "Windows Server 2016 Datacenter"\n', - example="Alpine Linux v3.5", + examples=["Alpine Linux v3.5"], ) OSVersion: str | None = Field( None, description="Version of the host's operating system\n\n


\n\n> **Note**: The information returned in this field, including its\n> very existence, and the formatting of values, should not be considered\n> stable, and may change without notice.\n", - example="16.04", + examples=["16.04"], ) OSType: str | None = Field( None, description='Generic type of the operating system of the host, as returned by the\nGo runtime (`GOOS`).\n\nCurrently returned values are "linux" and "windows". A full list of\npossible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment).\n', - example="linux", + examples=["linux"], ) Architecture: str | None = Field( None, description="Hardware architecture of the host, as returned by the Go runtime\n(`GOARCH`).\n\nA full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment).\n", - example="x86_64", + examples=["x86_64"], ) NCPU: int | None = Field( None, description="The number of logical CPUs usable by the daemon.\n\nThe number of available CPUs is checked by querying the operating\nsystem when the daemon starts. Changes to operating system CPU\nallocation after the daemon is started are not reflected.\n", - example=4, + examples=[4], ) MemTotal: int | None = Field( None, description="Total amount of physical memory available on the host, in bytes.\n", - example=2095882240, + examples=[2095882240], ) IndexServerAddress: str | None = Field( "https://index.docker.io/v1/", description="Address / URL of the index server that is used for image search,\nand as a default for user authentication for Docker Hub and Docker Cloud.\n", - example="https://index.docker.io/v1/", + examples=["https://index.docker.io/v1/"], ) RegistryConfig: RegistryServiceConfig | None = None - GenericResources: GenericResources | None = None + GenericResources_: GenericResources | None = Field(None, alias="GenericResources") HttpProxy: str | None = Field( None, description="HTTP-proxy configured for the daemon. This value is obtained from the\n[`HTTP_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html) environment variable.\nCredentials ([user info component](https://tools.ietf.org/html/rfc3986#section-3.2.1)) in the proxy URL\nare masked in the API response.\n\nContainers do not automatically inherit this configuration.\n", - example="http://xxxxx:xxxxx@proxy.corp.example.com:8080", + examples=["http://xxxxx:xxxxx@proxy.corp.example.com:8080"], ) HttpsProxy: str | None = Field( None, description="HTTPS-proxy configured for the daemon. This value is obtained from the\n[`HTTPS_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html) environment variable.\nCredentials ([user info component](https://tools.ietf.org/html/rfc3986#section-3.2.1)) in the proxy URL\nare masked in the API response.\n\nContainers do not automatically inherit this configuration.\n", - example="https://xxxxx:xxxxx@proxy.corp.example.com:4443", + examples=["https://xxxxx:xxxxx@proxy.corp.example.com:4443"], ) NoProxy: str | None = Field( None, description="Comma-separated list of domain extensions for which no proxy should be\nused. This value is obtained from the [`NO_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html)\nenvironment variable.\n\nContainers do not automatically inherit this configuration.\n", - example="*.local, 169.254/16", + examples=["*.local, 169.254/16"], ) Name: str | None = Field( - None, description="Hostname of the host.", example="node5.corp.example.com" + None, description="Hostname of the host.", examples=["node5.corp.example.com"] ) Labels: list[str] | None = Field( None, description="User-defined labels (key/value metadata) as set on the daemon.\n\n


\n\n> **Note**: When part of a Swarm, nodes can both have _daemon_ labels,\n> set through the daemon configuration, and _node_ labels, set from a\n> manager node in the Swarm. Node labels are not included in this\n> field. Node labels can be retrieved using the `/nodes/(id)` endpoint\n> on a manager node in the Swarm.\n", - example=["storage=ssd", "production"], + examples=[["storage=ssd", "production"]], ) ExperimentalBuild: bool | None = Field( None, description="Indicates if experimental features are enabled on the daemon.\n", - example=True, + examples=[True], ) ServerVersion: str | None = Field( None, description="Version string of the daemon.\n\n> **Note**: the [standalone Swarm API](/swarm/swarm-api/)\n> returns the Swarm version instead of the daemon version, for example\n> `swarm/1.2.8`.\n", - example="17.06.0-ce", + examples=["17.06.0-ce"], ) ClusterStore: str | None = Field( None, description="URL of the distributed storage backend.\n\n\nThe storage backend is used for multihost networking (to store\nnetwork and endpoint information) and by the node discovery mechanism.\n\n


\n\n> **Deprecated**: This field is only propagated when using standalone Swarm\n> mode, and overlay networking using an external k/v store. Overlay\n> networks with Swarm mode enabled use the built-in raft store, and\n> this field will be empty.\n", - example="consul://consul.corp.example.com:8600/some/path", + examples=["consul://consul.corp.example.com:8600/some/path"], ) ClusterAdvertise: str | None = Field( None, description="The network endpoint that the Engine advertises for the purpose of\nnode discovery. ClusterAdvertise is a `host:port` combination on which\nthe daemon is reachable by other hosts.\n\n


\n\n> **Deprecated**: This field is only propagated when using standalone Swarm\n> mode, and overlay networking using an external k/v store. Overlay\n> networks with Swarm mode enabled use the built-in raft store, and\n> this field will be empty.\n", - example="node5.corp.example.com:8000", + examples=["node5.corp.example.com:8000"], ) Runtimes: dict[str, Runtime] | None = Field( {"runc": {"path": "runc"}}, description='List of [OCI compliant](https://github.com/opencontainers/runtime-spec)\nruntimes configured on the daemon. Keys hold the "name" used to\nreference the runtime.\n\nThe Docker daemon relies on an OCI compliant runtime (invoked via the\n`containerd` daemon) as its interface to the Linux kernel namespaces,\ncgroups, and SELinux.\n\nThe default runtime is `runc`, and automatically configured. Additional\nruntimes can be configured by the user and will be listed here.\n', - example={ - "runc": {"path": "runc"}, - "runc-master": {"path": "/go/bin/runc"}, - "custom": { - "path": "/usr/local/bin/my-oci-runtime", - "runtimeArgs": ["--debug", "--systemd-cgroup=false"], - }, - }, + examples=[ + { + "runc": {"path": "runc"}, + "runc-master": {"path": "/go/bin/runc"}, + "custom": { + "path": "/usr/local/bin/my-oci-runtime", + "runtimeArgs": ["--debug", "--systemd-cgroup=false"], + }, + } + ], ) DefaultRuntime: str | None = Field( "runc", description="Name of the default OCI runtime that is used when starting containers.\n\nThe default can be overridden per-container at create time.\n", - example="runc", + examples=["runc"], ) Swarm: SwarmInfo | None = None LiveRestoreEnabled: bool | None = Field( False, description="Indicates if live restore is enabled.\n\nIf enabled, containers are kept running when the daemon is shutdown\nor upon daemon start if running containers are detected.\n", - example=False, + examples=[False], ) Isolation: Isolation2 | None = Field( Isolation2.default, @@ -3964,7 +4125,7 @@ class SystemInfo(BaseModel): InitBinary: str | None = Field( None, description="Name and, optional, path of the `docker-init` binary.\n\nIf the path is omitted, the daemon searches the host's `$PATH` for the\nbinary and uses the first result.\n", - example="docker-init", + examples=["docker-init"], ) ContainerdCommit: Commit | None = None RuncCommit: Commit | None = None @@ -3972,18 +4133,20 @@ class SystemInfo(BaseModel): SecurityOptions: list[str] | None = Field( None, description="List of security features that are enabled on the daemon, such as\napparmor, seccomp, SELinux, user-namespaces (userns), and rootless.\n\nAdditional configuration options for each security feature may\nbe present, and are included as a comma-separated list of key/value\npairs.\n", - example=[ - "name=apparmor", - "name=seccomp,profile=default", - "name=selinux", - "name=userns", - "name=rootless", + examples=[ + [ + "name=apparmor", + "name=seccomp,profile=default", + "name=selinux", + "name=userns", + "name=rootless", + ] ], ) ProductLicense: str | None = Field( None, description="Reports a summary of the product license on the daemon.\n\nIf a commercial license has been applied to the daemon, information\nsuch as number of nodes, and expiration are included.\n", - example="Community Engine", + examples=["Community Engine"], ) DefaultAddressPools: list[DefaultAddressPool] | None = Field( None, @@ -3992,9 +4155,11 @@ class SystemInfo(BaseModel): Warnings: list[str] | None = Field( None, description="List of warnings / informational messages about missing features, or\nissues related to the daemon configuration.\n\nThese messages can be printed by the client as information to the user.\n", - example=[ - "WARNING: No memory limit support", - "WARNING: bridge-nf-call-iptables is disabled", - "WARNING: bridge-nf-call-ip6tables is disabled", + examples=[ + [ + "WARNING: No memory limit support", + "WARNING: bridge-nf-call-iptables is disabled", + "WARNING: bridge-nf-call-ip6tables is disabled", + ] ], ) diff --git a/packages/models-library/src/models_library/generics.py b/packages/models-library/src/models_library/generics.py index 50d6f339810..753510d088b 100644 --- a/packages/models-library/src/models_library/generics.py +++ b/packages/models-library/src/models_library/generics.py @@ -1,66 +1,66 @@ from collections.abc import ItemsView, Iterable, Iterator, KeysView, ValuesView from typing import Any, Generic, TypeVar -from pydantic.generics import GenericModel +from pydantic import BaseModel, RootModel DictKey = TypeVar("DictKey") DictValue = TypeVar("DictValue") -class DictModel(GenericModel, Generic[DictKey, DictValue]): - __root__: dict[DictKey, DictValue] +class DictModel(RootModel[dict[DictKey, DictValue]], Generic[DictKey, DictValue]): + root: dict[DictKey, DictValue] def __getitem__(self, k: DictKey) -> DictValue: - return self.__root__.__getitem__(k) + return self.root.__getitem__(k) def __setitem__(self, k: DictKey, v: DictValue) -> None: - self.__root__.__setitem__(k, v) + self.root.__setitem__(k, v) def items(self) -> ItemsView[DictKey, DictValue]: - return self.__root__.items() + return self.root.items() def keys(self) -> KeysView[DictKey]: - return self.__root__.keys() + return self.root.keys() def values(self) -> ValuesView[DictValue]: - return self.__root__.values() + return self.root.values() def update(self, *s: Iterable[tuple[DictKey, DictValue]]) -> None: - return self.__root__.update(*s) + return self.root.update(*s) def __iter__(self) -> Iterator[DictKey]: # type: ignore - return self.__root__.__iter__() + return self.root.__iter__() def get(self, key: DictKey, default: DictValue | None = None): - return self.__root__.get(key, default) + return self.root.get(key, default) def setdefault(self, key: DictKey, default: DictValue): - return self.__root__.setdefault(key, default) + return self.root.setdefault(key, default) def __len__(self) -> int: - return self.__root__.__len__() + return self.root.__len__() DataT = TypeVar("DataT") -class ListModel(GenericModel, Generic[DataT]): - __root__: list[DataT] +class ListModel(RootModel[list[DataT]], Generic[DataT]): + root: list[DataT] def __iter__(self): - return iter(self.__root__) + return iter(self.root) def __getitem__(self, item): - return self.__root__[item] + return self.root[item] def __len__(self): - return len(self.__root__) + return len(self.root) -class Envelope(GenericModel, Generic[DataT]): +class Envelope(BaseModel, Generic[DataT]): data: DataT | None = None error: Any | None = None @classmethod def from_data(cls, obj: Any) -> "Envelope": - return cls.parse_obj({"data": obj}) + return cls.model_validate({"data": obj}) diff --git a/packages/models-library/src/models_library/groups.py b/packages/models-library/src/models_library/groups.py index 3be5d1663c8..488776b6d8e 100644 --- a/packages/models-library/src/models_library/groups.py +++ b/packages/models-library/src/models_library/groups.py @@ -1,7 +1,7 @@ import enum -from typing import Any, ClassVar, Final +from typing import Final -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic.types import PositiveInt from .utils.common_validators import create_enums_pre_validator @@ -28,16 +28,15 @@ class Group(BaseModel): group_type: GroupTypeInModel = Field(..., alias="type") thumbnail: str | None - _from_equivalent_enums = validator("group_type", allow_reuse=True, pre=True)( + _from_equivalent_enums = field_validator("group_type", mode="before")( create_enums_pre_validator(GroupTypeInModel) ) class GroupAtDB(Group): - class Config: - orm_mode = True - - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "example": { "gid": 218, "name": "Friends group", @@ -45,4 +44,5 @@ class Config: "type": "standard", "thumbnail": "https://image.flaticon.com/icons/png/512/23/23374.png", } - } + }, + ) diff --git a/packages/models-library/src/models_library/invitations.py b/packages/models-library/src/models_library/invitations.py index f7f8328b9be..b1a8e4154e9 100644 --- a/packages/models-library/src/models_library/invitations.py +++ b/packages/models-library/src/models_library/invitations.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from typing import Final -from pydantic import BaseModel, EmailStr, Field, PositiveInt, validator +from pydantic import BaseModel, EmailStr, Field, PositiveInt, field_validator from .products import ProductName @@ -35,7 +35,7 @@ class InvitationInputs(BaseModel): description="If None, it will use INVITATIONS_DEFAULT_PRODUCT", ) - @validator("issuer", pre=True) + @field_validator("issuer", mode="before") @classmethod def trim_long_issuers_to_max_length(cls, v): if v and isinstance(v, str): @@ -57,7 +57,7 @@ def create_from_inputs( cls, invitation_inputs: InvitationInputs, default_product: ProductName ) -> "InvitationContent": - kwargs = invitation_inputs.dict(exclude_none=True) + kwargs = invitation_inputs.model_dump(exclude_none=True) kwargs.setdefault("product", default_product) return cls( created=datetime.now(tz=timezone.utc), diff --git a/packages/models-library/src/models_library/osparc_variable_identifier.py b/packages/models-library/src/models_library/osparc_variable_identifier.py index 71e4779b2ad..a4b0d4e882b 100644 --- a/packages/models-library/src/models_library/osparc_variable_identifier.py +++ b/packages/models-library/src/models_library/osparc_variable_identifier.py @@ -1,30 +1,24 @@ from copy import deepcopy from typing import Any, TypeVar -from pydantic import BaseModel, Field -from pydantic.errors import PydanticErrorMixin +from models_library.basic_types import ConstrainedStr +from models_library.errors_classes import OsparcErrorMixin +from pydantic import BaseModel from .utils.string_substitution import OSPARC_IDENTIFIER_PREFIX T = TypeVar("T") -class OsparcVariableIdentifier(BaseModel): +class OsparcVariableIdentifier(ConstrainedStr): # NOTE: To allow parametrized value, set the type to Union[OsparcVariableIdentifier, ...] # NOTE: When dealing with str types, to avoid unexpected behavior, the following # order is suggested `OsparcVariableIdentifier | str` - __root__: str = Field( - ..., - # NOTE: in below regex `{`` and `}` are respectively escaped with `{{` and `}}` - regex=rf"^\${{1,2}}(?:\{{)?{OSPARC_IDENTIFIER_PREFIX}[A-Za-z0-9_]+(?:\}})?(:-.+)?$", + # NOTE: in below regex `{`` and `}` are respectively escaped with `{{` and `}}` + pattern = ( + rf"^\${{1,2}}(?:\{{)?{OSPARC_IDENTIFIER_PREFIX}[A-Za-z0-9_]+(?:\}})?(:-.+)?$" ) - def __hash__(self): - return hash(str(self.__root__)) - - def __eq__(self, other): - return self.__root__ == other.__root__ - def _get_without_template_markers(self) -> str: # $VAR # ${VAR} @@ -32,7 +26,7 @@ def _get_without_template_markers(self) -> str: # ${VAR:-default} # ${VAR:-{}} return ( - self.__root__.removeprefix("$$") + self.removeprefix("$$") .removeprefix("$") .removeprefix("{") .removesuffix("}") @@ -48,7 +42,7 @@ def default_value(self) -> str | None: return parts[1] if len(parts) > 1 else None -class UnresolvedOsparcVariableIdentifierError(PydanticErrorMixin, TypeError): +class UnresolvedOsparcVariableIdentifierError(OsparcErrorMixin, TypeError): msg_template = "Provided argument is unresolved: value={value}" diff --git a/packages/models-library/src/models_library/payments.py b/packages/models-library/src/models_library/payments.py index 7a4ec846575..ff704ab7d2e 100644 --- a/packages/models-library/src/models_library/payments.py +++ b/packages/models-library/src/models_library/payments.py @@ -1,7 +1,7 @@ from decimal import Decimal -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from .emails import LowerCaseEmailStr from .products import StripePriceID, StripeTaxRateID @@ -19,15 +19,8 @@ class UserInvoiceAddress(BaseModel): description="Currently validated in webserver via pycountry library. Two letter country code alpha_2 expected.", ) - @validator("*", pre=True) - @classmethod - def parse_empty_string_as_null(cls, v): - if isinstance(v, str) and len(v.strip()) == 0: - return None - return v - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "line1": None, @@ -38,6 +31,14 @@ class Config: }, ] } + ) + + @field_validator("*", mode="before") + @classmethod + def parse_empty_string_as_null(cls, v): + if isinstance(v, str) and len(v.strip()) == 0: + return None + return v class InvoiceDataGet(BaseModel): @@ -48,18 +49,17 @@ class InvoiceDataGet(BaseModel): user_display_name: str user_email: LowerCaseEmailStr - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { - "credit_amount": Decimal(15.5), + "credit_amount": Decimal(15.5), # type: ignore[dict-item] "stripe_price_id": "stripe-price-id", "stripe_tax_rate_id": "stripe-tax-rate-id", - "user_invoice_address": UserInvoiceAddress.Config.schema_extra[ - "examples" - ][0], + "user_invoice_address": UserInvoiceAddress.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "user_display_name": "My Name", - "user_email": LowerCaseEmailStr("email@example.itis"), + "user_email": "email@example.itis", }, ] } + ) diff --git a/packages/models-library/src/models_library/products.py b/packages/models-library/src/models_library/products.py index c38281d9f6a..51c44a83d47 100644 --- a/packages/models-library/src/models_library/products.py +++ b/packages/models-library/src/models_library/products.py @@ -1,7 +1,7 @@ from decimal import Decimal -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field ProductName: TypeAlias = str StripePriceID: TypeAlias = str @@ -12,20 +12,20 @@ class CreditResultGet(BaseModel): product_name: ProductName credit_amount: Decimal = Field(..., description="") - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ - {"product_name": "s4l", "credit_amount": Decimal(15.5)}, + {"product_name": "s4l", "credit_amount": Decimal(15.5)}, # type: ignore[dict-item] ] } + ) class ProductStripeInfoGet(BaseModel): stripe_price_id: StripePriceID stripe_tax_rate_id: StripeTaxRateID - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "stripe_price_id": "stripe-price-id", @@ -33,3 +33,4 @@ class Config: }, ] } + ) diff --git a/packages/models-library/src/models_library/progress_bar.py b/packages/models-library/src/models_library/progress_bar.py index 788331b103e..da2829b0c94 100644 --- a/packages/models-library/src/models_library/progress_bar.py +++ b/packages/models-library/src/models_library/progress_bar.py @@ -1,6 +1,6 @@ -from typing import Any, ClassVar, Literal, TypeAlias +from typing import Literal, TypeAlias -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from .basic_types import IDStr @@ -15,8 +15,8 @@ class ProgressStructuredMessage(BaseModel): unit: str | None sub: "ProgressStructuredMessage | None" - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "description": "some description", @@ -42,6 +42,7 @@ class Config: }, ] } + ) UNITLESS = None @@ -77,9 +78,9 @@ def composed_message(self) -> str: return msg - class Config: - frozen = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + frozen=True, + json_schema_extra={ "examples": [ # typical percent progress (no units) { @@ -96,9 +97,8 @@ class Config: { "actual_value": 0.3, "total": 1.0, - "message": ProgressStructuredMessage.Config.schema_extra[ - "examples" - ][2], + "message": ProgressStructuredMessage.model_config["json_schema_extra"]["examples"][2], # type: ignore [index] }, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 6c2036caa5a..440bfe915bb 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -1,18 +1,17 @@ """ Models a study's project document """ -import re from copy import deepcopy from datetime import datetime from enum import Enum from typing import Any, Final, TypeAlias from uuid import UUID +from models_library.basic_types import ConstrainedStr from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, ConstrainedStr, Extra, Field, validator +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator from .basic_regex import DATE_RE, UUID_RE_BASE -from .basic_types import HttpUrlWithCustomMinLength from .emails import LowerCaseEmailStr from .projects_access import AccessRights, GroupIDStr from .projects_nodes import Node @@ -32,17 +31,11 @@ class ProjectIDStr(ConstrainedStr): - regex = re.compile(UUID_RE_BASE) - - class Config: - frozen = True + pattern = UUID_RE_BASE class DateTimeStr(ConstrainedStr): - regex = re.compile(DATE_RE) - - class Config: - frozen = True + pattern = DATE_RE @classmethod def to_datetime(cls, s: "DateTimeStr"): @@ -73,7 +66,7 @@ class BaseProjectModel(BaseModel): description="longer one-line description about the project", examples=["Dabbling in temporal transitions ..."], ) - thumbnail: HttpUrlWithCustomMinLength | None = Field( + thumbnail: HttpUrl | None = Field( ..., description="url of the project thumbnail", examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"], @@ -86,11 +79,11 @@ class BaseProjectModel(BaseModel): workbench: NodesDict = Field(..., description="Project's pipeline") # validators - _empty_thumbnail_is_none = validator("thumbnail", allow_reuse=True, pre=True)( + _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) - _none_description_is_empty = validator("description", allow_reuse=True, pre=True)( + _none_description_is_empty = field_validator("description", mode="before")( none_to_empty_str_pre_validator ) @@ -108,17 +101,16 @@ class ProjectAtDB(BaseProjectModel): False, description="Defines if a study is available publicly" ) - @validator("project_type", pre=True) + @field_validator("project_type", mode="before") @classmethod def convert_sql_alchemy_enum(cls, v): if isinstance(v, Enum): return v.value return v - class Config: - orm_mode = True - use_enum_values = True - allow_population_by_field_name = True + model_config = ConfigDict( + from_attributes=True, use_enum_values=True, populate_by_name=True + ) class Project(BaseProjectModel): @@ -180,18 +172,16 @@ class Project(BaseProjectModel): alias="workspaceId", ) - class Config: - description = "Document that stores metadata, pipeline and UI setup of a study" - title = "osparc-simcore project" - extra = Extra.forbid - - @staticmethod - def schema_extra(schema: dict, _model: "Project"): - # pylint: disable=unsubscriptable-object - - # Patch to allow jsonschema nullable - # SEE https://github.com/samuelcolvin/pydantic/issues/990#issuecomment-645961530 - state_pydantic_schema = deepcopy(schema["properties"]["state"]) - schema["properties"]["state"] = { - "anyOf": [{"type": "null"}, state_pydantic_schema] - } + def _patch_json_schema_extra(self, schema: dict) -> None: + # Patch to allow jsonschema nullable + # SEE https://github.com/samuelcolvin/pydantic/issues/990#issuecomment-645961530 + state_pydantic_schema = deepcopy(schema["properties"]["state"]) + schema["properties"]["state"] = { + "anyOf": [{"type": "null"}, state_pydantic_schema] + } + + model_config = ConfigDict( + title="osparc-simcore project", + extra="forbid", + json_schema_extra=_patch_json_schema_extra, # type: ignore[typeddict-item] + ) diff --git a/packages/models-library/src/models_library/projects_access.py b/packages/models-library/src/models_library/projects_access.py index 1b800c6b0ae..29ca6c9f592 100644 --- a/packages/models-library/src/models_library/projects_access.py +++ b/packages/models-library/src/models_library/projects_access.py @@ -3,9 +3,8 @@ """ from enum import Enum -from typing import Any, ClassVar -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from pydantic.types import PositiveInt from .basic_types import IDStr @@ -26,33 +25,22 @@ class AccessRights(BaseModel): read: bool = Field(..., description="has read access") write: bool = Field(..., description="has write access") delete: bool = Field(..., description="has deletion rights") - - class Config: - extra = Extra.forbid - - -class PositiveIntWithExclusiveMinimumRemoved(PositiveInt): - # As we are trying to match this Pydantic model to a historical json schema "project-v0.0.1" we need to remove this - # Pydantic does not support exclusiveMinimum boolean https://github.com/pydantic/pydantic/issues/4108 - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.pop("exclusiveMinimum", None) + model_config = ConfigDict(extra="forbid") class Owner(BaseModel): - user_id: PositiveIntWithExclusiveMinimumRemoved = Field( - ..., description="Owner's user id" - ) + user_id: PositiveInt = Field(..., description="Owner's user id") first_name: FirstNameStr | None = Field(..., description="Owner's first name") last_name: LastNameStr | None = Field(..., description="Owner's last name") - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ # NOTE: None and empty string are both defining an undefined value {"user_id": 1, "first_name": None, "last_name": None}, {"user_id": 2, "first_name": "", "last_name": ""}, {"user_id": 3, "first_name": "John", "last_name": "Smith"}, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/projects_comments.py b/packages/models-library/src/models_library/projects_comments.py index 234ec638a4a..88937d83d78 100644 --- a/packages/models-library/src/models_library/projects_comments.py +++ b/packages/models-library/src/models_library/projects_comments.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import TypeAlias -from pydantic import BaseModel, Extra, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt from .projects import ProjectID from .users import UserID @@ -33,12 +33,8 @@ class _ProjectsCommentsBase(BaseModel): class ProjectsCommentsDB(_ProjectsCommentsBase): - class Config: - extra = Extra.forbid - validation = False + model_config = ConfigDict(extra="forbid") class ProjectsCommentsAPI(_ProjectsCommentsBase): - class Config: - extra = Extra.forbid - validation = False + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/projects_networks.py b/packages/models-library/src/models_library/projects_networks.py index e0775ccb5d5..ee255dd80ff 100644 --- a/packages/models-library/src/models_library/projects_networks.py +++ b/packages/models-library/src/models_library/projects_networks.py @@ -1,7 +1,7 @@ import re -from typing import Any, ClassVar, Final +from typing import Annotated, Final, TypeAlias -from pydantic import BaseModel, ConstrainedStr, Field +from pydantic import BaseModel, ConfigDict, Field, StringConstraints from .generics import DictModel from .projects import ProjectID @@ -12,12 +12,9 @@ PROJECT_NETWORK_PREFIX: Final[str] = "prj-ntwrk" -class DockerNetworkName(ConstrainedStr): - regex = SERVICE_NETWORK_RE +DockerNetworkName: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_NETWORK_RE)] - -class DockerNetworkAlias(ConstrainedStr): - regex = SERVICE_NETWORK_RE +DockerNetworkAlias: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_NETWORK_RE)] class ContainerAliases(DictModel[NodeIDStr, DockerNetworkAlias]): @@ -25,8 +22,8 @@ class ContainerAliases(DictModel[NodeIDStr, DockerNetworkAlias]): class NetworksWithAliases(DictModel[DockerNetworkName, ContainerAliases]): - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "network_one": { @@ -36,6 +33,7 @@ class Config: }, ] } + ) class ProjectsNetworks(BaseModel): @@ -47,10 +45,9 @@ class ProjectsNetworks(BaseModel): "is given a user defined alias by which it is identified on the network." ), ) - - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "example": { "project_uuid": "ec5cdfea-f24e-4aa1-83b8-6dccfdc8cf4d", "networks_with_aliases": { @@ -60,4 +57,5 @@ class Config: } }, } - } + }, + ) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 318f7149ab4..b074dd4dec6 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -3,21 +3,22 @@ """ from copy import deepcopy -from typing import Any, ClassVar, TypeAlias, Union +from typing import Annotated, Any, TypeAlias, Union from pydantic import ( BaseModel, - ConstrainedStr, - Extra, + ConfigDict, Field, + HttpUrl, Json, StrictBool, StrictFloat, StrictInt, - validator, + StringConstraints, + field_validator, ) -from .basic_types import EnvVarKey, HttpUrlWithCustomMinLength, KeyIDStr +from .basic_types import EnvVarKey, KeyIDStr from .projects_access import AccessEnum from .projects_nodes_io import ( DatCoreFileLink, @@ -58,12 +59,15 @@ InputID: TypeAlias = KeyIDStr OutputID: TypeAlias = KeyIDStr -InputsDict: TypeAlias = dict[InputID, InputTypes] -OutputsDict: TypeAlias = dict[OutputID, OutputTypes] - +# union_mode="smart" by default for Pydantic>=2: https://docs.pydantic.dev/latest/concepts/unions/#union-modes +InputsDict: TypeAlias = dict[ + InputID, Annotated[InputTypes, Field(union_mode="left_to_right")] +] +OutputsDict: TypeAlias = dict[ + OutputID, Annotated[OutputTypes, Field(union_mode="left_to_right")] +] -class UnitStr(ConstrainedStr): - strip_whitespace = True +UnitStr: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True)] class NodeState(BaseModel): @@ -85,10 +89,9 @@ class NodeState(BaseModel): le=1.0, description="current progress of the task if available (None if not started or not a computational task)", ) - - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { "modified": True, @@ -106,7 +109,8 @@ class Config: "currentStatus": "SUCCESS", }, ] - } + }, + ) class Node(BaseModel): @@ -134,7 +138,7 @@ class Node(BaseModel): description="the node progress value (deprecated in DB, still used for API only)", deprecated=True, ) - thumbnail: HttpUrlWithCustomMinLength | None = Field( + thumbnail: Annotated[str, HttpUrl] | None = Field( default=None, description="url of the latest screenshot of the node", examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"], @@ -208,7 +212,7 @@ class Node(BaseModel): ), ) - @validator("thumbnail", pre=True) + @field_validator("thumbnail", mode="before") @classmethod def convert_empty_str_to_none(cls, v): if isinstance(v, str) and v == "": @@ -221,7 +225,7 @@ def convert_old_enum_name(cls, v) -> RunningState: return RunningState.FAILED return RunningState(v) - @validator("state", pre=True) + @field_validator("state", mode="before") @classmethod def convert_from_enum(cls, v): if isinstance(v, str): @@ -230,16 +234,16 @@ def convert_from_enum(cls, v): return NodeState(currentStatus=running_state_value) return v - class Config: - extra = Extra.forbid - + def _patch_json_schema_extra(self, schema: dict) -> None: # NOTE: exporting without this trick does not make runHash as nullable. # It is a Pydantic issue see https://github.com/samuelcolvin/pydantic/issues/1270 - @staticmethod - def schema_extra(schema, _model: "Node"): - # SEE https://swagger.io/docs/specification/data-models/data-types/#Null - for prop_name in ["parent", "runHash"]: - if prop_name in schema.get("properties", {}): - prop = deepcopy(schema["properties"][prop_name]) - prop["nullable"] = True - schema["properties"][prop_name] = prop + for prop_name in ["parent", "runHash"]: + if prop_name in schema.get("properties", {}): + prop = deepcopy(schema["properties"][prop_name]) + prop["nullable"] = True + schema["properties"][prop_name] = prop + + model_config = ConfigDict( + extra="forbid", + json_schema_extra=_patch_json_schema_extra, # type: ignore[typeddict-item] + ) diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index b2d88485489..876c2f71744 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -8,18 +8,19 @@ import re from pathlib import Path -from typing import Any, ClassVar, TypeAlias +from typing import Annotated, TypeAlias from uuid import UUID -from models_library.basic_types import KeyIDStr +from models_library.basic_types import ConstrainedStr, KeyIDStr from pydantic import ( AnyUrl, BaseModel, - ConstrainedStr, - Extra, + ConfigDict, Field, - parse_obj_as, - validator, + StringConstraints, + TypeAdapter, + ValidationInfo, + field_validator, ) from .basic_regex import ( @@ -31,10 +32,7 @@ NodeID = UUID - -class UUIDStr(ConstrainedStr): - regex: re.Pattern[str] | None = re.compile(UUID_RE) - +UUIDStr: TypeAlias = Annotated[str, StringConstraints(pattern=UUID_RE)] NodeIDStr = UUIDStr @@ -43,7 +41,7 @@ class UUIDStr(ConstrainedStr): class SimcoreS3FileID(ConstrainedStr): - regex: re.Pattern[str] | None = re.compile(SIMCORE_S3_FILE_ID_RE) + pattern: re.Pattern[str] | None = re.compile(SIMCORE_S3_FILE_ID_RE) class SimcoreS3DirectoryID(ConstrainedStr): @@ -52,7 +50,7 @@ class SimcoreS3DirectoryID(ConstrainedStr): `{project_id}/{node_id}/simcore-dir-name/` """ - regex: re.Pattern[str] | None = re.compile(SIMCORE_S3_DIRECTORY_ID_RE) + pattern: re.Pattern[str] | None = re.compile(SIMCORE_S3_DIRECTORY_ID_RE) @staticmethod def _get_parent(s3_object: str, *, parent_index: int) -> str: @@ -72,8 +70,8 @@ def _get_parent(s3_object: str, *, parent_index: int) -> str: raise ValueError(msg) from err @classmethod - def validate(cls, value: str) -> str: - value = super().validate(value) + def _validate(cls, __input_value: str) -> str: + value = super()._validate(__input_value) value = value.rstrip("/") parent = cls._get_parent(value, parent_index=3) @@ -86,7 +84,7 @@ def validate(cls, value: str) -> str: @classmethod def from_simcore_s3_object(cls, s3_object: str) -> "SimcoreS3DirectoryID": parent_path: str = cls._get_parent(s3_object, parent_index=4) - return parse_obj_as(cls, f"{parent_path}/") + return TypeAdapter(cls).validate_python(f"{parent_path}/") class DatCoreFileID(ConstrainedStr): @@ -108,10 +106,9 @@ class PortLink(BaseModel): ..., description="The port key in the node given by nodeUuid", ) - - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ # minimal { @@ -119,25 +116,26 @@ class Config: "output": "out_2", } ], - } + }, + ) class DownloadLink(BaseModel): """I/O port type to hold a generic download link to a file (e.g. S3 pre-signed link, etc)""" - download_link: AnyUrl = Field(..., alias="downloadLink") + download_link: Annotated[str, AnyUrl] = Field(..., alias="downloadLink") label: str | None = Field(default=None, description="Display name") - - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ # minimal { "downloadLink": "https://fakeimg.pl/250x100/", } ], - } + }, + ) ## CUSTOM STORAGE SERVICES ----------- @@ -155,8 +153,7 @@ class BaseFileLink(BaseModel): ) label: str | None = Field( - default=None, - description="The real file name", + default=None, description="The real file name", validate_default=True ) e_tag: str | None = Field( @@ -165,7 +162,7 @@ class BaseFileLink(BaseModel): alias="eTag", ) - @validator("store", pre=True) + @field_validator("store", mode="before") @classmethod def legacy_enforce_str_to_int(cls, v): # SEE example 'legacy: store as string' @@ -182,7 +179,7 @@ class SimCoreFileLink(BaseFileLink): deprecated=True, ) - @validator("store", always=True) + @field_validator("store") @classmethod def check_discriminator(cls, v): """Used as discriminator to cast to this class""" @@ -191,16 +188,16 @@ def check_discriminator(cls, v): raise ValueError(msg) return 0 - @validator("label", always=True, pre=True) + @field_validator("label", mode="before") @classmethod - def pre_fill_label_with_filename_ext(cls, v, values): - if v is None and "path" in values: - return Path(values["path"]).name + def pre_fill_label_with_filename_ext(cls, v, info: ValidationInfo): + if v is None and "path" in info.data: + return Path(info.data["path"]).name return v - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { "store": 0, @@ -225,7 +222,8 @@ class Config: "path": "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt", }, ], - } + }, + ) class DatCoreFileLink(BaseFileLink): @@ -241,7 +239,7 @@ class DatCoreFileLink(BaseFileLink): description="Unique identifier to access the dataset on datcore (REQUIRED for datcore)", ) - @validator("store", always=True) + @field_validator("store") @classmethod def check_discriminator(cls, v): """Used as discriminator to cast to this class""" @@ -251,9 +249,9 @@ def check_discriminator(cls, v): raise ValueError(msg) return 1 - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { # minimal @@ -270,7 +268,8 @@ class Config: "label": "initial_WTstates", }, ], - } + }, + ) # Bundles all model links to a file vs PortLink diff --git a/packages/models-library/src/models_library/projects_nodes_ui.py b/packages/models-library/src/models_library/projects_nodes_ui.py index aa55332ccba..e14f2b21a28 100644 --- a/packages/models-library/src/models_library/projects_nodes_ui.py +++ b/packages/models-library/src/models_library/projects_nodes_ui.py @@ -2,20 +2,18 @@ Models node UI (legacy model, use instead projects.ui.py) """ -from pydantic import BaseModel, Extra, Field -from pydantic.color import Color +from pydantic import BaseModel, ConfigDict, Field +from pydantic_extra_types.color import Color class Position(BaseModel): - x: int = Field(..., description="The x position", example=["12"]) - y: int = Field(..., description="The y position", example=["15"]) + x: int = Field(..., description="The x position", examples=[["12"]]) + y: int = Field(..., description="The y position", examples=[["15"]]) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class Marker(BaseModel): color: Color = Field(...) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/projects_pipeline.py b/packages/models-library/src/models_library/projects_pipeline.py index 2139d182043..975d4726b4e 100644 --- a/packages/models-library/src/models_library/projects_pipeline.py +++ b/packages/models-library/src/models_library/projects_pipeline.py @@ -1,9 +1,8 @@ import datetime -from typing import Any, ClassVar from uuid import UUID import arrow -from pydantic import BaseModel, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt from .clusters import ClusterID from .projects_nodes import NodeState @@ -58,8 +57,8 @@ class ComputationTask(BaseModel): description="task last modification timestamp or None if the there is no task", ) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "id": "42838344-03de-4ce2-8d93-589a5dcdfd05", @@ -89,9 +88,9 @@ class Config: }, "iteration": None, "cluster_id": None, - "started": arrow.utcnow().shift(minutes=-50).datetime, + "started": arrow.utcnow().shift(minutes=-50).datetime, # type: ignore[dict-item] "stopped": None, - "submitted": arrow.utcnow().shift(hours=-1).datetime, + "submitted": arrow.utcnow().shift(hours=-1).datetime, # type: ignore[dict-item] }, { "id": "f81d7994-9ccc-4c95-8c32-aa70d6bbb1b0", @@ -121,9 +120,10 @@ class Config: }, "iteration": 2, "cluster_id": 0, - "started": arrow.utcnow().shift(minutes=-50).datetime, - "stopped": arrow.utcnow().shift(minutes=-20).datetime, - "submitted": arrow.utcnow().shift(hours=-1).datetime, + "started": arrow.utcnow().shift(minutes=-50).datetime, # type: ignore[dict-item] + "stopped": arrow.utcnow().shift(minutes=-20).datetime, # type: ignore[dict-item] + "submitted": arrow.utcnow().shift(hours=-1).datetime, # type: ignore[dict-item] }, ] } + ) diff --git a/packages/models-library/src/models_library/projects_state.py b/packages/models-library/src/models_library/projects_state.py index 38c68d5d4a4..45a920a464c 100644 --- a/packages/models-library/src/models_library/projects_state.py +++ b/packages/models-library/src/models_library/projects_state.py @@ -3,9 +3,8 @@ """ from enum import Enum, unique -from typing import Any, ClassVar -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator from .projects_access import Owner @@ -57,14 +56,15 @@ class ProjectStatus(str, Enum): class ProjectLocked(BaseModel): value: bool = Field(..., description="True if the project is locked") owner: Owner | None = Field( - default=None, description="If locked, the user that owns the lock" + default=None, + description="If locked, the user that owns the lock", + validate_default=True, ) status: ProjectStatus = Field(..., description="The status of the project") - - class Config: - extra = Extra.forbid - use_enum_values = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + use_enum_values=True, + json_schema_extra={ "examples": [ {"value": False, "status": ProjectStatus.CLOSED}, { @@ -77,24 +77,25 @@ class Config: }, }, ] - } + }, + ) - @validator("owner", pre=True, always=True) + @field_validator("owner", mode="before") @classmethod - def check_not_null(cls, v, values): - if values["value"] is True and v is None: + def check_not_null(cls, v, info: ValidationInfo): + if info.data["value"] is True and v is None: msg = "value cannot be None when project is locked" raise ValueError(msg) return v - @validator("status", always=True) + @field_validator("status") @classmethod - def check_status_compatible(cls, v, values): - if values["value"] is False and v not in ["CLOSED", "OPENED"]: - msg = f"status is set to {v} and lock is set to {values['value']}!" + def check_status_compatible(cls, v, info: ValidationInfo): + if info.data["value"] is False and v not in ["CLOSED", "OPENED"]: + msg = f"status is set to {v} and lock is set to {info.data['value']}!" raise ValueError(msg) - if values["value"] is True and v == "CLOSED": - msg = f"status is set to {v} and lock is set to {values['value']}!" + if info.data["value"] is True and v == "CLOSED": + msg = f"status is set to {v} and lock is set to {info.data['value']}!" raise ValueError(msg) return v @@ -104,13 +105,11 @@ class ProjectRunningState(BaseModel): ..., description="The running state of the project", examples=["STARTED"] ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class ProjectState(BaseModel): locked: ProjectLocked = Field(..., description="The project lock state") state: ProjectRunningState = Field(..., description="The project running state") - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/projects_ui.py b/packages/models-library/src/models_library/projects_ui.py index 154007a2a6d..93aa68d628b 100644 --- a/packages/models-library/src/models_library/projects_ui.py +++ b/packages/models-library/src/models_library/projects_ui.py @@ -2,10 +2,11 @@ Models Front-end UI """ -from typing import Any, ClassVar, Literal, TypedDict +from typing import Literal -from pydantic import BaseModel, Extra, Field, validator -from pydantic.color import Color +from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic_extra_types.color import Color +from typing_extensions import TypedDict from .projects_nodes_io import NodeID, NodeIDStr from .projects_nodes_ui import Marker, Position @@ -15,9 +16,7 @@ class WorkbenchUI(BaseModel): position: Position = Field(..., description="The node position in the workbench") marker: Marker | None = None - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _SlideshowRequired(TypedDict): @@ -32,10 +31,9 @@ class Annotation(BaseModel): type: Literal["note", "rect", "text"] = Field(...) color: Color = Field(...) attributes: dict = Field(..., description="svg attributes") - - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { "type": "note", @@ -60,7 +58,8 @@ class Config: "attributes": {"x": 415, "y": 100, "text": "Hey!"}, }, ] - } + }, + ) class StudyUI(BaseModel): @@ -69,9 +68,8 @@ class StudyUI(BaseModel): current_node_id: NodeID | None = Field(default=None, alias="currentNodeId") annotations: dict[NodeIDStr, Annotation] | None = None - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") - _empty_is_none = validator("*", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("*", mode="before")( empty_str_to_none_pre_validator ) diff --git a/packages/models-library/src/models_library/rabbitmq_basic_types.py b/packages/models-library/src/models_library/rabbitmq_basic_types.py index 022b66b9a9d..e8ae694b8be 100644 --- a/packages/models-library/src/models_library/rabbitmq_basic_types.py +++ b/packages/models-library/src/models_library/rabbitmq_basic_types.py @@ -1,15 +1,15 @@ -import re from typing import Final -from pydantic import ConstrainedStr, parse_obj_as +from models_library.basic_types import ConstrainedStr +from pydantic import TypeAdapter REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS: Final[str] = r"^[\w\-\.]*$" class RPCNamespace(ConstrainedStr): + pattern = REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS min_length: int = 1 max_length: int = 252 - regex: re.Pattern[str] | None = re.compile(REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS) @classmethod def from_entries(cls, entries: dict[str, str]) -> "RPCNamespace": @@ -18,10 +18,10 @@ def from_entries(cls, entries: dict[str, str]) -> "RPCNamespace": Keeping this to a predefined length """ composed_string = "-".join(f"{k}_{v}" for k, v in sorted(entries.items())) - return parse_obj_as(cls, composed_string) + return TypeAdapter(cls).validate_python(composed_string) class RPCMethodName(ConstrainedStr): + pattern = REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS min_length: int = 1 max_length: int = 252 - regex: re.Pattern[str] | None = re.compile(REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS) diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 07dab962281..13ecda316ae 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -30,12 +30,12 @@ class RabbitEventMessageType(str, Enum): class RabbitMessageBase(BaseModel): - channel_name: str = Field(..., const=True) + channel_name: str @classmethod def get_channel_name(cls) -> str: # NOTE: this returns the channel type name - name: str = cls.__fields__["channel_name"].default + name: str = cls.model_fields["channel_name"].default return name @abstractmethod @@ -132,9 +132,7 @@ def routing_key(self) -> str | None: class _RabbitAutoscalingBaseMessage(RabbitMessageBase): - channel_name: Literal["io.simcore.autoscaling"] = Field( - default="io.simcore.autoscaling", const=True - ) + channel_name: Literal["io.simcore.autoscaling"] = "io.simcore.autoscaling" origin: str = Field( ..., description="autoscaling app type, in case there would be more than one" ) @@ -177,9 +175,7 @@ class RabbitResourceTrackingMessageType(StrAutoEnum): class RabbitResourceTrackingBaseMessage(RabbitMessageBase): - channel_name: Literal["io.simcore.service.tracking"] = Field( - default="io.simcore.service.tracking", const=True - ) + channel_name: Literal["io.simcore.service.tracking"] = "io.simcore.service.tracking" service_run_id: str = Field( ..., description="uniquely identitifies the service run" @@ -194,9 +190,9 @@ def routing_key(self) -> str | None: class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage): - message_type: RabbitResourceTrackingMessageType = Field( - default=RabbitResourceTrackingMessageType.TRACKING_STARTED, const=True - ) + message_type: Literal[ + RabbitResourceTrackingMessageType.TRACKING_STARTED + ] = RabbitResourceTrackingMessageType.TRACKING_STARTED wallet_id: WalletID | None wallet_name: str | None @@ -234,9 +230,9 @@ class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage): class RabbitResourceTrackingHeartbeatMessage(RabbitResourceTrackingBaseMessage): - message_type: RabbitResourceTrackingMessageType = Field( - default=RabbitResourceTrackingMessageType.TRACKING_HEARTBEAT, const=True - ) + message_type: Literal[ + RabbitResourceTrackingMessageType.TRACKING_HEARTBEAT + ] = RabbitResourceTrackingMessageType.TRACKING_HEARTBEAT class SimcorePlatformStatus(StrAutoEnum): @@ -245,9 +241,9 @@ class SimcorePlatformStatus(StrAutoEnum): class RabbitResourceTrackingStoppedMessage(RabbitResourceTrackingBaseMessage): - message_type: RabbitResourceTrackingMessageType = Field( - default=RabbitResourceTrackingMessageType.TRACKING_STOPPED, const=True - ) + message_type: Literal[ + RabbitResourceTrackingMessageType.TRACKING_STOPPED + ] = RabbitResourceTrackingMessageType.TRACKING_STOPPED simcore_platform_status: SimcorePlatformStatus = Field( ..., @@ -263,9 +259,7 @@ class RabbitResourceTrackingStoppedMessage(RabbitResourceTrackingBaseMessage): class WalletCreditsMessage(RabbitMessageBase): - channel_name: Literal["io.simcore.service.wallets"] = Field( - default="io.simcore.service.wallets", const=True - ) + channel_name: Literal["io.simcore.service.wallets"] = "io.simcore.service.wallets" created_at: datetime.datetime = Field( default_factory=lambda: arrow.utcnow().datetime, description="message creation datetime", @@ -283,9 +277,9 @@ class CreditsLimit(IntEnum): class WalletCreditsLimitReachedMessage(RabbitMessageBase): - channel_name: Literal["io.simcore.service.wallets-credit-limit-reached"] = Field( - default="io.simcore.service.wallets-credit-limit-reached", const=True - ) + channel_name: Literal[ + "io.simcore.service.wallets-credit-limit-reached" + ] = "io.simcore.service.wallets-credit-limit-reached" created_at: datetime.datetime = Field( default_factory=lambda: arrow.utcnow().datetime, description="message creation datetime", diff --git a/packages/models-library/src/models_library/resource_tracker.py b/packages/models-library/src/models_library/resource_tracker.py index 13c92e161ed..c3b42a08795 100644 --- a/packages/models-library/src/models_library/resource_tracker.py +++ b/packages/models-library/src/models_library/resource_tracker.py @@ -2,16 +2,16 @@ from datetime import datetime, timezone from decimal import Decimal from enum import IntEnum, auto -from typing import Any, ClassVar, NamedTuple, TypeAlias +from typing import NamedTuple, TypeAlias from pydantic import ( BaseModel, ByteSize, - Extra, + ConfigDict, Field, NonNegativeInt, PositiveInt, - validator, + field_validator, ) from .products import ProductName @@ -59,26 +59,28 @@ class PricingInfo(BaseModel): pricing_unit_id: PricingUnitId pricing_unit_cost_id: PricingUnitCostId - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"pricing_plan_id": 1, "pricing_unit_id": 1, "pricing_unit_cost_id": 1} ] } + ) class HardwareInfo(BaseModel): aws_ec2_instances: list[str] - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ {"aws_ec2_instances": ["c6a.4xlarge"]}, {"aws_ec2_instances": []}, ] } + ) - @validator("aws_ec2_instances") + @field_validator("aws_ec2_instances") @classmethod def warn_if_too_many_instances_are_present(cls, v: list[str]) -> list[str]: if len(v) > 1: @@ -106,10 +108,9 @@ class StartedAt(BaseModel): from_: datetime | None = Field(None, alias="from") until: datetime | None = Field(None) - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) - @validator("from_", pre=True) + @field_validator("from_", mode="before") @classmethod def parse_from_filter(cls, v): """Parse the filters field.""" @@ -124,7 +125,7 @@ def parse_from_filter(cls, v): return from_ return v - @validator("until", pre=True) + @field_validator("until", mode="before") @classmethod def parse_until_filter(cls, v): """Parse the filters field.""" @@ -153,9 +154,8 @@ class PricingPlanCreate(BaseModel): description: str classification: PricingPlanClassification pricing_plan_key: str - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "product_name": "osparc", @@ -166,6 +166,7 @@ class Config: } ] } + ) class PricingPlanUpdate(BaseModel): @@ -174,8 +175,8 @@ class PricingPlanUpdate(BaseModel): description: str is_active: bool - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_plan_id": 1, @@ -185,6 +186,7 @@ class Config: } ] } + ) ## Pricing Units @@ -202,10 +204,10 @@ class UnitExtraInfo(BaseModel): RAM: ByteSize VRAM: ByteSize - class Config: - allow_population_by_field_name = True - extra = Extra.allow - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + populate_by_name=True, + extra="allow", + json_schema_extra={ "examples": [ { "CPU": 32, @@ -215,7 +217,8 @@ class Config: "custom key": "custom value", } ] - } + }, + ) class PricingUnitWithCostCreate(BaseModel): @@ -227,13 +230,13 @@ class PricingUnitWithCostCreate(BaseModel): cost_per_unit: Decimal comment: str - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_plan_id": 1, "unit_name": "My pricing plan", - "unit_extra_info": UnitExtraInfo.Config.schema_extra["examples"][0], + "unit_extra_info": UnitExtraInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "default": True, "specific_info": {"aws_ec2_instances": ["t3.medium"]}, "cost_per_unit": 10, @@ -241,6 +244,7 @@ class Config: } ] } + ) class PricingUnitCostUpdate(BaseModel): @@ -257,14 +261,14 @@ class PricingUnitWithCostUpdate(BaseModel): specific_info: SpecificInfo pricing_unit_cost_update: None | PricingUnitCostUpdate - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_plan_id": 1, "pricing_unit_id": 1, "unit_name": "My pricing plan", - "unit_extra_info": UnitExtraInfo.Config.schema_extra["examples"][0], + "unit_extra_info": UnitExtraInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "default": True, "specific_info": {"aws_ec2_instances": ["t3.medium"]}, "pricing_unit_cost_update": { @@ -276,13 +280,14 @@ class Config: "pricing_plan_id": 1, "pricing_unit_id": 1, "unit_name": "My pricing plan", - "unit_extra_info": UnitExtraInfo.Config.schema_extra["examples"][0], + "unit_extra_info": UnitExtraInfo.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] "default": True, "specific_info": {"aws_ec2_instances": ["t3.medium"]}, "pricing_unit_cost_update": None, }, ] } + ) class ServicesAggregatedUsagesType(StrAutoEnum): diff --git a/packages/models-library/src/models_library/rest_ordering.py b/packages/models-library/src/models_library/rest_ordering.py index c8a791343ee..7b1b6b39c39 100644 --- a/packages/models-library/src/models_library/rest_ordering.py +++ b/packages/models-library/src/models_library/rest_ordering.py @@ -1,6 +1,6 @@ from enum import Enum -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from .basic_types import IDStr @@ -16,5 +16,4 @@ class OrderBy(BaseModel): field: IDStr = Field() direction: OrderDirection = Field(default=OrderDirection.ASC) - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index 89c90cb1c2d..d8e3b9990b3 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -1,17 +1,16 @@ -from typing import Any, ClassVar, Final, Generic, TypeVar +from typing import Annotated, Final, Generic, TypeAlias, TypeVar from pydantic import ( AnyHttpUrl, BaseModel, - ConstrainedInt, - Extra, + ConfigDict, Field, NonNegativeInt, PositiveInt, - parse_obj_as, - validator, + TypeAdapter, + ValidationInfo, + field_validator, ) -from pydantic.generics import GenericModel from .utils.common_validators import none_to_empty_list_pre_validator @@ -21,19 +20,20 @@ MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE: Final[int] = 50 -class PageLimitInt(ConstrainedInt): - ge = 1 - lt = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +PageLimitInt: TypeAlias = Annotated[int, Field(ge=1, lt=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE)] - -DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = parse_obj_as(PageLimitInt, 20) +DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = TypeAdapter( + PageLimitInt +).validate_python(20) class PageQueryParameters(BaseModel): """Use as pagination options in query parameters""" limit: PageLimitInt = Field( - default=parse_obj_as(PageLimitInt, DEFAULT_NUMBER_OF_ITEMS_PER_PAGE), + default=TypeAdapter(PageLimitInt).validate_python( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + ), description="maximum number of items to return (pagination)", ) offset: NonNegativeInt = Field( @@ -47,38 +47,36 @@ class PageMetaInfoLimitOffset(BaseModel): offset: NonNegativeInt = 0 count: NonNegativeInt - @validator("offset") + @field_validator("offset") @classmethod - def _check_offset(cls, v, values): - if v > 0 and v >= values["total"]: - msg = f"offset {v} cannot be equal or bigger than total {values['total']}, please check" + def _check_offset(cls, v, info: ValidationInfo): + if v > 0 and v >= info.data["total"]: + msg = f"offset {v} cannot be equal or bigger than total {info.data['total']}, please check" raise ValueError(msg) return v - @validator("count") + @field_validator("count") @classmethod - def _check_count(cls, v, values): - if v > values["limit"]: - msg = f"count {v} bigger than limit {values['limit']}, please check" + def _check_count(cls, v, info: ValidationInfo): + if v > info.data["limit"]: + msg = f"count {v} bigger than limit {info.data['limit']}, please check" raise ValueError(msg) - if v > values["total"]: - msg = ( - f"count {v} bigger than expected total {values['total']}, please check" - ) + if v > info.data["total"]: + msg = f"count {v} bigger than expected total {info.data['total']}, please check" raise ValueError(msg) - if "offset" in values and (values["offset"] + v) > values["total"]: - msg = f"offset {values['offset']} + count {v} is bigger than allowed total {values['total']}, please check" + if "offset" in info.data and (info.data["offset"] + v) > info.data["total"]: + msg = f"offset {info.data['offset']} + count {v} is bigger than allowed total {info.data['total']}, please check" raise ValueError(msg) return v - class Config: - extra = Extra.forbid - - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ {"total": 7, "count": 4, "limit": 4, "offset": 0}, ] - } + }, + ) RefT = TypeVar("RefT") @@ -91,18 +89,17 @@ class PageRefs(BaseModel, Generic[RefT]): next: RefT | None last: RefT - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") -class PageLinks(PageRefs[AnyHttpUrl]): +class PageLinks(PageRefs[Annotated[str, AnyHttpUrl]]): ... ItemT = TypeVar("ItemT") -class Page(GenericModel, Generic[ItemT]): +class Page(BaseModel, Generic[ItemT]): """ Paginated response model of ItemTs """ @@ -111,26 +108,25 @@ class Page(GenericModel, Generic[ItemT]): links: PageLinks = Field(alias="_links") data: list[ItemT] - _none_is_empty = validator("data", allow_reuse=True, pre=True)( + _none_is_empty = field_validator("data", mode="before")( none_to_empty_list_pre_validator ) - @validator("data") + @field_validator("data") @classmethod - def _check_data_compatible_with_meta(cls, v, values): - if "meta" not in values: + def _check_data_compatible_with_meta(cls, v, info: ValidationInfo): + if "meta" not in info.data: # if the validation failed in meta this happens msg = "meta not in values" raise ValueError(msg) - if len(v) != values["meta"].count: - msg = f"container size [{len(v)}] must be equal to count [{values['meta'].count}]" + if len(v) != info.data["meta"].count: + msg = f"container size [{len(v)}] must be equal to count [{info.data['meta'].count}]" raise ValueError(msg) return v - class Config: - extra = Extra.forbid - - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ # first page Page[str] { @@ -157,4 +153,5 @@ class Config: "data": ["data 5", "data 6", "data 7"], }, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index ec9cedf0a54..8d901d50f30 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -1,8 +1,9 @@ from math import ceil from typing import Any, Protocol, TypedDict, Union, runtime_checkable -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import parse_obj_as +from .basic_types import AnyHttpUrl from .rest_pagination import PageLinks, PageMetaInfoLimitOffset # NOTE: In this repo we use two type of URL-like data structures: diff --git a/packages/models-library/src/models_library/rpc_pagination.py b/packages/models-library/src/models_library/rpc_pagination.py index 34eeb997990..0ec454cc9fd 100644 --- a/packages/models-library/src/models_library/rpc_pagination.py +++ b/packages/models-library/src/models_library/rpc_pagination.py @@ -1,8 +1,8 @@ # mypy: disable-error-code=truthy-function from math import ceil -from typing import Any, ClassVar, Generic +from typing import Any, Generic -from pydantic import Extra, Field +from pydantic import ConfigDict, Field from .rest_pagination import ( DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, @@ -74,10 +74,9 @@ def create( data=chunk, ) - class Config: - extra = Extra.forbid - - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ # first page Page[str] { @@ -104,4 +103,5 @@ class Config: "data": ["data 5", "data 6", "data 7"], }, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/service_settings_labels.py b/packages/models-library/src/models_library/service_settings_labels.py index 95bc13a0b09..8f954737231 100644 --- a/packages/models-library/src/models_library/service_settings_labels.py +++ b/packages/models-library/src/models_library/service_settings_labels.py @@ -3,19 +3,20 @@ from enum import Enum from functools import cached_property from pathlib import Path -from typing import Any, ClassVar, Literal, TypeAlias +from typing import Any, Literal, TypeAlias from pydantic import ( BaseModel, ByteSize, - Extra, + ConfigDict, Field, Json, PrivateAttr, ValidationError, + ValidationInfo, + field_validator, + model_validator, parse_obj_as, - root_validator, - validator, ) from .callbacks_mapping import CallbacksMapping @@ -24,11 +25,9 @@ from .services_resources import DEFAULT_SINGLE_SERVICE_NAME from .utils.json_serialization import json_dumps - -class _BaseConfig: - arbitrary_types_allowed = True - extra = Extra.forbid - keep_untouched = (cached_property,) +_BaseConfig = ConfigDict( + extra="forbid", arbitrary_types_allowed=True, ignored_types=(cached_property,) +) class ContainerSpec(BaseModel): @@ -40,18 +39,20 @@ class ContainerSpec(BaseModel): alias="Command", description="Used to override the container's command", # NOTE: currently constraint to our use cases. Might mitigate some security issues. - min_items=1, - max_items=2, + min_length=1, + max_length=2, ) - class Config(_BaseConfig): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + **_BaseConfig, + json_schema_extra={ "examples": [ {"Command": ["executable"]}, {"Command": ["executable", "subcommand"]}, {"Command": ["ofs", "linear-regression"]}, ] - } + }, + ) class SimcoreServiceSettingLabelEntry(BaseModel): @@ -93,7 +94,7 @@ def get_destination_containers(self) -> list[str]: # as fields return self._destination_containers - @validator("setting_type", pre=True) + @field_validator("setting_type", mode="before") @classmethod def ensure_backwards_compatible_setting_type(cls, v): if v == "resources": @@ -101,9 +102,10 @@ def ensure_backwards_compatible_setting_type(cls, v): return "Resources" return v - class Config(_BaseConfig): - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + **_BaseConfig, + populate_by_name=True, + json_schema_extra={ "examples": [ # constraints { @@ -157,7 +159,8 @@ class Config(_BaseConfig): }, }, ] - } + }, + ) SimcoreServiceSettingsLabel = ListModel[SimcoreServiceSettingLabelEntry] @@ -191,9 +194,9 @@ class PathMappingsLabel(BaseModel): ), ) - @validator("volume_size_limits") + @field_validator("volume_size_limits") @classmethod - def validate_volume_limits(cls, v, values) -> str | None: + def validate_volume_limits(cls, v, info: ValidationInfo) -> str | None: if v is None: return v @@ -205,9 +208,9 @@ def validate_volume_limits(cls, v, values) -> str | None: msg = f"Provided size='{size_str}' contains invalid charactes: {e!s}" raise ValueError(msg) from e - inputs_path: Path | None = values.get("inputs_path") - outputs_path: Path | None = values.get("outputs_path") - state_paths: list[Path] | None = values.get("state_paths") + inputs_path: Path | None = info.data.get("inputs_path") + outputs_path: Path | None = info.data.get("outputs_path") + state_paths: list[Path] | None = info.data.get("state_paths") path = Path(path_str) if not ( path in (inputs_path, outputs_path) @@ -218,8 +221,9 @@ def validate_volume_limits(cls, v, values) -> str | None: output: str | None = v return output - class Config(_BaseConfig): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + **_BaseConfig, + json_schema_extra={ "examples": [ { "outputs_path": "/tmp/outputs", # noqa: S108 nosec @@ -249,7 +253,8 @@ class Config(_BaseConfig): }, }, ] - } + }, + ) ComposeSpecLabelDict: TypeAlias = dict[str, Any] @@ -292,6 +297,7 @@ class DynamicSidecarServiceLabels(BaseModel): "specified. Required by dynamic-sidecar when " "compose_spec is set." ), + validate_default=True, ) user_preferences_path: Path | None = Field( @@ -339,25 +345,29 @@ def needs_dynamic_sidecar(self) -> bool: """if paths mapping is present the service needs to be ran via dynamic-sidecar""" return self.paths_mapping is not None - @validator("container_http_entry", always=True) + @field_validator("container_http_entry") @classmethod - def compose_spec_requires_container_http_entry(cls, v, values) -> str | None: + def compose_spec_requires_container_http_entry( + cls, v, info: ValidationInfo + ) -> str | None: v = None if v == "" else v - if v is None and values.get("compose_spec") is not None: + if v is None and info.data.get("compose_spec") is not None: msg = "Field `container_http_entry` must be defined but is missing" raise ValueError(msg) - if v is not None and values.get("compose_spec") is None: + if v is not None and info.data.get("compose_spec") is None: msg = "`container_http_entry` not allowed if `compose_spec` is missing" raise ValueError(msg) return f"{v}" if v else v - @validator("containers_allowed_outgoing_permit_list") + @field_validator("containers_allowed_outgoing_permit_list") @classmethod - def _containers_allowed_outgoing_permit_list_in_compose_spec(cls, v, values): + def _containers_allowed_outgoing_permit_list_in_compose_spec( + cls, v, info: ValidationInfo + ): if v is None: return v - compose_spec: dict | None = values.get("compose_spec") + compose_spec: dict | None = info.data.get("compose_spec") if compose_spec is None: keys = set(v.keys()) if len(keys) != 1 or DEFAULT_SINGLE_SERVICE_NAME not in keys: @@ -372,13 +382,15 @@ def _containers_allowed_outgoing_permit_list_in_compose_spec(cls, v, values): return v - @validator("containers_allowed_outgoing_internet") + @field_validator("containers_allowed_outgoing_internet") @classmethod - def _containers_allowed_outgoing_internet_in_compose_spec(cls, v, values): + def _containers_allowed_outgoing_internet_in_compose_spec( + cls, v, info: ValidationInfo + ): if v is None: - return v + return None - compose_spec: dict | None = values.get("compose_spec") + compose_spec: dict | None = info.data.get("compose_spec") if compose_spec is None: if {DEFAULT_SINGLE_SERVICE_NAME} != v: err_msg = ( @@ -393,10 +405,10 @@ def _containers_allowed_outgoing_internet_in_compose_spec(cls, v, values): raise ValueError(err_msg) return v - @validator("callbacks_mapping") + @field_validator("callbacks_mapping") @classmethod def _ensure_callbacks_mapping_container_names_defined_in_compose_spec( - cls, v: CallbacksMapping, values + cls, v: CallbacksMapping, info: ValidationInfo ): if v is None: return {} @@ -408,7 +420,7 @@ def _ensure_callbacks_mapping_container_names_defined_in_compose_spec( if len(defined_services) == 0: return v - compose_spec: dict | None = values.get("compose_spec") + compose_spec: dict | None = info.data.get("compose_spec") if compose_spec is None: if {DEFAULT_SINGLE_SERVICE_NAME} != defined_services: err_msg = f"Expected only 1 entry '{DEFAULT_SINGLE_SERVICE_NAME}' not '{defined_services}'" @@ -421,17 +433,17 @@ def _ensure_callbacks_mapping_container_names_defined_in_compose_spec( raise ValueError(err_msg) return v - @validator("user_preferences_path", pre=True) + @field_validator("user_preferences_path", mode="before") @classmethod def _deserialize_from_json(cls, v): return f"{v}".removeprefix('"').removesuffix('"') - @validator("user_preferences_path") + @field_validator("user_preferences_path") @classmethod def _user_preferences_path_no_included_in_other_volumes( - cls, v: CallbacksMapping, values + cls, v: CallbacksMapping, info: ValidationInfo ): - paths_mapping: PathMappingsLabel | None = values.get("paths_mapping", None) + paths_mapping: PathMappingsLabel | None = info.data.get("paths_mapping", None) if paths_mapping is None: return v @@ -445,33 +457,24 @@ def _user_preferences_path_no_included_in_other_volumes( raise ValueError(msg) return v - @root_validator - @classmethod - def _not_allowed_in_both_specs(cls, values): + @model_validator(mode="after") + def _not_allowed_in_both_specs(self): match_keys = { "containers_allowed_outgoing_internet", "containers_allowed_outgoing_permit_list", } - if match_keys & set(values.keys()) != match_keys: - err_msg = ( - f"Expected the following keys {match_keys} to be present {values=}" - ) + if match_keys & set(self.model_fields) != match_keys: + err_msg = f"Expected the following keys {match_keys} to be present {self.model_fields=}" raise ValueError(err_msg) - containers_allowed_outgoing_internet = values[ - "containers_allowed_outgoing_internet" - ] - containers_allowed_outgoing_permit_list = values[ - "containers_allowed_outgoing_permit_list" - ] if ( - containers_allowed_outgoing_internet is None - or containers_allowed_outgoing_permit_list is None + self.containers_allowed_outgoing_internet is None + or self.containers_allowed_outgoing_permit_list is None ): - return values + return self - common_containers = set(containers_allowed_outgoing_internet) & set( - containers_allowed_outgoing_permit_list.keys() + common_containers = set(self.containers_allowed_outgoing_internet) & set( + self.containers_allowed_outgoing_permit_list.keys() ) if len(common_containers) > 0: err_msg = ( @@ -481,10 +484,9 @@ def _not_allowed_in_both_specs(cls, values): ) raise ValueError(err_msg) - return values + return self - class Config(_BaseConfig): - ... + model_config = _BaseConfig class SimcoreServiceLabels(DynamicSidecarServiceLabels): @@ -513,24 +515,32 @@ class SimcoreServiceLabels(DynamicSidecarServiceLabels): ), ) - class Config(_BaseConfig): - extra = Extra.allow - schema_extra: ClassVar[dict[str, Any]] = { + model_config = _BaseConfig | ConfigDict( + extra="allow", + json_schema_extra={ "examples": [ # WARNING: do not change order. Used in tests! # legacy service { "simcore.service.settings": json_dumps( - SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"] + SimcoreServiceSettingLabelEntry.model_config[ + "json_schema_extra" + ][ + "examples" + ] # type: ignore[index] ) }, # dynamic-service { "simcore.service.settings": json_dumps( - SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"] + SimcoreServiceSettingLabelEntry.model_config[ + "json_schema_extra" + ][ + "examples" + ] # type: ignore[index] ), "simcore.service.paths-mapping": json_dumps( - PathMappingsLabel.Config.schema_extra["examples"][0] + PathMappingsLabel.model_config["json_schema_extra"]["examples"][0] # type: ignore [index] ), "simcore.service.restart-policy": RestartPolicy.NO_RESTART.value, "simcore.service.callbacks-mapping": json_dumps( @@ -549,10 +559,14 @@ class Config(_BaseConfig): # dynamic-service with compose spec { "simcore.service.settings": json_dumps( - SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"] + SimcoreServiceSettingLabelEntry.model_config[ + "json_schema_extra" + ][ + "examples" + ] # type: ignore[index] ), "simcore.service.paths-mapping": json_dumps( - PathMappingsLabel.Config.schema_extra["examples"][0] + PathMappingsLabel.model_config["json_schema_extra"]["examples"][0], # type: ignore[index] ), "simcore.service.compose-spec": json_dumps( { @@ -580,8 +594,9 @@ class Config(_BaseConfig): "simcore.service.container-http-entrypoint": "rt-web", "simcore.service.restart-policy": RestartPolicy.ON_INPUTS_DOWNLOADED.value, "simcore.service.callbacks-mapping": json_dumps( - CallbacksMapping.Config.schema_extra["examples"][3] + CallbacksMapping.model_config["json_schema_extra"]["examples"][3] # type: ignore [index] ), }, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/service_settings_nat_rule.py b/packages/models-library/src/models_library/service_settings_nat_rule.py index bcdf0604eec..ee937c81254 100644 --- a/packages/models-library/src/models_library/service_settings_nat_rule.py +++ b/packages/models-library/src/models_library/service_settings_nat_rule.py @@ -1,7 +1,14 @@ from collections.abc import Generator -from typing import Any, ClassVar, Final +from typing import Final -from pydantic import BaseModel, Extra, Field, parse_obj_as, validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + parse_obj_as, +) from .basic_types import PortInt from .osparc_variable_identifier import OsparcVariableIdentifier, raise_if_unresolved @@ -17,14 +24,14 @@ class _PortRange(BaseModel): lower: PortInt | OsparcVariableIdentifier upper: PortInt | OsparcVariableIdentifier - @validator("upper") + @field_validator("upper") @classmethod - def lower_less_than_upper(cls, v, values) -> PortInt: + def lower_less_than_upper(cls, v, info: ValidationInfo) -> PortInt: if isinstance(v, OsparcVariableIdentifier): return v # type: ignore # bypass validation if unresolved upper = v - lower: PortInt | OsparcVariableIdentifier | None = values.get("lower") + lower: PortInt | OsparcVariableIdentifier | None = info.data.get("lower") if lower and isinstance(lower, OsparcVariableIdentifier): return v # type: ignore # bypass validation if unresolved @@ -34,9 +41,7 @@ def lower_less_than_upper(cls, v, values) -> PortInt: raise ValueError(msg) return PortInt(v) - class Config: - arbitrary_types_allowed = True - validate_assignment = True + model_config = ConfigDict(arbitrary_types_allowed=True, validate_assignment=True) class DNSResolver(BaseModel): @@ -45,16 +50,17 @@ class DNSResolver(BaseModel): ) port: PortInt | OsparcVariableIdentifier - class Config: - arbitrary_types_allowed = True - validate_assignment = True - extra = Extra.allow - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + arbitrary_types_allowed=True, + validate_assignment=True, + extra="allow", + json_schema_extra={ "examples": [ {"address": "1.1.1.1", "port": 53}, # NOSONAR {"address": "ns1.example.com", "port": 53}, ] - } + }, + ) class NATRule(BaseModel): @@ -69,6 +75,8 @@ class NATRule(BaseModel): description="specify a DNS resolver address and port", ) + model_config = ConfigDict(arbitrary_types_allowed=True, validate_assignment=True) + def iter_tcp_ports(self) -> Generator[PortInt, None, None]: for port in self.tcp_ports: if isinstance(port, _PortRange): @@ -81,7 +89,3 @@ def iter_tcp_ports(self) -> Generator[PortInt, None, None]: ) else: yield raise_if_unresolved(port) - - class Config: - arbitrary_types_allowed = True - validate_assignment = True diff --git a/packages/models-library/src/models_library/services_access.py b/packages/models-library/src/models_library/services_access.py index 8bc6786c695..84dbd7d17a0 100644 --- a/packages/models-library/src/models_library/services_access.py +++ b/packages/models-library/src/models_library/services_access.py @@ -2,7 +2,7 @@ """ -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from .users import GroupID from .utils.change_case import snake_to_camel @@ -22,10 +22,9 @@ class ServiceGroupAccessRightsV2(BaseModel): execute: bool = False write: bool = False - class Config: - alias_generator = snake_to_camel - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict( + alias_generator=snake_to_camel, populate_by_name=True, extra="forbid" + ) class ServiceAccessRights(BaseModel): diff --git a/packages/models-library/src/models_library/services_authoring.py b/packages/models-library/src/models_library/services_authoring.py index 18673319f46..05b5197994c 100644 --- a/packages/models-library/src/models_library/services_authoring.py +++ b/packages/models-library/src/models_library/services_authoring.py @@ -1,6 +1,4 @@ -from typing import Any, ClassVar - -from pydantic import BaseModel, Field, HttpUrl +from pydantic import BaseModel, ConfigDict, Field, HttpUrl from .emails import LowerCaseEmailStr @@ -18,15 +16,15 @@ class Badge(BaseModel): ..., description="Link to the status", ) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "osparc.io", "image": "https://img.shields.io/website-up-down-green-red/https/itisfoundation.github.io.svg?label=documentation", "url": "https://itisfoundation.github.io/", } } + ) class Author(BaseModel): @@ -39,9 +37,8 @@ class Author(BaseModel): description="Email address", ) affiliation: str | None = Field(None) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "name": "Jim Knopf", @@ -54,3 +51,4 @@ class Config: }, ] } + ) diff --git a/packages/models-library/src/models_library/services_base.py b/packages/models-library/src/models_library/services_base.py index d80fc59df24..5f92d6e46b6 100644 --- a/packages/models-library/src/models_library/services_base.py +++ b/packages/models-library/src/models_library/services_base.py @@ -1,4 +1,5 @@ -from pydantic import BaseModel, Field, HttpUrl, validator +from typing import Annotated +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator from .services_types import ServiceKey, ServiceVersion from .utils.common_validators import empty_str_to_none_pre_validator @@ -16,22 +17,22 @@ class ServiceKeyVersion(BaseModel): description="service version number", ) - class Config: - frozen = True + model_config = ConfigDict(frozen=True) class ServiceBaseDisplay(BaseModel): name: str = Field( ..., description="Display name: short, human readable name for the node", - example="Fast Counter", + examples=["Fast Counter"], ) - thumbnail: HttpUrl | None = Field( + thumbnail: Annotated[str, HttpUrl] | None = Field( None, description="url to the thumbnail", examples=[ "https://user-images.githubusercontent.com/32800795/61083844-ff48fb00-a42c-11e9-8e63-fa2d709c8baf.png" ], + validate_default=True, ) description: str = Field( ..., @@ -53,6 +54,6 @@ class ServiceBaseDisplay(BaseModel): " This name is not used for version comparison but is useful for communication and documentation purposes.", ) - _empty_is_none = validator("thumbnail", allow_reuse=True, pre=True, always=False)( + _empty_is_none = field_validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) diff --git a/packages/models-library/src/models_library/services_creation.py b/packages/models-library/src/models_library/services_creation.py index e2102efe075..c1c2c5172fc 100644 --- a/packages/models-library/src/models_library/services_creation.py +++ b/packages/models-library/src/models_library/services_creation.py @@ -1,6 +1,6 @@ -from typing import Any, ClassVar +from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from .services import ServiceKey, ServiceVersion from .services_resources import ServiceResourcesDict @@ -23,8 +23,8 @@ class CreateServiceMetricsAdditionalParams(BaseModel): service_resources: ServiceResourcesDict service_additional_metadata: dict[str, Any] - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "wallet_id": 1, "wallet_name": "a private wallet for me", @@ -42,3 +42,4 @@ class Config: "service_additional_metadata": {}, } } + ) diff --git a/packages/models-library/src/models_library/services_history.py b/packages/models-library/src/models_library/services_history.py index 70f4e513c15..b38f5f2e783 100644 --- a/packages/models-library/src/models_library/services_history.py +++ b/packages/models-library/src/models_library/services_history.py @@ -1,7 +1,7 @@ from datetime import datetime -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from .services_types import ServiceKey, ServiceVersion from .utils.change_case import snake_to_camel @@ -21,9 +21,7 @@ class Compatibility(BaseModel): ..., description="Latest compatible service at this moment" ) - class Config: - alias_generator = snake_to_camel - allow_population_by_field_name = True + model_config = ConfigDict(alias_generator=snake_to_camel, populate_by_name=True) class ServiceRelease(BaseModel): @@ -46,10 +44,10 @@ class ServiceRelease(BaseModel): default=None, description="Compatibility with other releases at this moment" ) - class Config: - alias_generator = snake_to_camel - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + alias_generator=snake_to_camel, + populate_by_name=True, + json_schema_extra={ "examples": [ # minimal { @@ -69,7 +67,8 @@ class Config: }, }, ] - } + }, + ) ReleaseHistory: TypeAlias = list[ServiceRelease] diff --git a/packages/models-library/src/models_library/services_io.py b/packages/models-library/src/models_library/services_io.py index 52c09fa241d..49264f19799 100644 --- a/packages/models-library/src/models_library/services_io.py +++ b/packages/models-library/src/models_library/services_io.py @@ -1,15 +1,15 @@ -import re -from typing import Any, ClassVar +from typing import Annotated, Any, TypeAlias from pydantic import ( BaseModel, - ConstrainedStr, - Extra, + ConfigDict, Field, StrictBool, StrictFloat, StrictInt, - validator, + StringConstraints, + ValidationInfo, + field_validator, ) from .services_constants import ANY_FILETYPE @@ -22,12 +22,7 @@ jsonschema_validate_schema, ) - -class PropertyTypeStr(ConstrainedStr): - regex = re.compile(PROPERTY_TYPE_RE) - - class Config: - frozen = True +PropertyTypeStr: TypeAlias = Annotated[str, StringConstraints(pattern=PROPERTY_TYPE_RE)] class BaseServiceIOModel(BaseModel): @@ -45,11 +40,11 @@ class BaseServiceIOModel(BaseModel): description="DEPRECATED: new display order is taken from the item position. This will be removed.", ) - label: str = Field(..., description="short name for the property", example="Age") + label: str = Field(..., description="short name for the property", examples=["Age"]) description: str = Field( ..., description="description of the property", - example="Age in seconds since 1970", + examples=["Age in seconds since 1970"], ) # mathematical and physics descriptors @@ -92,18 +87,20 @@ class BaseServiceIOModel(BaseModel): deprecated=True, # add x_unit in content_schema instead ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") - @validator("content_schema") + @field_validator("content_schema") @classmethod - def _check_type_is_set_to_schema(cls, v, values): - if v is not None and (ptype := values["property_type"]) != "ref_contentSchema": + def _check_type_is_set_to_schema(cls, v, info: ValidationInfo): + if ( + v is not None + and (ptype := info.data["property_type"]) != "ref_contentSchema" + ): msg = f"content_schema is defined but set the wrong type. Expected type=ref_contentSchema but got ={ptype}." raise ValueError(msg) return v - @validator("content_schema") + @field_validator("content_schema") @classmethod def _check_valid_json_schema(cls, v): if v is not None: @@ -151,8 +148,9 @@ class ServiceInput(BaseServiceIOModel): description="custom widget to use instead of the default one determined from the data-type", ) - class Config(BaseServiceIOModel.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + **BaseServiceIOModel.model_config, + json_schema_extra={ "examples": [ # file-wo-widget: { @@ -206,7 +204,8 @@ class Config(BaseServiceIOModel.Config): }, }, ], - } + }, + ) @classmethod def from_json_schema(cls, port_schema: dict[str, Any]) -> "ServiceInput": @@ -222,8 +221,9 @@ class ServiceOutput(BaseServiceIOModel): deprecated=True, ) - class Config(BaseServiceIOModel.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + **BaseServiceIOModel.model_config, + json_schema_extra={ "examples": [ { "displayOrder": 2, @@ -251,7 +251,8 @@ class Config(BaseServiceIOModel.Config): "type": ANY_FILETYPE, }, ] - } + }, + ) @classmethod def from_json_schema(cls, port_schema: dict[str, Any]) -> "ServiceOutput": diff --git a/packages/models-library/src/models_library/services_metadata_editable.py b/packages/models-library/src/models_library/services_metadata_editable.py index 18d66483f1c..4ad106225c0 100644 --- a/packages/models-library/src/models_library/services_metadata_editable.py +++ b/packages/models-library/src/models_library/services_metadata_editable.py @@ -1,8 +1,8 @@ # mypy: disable-error-code=truthy-function from datetime import datetime -from typing import Any, ClassVar +from typing import Annotated, Any -from pydantic import Field, HttpUrl +from pydantic import ConfigDict, Field, HttpUrl from .services_base import ServiceBaseDisplay from .services_constants import LATEST_INTEGRATION_VERSION @@ -19,7 +19,7 @@ class ServiceMetaDataEditable(ServiceBaseDisplay): # Overrides ServiceBaseDisplay fields to Optional for a partial update name: str | None # type: ignore[assignment] - thumbnail: HttpUrl | None + thumbnail: Annotated[str, HttpUrl] | None description: str | None # type: ignore[assignment] description_ui: bool = False version_display: str | None = None @@ -35,8 +35,8 @@ class ServiceMetaDataEditable(ServiceBaseDisplay): classifiers: list[str] | None quality: dict[str, Any] = {} - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", @@ -62,3 +62,4 @@ class Config: }, } } + ) diff --git a/packages/models-library/src/models_library/services_metadata_published.py b/packages/models-library/src/models_library/services_metadata_published.py index b50d838d9d1..51fba05b7f4 100644 --- a/packages/models-library/src/models_library/services_metadata_published.py +++ b/packages/models-library/src/models_library/services_metadata_published.py @@ -1,7 +1,7 @@ from datetime import datetime -from typing import Any, ClassVar, Final, TypeAlias +from typing import Final, TypeAlias -from pydantic import Extra, Field, NonNegativeInt +from pydantic import ConfigDict, Field, NonNegativeInt from .basic_types import SemanticVersionStr from .boot_options import BootOption, BootOptions @@ -76,12 +76,8 @@ } }, "boot-options": { - "example_service_defined_boot_mode": BootOption.Config.schema_extra["examples"][ - 0 - ], - "example_service_defined_theme_selection": BootOption.Config.schema_extra[ - "examples" - ][1], + "example_service_defined_boot_mode": BootOption.model_config["json_schema_extra"]["examples"][0], # type: ignore [index] + "example_service_defined_theme_selection": BootOption.model_config["json_schema_extra"]["examples"][1], # type: ignore [index] }, "min-visible-inputs": 2, } @@ -120,7 +116,7 @@ class ServiceMetaDataPublished(ServiceKeyVersion, ServiceBaseDisplay): badges: list[Badge] | None = Field(None, deprecated=True) - authors: list[Author] = Field(..., min_items=1) + authors: list[Author] = Field(..., min_length=1) contact: LowerCaseEmailStr = Field( ..., description="email to correspond to the authors about the node", @@ -160,22 +156,21 @@ class ServiceMetaDataPublished(ServiceKeyVersion, ServiceBaseDisplay): description="Image manifest digest. Note that this is NOT injected as an image label", ) - class Config: - description = "Description of a simcore node 'class' with input and output" - extra = Extra.forbid - frozen = False # overrides config from ServiceKeyVersion. - allow_population_by_field_name = True - - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + frozen=False, + populate_by_name=True, + json_schema_extra={ "examples": [ - _EXAMPLE, - _EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER, + _EXAMPLE, # type: ignore[list-item] + _EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER, # type: ignore[list-item] # latest { - **_EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER, + **_EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER, # type: ignore[dict-item] "version_display": "Matterhorn Release", "description_ui": True, "release_date": "2024-05-31T13:45:30", }, ] - } + }, + ) diff --git a/packages/models-library/src/models_library/services_resources.py b/packages/models-library/src/models_library/services_resources.py index 7fe4f268f8c..ac7a6862754 100644 --- a/packages/models-library/src/models_library/services_resources.py +++ b/packages/models-library/src/models_library/services_resources.py @@ -1,15 +1,16 @@ import logging from enum import auto -from typing import Any, ClassVar, Final, TypeAlias +from typing import Any, Final, TypeAlias from pydantic import ( BaseModel, ByteSize, + ConfigDict, Field, StrictFloat, StrictInt, - parse_obj_as, - root_validator, + TypeAdapter, + model_validator, ) from .docker import DockerGenericTag @@ -23,13 +24,13 @@ # NOTE: replace hard coded `container` with function which can # extract the name from the `service_key` or `registry_address/service_key` -DEFAULT_SINGLE_SERVICE_NAME: Final[DockerGenericTag] = parse_obj_as( - DockerGenericTag, "container" -) +DEFAULT_SINGLE_SERVICE_NAME: Final[DockerGenericTag] = TypeAdapter( + DockerGenericTag +).validate_python("container") -MEMORY_50MB: Final[int] = parse_obj_as(ByteSize, "50mib") -MEMORY_250MB: Final[int] = parse_obj_as(ByteSize, "250mib") -MEMORY_1GB: Final[int] = parse_obj_as(ByteSize, "1gib") +MEMORY_50MB: Final[int] = TypeAdapter(ByteSize).validate_python("50mib") +MEMORY_250MB: Final[int] = TypeAdapter(ByteSize).validate_python("250mib") +MEMORY_1GB: Final[int] = TypeAdapter(ByteSize).validate_python("1gib") GIGA: Final[float] = 1e9 CPU_10_PERCENT: Final[int] = int(0.1 * GIGA) @@ -40,7 +41,7 @@ class ResourceValue(BaseModel): limit: StrictInt | StrictFloat | str reservation: StrictInt | StrictFloat | str - @root_validator() + @model_validator(mode="before") @classmethod def _ensure_limits_are_equal_or_above_reservations(cls, values): if isinstance(values["reservation"], str): @@ -59,8 +60,7 @@ def set_reservation_same_as_limit(self) -> None: def set_value(self, value: StrictInt | StrictFloat | str) -> None: self.limit = self.reservation = value - class Config: - validate_assignment = True + model_config = ConfigDict(validate_assignment=True) ResourcesDict = dict[ResourceName, ResourceValue] @@ -92,8 +92,8 @@ def set_reservation_same_as_limit(self) -> None: for resource in self.resources.values(): resource.set_reservation_same_as_limit() - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "image": "simcore/service/dynamic/pretty-intense:1.0.0", "resources": { @@ -108,6 +108,7 @@ class Config: }, } } + ) ServiceResourcesDict: TypeAlias = dict[DockerGenericTag, ImageResources] @@ -122,8 +123,7 @@ def create_from_single_service( ) -> ServiceResourcesDict: if boot_modes is None: boot_modes = [BootMode.CPU] - return parse_obj_as( - ServiceResourcesDict, + return TypeAdapter(ServiceResourcesDict).validate_python( { DEFAULT_SINGLE_SERVICE_NAME: { "image": image, @@ -140,8 +140,8 @@ def create_jsonable( output: dict[DockerGenericTag, Any] = jsonable_encoder(service_resources) return output - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ # no compose spec (majority of services) { @@ -150,8 +150,10 @@ class Config: "resources": { "CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), + "limit": TypeAdapter(ByteSize).validate_python("2Gib"), + "reservation": TypeAdapter(ByteSize).validate_python( + "2Gib" + ), }, }, "boot_modes": [BootMode.CPU], @@ -181,8 +183,10 @@ class Config: "resources": { "CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), + "limit": TypeAdapter(ByteSize).validate_python("2Gib"), + "reservation": TypeAdapter(ByteSize).validate_python( + "2Gib" + ), }, }, "boot_modes": [BootMode.CPU], @@ -195,8 +199,10 @@ class Config: "resources": { "CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), + "limit": TypeAdapter(ByteSize).validate_python("2Gib"), + "reservation": TypeAdapter(ByteSize).validate_python( + "2Gib" + ), }, }, "boot_modes": [BootMode.CPU], @@ -206,8 +212,10 @@ class Config: "resources": { "CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), + "limit": TypeAdapter(ByteSize).validate_python("2Gib"), + "reservation": TypeAdapter(ByteSize).validate_python( + "2Gib" + ), }, }, "boot_modes": [BootMode.CPU], @@ -215,3 +223,4 @@ class Config: }, ] } + ) diff --git a/packages/models-library/src/models_library/services_types.py b/packages/models-library/src/models_library/services_types.py index 366d8bc00c2..87559eff993 100644 --- a/packages/models-library/src/models_library/services_types.py +++ b/packages/models-library/src/models_library/services_types.py @@ -1,8 +1,8 @@ -import re +from typing import Annotated, TypeAlias from uuid import uuid4 import arrow -from pydantic import ConstrainedStr +from pydantic import StringConstraints from .basic_regex import PROPERTY_KEY_RE, SIMPLE_VERSION_RE from .services_regex import ( @@ -13,48 +13,21 @@ SERVICE_KEY_RE, ) +ServicePortKey: TypeAlias = Annotated[str, StringConstraints(pattern=PROPERTY_KEY_RE)] -class ServicePortKey(ConstrainedStr): - regex = re.compile(PROPERTY_KEY_RE) +FileName: TypeAlias = Annotated[str, StringConstraints(pattern=FILENAME_RE)] - class Config: - frozen = True +ServiceKey: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_KEY_RE)] +ServiceKeyEncoded: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_ENCODED_KEY_RE)] -class FileName(ConstrainedStr): - regex = re.compile(FILENAME_RE) +DynamicServiceKey: TypeAlias = Annotated[str, StringConstraints(pattern=DYNAMIC_SERVICE_KEY_RE)] - class Config: - frozen = True +ComputationalServiceKey: TypeAlias = Annotated[ + str, StringConstraints(pattern=COMPUTATIONAL_SERVICE_KEY_RE) +] - -class ServiceKey(ConstrainedStr): - regex = SERVICE_KEY_RE - - class Config: - frozen = True - - -class ServiceKeyEncoded(ConstrainedStr): - regex = re.compile(SERVICE_ENCODED_KEY_RE) - - class Config: - frozen = True - - -class DynamicServiceKey(ServiceKey): - regex = DYNAMIC_SERVICE_KEY_RE - - -class ComputationalServiceKey(ServiceKey): - regex = COMPUTATIONAL_SERVICE_KEY_RE - - -class ServiceVersion(ConstrainedStr): - regex = re.compile(SIMPLE_VERSION_RE) - - class Config: - frozen = True +ServiceVersion: TypeAlias = Annotated[str, StringConstraints(pattern=SIMPLE_VERSION_RE)] class RunID(str): diff --git a/packages/models-library/src/models_library/services_ui.py b/packages/models-library/src/models_library/services_ui.py index 22196693334..055fa58fd7b 100644 --- a/packages/models-library/src/models_library/services_ui.py +++ b/packages/models-library/src/models_library/services_ui.py @@ -1,6 +1,6 @@ from enum import Enum -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from pydantic.types import PositiveInt @@ -14,23 +14,20 @@ class TextArea(BaseModel): ..., alias="minHeight", description="minimum Height of the textarea" ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class Structure(BaseModel): key: str | bool | float label: str - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class SelectBox(BaseModel): - structure: list[Structure] = Field(..., min_items=1) + structure: list[Structure] = Field(..., min_length=1) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class Widget(BaseModel): @@ -39,5 +36,4 @@ class Widget(BaseModel): ) details: TextArea | SelectBox - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/user_preferences.py b/packages/models-library/src/models_library/user_preferences.py index 14d6b4e53f8..2680c10223d 100644 --- a/packages/models-library/src/models_library/user_preferences.py +++ b/packages/models-library/src/models_library/user_preferences.py @@ -1,15 +1,12 @@ from enum import auto -from typing import Annotated, Any, ClassVar, TypeAlias +from typing import Annotated, Any, ClassVar, Literal, TypeAlias, get_args from pydantic import BaseModel, Field -from pydantic.main import ModelMetaclass +from pydantic._internal._model_construction import ModelMetaclass from .services import ServiceKey, ServiceVersion from .utils.enums import StrAutoEnum -# NOTE: for pydantic-2 from pydantic._internal.import _model_construction -# use _model_construction.ModelMetaclass instead! - class _AutoRegisterMeta(ModelMetaclass): registered_user_preference_classes: ClassVar[dict[str, type]] = {} @@ -77,14 +74,14 @@ def get_preference_name(cls) -> PreferenceName: @classmethod def get_default_value(cls) -> Any: return ( - cls.__fields__["value"].default_factory() - if cls.__fields__["value"].default_factory - else cls.__fields__["value"].default + cls.model_fields["value"].default_factory() + if cls.model_fields["value"].default_factory + else cls.model_fields["value"].default ) class FrontendUserPreference(_BaseUserPreferenceModel): - preference_type: PreferenceType = Field(default=PreferenceType.FRONTEND, const=True) + preference_type: Literal[PreferenceType.FRONTEND] = PreferenceType.FRONTEND preference_identifier: PreferenceIdentifier = Field( ..., description="used by the frontend" @@ -93,11 +90,15 @@ class FrontendUserPreference(_BaseUserPreferenceModel): value: Any def to_db(self) -> dict: - return self.dict(exclude={"preference_identifier", "preference_type"}) + return self.model_dump(exclude={"preference_identifier", "preference_type"}) @classmethod def update_preference_default_value(cls, new_default: Any) -> None: - expected_type = cls.__fields__["value"].type_ + expected_type = ( + t[0] + if (t := get_args(cls.model_fields["value"].annotation)) + else cls.model_fields["value"].annotation + ) detected_type = type(new_default) if expected_type != detected_type: msg = ( @@ -105,14 +106,14 @@ def update_preference_default_value(cls, new_default: Any) -> None: ) raise TypeError(msg) - if cls.__fields__["value"].default is None: - cls.__fields__["value"].default_factory = lambda: new_default + if cls.model_fields["value"].default is None: + cls.model_fields["value"].default_factory = lambda: new_default else: - cls.__fields__["value"].default = new_default + cls.model_fields["value"].default = new_default class UserServiceUserPreference(_BaseUserPreferenceModel): - preference_type: PreferenceType = Field(PreferenceType.USER_SERVICE, const=True) + preference_type: Literal[PreferenceType.USER_SERVICE] = PreferenceType.USER_SERVICE service_key: ServiceKey = Field( ..., description="the service which manages the preferences" @@ -122,7 +123,7 @@ class UserServiceUserPreference(_BaseUserPreferenceModel): ) def to_db(self) -> dict: - return self.dict(exclude={"preference_type"}) + return self.model_dump(exclude={"preference_type"}) AnyUserPreference: TypeAlias = Annotated[ diff --git a/packages/models-library/src/models_library/users.py b/packages/models-library/src/models_library/users.py index 31ca948a1b8..8cb7793d2f8 100644 --- a/packages/models-library/src/models_library/users.py +++ b/packages/models-library/src/models_library/users.py @@ -1,18 +1,14 @@ -from typing import TypeAlias +from typing import Annotated, TypeAlias -from pydantic import BaseModel, ConstrainedStr, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt, StringConstraints UserID: TypeAlias = PositiveInt GroupID: TypeAlias = PositiveInt -class FirstNameStr(ConstrainedStr): - strip_whitespace = True - max_length = 255 +FirstNameStr: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, max_length=255)] - -class LastNameStr(FirstNameStr): - ... +LastNameStr: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, max_length=255)] class UserBillingDetails(BaseModel): @@ -26,5 +22,4 @@ class UserBillingDetails(BaseModel): postal_code: str | None phone: str | None - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py b/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py index 6f49b02a5ba..b48628b9ab9 100644 --- a/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py +++ b/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py @@ -3,22 +3,25 @@ # wget https://raw.githubusercontent.com/tiangolo/fastapi/master/fastapi/encoders.py --output-document=_original_fastapi_encoders # import dataclasses -from collections import defaultdict -from collections.abc import Callable +from collections import defaultdict, deque from enum import Enum from pathlib import PurePath from types import GeneratorType -from typing import Any +from typing import Any, Callable, Union +from models_library.utils.json_serialization import ENCODERS_BY_TYPE from pydantic import BaseModel -from pydantic.json import ENCODERS_BY_TYPE +from pydantic_core import PydanticUndefined, PydanticUndefinedType +from typing_extensions import Annotated, Doc -SetIntStr = set[int | str] -DictIntStrAny = dict[int | str, Any] +Undefined = PydanticUndefined +UndefinedType = PydanticUndefinedType + +IncEx = Union[set[int], set[str], dict[int, Any], dict[str, Any]] def generate_encoders_by_class_tuples( - type_encoder_map: dict[Any, Callable[[Any], Any]] + type_encoder_map: dict[Any, Callable[[Any], Any]], ) -> dict[Callable[[Any], Any], tuple[Any, ...]]: encoders_by_class_tuples: dict[Callable[[Any], Any], tuple[Any, ...]] = defaultdict( tuple @@ -32,32 +35,123 @@ def generate_encoders_by_class_tuples( def jsonable_encoder( - obj: Any, - include: SetIntStr | DictIntStrAny | None = None, - exclude: SetIntStr | DictIntStrAny | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - custom_encoder: dict[Any, Callable[[Any], Any]] | None = None, - sqlalchemy_safe: bool = True, + obj: Annotated[ + Any, + Doc( + """ + The input object to convert to JSON. + """ + ), + ], + include: Annotated[ + IncEx | None, + Doc( + """ + Pydantic's `include` parameter, passed to Pydantic models to set the + fields to include. + """ + ), + ] = None, + exclude: Annotated[ + IncEx | None, + Doc( + """ + Pydantic's `exclude` parameter, passed to Pydantic models to set the + fields to exclude. + """ + ), + ] = None, + by_alias: Annotated[ + bool, + Doc( + """ + Pydantic's `by_alias` parameter, passed to Pydantic models to define if + the output should use the alias names (when provided) or the Python + attribute names. In an API, if you set an alias, it's probably because you + want to use it in the result, so you probably want to leave this set to + `True`. + """ + ), + ] = True, + exclude_unset: Annotated[ + bool, + Doc( + """ + Pydantic's `exclude_unset` parameter, passed to Pydantic models to define + if it should exclude from the output the fields that were not explicitly + set (and that only had their default values). + """ + ), + ] = False, + exclude_defaults: Annotated[ + bool, + Doc( + """ + Pydantic's `exclude_defaults` parameter, passed to Pydantic models to define + if it should exclude from the output the fields that had the same default + value, even when they were explicitly set. + """ + ), + ] = False, + exclude_none: Annotated[ + bool, + Doc( + """ + Pydantic's `exclude_none` parameter, passed to Pydantic models to define + if it should exclude from the output any fields that have a `None` value. + """ + ), + ] = False, + custom_encoder: Annotated[ + dict[Any, Callable[[Any], Any]] | None, + Doc( + """ + Pydantic's `custom_encoder` parameter, passed to Pydantic models to define + a custom encoder. + """ + ), + ] = None, + sqlalchemy_safe: Annotated[ + bool, + Doc( + """ + Exclude from the output any fields that start with the name `_sa`. + + This is mainly a hack for compatibility with SQLAlchemy objects, they + store internal SQLAlchemy-specific state in attributes named with `_sa`, + and those objects can't (and shouldn't be) serialized to JSON. + """ + ), + ] = True, ) -> Any: + """ + Convert any object to something that can be encoded in JSON. + + This is used internally by FastAPI to make sure anything you return can be + encoded as JSON before it is sent to the client. + + You can also use it yourself, for example to convert objects before saving them + in a database that supports only JSON. + + Read more about it in the + [FastAPI docs for JSON Compatible Encoder](https://fastapi.tiangolo.com/tutorial/encoder/). + """ custom_encoder = custom_encoder or {} if custom_encoder: if type(obj) in custom_encoder: return custom_encoder[type(obj)](obj) - for encoder_type, encoder_instance in custom_encoder.items(): - if isinstance(obj, encoder_type): - return encoder_instance(obj) - if include is not None and not isinstance(include, set | dict): + else: + for encoder_type, encoder_instance in custom_encoder.items(): + if isinstance(obj, encoder_type): + return encoder_instance(obj) + if include is not None and not isinstance(include, (set, dict)): include = set(include) - if exclude is not None and not isinstance(exclude, set | dict): + if exclude is not None and not isinstance(exclude, (set, dict)): exclude = set(exclude) if isinstance(obj, BaseModel): - encoder = getattr(obj.__config__, "json_encoders", {}) - if custom_encoder: - encoder.update(custom_encoder) - obj_dict = obj.dict( + obj_dict = BaseModel.model_dump( + obj, + mode="json", include=include, exclude=exclude, by_alias=by_alias, @@ -71,7 +165,6 @@ def jsonable_encoder( obj_dict, exclude_none=exclude_none, exclude_defaults=exclude_defaults, - custom_encoder=encoder, sqlalchemy_safe=sqlalchemy_safe, ) if dataclasses.is_dataclass(obj): @@ -91,8 +184,10 @@ def jsonable_encoder( return obj.value if isinstance(obj, PurePath): return str(obj) - if isinstance(obj, str | int | float | type(None)): + if isinstance(obj, (str, int, float, type(None))): return obj + if isinstance(obj, UndefinedType): + return None if isinstance(obj, dict): encoded_dict = {} allowed_keys = set(obj.keys()) @@ -128,7 +223,7 @@ def jsonable_encoder( ) encoded_dict[encoded_key] = encoded_value return encoded_dict - if isinstance(obj, list | set | frozenset | GeneratorType | tuple): + if isinstance(obj, (list, set, frozenset, GeneratorType, tuple, deque)): encoded_list = [] for item in obj: encoded_list.append( @@ -161,7 +256,7 @@ def jsonable_encoder( data = vars(obj) except Exception as e: errors.append(e) - raise ValueError(errors) + raise ValueError(errors) from e return jsonable_encoder( data, include=include, diff --git a/packages/models-library/src/models_library/utils/json_serialization.py b/packages/models-library/src/models_library/utils/json_serialization.py index cc87c686041..69ffb00572d 100644 --- a/packages/models-library/src/models_library/utils/json_serialization.py +++ b/packages/models-library/src/models_library/utils/json_serialization.py @@ -3,12 +3,28 @@ - implemented using orjson, which performs better. SEE https://github.com/ijl/orjson?tab=readme-ov-file#performance """ +import datetime +from collections import deque from collections.abc import Callable +from decimal import Decimal +from enum import Enum +from ipaddress import ( + IPv4Address, + IPv4Interface, + IPv4Network, + IPv6Address, + IPv6Interface, + IPv6Network, +) +from pathlib import Path +from re import Pattern +from types import GeneratorType from typing import Any, Final, NamedTuple +from uuid import UUID import orjson -from pydantic.json import ENCODERS_BY_TYPE, pydantic_encoder -from pydantic.types import ConstrainedFloat +from pydantic import NameEmail, SecretBytes, SecretStr +from pydantic_extra_types.color import Color class SeparatorTuple(NamedTuple): @@ -16,12 +32,86 @@ class SeparatorTuple(NamedTuple): key_separator: str -# Extends encoders for pydantic_encoder -ENCODERS_BY_TYPE[ConstrainedFloat] = float - _orjson_default_separator: Final = SeparatorTuple(item_separator=",", key_separator=":") +def isoformat(o: datetime.date | datetime.time) -> str: + return o.isoformat() + + +def decimal_encoder(dec_value: Decimal) -> int | float: + """ + Encodes a Decimal as int of there's no exponent, otherwise float + + This is useful when we use ConstrainedDecimal to represent Numeric(x,0) + where a integer (but not int typed) is used. Encoding this as a float + results in failed round-tripping between encode and parse. + Our Id type is a prime example of this. + + >>> decimal_encoder(Decimal("1.0")) + 1.0 + + >>> decimal_encoder(Decimal("1")) + 1 + """ + if dec_value.as_tuple().exponent >= 0: # type: ignore[operator] + return int(dec_value) + else: + return float(dec_value) + + +ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { + bytes: lambda o: o.decode(), + Color: str, + datetime.date: isoformat, + datetime.datetime: isoformat, + datetime.time: isoformat, + datetime.timedelta: lambda td: td.total_seconds(), + Decimal: decimal_encoder, + Enum: lambda o: o.value, + frozenset: list, + deque: list, + GeneratorType: list, + IPv4Address: str, + IPv4Interface: str, + IPv4Network: str, + IPv6Address: str, + IPv6Interface: str, + IPv6Network: str, + NameEmail: str, + Path: str, + Pattern: lambda o: o.pattern, + SecretBytes: str, + SecretStr: str, + set: list, + UUID: str, +} + + +def pydantic_encoder(obj: Any) -> Any: + from dataclasses import asdict, is_dataclass + + from pydantic.main import BaseModel + + if isinstance(obj, BaseModel): + return obj.model_dump() + elif is_dataclass(obj): + return asdict(obj) # type: ignore[call-overload] + + # Check the class type and its superclasses for a matching encoder + for base in obj.__class__.__mro__[:-1]: + try: + encoder = ENCODERS_BY_TYPE[base] + except KeyError: + continue + return encoder(obj) + + # We have exited the for loop without finding a suitable encoder + raise TypeError( + f"Object of type '{obj.__class__.__name__}' is not JSON serializable" + ) + + def json_dumps( obj: Any, *, diff --git a/packages/models-library/src/models_library/utils/pydantic_tools_extension.py b/packages/models-library/src/models_library/utils/pydantic_tools_extension.py index 08e70fb92aa..1078120a0a6 100644 --- a/packages/models-library/src/models_library/utils/pydantic_tools_extension.py +++ b/packages/models-library/src/models_library/utils/pydantic_tools_extension.py @@ -1,15 +1,14 @@ import functools from typing import Final, TypeVar -from pydantic import Field, ValidationError -from pydantic.tools import parse_obj_as +from pydantic import Field, TypeAdapter, ValidationError T = TypeVar("T") def parse_obj_or_none(type_: type[T], obj) -> T | None: try: - return parse_obj_as(type_, obj) + return TypeAdapter(type_).validate_python(obj) except ValidationError: return None diff --git a/packages/models-library/src/models_library/utils/specs_substitution.py b/packages/models-library/src/models_library/utils/specs_substitution.py index f12968136f6..73260237b22 100644 --- a/packages/models-library/src/models_library/utils/specs_substitution.py +++ b/packages/models-library/src/models_library/utils/specs_substitution.py @@ -1,7 +1,7 @@ from typing import Any, NamedTuple, TypeAlias, cast +from models_library.errors_classes import OsparcErrorMixin from pydantic import StrictBool, StrictFloat, StrictInt -from pydantic.errors import PydanticErrorMixin from .json_serialization import json_dumps, json_loads from .string_substitution import ( @@ -15,7 +15,7 @@ SubstitutionValue: TypeAlias = StrictBool | StrictInt | StrictFloat | str -class IdentifierSubstitutionError(PydanticErrorMixin, KeyError): +class IdentifierSubstitutionError(OsparcErrorMixin, KeyError): msg_template: str = ( "Was not able to substitute identifier " "'{name}'. It was not found in: {substitutions}" diff --git a/packages/models-library/src/models_library/wallets.py b/packages/models-library/src/models_library/wallets.py index 08651353daa..29d12226972 100644 --- a/packages/models-library/src/models_library/wallets.py +++ b/packages/models-library/src/models_library/wallets.py @@ -1,9 +1,9 @@ from datetime import datetime from decimal import Decimal from enum import auto -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias -from pydantic import BaseModel, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt from .utils.enums import StrAutoEnum @@ -20,16 +20,17 @@ class WalletInfo(BaseModel): wallet_name: str wallet_credit_amount: Decimal - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "wallet_id": 1, "wallet_name": "My Wallet", - "wallet_credit_amount": Decimal(10), + "wallet_credit_amount": Decimal(10), # type: ignore[dict-item] } ] } + ) ZERO_CREDITS = Decimal(0) diff --git a/packages/models-library/src/models_library/workspaces.py b/packages/models-library/src/models_library/workspaces.py index c08e02501cb..e1d0f8d17fd 100644 --- a/packages/models-library/src/models_library/workspaces.py +++ b/packages/models-library/src/models_library/workspaces.py @@ -3,7 +3,7 @@ from models_library.access_rights import AccessRights from models_library.users import GroupID -from pydantic import BaseModel, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt WorkspaceID: TypeAlias = PositiveInt @@ -31,13 +31,11 @@ class WorkspaceDB(BaseModel): description="Timestamp of last modification", ) - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class UserWorkspaceAccessRightsDB(WorkspaceDB): my_access_rights: AccessRights access_rights: dict[GroupID, AccessRights] - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/packages/models-library/tests/test__pydantic_models.py b/packages/models-library/tests/test__pydantic_models.py index 716cf9f7906..548d34f6569 100644 --- a/packages/models-library/tests/test__pydantic_models.py +++ b/packages/models-library/tests/test__pydantic_models.py @@ -11,8 +11,9 @@ import pytest from models_library.projects_nodes import InputTypes, OutputTypes from models_library.projects_nodes_io import SimCoreFileLink -from pydantic import BaseModel, ValidationError, schema_json_of +from pydantic import BaseModel, Field, ValidationError, schema_json_of from pydantic.types import Json +from pydantic.version import version_short # NOTE: pydantic at a glance (just a few key features): # @@ -66,26 +67,31 @@ class ArgumentAnnotation(BaseModel): ArgumentAnnotation(**x_annotation.dict()) assert exc_info.value.errors()[0] == { + "input": {"items": {"type": "integer"}, "title": "schema[x]", "type": "array"}, "loc": ("data_schema",), - "msg": "JSON object must be str, bytes or bytearray", - "type": "type_error.json", + "msg": "JSON input should be string, bytes or bytearray", + "type": "json_type", + "url": f"https://errors.pydantic.dev/{version_short()}/v/json_type", } with pytest.raises(ValidationError) as exc_info: ArgumentAnnotation(name="foo", data_schema="invalid-json") assert exc_info.value.errors()[0] == { + "ctx": {"error": "expected value at line 1 column 1"}, + "input": "invalid-json", "loc": ("data_schema",), - "msg": "Invalid JSON", - "type": "value_error.json", + "msg": "Invalid JSON: expected value at line 1 column 1", + "type": "json_invalid", + "url": f"https://errors.pydantic.dev/{version_short()}/v/json_invalid", } def test_union_types_coercion(): # SEE https://pydantic-docs.helpmanual.io/usage/types/#unions class Func(BaseModel): - input: InputTypes - output: OutputTypes + input: InputTypes = Field(union_mode="left_to_right") + output: OutputTypes = Field(union_mode="left_to_right") assert get_origin(InputTypes) is Union assert get_origin(OutputTypes) is Union @@ -94,45 +100,49 @@ class Func(BaseModel): # NOTE: it is recommended that, when defining Union annotations, the most specific type is included first and followed by less specific types. # - assert Func.schema()["properties"]["input"] == { + assert Func.model_json_schema()["properties"]["input"] == { "title": "Input", "anyOf": [ {"type": "boolean"}, {"type": "integer"}, {"type": "number"}, - {"format": "json-string", "type": "string"}, + { + "contentMediaType": "application/json", + "contentSchema": {}, + "type": "string", + }, {"type": "string"}, - {"$ref": "#/definitions/PortLink"}, - {"$ref": "#/definitions/SimCoreFileLink"}, - {"$ref": "#/definitions/DatCoreFileLink"}, - {"$ref": "#/definitions/DownloadLink"}, + {"$ref": "#/$defs/PortLink"}, + {"$ref": "#/$defs/SimCoreFileLink"}, + {"$ref": "#/$defs/DatCoreFileLink"}, + {"$ref": "#/$defs/DownloadLink"}, {"type": "array", "items": {}}, {"type": "object"}, ], } # integers ------------------------ - model = Func.parse_obj({"input": "0", "output": 1}) - print(model.json(indent=1)) + model = Func.model_validate({"input": "0", "output": 1}) + print(model.model_dump_json(indent=1)) assert model.input == 0 assert model.output == 1 # numbers and bool ------------------------ - model = Func.parse_obj({"input": "0.5", "output": "false"}) - print(model.json(indent=1)) + model = Func.model_validate({"input": "0.5", "output": "false"}) + print(model.model_dump_json(indent=1)) assert model.input == 0.5 assert model.output is False # (undefined) json string vs string ------------------------ - model = Func.parse_obj( + model = Func.model_validate( { "input": '{"w": 42, "z": false}', # NOTE: this is a raw json string "output": "some/path/or/string", } ) - print(model.json(indent=1)) + print(model.model_dump_json(indent=1)) assert model.input == {"w": 42, "z": False} assert model.output == "some/path/or/string" @@ -140,24 +150,26 @@ class Func(BaseModel): # (undefined) json string vs SimCoreFileLink.dict() ------------ MINIMAL = 2 # <--- index of the example with the minimum required fields assert SimCoreFileLink in get_args(OutputTypes) - example = SimCoreFileLink.parse_obj( - SimCoreFileLink.Config.schema_extra["examples"][MINIMAL] + example = SimCoreFileLink.model_validate( + SimCoreFileLink.model_config["json_schema_extra"]["examples"][MINIMAL] ) - model = Func.parse_obj( + model = Func.model_validate( { "input": '{"w": 42, "z": false}', - "output": example.dict( + "output": example.model_dump( exclude_unset=True ), # NOTE: this is NOT a raw json string } ) - print(model.json(indent=1)) + print(model.model_dump_json(indent=1)) assert model.input == {"w": 42, "z": False} assert model.output == example assert isinstance(model.output, SimCoreFileLink) # json array and objects - model = Func.parse_obj({"input": {"w": 42, "z": False}, "output": [1, 2, 3, None]}) - print(model.json(indent=1)) + model = Func.model_validate( + {"input": {"w": 42, "z": False}, "output": [1, 2, 3, None]} + ) + print(model.model_dump_json(indent=1)) assert model.input == {"w": 42, "z": False} assert model.output == [1, 2, 3, None] diff --git a/packages/models-library/tests/test__pydantic_models_and_enums.py b/packages/models-library/tests/test__pydantic_models_and_enums.py index 51b4151fecb..51f0226ee80 100644 --- a/packages/models-library/tests/test__pydantic_models_and_enums.py +++ b/packages/models-library/tests/test__pydantic_models_and_enums.py @@ -110,8 +110,8 @@ def test_parsing_strenum_in_pydantic(): def test_parsing_str_and_enum_in_pydantic(): # Can still NOT parse equilalent enum(-only) - with pytest.raises(ValidationError): - parse_obj_as(ModelStrAndEnum, {"color": Color1.RED}) + # with pytest.raises(ValidationError): + # parse_obj_as(ModelStrAndEnum, {"color": Color1.RED}) # And the opposite? NO!!! with pytest.raises(ValidationError): diff --git a/packages/models-library/tests/test_basic_types.py b/packages/models-library/tests/test_basic_types.py index e2077d173d1..dbd847246cf 100644 --- a/packages/models-library/tests/test_basic_types.py +++ b/packages/models-library/tests/test_basic_types.py @@ -6,15 +6,15 @@ IDStr, MD5Str, SHA1Str, + ShortTruncatedStr, UUIDStr, VersionTag, ) -from pydantic import ConstrainedStr, ValidationError -from pydantic.tools import parse_obj_as +from pydantic import TypeAdapter, ValidationError class _Example(NamedTuple): - constr: type[ConstrainedStr] + constr: type[str] good: str bad: str @@ -49,27 +49,43 @@ class _Example(NamedTuple): "constraint_str_type,sample", [(p.constr, p.good) for p in _EXAMPLES], ) -def test_constrained_str_succeeds( - constraint_str_type: type[ConstrainedStr], sample: str -): - assert parse_obj_as(constraint_str_type, sample) == sample +def test_constrained_str_succeeds(constraint_str_type: type[str], sample: str): + assert TypeAdapter(constraint_str_type).validate_python(sample) == sample @pytest.mark.parametrize( "constraint_str_type,sample", [(p.constr, p.bad) for p in _EXAMPLES], ) -def test_constrained_str_fails(constraint_str_type: type[ConstrainedStr], sample: str): +def test_constrained_str_fails(constraint_str_type: type[str], sample: str): with pytest.raises(ValidationError): - parse_obj_as(constraint_str_type, sample) + TypeAdapter(constraint_str_type).validate_python(sample) def test_string_identifier_constraint_type(): # strip spaces - assert parse_obj_as(IDStr, " 123 trim spaces ") == "123 trim spaces" + assert ( + TypeAdapter(IDStr).validate_python(" 123 trim spaces ") == "123 trim spaces" + ) # limited to 100! - parse_obj_as(IDStr, "X" * 100) + TypeAdapter(IDStr).validate_python("X" * IDStr.max_length) with pytest.raises(ValidationError): - parse_obj_as(IDStr, "X" * 101) + TypeAdapter(IDStr).validate_python("X" * (IDStr.max_length + 1)) + + +def test_short_truncated_string(): + assert ( + TypeAdapter(ShortTruncatedStr).validate_python( + "X" * ShortTruncatedStr.curtail_length + ) + == "X" * ShortTruncatedStr.curtail_length + ) + + assert ( + TypeAdapter(ShortTruncatedStr).validate_python( + "X" * (ShortTruncatedStr.curtail_length + 1) + ) + == "X" * ShortTruncatedStr.curtail_length + ) diff --git a/packages/models-library/tests/test_docker.py b/packages/models-library/tests/test_docker.py index 2fddd55419a..55cb9419bbc 100644 --- a/packages/models-library/tests/test_docker.py +++ b/packages/models-library/tests/test_docker.py @@ -103,7 +103,7 @@ def test_docker_generic_tag(image_name: str, valid: bool): @pytest.mark.parametrize( "obj_data", - StandardSimcoreDockerLabels.Config.schema_extra["examples"], + StandardSimcoreDockerLabels.model_config["json_schema_extra"]["examples"], ids=str, ) def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]): diff --git a/packages/models-library/tests/test_emails.py b/packages/models-library/tests/test_emails.py index 42ae8c84f1f..f2b431c55d3 100644 --- a/packages/models-library/tests/test_emails.py +++ b/packages/models-library/tests/test_emails.py @@ -1,14 +1,21 @@ import pytest from models_library.emails import LowerCaseEmailStr -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError + + +class Profile(BaseModel): + email: LowerCaseEmailStr @pytest.mark.parametrize( "email_input", ["bla@gmail.com", "BlA@gMaIL.com", "BLA@GMAIL.COM"] ) def test_lowercase_email(email_input: str): - class Profile(BaseModel): - email: LowerCaseEmailStr - data = Profile(email=email_input) assert data.email == "bla@gmail.com" + + +@pytest.mark.parametrize("email_input", ["blagmail.com", "BlA@.com", "bLA@", ""]) +def test_malformed_email(email_input: str): + with pytest.raises(ValidationError): + Profile(email=email_input) diff --git a/packages/models-library/tests/test_errors.py b/packages/models-library/tests/test_errors.py index 6b10f6bcbdd..82cf979e463 100644 --- a/packages/models-library/tests/test_errors.py +++ b/packages/models-library/tests/test_errors.py @@ -5,7 +5,9 @@ import pytest from models_library.errors import ErrorDict -from pydantic import BaseModel, ValidationError, conint +from pydantic import BaseModel, Field, ValidationError +from pydantic.version import version_short +from typing_extensions import Annotated def test_pydantic_error_dict(): @@ -13,7 +15,7 @@ class B(BaseModel): y: list[int] class A(BaseModel): - x: conint(ge=2) + x: Annotated[int, Field(ge=2)] b: B with pytest.raises(ValidationError) as exc_info: @@ -34,13 +36,15 @@ def _copy(d, exclude): return {k: v for k, v in d.items() if k not in exclude} assert _copy(errors[0], exclude={"msg"}) == { + "ctx": {"ge": 2}, + "input": -1, "loc": ("x",), - # "msg": "ensure this value is...equal to 2", - "type": "value_error.number.not_ge", - "ctx": {"limit_value": 2}, + "type": "greater_than_equal", + "url": f"https://errors.pydantic.dev/{version_short()}/v/greater_than_equal", } assert _copy(errors[1], exclude={"msg"}) == { + "input": "wrong", "loc": ("b", "y", 1), - # "msg": "value is not a valid integer", - "type": "type_error.integer", + "type": "int_parsing", + "url": f"https://errors.pydantic.dev/{version_short()}/v/int_parsing", } diff --git a/packages/models-library/tests/test_errors_classes.py b/packages/models-library/tests/test_errors_classes.py index 4372c4eb8e5..754367805e2 100644 --- a/packages/models-library/tests/test_errors_classes.py +++ b/packages/models-library/tests/test_errors_classes.py @@ -10,7 +10,6 @@ import pytest from models_library.errors_classes import OsparcErrorMixin -from pydantic.errors import PydanticErrorMixin def test_get_full_class_name(): @@ -138,13 +137,7 @@ class MyError(OsparcErrorMixin, ValueError): def test_missing_keys_in_msg_template_does_not_raise(): - class MyErrorBefore(PydanticErrorMixin, ValueError): - msg_template = "{value} and {missing}" - - with pytest.raises(KeyError, match="missing"): - str(MyErrorBefore(value=42)) - - class MyErrorAfter(OsparcErrorMixin, ValueError): + class MyError(OsparcErrorMixin, ValueError): msg_template = "{value} and {missing}" - assert str(MyErrorAfter(value=42)) == "42 and 'missing=?'" + assert str(MyError(value=42)) == "42 and 'missing=?'" diff --git a/packages/models-library/tests/test_function_services_catalog.py b/packages/models-library/tests/test_function_services_catalog.py index 0844ed29a4e..b5f0c21b0bc 100644 --- a/packages/models-library/tests/test_function_services_catalog.py +++ b/packages/models-library/tests/test_function_services_catalog.py @@ -31,7 +31,7 @@ def test_catalog_frontend_services_registry(): registry = {(s.key, s.version): s for s in iter_service_docker_data()} for s in registry.values(): - print(s.json(exclude_unset=True, indent=1)) + print(s.model_dump_json(exclude_unset=True, indent=1)) # one version per front-end service? versions_per_service = defaultdict(list) diff --git a/packages/models-library/tests/test_generics.py b/packages/models-library/tests/test_generics.py index a1201701fd8..b778cd4a490 100644 --- a/packages/models-library/tests/test_generics.py +++ b/packages/models-library/tests/test_generics.py @@ -11,6 +11,7 @@ from faker import Faker from models_library.generics import DictModel, Envelope from pydantic import BaseModel, ValidationError +from pydantic.version import version_short def test_dict_base_model(): @@ -86,7 +87,7 @@ def test_enveloped_data_builtin(builtin_type: type, builtin_value: Any): def test_enveloped_data_model(): class User(BaseModel): idr: int - name = "Jane Doe" + name: str = "Jane Doe" enveloped = Envelope[User](data={"idr": 3}) @@ -102,9 +103,11 @@ def test_enveloped_data_dict(): error: ValidationError = err_info.value assert error.errors() == [ { + "input": "not-a-dict", "loc": ("data",), - "msg": "value is not a valid dict", - "type": "type_error.dict", + "msg": "Input should be a valid dictionary", + "type": "dict_type", + "url": f"https://errors.pydantic.dev/{version_short()}/v/dict_type", } ] @@ -122,9 +125,11 @@ def test_enveloped_data_list(): error: ValidationError = err_info.value assert error.errors() == [ { + "input": "not-a-list", "loc": ("data",), - "msg": "value is not a valid list", - "type": "type_error.list", + "msg": "Input should be a valid list", + "type": "list_type", + "url": f"https://errors.pydantic.dev/{version_short()}/v/list_type", } ] diff --git a/packages/models-library/tests/test_project_nodes_io.py b/packages/models-library/tests/test_project_nodes_io.py index 992c4d1f604..aac9568eccb 100644 --- a/packages/models-library/tests/test_project_nodes_io.py +++ b/packages/models-library/tests/test_project_nodes_io.py @@ -96,9 +96,15 @@ def test_store_discriminator(): }, } - datacore_node = Node.parse_obj(workbench["89f95b67-a2a3-4215-a794-2356684deb61"]) - rawgraph_node = Node.parse_obj(workbench["88119776-e869-4df2-a529-4aae9d9fa35c"]) - simcore_node = Node.parse_obj(workbench["75c1707c-ec1c-49ac-a7bf-af6af9088f38"]) + datacore_node = Node.model_validate( + workbench["89f95b67-a2a3-4215-a794-2356684deb61"] + ) + rawgraph_node = Node.model_validate( + workbench["88119776-e869-4df2-a529-4aae9d9fa35c"] + ) + simcore_node = Node.model_validate( + workbench["75c1707c-ec1c-49ac-a7bf-af6af9088f38"] + ) # must cast to the right subclass within project_nodes.py's InputTypes and OutputTypes unions assert datacore_node.outputs diff --git a/packages/models-library/tests/test_rabbit_messages.py b/packages/models-library/tests/test_rabbit_messages.py index 8c95af75e67..519d54c43e8 100644 --- a/packages/models-library/tests/test_rabbit_messages.py +++ b/packages/models-library/tests/test_rabbit_messages.py @@ -8,7 +8,7 @@ ProgressRabbitMessageProject, ProgressType, ) -from pydantic import parse_raw_as +from pydantic import TypeAdapter faker = Faker() @@ -19,29 +19,28 @@ pytest.param( ProgressRabbitMessageNode( project_id=faker.uuid4(cast_to=None), - user_id=faker.uuid4(cast_to=None), + user_id=faker.pyint(min_value=1), node_id=faker.uuid4(cast_to=None), progress_type=ProgressType.SERVICE_OUTPUTS_PULLING, report=ProgressReport(actual_value=0.4, total=1), - ).json(), + ).model_dump_json(), ProgressRabbitMessageNode, id="node_progress", ), pytest.param( ProgressRabbitMessageProject( project_id=faker.uuid4(cast_to=None), - user_id=faker.uuid4(cast_to=None), + user_id=faker.pyint(min_value=1), progress_type=ProgressType.PROJECT_CLOSING, report=ProgressReport(actual_value=0.4, total=1), - ).json(), + ).model_dump_json(), ProgressRabbitMessageProject, id="project_progress", ), ], ) async def test_raw_message_parsing(raw_data: str, class_type: type): - result = parse_raw_as( - Union[ProgressRabbitMessageNode, ProgressRabbitMessageProject], - raw_data, - ) + result = TypeAdapter( + Union[ProgressRabbitMessageNode, ProgressRabbitMessageProject] + ).validate_json(raw_data) assert type(result) == class_type diff --git a/packages/models-library/tests/test_rest_pagination.py b/packages/models-library/tests/test_rest_pagination.py index a9da9db2f1b..d0f5f9f7d92 100644 --- a/packages/models-library/tests/test_rest_pagination.py +++ b/packages/models-library/tests/test_rest_pagination.py @@ -7,7 +7,7 @@ @pytest.mark.parametrize("cls_model", [Page[str], PageMetaInfoLimitOffset]) def test_page_response_limit_offset_models(cls_model: BaseModel): - examples = cls_model.Config.schema_extra["examples"] + examples = cls_model.model_config["json_schema_extra"]["examples"] for index, example in enumerate(examples): print(f"{index:-^10}:\n", example) @@ -35,14 +35,14 @@ def test_invalid_count(count: int, offset: int): def test_data_size_does_not_fit_count(): - example = deepcopy(Page[str].Config.schema_extra["examples"][0]) + example = deepcopy(Page[str].model_config["json_schema_extra"]["examples"][0]) example["_meta"]["count"] = len(example["data"]) - 1 with pytest.raises(ValueError): Page[str](**example) def test_empty_data_is_converted_to_list(): - example = deepcopy(Page[str].Config.schema_extra["examples"][0]) + example = deepcopy(Page[str].model_config["json_schema_extra"]["examples"][0]) example["data"] = None example["_meta"]["count"] = 0 model_instance = Page[str](**example) diff --git a/packages/models-library/tests/test_rpc_pagination.py b/packages/models-library/tests/test_rpc_pagination.py index 787aba4daa9..26931b9032e 100644 --- a/packages/models-library/tests/test_rpc_pagination.py +++ b/packages/models-library/tests/test_rpc_pagination.py @@ -4,10 +4,12 @@ from models_library.rpc_pagination import PageRpc -@pytest.mark.parametrize("example", PageRpc.Config.schema_extra["examples"]) +@pytest.mark.parametrize( + "example", PageRpc.model_config["json_schema_extra"]["examples"] +) def test_create_page_rpc(example: dict[str, Any]): - expected = PageRpc.parse_obj(example) + expected = PageRpc.model_validate(example) assert PageRpc[str].create( expected.data, diff --git a/packages/models-library/tests/test_service_resources.py b/packages/models-library/tests/test_service_resources.py index c119a33e898..2bc0ccf7483 100644 --- a/packages/models-library/tests/test_service_resources.py +++ b/packages/models-library/tests/test_service_resources.py @@ -13,7 +13,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter @pytest.mark.parametrize( @@ -27,19 +27,19 @@ ), ) def test_compose_image(example: str) -> None: - parse_obj_as(DockerGenericTag, example) + TypeAdapter(DockerGenericTag).validate_python(example) @pytest.fixture def resources_dict() -> ResourcesDict: - return parse_obj_as( - ResourcesDict, ImageResources.Config.schema_extra["example"]["resources"] + return TypeAdapter(ResourcesDict).validate_python( + ImageResources.model_config["json_schema_extra"]["example"]["resources"] ) @pytest.fixture def compose_image() -> DockerGenericTag: - return parse_obj_as(DockerGenericTag, "image:latest") + return TypeAdapter(DockerGenericTag).validate_python("image:latest") def _ensure_resource_value_is_an_object(data: ResourcesDict) -> None: @@ -56,21 +56,21 @@ def test_resources_dict_parsed_as_expected(resources_dict: ResourcesDict) -> Non def test_image_resources_parsed_as_expected() -> None: - result: ImageResources = ImageResources.parse_obj( - ImageResources.Config.schema_extra["example"] + result: ImageResources = ImageResources.model_validate( + ImageResources.model_config["json_schema_extra"]["example"] ) _ensure_resource_value_is_an_object(result.resources) assert type(result) == ImageResources - result: ImageResources = parse_obj_as( - ImageResources, ImageResources.Config.schema_extra["example"] + result: ImageResources = TypeAdapter(ImageResources).validate_python( + ImageResources.model_config["json_schema_extra"]["example"] ) assert type(result) == ImageResources _ensure_resource_value_is_an_object(result.resources) @pytest.mark.parametrize( - "example", ServiceResourcesDictHelpers.Config.schema_extra["examples"] + "example", ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"] ) def test_service_resource_parsed_as_expected( example: dict[DockerGenericTag, Any], compose_image: DockerGenericTag @@ -84,27 +84,27 @@ def _assert_service_resources_dict( for image_resources in service_resources_dict.values(): _ensure_resource_value_is_an_object(image_resources.resources) - service_resources_dict: ServiceResourcesDict = parse_obj_as( - ServiceResourcesDict, example - ) + service_resources_dict: ServiceResourcesDict = TypeAdapter( + ServiceResourcesDict + ).validate_python(example) _assert_service_resources_dict(service_resources_dict) for image_resources in example.values(): service_resources_dict_from_single_service = ( ServiceResourcesDictHelpers.create_from_single_service( image=compose_image, - resources=ImageResources.parse_obj(image_resources).resources, + resources=ImageResources.model_validate(image_resources).resources, ) ) _assert_service_resources_dict(service_resources_dict_from_single_service) @pytest.mark.parametrize( - "example", ServiceResourcesDictHelpers.Config.schema_extra["examples"] + "example", ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"] ) def test_create_jsonable_dict(example: dict[DockerGenericTag, Any]) -> None: - service_resources_dict: ServiceResourcesDict = parse_obj_as( - ServiceResourcesDict, example - ) + service_resources_dict: ServiceResourcesDict = TypeAdapter( + ServiceResourcesDict + ).validate_python(example) result = ServiceResourcesDictHelpers.create_jsonable(service_resources_dict) assert example == result diff --git a/packages/models-library/tests/test_service_settings_labels.py b/packages/models-library/tests/test_service_settings_labels.py index a564c1be88f..f9c096804cc 100644 --- a/packages/models-library/tests/test_service_settings_labels.py +++ b/packages/models-library/tests/test_service_settings_labels.py @@ -31,7 +31,7 @@ ) from models_library.services_resources import DEFAULT_SINGLE_SERVICE_NAME from models_library.utils.string_substitution import TextTemplate -from pydantic import BaseModel, ValidationError, parse_obj_as, parse_raw_as +from pydantic import BaseModel, TypeAdapter, ValidationError, parse_obj_as from pydantic.json import pydantic_encoder @@ -43,17 +43,17 @@ class _Parametrization(NamedTuple): SIMCORE_SERVICE_EXAMPLES = { "legacy": _Parametrization( - example=SimcoreServiceLabels.Config.schema_extra["examples"][0], + example=SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][0], items=1, uses_dynamic_sidecar=False, ), "dynamic-service": _Parametrization( - example=SimcoreServiceLabels.Config.schema_extra["examples"][1], + example=SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][1], items=5, uses_dynamic_sidecar=True, ), "dynamic-service-with-compose-spec": _Parametrization( - example=SimcoreServiceLabels.Config.schema_extra["examples"][2], + example=SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][2], items=6, uses_dynamic_sidecar=True, ), @@ -66,7 +66,7 @@ class _Parametrization(NamedTuple): ids=list(SIMCORE_SERVICE_EXAMPLES.keys()), ) def test_simcore_service_labels(example: dict, items: int, uses_dynamic_sidecar: bool): - simcore_service_labels = SimcoreServiceLabels.parse_obj(example) + simcore_service_labels = SimcoreServiceLabels.model_validate(example) assert simcore_service_labels assert len(simcore_service_labels.dict(exclude_unset=True)) == items @@ -75,11 +75,11 @@ def test_simcore_service_labels(example: dict, items: int, uses_dynamic_sidecar: def test_service_settings(): simcore_settings_settings_label = SimcoreServiceSettingsLabel.parse_obj( - SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"] + SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"] ) assert simcore_settings_settings_label assert len(simcore_settings_settings_label) == len( - SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"] + SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"] ) assert simcore_settings_settings_label[0] @@ -95,7 +95,7 @@ def test_correctly_detect_dynamic_sidecar_boot( ): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) - model_instance = parse_obj_as(model_cls, example) + model_instance = TypeAdapter(model_cls).validate_python(example) assert model_instance.callbacks_mapping is not None assert model_instance.needs_dynamic_sidecar == ( "simcore.service.paths-mapping" in example @@ -104,7 +104,7 @@ def test_correctly_detect_dynamic_sidecar_boot( def test_raises_error_if_http_entrypoint_is_missing(): simcore_service_labels: dict[str, Any] = deepcopy( - SimcoreServiceLabels.Config.schema_extra["examples"][2] + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][2] ) del simcore_service_labels["simcore.service.container-http-entrypoint"] @@ -113,22 +113,27 @@ def test_raises_error_if_http_entrypoint_is_missing(): def test_path_mappings_none_state_paths(): - sample_data = deepcopy(PathMappingsLabel.Config.schema_extra["examples"][0]) + sample_data = deepcopy( + PathMappingsLabel.model_config["json_schema_extra"]["examples"][0] + ) sample_data["state_paths"] = None with pytest.raises(ValidationError): PathMappingsLabel(**sample_data) def test_path_mappings_json_encoding(): - for example in PathMappingsLabel.Config.schema_extra["examples"]: - path_mappings = PathMappingsLabel.parse_obj(example) + for example in PathMappingsLabel.model_config["json_schema_extra"]["examples"]: + path_mappings = PathMappingsLabel.model_validate(example) print(path_mappings) - assert PathMappingsLabel.parse_raw(path_mappings.json()) == path_mappings + assert ( + PathMappingsLabel.parse_raw(path_mappings.model_dump_json()) + == path_mappings + ) def test_simcore_services_labels_compose_spec_null_container_http_entry_provided(): sample_data: dict[str, Any] = deepcopy( - SimcoreServiceLabels.Config.schema_extra["examples"][2] + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][2] ) assert sample_data["simcore.service.container-http-entrypoint"] @@ -140,7 +145,7 @@ def test_simcore_services_labels_compose_spec_null_container_http_entry_provided def test_raises_error_wrong_restart_policy(): simcore_service_labels: dict[str, Any] = deepcopy( - SimcoreServiceLabels.Config.schema_extra["examples"][2] + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][2] ) simcore_service_labels["simcore.service.restart-policy"] = "__not_a_valid_policy__" @@ -150,7 +155,7 @@ def test_raises_error_wrong_restart_policy(): def test_path_mappings_label_unsupported_size_constraints(): with pytest.raises(ValidationError) as exec_into: - PathMappingsLabel.parse_obj( + PathMappingsLabel.model_validate( { "outputs_path": "/ok_input_path", "inputs_path": "/ok_output_path", @@ -163,7 +168,7 @@ def test_path_mappings_label_unsupported_size_constraints(): def test_path_mappings_label_defining_constraing_on_missing_path(): with pytest.raises(ValidationError) as exec_into: - PathMappingsLabel.parse_obj( + PathMappingsLabel.model_validate( { "outputs_path": "/ok_input_path", "inputs_path": "/ok_output_path", @@ -177,10 +182,10 @@ def test_path_mappings_label_defining_constraing_on_missing_path(): ) -PORT_1: Final[PortInt] = parse_obj_as(PortInt, 1) -PORT_3: Final[PortInt] = parse_obj_as(PortInt, 3) -PORT_20: Final[PortInt] = parse_obj_as(PortInt, 20) -PORT_99: Final[PortInt] = parse_obj_as(PortInt, 99) +PORT_1: Final[PortInt] = TypeAdapter(PortInt).validate_python(1) +PORT_3: Final[PortInt] = TypeAdapter(PortInt).validate_python(3) +PORT_20: Final[PortInt] = TypeAdapter(PortInt).validate_python(20) +PORT_99: Final[PortInt] = TypeAdapter(PortInt).validate_python(99) def test_port_range(): @@ -580,7 +585,7 @@ def test_resolving_some_service_labels_at_load_time( ) assert template.is_valid() resolved_label: str = template.safe_substitute(vendor_environments) - to_restore = parse_raw_as(pydantic_model, resolved_label) + to_restore = TypeAdapter(pydantic_model).validate_json(resolved_label) setattr(service_meta, attribute_name, to_restore) print(json.dumps(service_labels, indent=1)) @@ -590,8 +595,8 @@ def test_resolving_some_service_labels_at_load_time( # NOTE: vendor values are in the database and therefore are available at this point labels = SimcoreServiceLabels.parse_obj(service_labels) - print("After", labels.json(indent=1)) - formatted_json = service_meta.json(indent=1) + print("After", labels.model_dump_json(indent=1)) + formatted_json = service_meta.model_dump_json(indent=1) print("After", formatted_json) for entry in vendor_environments: print(entry) @@ -601,7 +606,7 @@ def test_resolving_some_service_labels_at_load_time( def test_user_preferences_path_is_part_of_exiting_volume(): labels_data = { "simcore.service.paths-mapping": json.dumps( - PathMappingsLabel.Config.schema_extra["examples"][0] + PathMappingsLabel.model_config["json_schema_extra"]["examples"][0] ), "simcore.service.user-preferences-path": json.dumps( "/tmp/outputs" # noqa: S108 diff --git a/packages/models-library/tests/test_services.py b/packages/models-library/tests/test_services.py index 54906743db1..c7b7562eaa6 100644 --- a/packages/models-library/tests/test_services.py +++ b/packages/models-library/tests/test_services.py @@ -182,7 +182,7 @@ def _find_pattern_entry(obj: dict[str, Any], key: str) -> Any: def test_boot_option_wrong_default() -> None: - for example in [deepcopy(x) for x in BootOption.Config.schema_extra["examples"]]: + for example in [deepcopy(x) for x in BootOption.model_config["json_schema_extra"]["examples"]]: with pytest.raises(ValueError): example["default"] = "__undefined__" assert BootOption(**example) @@ -201,11 +201,12 @@ def test_service_docker_data_labels_convesion(): # we want labels to look like io.simcore.a_label_property convension_breaking_fields: set[tuple[str, str]] = set() - fiedls_with_aliases: list[tuple[str, str]] = [ - (x.name, x.alias) for x in ServiceMetaDataPublished.__fields__.values() + fields_with_aliases: list[tuple[str, str]] = [ + (name, info.alias) for name, info in ServiceMetaDataPublished.model_fields.items() + if info.alias is not None ] - for name, alias in fiedls_with_aliases: + for name, alias in fields_with_aliases: if alias in FIELD_NAME_EXCEPTIONS: continue # check dashes and uppercase diff --git a/packages/models-library/tests/test_utils_common_validators.py b/packages/models-library/tests/test_utils_common_validators.py index d7fe367ab5d..d4c7cb5409f 100644 --- a/packages/models-library/tests/test_utils_common_validators.py +++ b/packages/models-library/tests/test_utils_common_validators.py @@ -7,7 +7,7 @@ none_to_empty_str_pre_validator, null_or_none_str_to_none_validator, ) -from pydantic import BaseModel, ValidationError, validator +from pydantic import BaseModel, ValidationError, field_validator def test_enums_pre_validator(): @@ -20,13 +20,14 @@ class Model(BaseModel): class ModelWithPreValidator(BaseModel): color: Enum1 - _from_equivalent_enums = validator("color", allow_reuse=True, pre=True)( + _from_equivalent_enums = field_validator("color", mode="before")( create_enums_pre_validator(Enum1) ) # with Enum1 model = Model(color=Enum1.RED) - assert ModelWithPreValidator(color=Enum1.RED) == model + # See: https://docs.pydantic.dev/latest/migration/#changes-to-pydanticbasemodel + assert ModelWithPreValidator(color=Enum1.RED).model_dump() == model.model_dump() # with Enum2 class Enum2(Enum): @@ -35,14 +36,15 @@ class Enum2(Enum): with pytest.raises(ValidationError): Model(color=Enum2.RED) - assert ModelWithPreValidator(color=Enum2.RED) == model + # See: https://docs.pydantic.dev/latest/migration/#changes-to-pydanticbasemodel + assert ModelWithPreValidator(color=Enum2.RED).model_dump() == model.model_dump() def test_empty_str_to_none_pre_validator(): class Model(BaseModel): nullable_message: str | None - _empty_is_none = validator("nullable_message", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("nullable_message", mode="before")( empty_str_to_none_pre_validator ) @@ -54,7 +56,7 @@ def test_none_to_empty_str_pre_validator(): class Model(BaseModel): message: str - _none_is_empty = validator("message", allow_reuse=True, pre=True)( + _none_is_empty = field_validator("message", mode="before")( none_to_empty_str_pre_validator ) @@ -66,9 +68,9 @@ def test_null_or_none_str_to_none_validator(): class Model(BaseModel): message: str | None - _null_or_none_str_to_none_validator = validator( - "message", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) + _null_or_none_str_to_none_validator = field_validator("message", mode="before")( + null_or_none_str_to_none_validator + ) model = Model.parse_obj({"message": "none"}) assert model == Model.parse_obj({"message": None}) diff --git a/packages/models-library/tests/test_utils_nodes.py b/packages/models-library/tests/test_utils_nodes.py index 47465ce236d..87831ac8807 100644 --- a/packages/models-library/tests/test_utils_nodes.py +++ b/packages/models-library/tests/test_utils_nodes.py @@ -14,7 +14,6 @@ SimcoreS3FileID, ) from models_library.utils.nodes import compute_node_hash -from pydantic import AnyUrl, parse_obj_as @pytest.fixture() @@ -45,9 +44,7 @@ def node_id() -> NodeID: "input_bool": True, "input_string": "string", "input_downloadlink": DownloadLink( - downloadLink=parse_obj_as( - AnyUrl, "http://httpbin.org/image/jpeg" - ) + downloadLink="http://httpbin.org/image/jpeg" ), "input_simcorelink": SimCoreFileLink( store=0, diff --git a/packages/models-library/tests/test_utils_pydantic_tools_extension.py b/packages/models-library/tests/test_utils_pydantic_tools_extension.py index 95a50099c02..34cbb528dbb 100644 --- a/packages/models-library/tests/test_utils_pydantic_tools_extension.py +++ b/packages/models-library/tests/test_utils_pydantic_tools_extension.py @@ -14,15 +14,20 @@ class MyModel(BaseModel): def test_schema(): - assert MyModel.schema() == { + assert MyModel.model_json_schema() == { "title": "MyModel", "type": "object", "properties": { "a": {"title": "A", "type": "integer"}, - "b": {"title": "B", "type": "integer"}, + "b": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "B"}, "c": {"title": "C", "default": 42, "type": "integer"}, - "d": {"title": "D", "type": "integer"}, + "d": { + "anyOf": [{"type": "integer"}, {"type": "null"}], + "default": None, + "title": "D", + }, "e": { + "default": None, "title": "E", "type": "integer", "description": "optional non-nullable", diff --git a/packages/models-library/tests/test_utils_service_io.py b/packages/models-library/tests/test_utils_service_io.py index 2bf58a28869..51e4324e2be 100644 --- a/packages/models-library/tests/test_utils_service_io.py +++ b/packages/models-library/tests/test_utils_service_io.py @@ -20,10 +20,10 @@ from pydantic import parse_obj_as example_inputs_labels = [ - e for e in ServiceInput.Config.schema_extra["examples"] if e["label"] + e for e in ServiceInput.model_config["json_schema_extra"]["examples"] if e["label"] ] example_outputs_labels = [ - e for e in ServiceOutput.Config.schema_extra["examples"] if e["label"] + e for e in ServiceOutput.model_config["json_schema_extra"]["examples"] if e["label"] ] @@ -31,16 +31,16 @@ def service_port(request: pytest.FixtureRequest) -> ServiceInput | ServiceOutput: try: index = example_inputs_labels.index(request.param) - example = ServiceInput.Config.schema_extra["examples"][index] + example = ServiceInput.model_config["json_schema_extra"]["examples"][index] return ServiceInput.parse_obj(example) except ValueError: index = example_outputs_labels.index(request.param) - example = ServiceOutput.Config.schema_extra["examples"][index] + example = ServiceOutput.model_config["json_schema_extra"]["examples"][index] return ServiceOutput.parse_obj(example) def test_get_schema_from_port(service_port: ServiceInput | ServiceOutput): - print(service_port.json(indent=2)) + print(service_port.model_dump_json(indent=2)) # get schema = get_service_io_json_schema(service_port) diff --git a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py index 7cfbf13df11..297e9a9ab13 100644 --- a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py @@ -82,7 +82,7 @@ def _is_model_cls(obj) -> bool: for model_name, model_cls in inspect.getmembers(module, _is_model_cls): assert model_name # nosec if ( - (config_cls := model_cls.Config) + (config_cls := model_cls.model_config) and inspect.isclass(config_cls) and is_strict_inner(model_cls, config_cls) and (schema_extra := getattr(config_cls, "schema_extra", {})) @@ -121,7 +121,9 @@ def model_cls_examples(model_cls: type[BaseModel]) -> dict[str, dict[str, Any]]: ) # checks exampleS setup in schema_extra - examples_list = copy.deepcopy(model_cls.Config.schema_extra.get("examples", [])) + examples_list = copy.deepcopy( + model_cls.model_config["json_schema_extra"].get("examples", []) + ) assert isinstance(examples_list, list), ( "OpenAPI and json-schema differ regarding the format for exampleS." "The former is a dict and the latter an array. " @@ -131,7 +133,7 @@ def model_cls_examples(model_cls: type[BaseModel]) -> dict[str, dict[str, Any]]: ) # check example in schema_extra - example = copy.deepcopy(model_cls.Config.schema_extra.get("example")) + example = copy.deepcopy(model_cls.model_config["json_schema_extra"].get("example")) # collect all examples and creates fixture -> {example-name: example, ...} examples = { diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 3316f4276ed..3e40b2694d4 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -32,11 +32,6 @@ urllib3>=1.26.5 # https://github.com/advisories/GH # Breaking changes ----------------------------------------------------------------------------------------- # - -# SEE https://github.com/ITISFoundation/osparc-simcore/issues/4481 -fastapi<0.100.0 -pydantic<2.0 - # with new released version 1.0.0 (https://github.com/aio-libs/aiozipkin/releases). # TODO: includes async features https://docs.sqlalchemy.org/en/14/changelog/migration_20.html sqlalchemy<2.0 diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py index 889b8cfcd1c..285d08ef58e 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/s3.py @@ -4,6 +4,7 @@ from aws_library.s3 import S3NotConnectedError, SimcoreS3API from fastapi import FastAPI from models_library.api_schemas_storage import S3BucketName +from pydantic import TypeAdapter from settings_library.s3 import S3Settings from tenacity import ( AsyncRetrying, @@ -36,7 +37,9 @@ async def on_startup() -> None: ): with attempt: connected = await client.http_check_bucket_connected( - bucket=S3BucketName(settings.S3_BUCKET_NAME) + bucket=TypeAdapter(S3BucketName).validate_python( + settings.S3_BUCKET_NAME + ) ) if not connected: raise S3NotConnectedError # pragma: no cover diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index 75e17e28c96..8977d6fa5a0 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -18,7 +18,7 @@ from models_library.rest_ordering import OrderBy from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import AnyUrl, PositiveInt +from pydantic import AnyUrl, PositiveInt, TypeAdapter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, ) @@ -158,7 +158,7 @@ async def export_service_runs( started_until = filters.started_at.until if filters else None # Create S3 key name - s3_bucket_name = S3BucketName(bucket_name) + s3_bucket_name = TypeAdapter(S3BucketName).validate_python(bucket_name) # NOTE: su stands for "service usage" file_name = f"su_{shortuuid.uuid()}.csv" s3_object_key = f"resource-usage-tracker-service-runs/{datetime.now(tz=timezone.utc).date()}/{file_name}" diff --git a/services/storage/src/simcore_service_storage/handlers_health.py b/services/storage/src/simcore_service_storage/handlers_health.py index 4ae743afef9..eb94feb9bb4 100644 --- a/services/storage/src/simcore_service_storage/handlers_health.py +++ b/services/storage/src/simcore_service_storage/handlers_health.py @@ -11,6 +11,7 @@ from models_library.api_schemas_storage import HealthCheck, S3BucketName from models_library.app_diagnostics import AppStatusCheck from models_library.utils.json_serialization import json_dumps +from pydantic import TypeAdapter from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ._meta import API_VERSION, API_VTAG, PROJECT_NAME, VERSION @@ -53,7 +54,9 @@ async def get_status(request: web.Request) -> web.Response: s3_state = ( "connected" if await get_s3_client(request.app).bucket_exists( - bucket=S3BucketName(app_settings.STORAGE_S3.S3_BUCKET_NAME) + bucket=TypeAdapter(S3BucketName).validate_python( + app_settings.STORAGE_S3.S3_BUCKET_NAME + ) ) else "no access to S3 bucket" ) diff --git a/services/storage/tests/unit/test_dsm_soft_links.py b/services/storage/tests/unit/test_dsm_soft_links.py index d2d1c6acd65..ed0e01ea7f0 100644 --- a/services/storage/tests/unit/test_dsm_soft_links.py +++ b/services/storage/tests/unit/test_dsm_soft_links.py @@ -13,7 +13,7 @@ from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize +from pydantic import ByteSize, TypeAdapter from simcore_postgres_database.storage_models import file_meta_data from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -34,7 +34,7 @@ async def output_file( file = FileMetaData.from_simcore_node( user_id=user_id, file_id=SimcoreS3FileID(f"{project_id}/{node_id}/filename.txt"), - bucket=S3BucketName("master-simcore"), + bucket=TypeAdapter(S3BucketName).validate_python("master-simcore"), location_id=SimcoreS3DataManager.get_location_id(), location_name=SimcoreS3DataManager.get_location_name(), sha256_checksum=faker.sha256(), diff --git a/services/storage/tests/unit/test_models.py b/services/storage/tests/unit/test_models.py index 82bd900b772..0dbab6821d2 100644 --- a/services/storage/tests/unit/test_models.py +++ b/services/storage/tests/unit/test_models.py @@ -4,7 +4,7 @@ from models_library.api_schemas_storage import S3BucketName from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID, StorageFileID -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError from simcore_service_storage.models import FileMetaData from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -15,7 +15,7 @@ ) def test_file_id_raises_error(file_id: str): with pytest.raises(ValidationError): - parse_obj_as(StorageFileID, file_id) + TypeAdapter(StorageFileID).validate_python(file_id) @pytest.mark.parametrize( @@ -38,17 +38,17 @@ def test_file_id_raises_error(file_id: str): ], ) def test_file_id(file_id: str): - parsed_file_id = parse_obj_as(StorageFileID, file_id) + parsed_file_id = TypeAdapter(StorageFileID).validate_python(file_id) assert parsed_file_id assert parsed_file_id == file_id def test_fmd_build(): - file_id = parse_obj_as(SimcoreS3FileID, f"api/{uuid.uuid4()}/xx.dat") + file_id = TypeAdapter(SimcoreS3FileID).validate_python(f"api/{uuid.uuid4()}/xx.dat") fmd = FileMetaData.from_simcore_node( user_id=12, file_id=file_id, - bucket=S3BucketName("test-bucket"), + bucket=TypeAdapter(S3BucketName).validate_python("test-bucket"), location_id=SimcoreS3DataManager.get_location_id(), location_name=SimcoreS3DataManager.get_location_name(), sha256_checksum=None, @@ -64,11 +64,13 @@ def test_fmd_build(): assert fmd.location_id == SimcoreS3DataManager.get_location_id() assert fmd.bucket_name == "test-bucket" - file_id = parse_obj_as(SimcoreS3FileID, f"{uuid.uuid4()}/{uuid.uuid4()}/xx.dat") + file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{uuid.uuid4()}/{uuid.uuid4()}/xx.dat" + ) fmd = FileMetaData.from_simcore_node( user_id=12, file_id=file_id, - bucket=S3BucketName("test-bucket"), + bucket=TypeAdapter(S3BucketName).validate_python("test-bucket"), location_id=SimcoreS3DataManager.get_location_id(), location_name=SimcoreS3DataManager.get_location_name(), sha256_checksum=None, diff --git a/services/storage/tests/unit/test_utils.py b/services/storage/tests/unit/test_utils.py index 1b71a1c29d5..90dca22d42d 100644 --- a/services/storage/tests/unit/test_utils.py +++ b/services/storage/tests/unit/test_utils.py @@ -16,7 +16,7 @@ from faker import Faker from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from pydantic import ByteSize, HttpUrl, parse_obj_as +from pydantic import ByteSize, HttpUrl, TypeAdapter, parse_obj_as from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID from simcore_service_storage.models import ETag, FileMetaData, S3BucketName, UploadID from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -79,7 +79,7 @@ def test_file_entry_valid( fmd = FileMetaData.from_simcore_node( user_id=faker.pyint(min_value=1), file_id=file_id, - bucket=S3BucketName("pytest-bucket"), + bucket=TypeAdapter(S3BucketName).validate_python("pytest-bucket"), location_id=SimcoreS3DataManager.get_location_id(), location_name=SimcoreS3DataManager.get_location_name(), sha256_checksum=None, diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 8613fbc8319..4f6b3c3cf1f 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -27,7 +27,7 @@ ) from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects import ProjectID -from models_library.projects_access import Owner, PositiveIntWithExclusiveMinimumRemoved +from models_library.projects_access import Owner from models_library.projects_state import ( ProjectLocked, ProjectRunningState, @@ -41,6 +41,7 @@ ServiceResourcesDictHelpers, ) from models_library.utils.fastapi_encoders import jsonable_encoder +from pydantic import PositiveInt from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict, log_client_in @@ -1308,7 +1309,7 @@ async def test_open_shared_project_2_users_locked( expected_project_state_client_2.locked.value = True expected_project_state_client_2.locked.status = ProjectStatus.OPENED owner2 = Owner( - user_id=PositiveIntWithExclusiveMinimumRemoved(user_2["id"]), + user_id=PositiveInt(user_2["id"]), first_name=user_2.get("first_name", None), last_name=user_2.get("last_name", None), ) From 19e3923b2d94ea0a46ad61d559662ea2f8ad6848 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Oct 2024 16:32:55 +0200 Subject: [PATCH 02/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Add=20common=20libra?= =?UTF-8?q?ry=20(#6495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/CODEOWNERS | 1 + .github/workflows/ci-testing-deploy.yml | 45 +++++++++++ ci/github/unit-testing/common-library.bash | 43 ++++++++++ packages/aws-library/requirements/_base.in | 1 + packages/common-library/.gitignore | 0 packages/common-library/Makefile | 49 ++++++++++++ packages/common-library/README.md | 42 ++++++++++ packages/common-library/VERSION | 1 + packages/common-library/requirements/Makefile | 6 ++ packages/common-library/requirements/_base.in | 6 ++ .../common-library/requirements/_base.txt | 12 +++ packages/common-library/requirements/_test.in | 21 +++++ .../common-library/requirements/_test.txt | 55 +++++++++++++ .../common-library/requirements/_tools.in | 5 ++ .../common-library/requirements/_tools.txt | 79 +++++++++++++++++++ packages/common-library/requirements/ci.txt | 17 ++++ packages/common-library/requirements/dev.txt | 18 +++++ packages/common-library/setup.cfg | 24 ++++++ packages/common-library/setup.py | 60 ++++++++++++++ .../src/common_library/__init__.py | 12 +++ .../src/common_library}/errors_classes.py | 5 +- .../src/common_library/py.typed | 0 .../pydantic_fields_extension.py | 22 ++++++ .../pydantic_networks_extension.py | 5 ++ .../src/common_library/serialization.py | 32 ++++++++ packages/common-library/tests/conftest.py | 33 ++++++++ .../tests/test_errors_classes.py | 2 +- .../tests/test_pydantic_fields_extension.py | 72 +++++++++++++++++ .../tests/test_pydantic_networks_extension.py | 20 +++++ .../tests/test_serialization.py | 30 +++++++ .../requirements/_base.in | 1 + .../requirements/ci.txt | 1 + .../requirements/dev.txt | 1 + packages/models-library/requirements/_base.in | 1 + packages/models-library/requirements/ci.txt | 1 + packages/models-library/requirements/dev.txt | 1 + .../osparc_variable_identifier.py | 3 +- .../utils/specs_substitution.py | 3 +- .../requirements/_base.in | 1 + .../notifications-library/requirements/ci.txt | 1 + .../requirements/dev.txt | 1 + .../src/notifications_library/errors.py | 2 +- .../service-integration/requirements/_base.in | 1 + .../service-integration/requirements/ci.txt | 1 + .../service-integration/requirements/dev.txt | 1 + .../service-library/requirements/_base.in | 1 + packages/service-library/requirements/ci.txt | 1 + .../requirements/ci[aiohttp].txt | 1 + .../service-library/requirements/ci[all].txt | 1 + .../requirements/ci[fastapi].txt | 1 + packages/service-library/requirements/dev.txt | 1 + .../requirements/dev[aiohttp].txt | 1 + .../service-library/requirements/dev[all].txt | 1 + .../requirements/dev[fastapi].txt | 1 + .../rabbitmq/rpc_interfaces/catalog/errors.py | 2 +- .../settings-library/requirements/_base.in | 5 +- packages/settings-library/requirements/ci.txt | 1 + .../settings-library/requirements/dev.txt | 1 + packages/simcore-sdk/requirements/_base.in | 2 + packages/simcore-sdk/requirements/ci.txt | 1 + packages/simcore-sdk/requirements/dev.txt | 1 + services/agent/requirements/_base.in | 1 + services/agent/requirements/ci.txt | 1 + services/agent/requirements/dev.txt | 1 + services/api-server/requirements/_base.in | 1 + services/api-server/requirements/ci.txt | 1 + services/api-server/requirements/dev.txt | 1 + services/autoscaling/requirements/_base.in | 1 + services/autoscaling/requirements/ci.txt | 1 + services/autoscaling/requirements/dev.txt | 1 + services/catalog/requirements/_base.in | 1 + services/catalog/requirements/ci.txt | 1 + services/catalog/requirements/dev.txt | 1 + .../clusters-keeper/requirements/_base.in | 1 + services/clusters-keeper/requirements/ci.txt | 1 + services/clusters-keeper/requirements/dev.txt | 1 + services/dask-sidecar/requirements/_base.in | 1 + services/dask-sidecar/requirements/ci.txt | 1 + services/dask-sidecar/requirements/dev.txt | 1 + .../datcore-adapter/requirements/_base.in | 1 + services/datcore-adapter/requirements/ci.txt | 1 + services/datcore-adapter/requirements/dev.txt | 1 + services/director-v2/requirements/_base.in | 1 + services/director-v2/requirements/ci.txt | 1 + services/director-v2/requirements/dev.txt | 1 + .../dynamic-scheduler/requirements/_base.in | 1 + .../dynamic-scheduler/requirements/ci.txt | 1 + .../dynamic-scheduler/requirements/dev.txt | 1 + .../dynamic-sidecar/requirements/_base.in | 1 + services/dynamic-sidecar/requirements/ci.txt | 1 + services/dynamic-sidecar/requirements/dev.txt | 1 + services/efs-guardian/requirements/_base.in | 1 + services/efs-guardian/requirements/ci.txt | 1 + services/efs-guardian/requirements/dev.txt | 1 + services/invitations/requirements/_base.in | 1 + services/invitations/requirements/ci.txt | 3 +- services/invitations/requirements/dev.txt | 4 +- services/payments/requirements/_base.in | 1 + services/payments/requirements/ci.txt | 2 + services/payments/requirements/dev.txt | 1 + .../requirements/_base.in | 1 + .../requirements/ci.txt | 1 + .../requirements/dev.txt | 1 + services/storage/requirements/_base.in | 1 + services/storage/requirements/ci.txt | 1 + services/storage/requirements/dev.txt | 1 + 106 files changed, 801 insertions(+), 10 deletions(-) create mode 100755 ci/github/unit-testing/common-library.bash create mode 100644 packages/common-library/.gitignore create mode 100644 packages/common-library/Makefile create mode 100644 packages/common-library/README.md create mode 100644 packages/common-library/VERSION create mode 100644 packages/common-library/requirements/Makefile create mode 100644 packages/common-library/requirements/_base.in create mode 100644 packages/common-library/requirements/_base.txt create mode 100644 packages/common-library/requirements/_test.in create mode 100644 packages/common-library/requirements/_test.txt create mode 100644 packages/common-library/requirements/_tools.in create mode 100644 packages/common-library/requirements/_tools.txt create mode 100644 packages/common-library/requirements/ci.txt create mode 100644 packages/common-library/requirements/dev.txt create mode 100644 packages/common-library/setup.cfg create mode 100644 packages/common-library/setup.py create mode 100644 packages/common-library/src/common_library/__init__.py rename packages/{models-library/src/models_library => common-library/src/common_library}/errors_classes.py (89%) create mode 100644 packages/common-library/src/common_library/py.typed create mode 100644 packages/common-library/src/common_library/pydantic_fields_extension.py create mode 100644 packages/common-library/src/common_library/pydantic_networks_extension.py create mode 100644 packages/common-library/src/common_library/serialization.py create mode 100644 packages/common-library/tests/conftest.py rename packages/{models-library => common-library}/tests/test_errors_classes.py (98%) create mode 100644 packages/common-library/tests/test_pydantic_fields_extension.py create mode 100644 packages/common-library/tests/test_pydantic_networks_extension.py create mode 100644 packages/common-library/tests/test_serialization.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 89701508fbf..faa7597a371 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ Makefile @pcrespov @sanderegg /api/ @sanderegg @pcrespov @matusdrobuliak66 /ci/ @sanderegg @pcrespov /docs/ @pcrespov +/packages/common-library/ @giancarloromeo /packages/models-library/ @sanderegg @pcrespov @matusdrobuliak66 /packages/postgres-database/ @matusdrobuliak66 /packages/pytest-simcore/ @pcrespov @sanderegg diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 1ec9dd55293..adbea2e8e41 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -54,6 +54,7 @@ jobs: aws-library: ${{ steps.filter.outputs.aws-library }} dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }} models-library: ${{ steps.filter.outputs.models-library }} + common-library: ${{ steps.filter.outputs.common-library }} notifications-library: ${{ steps.filter.outputs.notifications-library }} postgres-database: ${{ steps.filter.outputs.postgres-database }} service-integration: ${{ steps.filter.outputs.service-integration }} @@ -110,6 +111,8 @@ jobs: - 'services/docker-compose*' - 'scripts/mypy/*' - 'mypy.ini' + common-library: + - 'packages/common-library/**' notifications-library: - 'packages/notifications-library/**' - 'packages/postgres-database/**' @@ -1593,6 +1596,47 @@ jobs: with: flags: unittests #optional + unit-test-common-library: + needs: changes + if: ${{ needs.changes.outputs.common-library == 'true' || github.event_name == 'push' }} + timeout-minutes: 18 # if this timeout gets too small, then split the tests + name: "[unit] common-library" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-22.04] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: install uv + uses: yezz123/setup-uv@v4 + - uses: actions/cache@v4 + id: cache-uv + with: + path: ~/.cache/uv + key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: ./ci/github/unit-testing/common-library.bash install + - name: typecheck + run: ./ci/github/unit-testing/common-library.bash typecheck + - name: test + run: ./ci/github/unit-testing/common-library.bash test + - uses: codecov/codecov-action@v4.5.0 + with: + flags: unittests #optional + unit-test-notifications-library: needs: changes if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }} @@ -1704,6 +1748,7 @@ jobs: unit-test-efs-guardian, unit-test-frontend, unit-test-models-library, + unit-test-common-library, unit-test-notifications-library, unit-test-osparc-gateway-server, unit-test-payments, diff --git a/ci/github/unit-testing/common-library.bash b/ci/github/unit-testing/common-library.bash new file mode 100755 index 00000000000..71547174103 --- /dev/null +++ b/ci/github/unit-testing/common-library.bash @@ -0,0 +1,43 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/common-library + make install-ci + popd + uv pip list +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/common-library + make tests-ci + popd +} + +typecheck() { + # shellcheck source=/dev/null + source .venv/bin/activate + uv pip install mypy + pushd packages/common-library + make mypy + popd +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index 628cebcf110..d884806f703 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -2,6 +2,7 @@ # Specifies third-party dependencies for 'aws-library' # --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/packages/common-library/.gitignore b/packages/common-library/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/common-library/Makefile b/packages/common-library/Makefile new file mode 100644 index 00000000000..b554ec6f9c0 --- /dev/null +++ b/packages/common-library/Makefile @@ -0,0 +1,49 @@ +# +# Targets for DEVELOPMENT of common Library +# +include ../../scripts/common.Makefile +include ../../scripts/common-package.Makefile + +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE_C) requirements reqs + + +.PHONY: install-dev install-prod install-ci +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + @uv pip sync requirements/$(subst install-,,$@).txt + + +.PHONY: tests tests-ci +tests: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov=common_library \ + --durations=10 \ + --exitfirst \ + --failed-first \ + --pdb \ + -vv \ + $(CURDIR)/tests + +tests-ci: ## runs unit tests [ci-mode] + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-append \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov-report=xml \ + --cov=common_library \ + --durations=10 \ + --log-date-format="%Y-%m-%d %H:%M:%S" \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --verbose \ + -m "not heavy_load" \ + $(CURDIR)/tests diff --git a/packages/common-library/README.md b/packages/common-library/README.md new file mode 100644 index 00000000000..8e5c489787b --- /dev/null +++ b/packages/common-library/README.md @@ -0,0 +1,42 @@ +# simcore pydantic common library + +Contains the common classes, functions and in general utilities for use in the simcore platform. + +## Installation + +```console +make help +make install-dev +``` + +## Test + +```console +make help +make test-dev +``` + + +## Diagnostics + +How run diagnostics on the service metadata published in a docker registry? + +1. Setup environment +```bash +make devenv +source .venv/bin/activate + +cd packages/common-library +make install-dev +``` +2. Set ``REGISTRY_*`` env vars in ``.env`` (in the repository base folder) +3. Download test data, run diagnostics, archive tests-data, and cleanup +```bash +export DEPLOY_NAME=my-deploy + +make pull_test_data >$DEPLOY_NAME-registry-diagnostics.log 2>&1 +pytest -vv -m diagnostics >>$DEPLOY_NAME-registry-diagnostics.log 2>&1 +zip -r $DEPLOY_NAME-registry-test-data.zip tests/data/.downloaded-ignore +rm -r tests/data/.downloaded-ignore +``` +4. Move all ``$DEPLOY_NAME-*`` files to an archive diff --git a/packages/common-library/VERSION b/packages/common-library/VERSION new file mode 100644 index 00000000000..6e8bf73aa55 --- /dev/null +++ b/packages/common-library/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/packages/common-library/requirements/Makefile b/packages/common-library/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/packages/common-library/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/packages/common-library/requirements/_base.in b/packages/common-library/requirements/_base.in new file mode 100644 index 00000000000..73cc02bceb7 --- /dev/null +++ b/packages/common-library/requirements/_base.in @@ -0,0 +1,6 @@ +# +# Specifies third-party dependencies for 'common-library' +# +--constraint ../../../requirements/constraints.txt + +pydantic diff --git a/packages/common-library/requirements/_base.txt b/packages/common-library/requirements/_base.txt new file mode 100644 index 00000000000..32a8575abea --- /dev/null +++ b/packages/common-library/requirements/_base.txt @@ -0,0 +1,12 @@ +annotated-types==0.7.0 + # via pydantic +pydantic==2.9.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in +pydantic-core==2.23.4 + # via pydantic +typing-extensions==4.12.2 + # via + # pydantic + # pydantic-core diff --git a/packages/common-library/requirements/_test.in b/packages/common-library/requirements/_test.in new file mode 100644 index 00000000000..4454d79d36a --- /dev/null +++ b/packages/common-library/requirements/_test.in @@ -0,0 +1,21 @@ +# +# Specifies dependencies required to run 'common-library' +# +--constraint ../../../requirements/constraints.txt + +# Adds base AS CONSTRAINT specs, not requirement. +# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt +# +--constraint _base.txt + +coverage +faker +pytest +pytest-asyncio +pytest-cov +pytest-icdiff +pytest-instafail +pytest-mock +pytest-runner +pytest-sugar +python-dotenv diff --git a/packages/common-library/requirements/_test.txt b/packages/common-library/requirements/_test.txt new file mode 100644 index 00000000000..2354abd790d --- /dev/null +++ b/packages/common-library/requirements/_test.txt @@ -0,0 +1,55 @@ +coverage==7.6.1 + # via + # -r requirements/_test.in + # pytest-cov +faker==30.1.0 + # via -r requirements/_test.in +icdiff==2.0.7 + # via pytest-icdiff +iniconfig==2.0.0 + # via pytest +packaging==24.1 + # via + # pytest + # pytest-sugar +pluggy==1.5.0 + # via pytest +pprintpp==0.4.0 + # via pytest-icdiff +pytest==8.3.3 + # via + # -r requirements/_test.in + # pytest-asyncio + # pytest-cov + # pytest-icdiff + # pytest-instafail + # pytest-mock + # pytest-sugar +pytest-asyncio==0.23.8 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +pytest-cov==5.0.0 + # via -r requirements/_test.in +pytest-icdiff==0.9 + # via -r requirements/_test.in +pytest-instafail==0.5.0 + # via -r requirements/_test.in +pytest-mock==3.14.0 + # via -r requirements/_test.in +pytest-runner==6.0.1 + # via -r requirements/_test.in +pytest-sugar==1.0.0 + # via -r requirements/_test.in +python-dateutil==2.9.0.post0 + # via faker +python-dotenv==1.0.1 + # via -r requirements/_test.in +six==1.16.0 + # via python-dateutil +termcolor==2.5.0 + # via pytest-sugar +typing-extensions==4.12.2 + # via + # -c requirements/_base.txt + # faker diff --git a/packages/common-library/requirements/_tools.in b/packages/common-library/requirements/_tools.in new file mode 100644 index 00000000000..1def82c12a3 --- /dev/null +++ b/packages/common-library/requirements/_tools.in @@ -0,0 +1,5 @@ +--constraint ../../../requirements/constraints.txt +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt diff --git a/packages/common-library/requirements/_tools.txt b/packages/common-library/requirements/_tools.txt new file mode 100644 index 00000000000..a333bb822ae --- /dev/null +++ b/packages/common-library/requirements/_tools.txt @@ -0,0 +1,79 @@ +astroid==3.3.5 + # via pylint +black==24.8.0 + # via -r requirements/../../../requirements/devenv.txt +build==1.2.2.post1 + # via pip-tools +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.4.0 + # via pre-commit +click==8.1.7 + # via + # black + # pip-tools +dill==0.3.9 + # via pylint +distlib==0.3.8 + # via virtualenv +filelock==3.16.1 + # via virtualenv +identify==2.6.1 + # via pre-commit +isort==5.13.2 + # via + # -r requirements/../../../requirements/devenv.txt + # pylint +mccabe==0.7.0 + # via pylint +mypy==1.11.2 + # via -r requirements/../../../requirements/devenv.txt +mypy-extensions==1.0.0 + # via + # black + # mypy +nodeenv==1.9.1 + # via pre-commit +packaging==24.1 + # via + # -c requirements/_test.txt + # black + # build +pathspec==0.12.1 + # via black +pip==24.2 + # via pip-tools +pip-tools==7.4.1 + # via -r requirements/../../../requirements/devenv.txt +platformdirs==4.3.6 + # via + # black + # pylint + # virtualenv +pre-commit==4.0.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==3.3.1 + # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.2.0 + # via + # build + # pip-tools +pyyaml==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # pre-commit +ruff==0.6.9 + # via -r requirements/../../../requirements/devenv.txt +setuptools==75.1.0 + # via pip-tools +tomlkit==0.13.2 + # via pylint +typing-extensions==4.12.2 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # mypy +virtualenv==20.26.6 + # via pre-commit +wheel==0.44.0 + # via pip-tools diff --git a/packages/common-library/requirements/ci.txt b/packages/common-library/requirements/ci.txt new file mode 100644 index 00000000000..ed9eb3028e8 --- /dev/null +++ b/packages/common-library/requirements/ci.txt @@ -0,0 +1,17 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'common-library' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt + +# installs this repo's packages +pytest-simcore @ ../pytest-simcore + +# current module +simcore-common-library @ . diff --git a/packages/common-library/requirements/dev.txt b/packages/common-library/requirements/dev.txt new file mode 100644 index 00000000000..02718f95c3a --- /dev/null +++ b/packages/common-library/requirements/dev.txt @@ -0,0 +1,18 @@ +# Shortcut to install all packages needed to develop 'common-library' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +--editable ../pytest-simcore/ + +# current module +--editable . diff --git a/packages/common-library/setup.cfg b/packages/common-library/setup.cfg new file mode 100644 index 00000000000..b33be52008a --- /dev/null +++ b/packages/common-library/setup.cfg @@ -0,0 +1,24 @@ +[bumpversion] +current_version = 0.2.0 +commit = True +message = packages/common-library version: {current_version} → {new_version} +tag = False +commit_args = --no-verify + +[bumpversion:file:VERSION] + +[bdist_wheel] +universal = 1 + +[aliases] +test = pytest + +[tool:pytest] +asyncio_mode = auto +markers = + diagnostics: "can be used to run diagnostics against deployed data (e.g. database, registry etc)" + testit: "marks test to run during development" + +[mypy] +plugins = + pydantic.mypy diff --git a/packages/common-library/setup.py b/packages/common-library/setup.py new file mode 100644 index 00000000000..4e381f5bbc2 --- /dev/null +++ b/packages/common-library/setup.py @@ -0,0 +1,60 @@ +import re +import sys +from pathlib import Path + +from setuptools import find_packages, setup + + +def read_reqs(reqs_path: Path) -> set[str]: + return { + r + for r in re.findall( + r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", + reqs_path.read_text(), + re.MULTILINE, + ) + if isinstance(r, str) + } + + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +INSTALL_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_base.in") +) # WEAK requirements + +TEST_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_test.txt") +) # STRICK requirements + + +SETUP = { + "name": "simcore-common-library", + "version": Path(CURRENT_DIR / "VERSION").read_text().strip(), + "author": "Giancarlo Romeo (giancarloromeo)", + "description": "Core service library for simcore pydantic common", + "python_requires": "~=3.11", + "classifiers": [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.11", + ], + "long_description": Path(CURRENT_DIR / "README.md").read_text(), + "license": "MIT license", + "install_requires": INSTALL_REQUIREMENTS, + "packages": find_packages(where="src"), + "package_data": {"": ["py.typed"]}, + "package_dir": {"": "src"}, + "include_package_data": True, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "zip_safe": False, +} + + +if __name__ == "__main__": + setup(**SETUP) diff --git a/packages/common-library/src/common_library/__init__.py b/packages/common-library/src/common_library/__init__.py new file mode 100644 index 00000000000..dc0c65ff721 --- /dev/null +++ b/packages/common-library/src/common_library/__init__.py @@ -0,0 +1,12 @@ +""" osparc's service common library + +""" + +# +# NOTE: +# - "examples" = [ ...] keyword and NOT "example". See https://json-schema.org/understanding-json-schema/reference/generic.html#annotations +# + +from importlib.metadata import version + +__version__: str = version("simcore-common-library") diff --git a/packages/models-library/src/models_library/errors_classes.py b/packages/common-library/src/common_library/errors_classes.py similarity index 89% rename from packages/models-library/src/models_library/errors_classes.py rename to packages/common-library/src/common_library/errors_classes.py index 6f6d9e0c266..1438536d550 100644 --- a/packages/models-library/src/models_library/errors_classes.py +++ b/packages/common-library/src/common_library/errors_classes.py @@ -7,16 +7,17 @@ def __missing__(self, key): class OsparcErrorMixin(PydanticErrorMixin): + code: str # type: ignore[assignment] msg_template: str def __new__(cls, *_args, **_kwargs): if not hasattr(cls, "code"): - cls.code = cls._get_full_class_name() # type: ignore[assignment] + cls.code = cls._get_full_class_name() return super().__new__(cls) def __init__(self, *_args, **kwargs) -> None: self.__dict__ = kwargs - super().__init__(message=self._build_message(), code=self.code) + super().__init__(message=self._build_message(), code=self.code) # type: ignore[arg-type] def __str__(self) -> str: return self._build_message() diff --git a/packages/common-library/src/common_library/py.typed b/packages/common-library/src/common_library/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/common-library/src/common_library/pydantic_fields_extension.py b/packages/common-library/src/common_library/pydantic_fields_extension.py new file mode 100644 index 00000000000..f2f6d59a5f4 --- /dev/null +++ b/packages/common-library/src/common_library/pydantic_fields_extension.py @@ -0,0 +1,22 @@ +from types import UnionType +from typing import Any, Literal, get_args, get_origin + +from pydantic.fields import FieldInfo + + +def get_type(info: FieldInfo) -> Any: + field_type = info.annotation + if args := get_args(info.annotation): + field_type = next(a for a in args if a != type(None)) + return field_type + + +def is_literal(info: FieldInfo) -> bool: + return get_origin(info.annotation) is Literal + + +def is_nullable(info: FieldInfo) -> bool: + origin = get_origin(info.annotation) # X | None or Optional[X] will return Union + if origin is UnionType: + return any(x in get_args(info.annotation) for x in (type(None), Any)) + return False diff --git a/packages/common-library/src/common_library/pydantic_networks_extension.py b/packages/common-library/src/common_library/pydantic_networks_extension.py new file mode 100644 index 00000000000..b53a2bfc8ae --- /dev/null +++ b/packages/common-library/src/common_library/pydantic_networks_extension.py @@ -0,0 +1,5 @@ +from typing import Annotated, TypeAlias +from pydantic import AfterValidator, AnyHttpUrl + + +AnyHttpUrlLegacy: TypeAlias = Annotated[str, AnyHttpUrl, AfterValidator(lambda u: u.rstrip("/"))] diff --git a/packages/common-library/src/common_library/serialization.py b/packages/common-library/src/common_library/serialization.py new file mode 100644 index 00000000000..510bdf6a469 --- /dev/null +++ b/packages/common-library/src/common_library/serialization.py @@ -0,0 +1,32 @@ +from typing import Any + +from common_library.pydantic_fields_extension import get_type +from pydantic import BaseModel, SecretStr + + +def model_dump_with_secrets( + settings_obj: BaseModel, show_secrets: bool, **pydantic_export_options +) -> dict[str, Any]: + data = settings_obj.model_dump(**pydantic_export_options) + + for field_name in settings_obj.model_fields: + if field_name not in data: + continue + + field_data = data[field_name] + + if isinstance(field_data, SecretStr): + if show_secrets: + data[field_name] = field_data.get_secret_value() + else: + data[field_name] = str(field_data) + elif isinstance(field_data, dict): + field_type = get_type(settings_obj.model_fields[field_name]) + if issubclass(field_type, BaseModel): + data[field_name] = model_dump_with_secrets( + field_type.model_validate(field_data), + show_secrets, + **pydantic_export_options, + ) + + return data diff --git a/packages/common-library/tests/conftest.py b/packages/common-library/tests/conftest.py new file mode 100644 index 00000000000..46f09f86b46 --- /dev/null +++ b/packages/common-library/tests/conftest.py @@ -0,0 +1,33 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + +import sys +from pathlib import Path + +import common_library +import pytest + +pytest_plugins = [ + "pytest_simcore.pydantic_models", + "pytest_simcore.pytest_global_environs", + "pytest_simcore.repository_paths", + "pytest_simcore.schemas", +] + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +@pytest.fixture(scope="session") +def package_dir(): + pdir = Path(common_library.__file__).resolve().parent + assert pdir.exists() + return pdir + + +@pytest.fixture(scope="session") +def project_slug_dir() -> Path: + folder = CURRENT_DIR.parent + assert folder.exists() + assert any(folder.glob("src/common_library")) + return folder diff --git a/packages/models-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py similarity index 98% rename from packages/models-library/tests/test_errors_classes.py rename to packages/common-library/tests/test_errors_classes.py index 754367805e2..ae0ed8c1e3d 100644 --- a/packages/models-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -9,7 +9,7 @@ from typing import Any import pytest -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin def test_get_full_class_name(): diff --git a/packages/common-library/tests/test_pydantic_fields_extension.py b/packages/common-library/tests/test_pydantic_fields_extension.py new file mode 100644 index 00000000000..50ff5443c41 --- /dev/null +++ b/packages/common-library/tests/test_pydantic_fields_extension.py @@ -0,0 +1,72 @@ +from typing import Literal + +import pytest +from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable +from pydantic import BaseModel, Field + + +class MyModel(BaseModel): + a: int + b: float | None = Field(...) + c: str = "bla" + d: bool | None = None + e: Literal["bla"] + + +@pytest.mark.parametrize( + "fn,expected,name", + [ + ( + get_type, + int, + "a", + ), + ( + get_type, + float, + "b", + ), + ( + get_type, + str, + "c", + ), + (get_type, bool, "d"), + ( + is_literal, + False, + "a", + ), + ( + is_literal, + False, + "b", + ), + ( + is_literal, + False, + "c", + ), + (is_literal, False, "d"), + (is_literal, True, "e"), + ( + is_nullable, + False, + "a", + ), + ( + is_nullable, + True, + "b", + ), + ( + is_nullable, + False, + "c", + ), + (is_nullable, True, "d"), + (is_nullable, False, "e"), + ], +) +def test_field_fn(fn, expected, name): + assert expected == fn(MyModel.model_fields[name]) diff --git a/packages/common-library/tests/test_pydantic_networks_extension.py b/packages/common-library/tests/test_pydantic_networks_extension.py new file mode 100644 index 00000000000..3390f7c2acf --- /dev/null +++ b/packages/common-library/tests/test_pydantic_networks_extension.py @@ -0,0 +1,20 @@ +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy +from pydantic import AnyHttpUrl, TypeAdapter +from pydantic_core import Url + + +def test_any_http_url(): + url = TypeAdapter(AnyHttpUrl).validate_python( + "http://backgroud.testserver.io", + ) + + assert isinstance(url, Url) + assert f"{url}" == "http://backgroud.testserver.io/" # NOTE: trailing '/' added in Pydantic v2 + +def test_any_http_url_legacy(): + url = TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io", + ) + + assert isinstance(url, str) + assert url == "http://backgroud.testserver.io" diff --git a/packages/common-library/tests/test_serialization.py b/packages/common-library/tests/test_serialization.py new file mode 100644 index 00000000000..d53db58809c --- /dev/null +++ b/packages/common-library/tests/test_serialization.py @@ -0,0 +1,30 @@ +from typing import Final + +import pytest +from common_library.serialization import model_dump_with_secrets +from pydantic import BaseModel, SecretStr + + +class Credentials(BaseModel): + USERNAME: str | None = None + PASSWORD: SecretStr | None = None + + +ME: Final[Credentials] = Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42")) + + +@pytest.mark.parametrize( + "expected,show_secrets", + [ + ( + {"USERNAME": "DeepThought", "PASSWORD": "42"}, + True, + ), + ( + {"USERNAME": "DeepThought", "PASSWORD": "**********"}, + False, # hide secrets + ), + ], +) +def test_model_dump_with_secrets(expected: dict, show_secrets: bool): + assert expected == model_dump_with_secrets(ME, show_secrets=show_secrets) diff --git a/packages/dask-task-models-library/requirements/_base.in b/packages/dask-task-models-library/requirements/_base.in index 3cdef671c4b..f25da08947b 100644 --- a/packages/dask-task-models-library/requirements/_base.in +++ b/packages/dask-task-models-library/requirements/_base.in @@ -2,6 +2,7 @@ # Specifies third-party dependencies for 'dask-task-models-library' # --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/packages/dask-task-models-library/requirements/ci.txt b/packages/dask-task-models-library/requirements/ci.txt index 3e0c9323517..c8775e00d91 100644 --- a/packages/dask-task-models-library/requirements/ci.txt +++ b/packages/dask-task-models-library/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages pytest-simcore @ ../pytest-simcore +simcore-common-library @ ../common-library simcore-models-library @ ../models-library simcore-settings-library @ ../settings-library/ diff --git a/packages/dask-task-models-library/requirements/dev.txt b/packages/dask-task-models-library/requirements/dev.txt index 0edd20961ac..a9d9555b2e8 100644 --- a/packages/dask-task-models-library/requirements/dev.txt +++ b/packages/dask-task-models-library/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../pytest-simcore/ +--editable ../common-library/ --editable ../models-library/ --editable ../settings-library/ diff --git a/packages/models-library/requirements/_base.in b/packages/models-library/requirements/_base.in index e2f8eae097a..b33d20bdd6b 100644 --- a/packages/models-library/requirements/_base.in +++ b/packages/models-library/requirements/_base.in @@ -2,6 +2,7 @@ # Specifies third-party dependencies for 'models-library' # --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in arrow jsonschema diff --git a/packages/models-library/requirements/ci.txt b/packages/models-library/requirements/ci.txt index e8e9fd2aeab..caaa0bb1b31 100644 --- a/packages/models-library/requirements/ci.txt +++ b/packages/models-library/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library simcore-postgres-database[migration] @ ../postgres-database/ pytest-simcore @ ../pytest-simcore diff --git a/packages/models-library/requirements/dev.txt b/packages/models-library/requirements/dev.txt index 901530f3644..e8372a6f3f6 100644 --- a/packages/models-library/requirements/dev.txt +++ b/packages/models-library/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library/ --editable ../postgres-database/[migration] --editable ../pytest-simcore/ diff --git a/packages/models-library/src/models_library/osparc_variable_identifier.py b/packages/models-library/src/models_library/osparc_variable_identifier.py index a4b0d4e882b..80a8e6d0fc0 100644 --- a/packages/models-library/src/models_library/osparc_variable_identifier.py +++ b/packages/models-library/src/models_library/osparc_variable_identifier.py @@ -1,8 +1,9 @@ from copy import deepcopy from typing import Any, TypeVar +from common_library.errors_classes import OsparcErrorMixin from models_library.basic_types import ConstrainedStr -from models_library.errors_classes import OsparcErrorMixin + from pydantic import BaseModel from .utils.string_substitution import OSPARC_IDENTIFIER_PREFIX diff --git a/packages/models-library/src/models_library/utils/specs_substitution.py b/packages/models-library/src/models_library/utils/specs_substitution.py index 73260237b22..d8a7e9cf161 100644 --- a/packages/models-library/src/models_library/utils/specs_substitution.py +++ b/packages/models-library/src/models_library/utils/specs_substitution.py @@ -1,6 +1,7 @@ from typing import Any, NamedTuple, TypeAlias, cast -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin + from pydantic import StrictBool, StrictFloat, StrictInt from .json_serialization import json_dumps, json_loads diff --git a/packages/notifications-library/requirements/_base.in b/packages/notifications-library/requirements/_base.in index 4879a9d6fb6..047005b4a39 100644 --- a/packages/notifications-library/requirements/_base.in +++ b/packages/notifications-library/requirements/_base.in @@ -2,6 +2,7 @@ # Specifies third-party dependencies for 'notifications-library' # --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/packages/notifications-library/requirements/ci.txt b/packages/notifications-library/requirements/ci.txt index 2ac189e1a32..3432db2c649 100644 --- a/packages/notifications-library/requirements/ci.txt +++ b/packages/notifications-library/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library/ simcore-models-library @ ../models-library/ simcore-postgres-database @ ../postgres-database/ pytest-simcore @ ../pytest-simcore/ diff --git a/packages/notifications-library/requirements/dev.txt b/packages/notifications-library/requirements/dev.txt index 723de763080..0a010051348 100644 --- a/packages/notifications-library/requirements/dev.txt +++ b/packages/notifications-library/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library/ --editable ../models-library/ --editable ../postgres-database/ --editable ../pytest-simcore/ diff --git a/packages/notifications-library/src/notifications_library/errors.py b/packages/notifications-library/src/notifications_library/errors.py index 2ffaa461a02..21edbbb0dc7 100644 --- a/packages/notifications-library/src/notifications_library/errors.py +++ b/packages/notifications-library/src/notifications_library/errors.py @@ -1,6 +1,6 @@ from typing import Any -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class NotifierError(OsparcErrorMixin, Exception): diff --git a/packages/service-integration/requirements/_base.in b/packages/service-integration/requirements/_base.in index fee8aa856e2..dc7e5dd4a6c 100644 --- a/packages/service-integration/requirements/_base.in +++ b/packages/service-integration/requirements/_base.in @@ -3,6 +3,7 @@ # --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in click diff --git a/packages/service-integration/requirements/ci.txt b/packages/service-integration/requirements/ci.txt index 9b4e9f16e2b..46a2035bfc7 100644 --- a/packages/service-integration/requirements/ci.txt +++ b/packages/service-integration/requirements/ci.txt @@ -10,6 +10,7 @@ --requirement _base.txt --requirement _test.txt +simcore-common-library @ ../common-library simcore-models-library @ ../models-library pytest-simcore @ ../pytest-simcore diff --git a/packages/service-integration/requirements/dev.txt b/packages/service-integration/requirements/dev.txt index 9e2af0f7124..bbe3d832532 100644 --- a/packages/service-integration/requirements/dev.txt +++ b/packages/service-integration/requirements/dev.txt @@ -11,6 +11,7 @@ --requirement _test.txt --requirement _tools.txt +--editable ../common-library/ --editable ../models-library/ --editable ../pytest-simcore/ diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in index b806af36443..e0e928cee52 100644 --- a/packages/service-library/requirements/_base.in +++ b/packages/service-library/requirements/_base.in @@ -5,6 +5,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/packages/service-library/requirements/ci.txt b/packages/service-library/requirements/ci.txt index 4d4dea58c9f..b26845819f4 100644 --- a/packages/service-library/requirements/ci.txt +++ b/packages/service-library/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library simcore-models-library @ ../models-library simcore-settings-library @ ../settings-library/ pytest-simcore @ ../pytest-simcore diff --git a/packages/service-library/requirements/ci[aiohttp].txt b/packages/service-library/requirements/ci[aiohttp].txt index 721950755b5..ee41e3b69a6 100644 --- a/packages/service-library/requirements/ci[aiohttp].txt +++ b/packages/service-library/requirements/ci[aiohttp].txt @@ -12,6 +12,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library simcore-models-library @ ../models-library simcore-settings-library @ ../settings-library/ pytest-simcore @ ../pytest-simcore diff --git a/packages/service-library/requirements/ci[all].txt b/packages/service-library/requirements/ci[all].txt index f7610e97111..f43ee95908f 100644 --- a/packages/service-library/requirements/ci[all].txt +++ b/packages/service-library/requirements/ci[all].txt @@ -13,6 +13,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library simcore-models-library @ ../models-library simcore-settings-library @ ../settings-library/ pytest-simcore @ ../pytest-simcore diff --git a/packages/service-library/requirements/ci[fastapi].txt b/packages/service-library/requirements/ci[fastapi].txt index c2c337fd4c0..db051f4ef73 100644 --- a/packages/service-library/requirements/ci[fastapi].txt +++ b/packages/service-library/requirements/ci[fastapi].txt @@ -12,6 +12,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library simcore-models-library @ ../models-library simcore-settings-library @ ../settings-library/ pytest-simcore @ ../pytest-simcore diff --git a/packages/service-library/requirements/dev.txt b/packages/service-library/requirements/dev.txt index b4da8c10382..f814830c46b 100644 --- a/packages/service-library/requirements/dev.txt +++ b/packages/service-library/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library --editable ../models-library --editable ../settings-library --editable ../pytest-simcore diff --git a/packages/service-library/requirements/dev[aiohttp].txt b/packages/service-library/requirements/dev[aiohttp].txt index 5e0ae847c64..87748e35d29 100644 --- a/packages/service-library/requirements/dev[aiohttp].txt +++ b/packages/service-library/requirements/dev[aiohttp].txt @@ -13,6 +13,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library --editable ../models-library/ --editable ../settings-library/ --editable ../pytest-simcore/ diff --git a/packages/service-library/requirements/dev[all].txt b/packages/service-library/requirements/dev[all].txt index b372254b325..8b23b6105c5 100644 --- a/packages/service-library/requirements/dev[all].txt +++ b/packages/service-library/requirements/dev[all].txt @@ -14,6 +14,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library --editable ../models-library/ --editable ../settings-library/ --editable ../pytest-simcore/ diff --git a/packages/service-library/requirements/dev[fastapi].txt b/packages/service-library/requirements/dev[fastapi].txt index caea1c80fd5..d66370d7904 100644 --- a/packages/service-library/requirements/dev[fastapi].txt +++ b/packages/service-library/requirements/dev[fastapi].txt @@ -13,6 +13,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library --editable ../models-library/ --editable ../settings-library/ --editable ../pytest-simcore/ diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py index 65c403853ea..5837beddcae 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py @@ -1,6 +1,6 @@ from typing import Any -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class CatalogApiBaseError(OsparcErrorMixin, Exception): diff --git a/packages/settings-library/requirements/_base.in b/packages/settings-library/requirements/_base.in index ec1d848cc85..9672f593a7f 100644 --- a/packages/settings-library/requirements/_base.in +++ b/packages/settings-library/requirements/_base.in @@ -1,8 +1,11 @@ # -# Specifies third-party dependencies for 'models-library' +# Specifies third-party dependencies for 'settings-library' # --constraint ../../../requirements/constraints.txt +# intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in + pydantic>=1.9 diff --git a/packages/settings-library/requirements/ci.txt b/packages/settings-library/requirements/ci.txt index 9feda17bfaa..d950945b44b 100644 --- a/packages/settings-library/requirements/ci.txt +++ b/packages/settings-library/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages pytest-simcore @ ../pytest-simcore +simcore-common-library @ ../common-library/ # current module simcore-settings-library @ . diff --git a/packages/settings-library/requirements/dev.txt b/packages/settings-library/requirements/dev.txt index 32d383e9ccc..2168d7f36e1 100644 --- a/packages/settings-library/requirements/dev.txt +++ b/packages/settings-library/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../common-library/ --editable ../pytest-simcore/ # current module diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index a07a0b50b01..7999f81151a 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -7,6 +7,8 @@ --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/common-library/requirements/_base.in + aiocache aiofiles diff --git a/packages/simcore-sdk/requirements/ci.txt b/packages/simcore-sdk/requirements/ci.txt index 25fcdd73354..afeb04529b3 100644 --- a/packages/simcore-sdk/requirements/ci.txt +++ b/packages/simcore-sdk/requirements/ci.txt @@ -13,6 +13,7 @@ # installs this repo's packages simcore-postgres-database @ ../postgres-database pytest-simcore @ ../pytest-simcore +simcore-common-library @ ../common-library simcore-models-library @ ../models-library simcore-settings-library @ ../settings-library/ simcore-service-library @ ../service-library/ diff --git a/packages/simcore-sdk/requirements/dev.txt b/packages/simcore-sdk/requirements/dev.txt index b67f43d8690..c7e7f45b7ed 100644 --- a/packages/simcore-sdk/requirements/dev.txt +++ b/packages/simcore-sdk/requirements/dev.txt @@ -15,6 +15,7 @@ --editable ../pytest-simcore/ --editable ../postgres-database +--editable ../common-library/ --editable ../models-library/ --editable ../settings-library/ diff --git a/services/agent/requirements/_base.in b/services/agent/requirements/_base.in index 7064a551898..5fc14c5cfe2 100644 --- a/services/agent/requirements/_base.in +++ b/services/agent/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in diff --git a/services/agent/requirements/ci.txt b/services/agent/requirements/ci.txt index 394361bc0ae..f7cb0dd72a4 100644 --- a/services/agent/requirements/ci.txt +++ b/services/agent/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library @ ../../packages/service-library diff --git a/services/agent/requirements/dev.txt b/services/agent/requirements/dev.txt index a20e1ab941d..6f5e8cf7e24 100644 --- a/services/agent/requirements/dev.txt +++ b/services/agent/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library diff --git a/services/api-server/requirements/_base.in b/services/api-server/requirements/_base.in index f63ab332f47..30b633a2e10 100644 --- a/services/api-server/requirements/_base.in +++ b/services/api-server/requirements/_base.in @@ -7,6 +7,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/api-server/requirements/ci.txt b/services/api-server/requirements/ci.txt index 7ea96c9234e..844706d5fb2 100644 --- a/services/api-server/requirements/ci.txt +++ b/services/api-server/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database/ pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/services/api-server/requirements/dev.txt b/services/api-server/requirements/dev.txt index 2de1f4cc316..5afc552d753 100644 --- a/services/api-server/requirements/dev.txt +++ b/services/api-server/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database --editable ../../packages/pytest-simcore/ diff --git a/services/autoscaling/requirements/_base.in b/services/autoscaling/requirements/_base.in index ae362ec2744..231b8944c9d 100644 --- a/services/autoscaling/requirements/_base.in +++ b/services/autoscaling/requirements/_base.in @@ -7,6 +7,7 @@ --constraint ../../../services/dask-sidecar/requirements/_dask-distributed.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/aws-library/requirements/_base.in diff --git a/services/autoscaling/requirements/ci.txt b/services/autoscaling/requirements/ci.txt index 6ae3d42db4f..99c6675a719 100644 --- a/services/autoscaling/requirements/ci.txt +++ b/services/autoscaling/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library simcore-dask-task-models-library @ ../../packages/dask-task-models-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore diff --git a/services/autoscaling/requirements/dev.txt b/services/autoscaling/requirements/dev.txt index 432e7ef62e9..ab92769203f 100644 --- a/services/autoscaling/requirements/dev.txt +++ b/services/autoscaling/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../../packages/aws-library +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/catalog/requirements/_base.in b/services/catalog/requirements/_base.in index ef6f55597c3..1394dd65e5e 100644 --- a/services/catalog/requirements/_base.in +++ b/services/catalog/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ../../../requirements/constraints.txt --constraint constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/catalog/requirements/ci.txt b/services/catalog/requirements/ci.txt index 246c4b290a4..4ebc858a454 100644 --- a/services/catalog/requirements/ci.txt +++ b/services/catalog/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/services/catalog/requirements/dev.txt b/services/catalog/requirements/dev.txt index dccc4f79f39..c9df003398e 100644 --- a/services/catalog/requirements/dev.txt +++ b/services/catalog/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database --editable ../../packages/pytest-simcore/ diff --git a/services/clusters-keeper/requirements/_base.in b/services/clusters-keeper/requirements/_base.in index dc3b222d6db..558d68b67cc 100644 --- a/services/clusters-keeper/requirements/_base.in +++ b/services/clusters-keeper/requirements/_base.in @@ -7,6 +7,7 @@ --constraint ../../../services/dask-sidecar/requirements/_dask-distributed.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/aws-library/requirements/_base.in diff --git a/services/clusters-keeper/requirements/ci.txt b/services/clusters-keeper/requirements/ci.txt index 750d3b3a3d3..12125244536 100644 --- a/services/clusters-keeper/requirements/ci.txt +++ b/services/clusters-keeper/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/clusters-keeper/requirements/dev.txt b/services/clusters-keeper/requirements/dev.txt index 5324f4c79f7..faf4378c83d 100644 --- a/services/clusters-keeper/requirements/dev.txt +++ b/services/clusters-keeper/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../../packages/aws-library +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/dask-sidecar/requirements/_base.in b/services/dask-sidecar/requirements/_base.in index d2fbda72d85..4ff16158cdf 100644 --- a/services/dask-sidecar/requirements/_base.in +++ b/services/dask-sidecar/requirements/_base.in @@ -11,6 +11,7 @@ # - Added as constraints instead of requirements in order to avoid polluting base.txt # - Will be installed when prod.txt or dev.txt # +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/dask-task-models-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_base.in diff --git a/services/dask-sidecar/requirements/ci.txt b/services/dask-sidecar/requirements/ci.txt index 63335c18529..192b4ffcbf0 100644 --- a/services/dask-sidecar/requirements/ci.txt +++ b/services/dask-sidecar/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library/ simcore-dask-task-models-library @ ../../packages/dask-task-models-library/ simcore-models-library @ ../../packages/models-library/ pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/services/dask-sidecar/requirements/dev.txt b/services/dask-sidecar/requirements/dev.txt index 82fbeaefec6..6ad6237135b 100644 --- a/services/dask-sidecar/requirements/dev.txt +++ b/services/dask-sidecar/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library/ --editable ../../packages/dask-task-models-library/ --editable ../../packages/models-library/ --editable ../../packages/pytest-simcore/ diff --git a/services/datcore-adapter/requirements/_base.in b/services/datcore-adapter/requirements/_base.in index de131dd6430..bcedb3cc8ed 100644 --- a/services/datcore-adapter/requirements/_base.in +++ b/services/datcore-adapter/requirements/_base.in @@ -4,6 +4,7 @@ # NOTE: ALL version constraints MUST be commented --constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in # service-library[fastapi] diff --git a/services/datcore-adapter/requirements/ci.txt b/services/datcore-adapter/requirements/ci.txt index ddd345522ae..f68013921e7 100644 --- a/services/datcore-adapter/requirements/ci.txt +++ b/services/datcore-adapter/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/datcore-adapter/requirements/dev.txt b/services/datcore-adapter/requirements/dev.txt index 73afce79c61..04e2ca59025 100644 --- a/services/datcore-adapter/requirements/dev.txt +++ b/services/datcore-adapter/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/director-v2/requirements/_base.in b/services/director-v2/requirements/_base.in index 2198739ef70..dc173e2c2b6 100644 --- a/services/director-v2/requirements/_base.in +++ b/services/director-v2/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # NOTE: Make sure they are added in setup.install_requires +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/dask-task-models-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in diff --git a/services/director-v2/requirements/ci.txt b/services/director-v2/requirements/ci.txt index bda1da73082..67de20ae339 100644 --- a/services/director-v2/requirements/ci.txt +++ b/services/director-v2/requirements/ci.txt @@ -12,6 +12,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library/ simcore-dask-task-models-library @ ../../packages/dask-task-models-library/ simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database diff --git a/services/director-v2/requirements/dev.txt b/services/director-v2/requirements/dev.txt index 6d932514ae9..f183201fd55 100644 --- a/services/director-v2/requirements/dev.txt +++ b/services/director-v2/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library/ --editable ../../packages/dask-task-models-library/ --editable ../../packages/models-library --editable ../../packages/postgres-database/ diff --git a/services/dynamic-scheduler/requirements/_base.in b/services/dynamic-scheduler/requirements/_base.in index 74bc0519c82..12ae0b98af0 100644 --- a/services/dynamic-scheduler/requirements/_base.in +++ b/services/dynamic-scheduler/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/dynamic-scheduler/requirements/ci.txt b/services/dynamic-scheduler/requirements/ci.txt index cfd40e964d9..53b69dc323f 100644 --- a/services/dynamic-scheduler/requirements/ci.txt +++ b/services/dynamic-scheduler/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database pytest-simcore @ ../../packages/pytest-simcore diff --git a/services/dynamic-scheduler/requirements/dev.txt b/services/dynamic-scheduler/requirements/dev.txt index 89e4eb7519c..60cb7217e53 100644 --- a/services/dynamic-scheduler/requirements/dev.txt +++ b/services/dynamic-scheduler/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database --editable ../../packages/pytest-simcore diff --git a/services/dynamic-sidecar/requirements/_base.in b/services/dynamic-sidecar/requirements/_base.in index 251ca3cedfc..66b47a481b5 100644 --- a/services/dynamic-sidecar/requirements/_base.in +++ b/services/dynamic-sidecar/requirements/_base.in @@ -7,6 +7,7 @@ # NOTE: These input-requirements under packages are tested using latest updates # NOTE: Make sure these packages are added in setup.install_requires +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in # service-library[fastapi] diff --git a/services/dynamic-sidecar/requirements/ci.txt b/services/dynamic-sidecar/requirements/ci.txt index c6040c5bc77..3c2e2adeb92 100644 --- a/services/dynamic-sidecar/requirements/ci.txt +++ b/services/dynamic-sidecar/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/services/dynamic-sidecar/requirements/dev.txt b/services/dynamic-sidecar/requirements/dev.txt index 2d1c00661ed..ce064f44c52 100644 --- a/services/dynamic-sidecar/requirements/dev.txt +++ b/services/dynamic-sidecar/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database/ --editable ../../packages/pytest-simcore/ diff --git a/services/efs-guardian/requirements/_base.in b/services/efs-guardian/requirements/_base.in index 84e8460fa05..f8db1e50412 100644 --- a/services/efs-guardian/requirements/_base.in +++ b/services/efs-guardian/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/aws-library/requirements/_base.in diff --git a/services/efs-guardian/requirements/ci.txt b/services/efs-guardian/requirements/ci.txt index 85e9fca927f..850a17f5f00 100644 --- a/services/efs-guardian/requirements/ci.txt +++ b/services/efs-guardian/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/efs-guardian/requirements/dev.txt b/services/efs-guardian/requirements/dev.txt index 76ea75d980d..bc86a77da96 100644 --- a/services/efs-guardian/requirements/dev.txt +++ b/services/efs-guardian/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../../packages/aws-library +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/invitations/requirements/_base.in b/services/invitations/requirements/_base.in index 5ee2336503c..2a775800f93 100644 --- a/services/invitations/requirements/_base.in +++ b/services/invitations/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in # service-library[fastapi] diff --git a/services/invitations/requirements/ci.txt b/services/invitations/requirements/ci.txt index bbc14fe4596..4051e96b178 100644 --- a/services/invitations/requirements/ci.txt +++ b/services/invitations/requirements/ci.txt @@ -11,10 +11,11 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library +simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library -simcore-models-library @ ../../packages/models-library # installs current package simcore-service-invitations @ . diff --git a/services/invitations/requirements/dev.txt b/services/invitations/requirements/dev.txt index a45c4db8918..1de98a1f08a 100644 --- a/services/invitations/requirements/dev.txt +++ b/services/invitations/requirements/dev.txt @@ -12,10 +12,12 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library +--editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library ---editable ../../packages/models-library + # installs current package --editable . diff --git a/services/payments/requirements/_base.in b/services/payments/requirements/_base.in index da3813cc2bb..6c79c0abca7 100644 --- a/services/payments/requirements/_base.in +++ b/services/payments/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/payments/requirements/ci.txt b/services/payments/requirements/ci.txt index fb0d44c3a77..47e283fa955 100644 --- a/services/payments/requirements/ci.txt +++ b/services/payments/requirements/ci.txt @@ -11,10 +11,12 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library + # installs current package simcore-service-payments @ . diff --git a/services/payments/requirements/dev.txt b/services/payments/requirements/dev.txt index b20ade64f27..80aeaf26dbe 100644 --- a/services/payments/requirements/dev.txt +++ b/services/payments/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database --editable ../../packages/pytest-simcore diff --git a/services/resource-usage-tracker/requirements/_base.in b/services/resource-usage-tracker/requirements/_base.in index c71570d1fee..1ed4215b64f 100644 --- a/services/resource-usage-tracker/requirements/_base.in +++ b/services/resource-usage-tracker/requirements/_base.in @@ -7,6 +7,7 @@ # intra-repo required dependencies --requirement ../../../packages/aws-library/requirements/_base.in +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in diff --git a/services/resource-usage-tracker/requirements/ci.txt b/services/resource-usage-tracker/requirements/ci.txt index 9cfb2df141d..2f2df03f1d5 100644 --- a/services/resource-usage-tracker/requirements/ci.txt +++ b/services/resource-usage-tracker/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/resource-usage-tracker/requirements/dev.txt b/services/resource-usage-tracker/requirements/dev.txt index 4fc539932c0..253940c1800 100644 --- a/services/resource-usage-tracker/requirements/dev.txt +++ b/services/resource-usage-tracker/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../../packages/aws-library +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index 2db40016724..26ba2b4b835 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -6,6 +6,7 @@ --requirement ../../../packages/aws-library/requirements/_base.in +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/storage/requirements/ci.txt b/services/storage/requirements/ci.txt index f3d92884e0e..25d417f91c7 100644 --- a/services/storage/requirements/ci.txt +++ b/services/storage/requirements/ci.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library/ +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/services/storage/requirements/dev.txt b/services/storage/requirements/dev.txt index 0b2b3ae2938..97aefedee51 100644 --- a/services/storage/requirements/dev.txt +++ b/services/storage/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../../packages/aws-library/ +--editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database/ --editable ../../packages/pytest-simcore/ From 1d92881744a09171320df2c837c667c4017590df Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Oct 2024 14:24:43 +0200 Subject: [PATCH 03/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Upgrade=20libraries?= =?UTF-8?q?=20(pydantic=20v2)=20(#6366)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ci/helpers/requirements.txt | 30 +- packages/aws-library/requirements/_base.txt | 21 +- packages/aws-library/requirements/_test.txt | 15 +- packages/aws-library/requirements/ci.txt | 3 +- packages/aws-library/requirements/dev.txt | 3 +- .../src/aws_library/ec2/__init__.py | 8 + .../src/aws_library/ec2/_errors.py | 7 +- .../src/aws_library/ec2/_models.py | 84 +- .../aws-library/src/aws_library/s3/_client.py | 24 +- .../src/aws_library/s3/_constants.py | 14 +- .../aws-library/src/aws_library/s3/_errors.py | 14 +- .../aws-library/src/aws_library/s3/_utils.py | 4 +- .../src/aws_library/ssm/_errors.py | 4 +- packages/aws-library/tests/test_ec2_models.py | 22 +- packages/aws-library/tests/test_s3_client.py | 78 +- packages/aws-library/tests/test_s3_utils.py | 69 +- packages/common-library/requirements/_test.in | 1 + .../common-library/requirements/_test.txt | 23 +- .../common-library/requirements/_tools.txt | 2 +- .../src/common_library/errors_classes.py | 5 +- .../pydantic_networks_extension.py | 24 +- .../common_library/pydantic_type_adapters.py | 8 + .../src/common_library/serialization.py | 4 + .../tests/test_errors_classes.py | 3 +- .../tests/test_pydantic_fields_extension.py | 4 +- .../tests/test_pydantic_networks_extension.py | 27 +- .../tests/test_serialization.py | 7 +- .../requirements/_base.txt | 17 +- .../requirements/_test.txt | 6 - .../requirements/_tools.txt | 10 - .../container_tasks/docker.py | 11 +- .../container_tasks/errors.py | 14 +- .../container_tasks/events.py | 19 +- .../container_tasks/io.py | 46 +- .../container_tasks/protocol.py | 24 +- .../tests/container_tasks/test_docker.py | 2 +- .../tests/container_tasks/test_events.py | 4 +- .../tests/container_tasks/test_io.py | 18 +- .../tests/container_tasks/test_protocol.py | 6 +- .../models-library/requirements/_base.txt | 4 +- .../scripts/validate-pg-projects.py | 4 +- .../api_schemas_catalog/__init__.py | 6 +- .../api_schemas_clusters_keeper/__init__.py | 8 +- .../dynamic_services_service.py | 4 +- .../api_schemas_dynamic_scheduler/__init__.py | 8 +- .../api_schemas_efs_guardian/__init__.py | 8 +- .../api_schemas_payments/__init__.py | 6 +- .../__init__.py | 8 +- .../src/models_library/api_schemas_storage.py | 5 +- .../api_schemas_webserver/__init__.py | 6 +- .../api_schemas_webserver/projects_nodes.py | 2 +- .../src/models_library/basic_regex.py | 4 +- .../src/models_library/basic_types.py | 6 +- .../src/models_library/docker.py | 6 +- .../function_services_catalog/_utils.py | 8 +- .../services/demo_units.py | 2 +- .../services/file_picker.py | 2 +- .../services/iter_range.py | 2 +- .../services/iter_sensitivity.py | 2 +- .../services/nodes_group.py | 2 +- .../services/parameters.py | 4 +- .../services/probes.py | 6 +- .../src/models_library/projects.py | 10 - .../src/models_library/projects_nodes.py | 21 +- .../src/models_library/projects_nodes_io.py | 21 +- .../src/models_library/rabbitmq_messages.py | 2 +- .../src/models_library/rest_pagination.py | 16 +- .../models_library/rest_pagination_utils.py | 43 +- .../src/models_library/rpc_pagination.py | 2 +- .../models_library/service_settings_labels.py | 13 +- .../service_settings_nat_rule.py | 11 +- .../src/models_library/services_io.py | 4 +- .../utils/json_serialization.py | 7 +- .../src/models_library/utils/nodes.py | 6 +- .../tests/test__models_examples.py | 2 +- .../tests/test__pydantic_models.py | 6 +- .../tests/test__pydantic_models_and_enums.py | 24 +- .../tests/test_api_schemas_catalog.py | 8 +- .../test_api_schemas_webserver_projects.py | 26 +- .../tests/test_callbacks_mapping.py | 8 +- packages/models-library/tests/test_docker.py | 20 +- .../models-library/tests/test_generics.py | 10 +- .../tests/test_osparc_variable_identifier.py | 35 +- .../tests/test_project_networks.py | 16 +- .../tests/test_project_nodes.py | 4 +- .../tests/test_project_nodes_io.py | 14 +- .../models-library/tests/test_projects.py | 4 +- .../tests/test_projects_state.py | 4 +- .../tests/test_rest_pagination_utils.py | 10 +- .../tests/test_service_settings_labels.py | 43 +- .../tests/test_service_settings_nat_rule.py | 12 +- .../models-library/tests/test_services_io.py | 2 +- .../tests/test_sidecar_volumes.py | 2 +- .../tests/test_user_preferences.py | 32 +- .../tests/test_utils_common_validators.py | 32 +- .../test_utils_pydantic_tools_extension.py | 4 +- .../tests/test_utils_service_io.py | 14 +- .../tests/test_utils_specs_substitution.py | 4 +- .../requirements/_base.txt | 17 +- .../requirements/_test.txt | 11 +- .../requirements/_tools.txt | 10 - .../tests/email/test_email_events.py | 4 +- .../tests/with_db/conftest.py | 6 +- .../postgres-database/requirements/_base.in | 1 + .../postgres-database/requirements/_base.txt | 7 +- .../postgres-database/requirements/_test.txt | 7 - .../postgres-database/requirements/_tools.txt | 10 - .../postgres-database/requirements/ci.txt | 3 +- .../postgres-database/requirements/dev.txt | 4 +- .../utils_folders.py | 41 +- .../utils_projects_metadata.py | 11 +- .../utils_projects_nodes.py | 16 +- .../tests/test_utils_projects_nodes.py | 4 +- .../src/pytest_simcore/aws_s3_service.py | 2 +- .../src/pytest_simcore/aws_server.py | 18 +- .../src/pytest_simcore/docker_registry.py | 4 +- .../src/pytest_simcore/faker_payments_data.py | 16 +- .../src/pytest_simcore/faker_products_data.py | 8 +- .../src/pytest_simcore/faker_projects_data.py | 7 +- .../src/pytest_simcore/faker_users_data.py | 10 +- .../pytest_simcore/helpers/faker_factories.py | 2 +- .../helpers/httpx_calls_capture_openapi.py | 4 +- .../helpers/httpx_calls_capture_parameters.py | 19 +- .../helpers/httpx_client_base_dev.py | 14 +- .../helpers/parametrizations.py | 4 +- .../src/pytest_simcore/helpers/s3.py | 6 +- .../helpers/webserver_projects.py | 6 +- .../src/pytest_simcore/httpbin_service.py | 4 +- .../src/pytest_simcore/httpx_calls_capture.py | 8 +- .../src/pytest_simcore/minio_service.py | 2 +- .../src/pytest_simcore/pydantic_models.py | 4 +- .../src/pytest_simcore/rabbit_service.py | 2 +- .../services_api_mocks_for_aiohttp_clients.py | 61 +- .../pytest_simcore/simcore_storage_service.py | 6 +- .../requirements/_base.txt | 19 +- .../requirements/_test.txt | 9 - .../requirements/_tools.txt | 11 - .../_compose_spec_model_autogenerated.py | 226 ++- .../src/service_integration/cli/__init__.py | 2 +- .../service_integration/cli/_compose_spec.py | 2 +- .../src/service_integration/cli/_config.py | 6 +- .../src/service_integration/errors.py | 6 +- .../src/service_integration/oci_image_spec.py | 33 +- .../src/service_integration/osparc_config.py | 60 +- .../service_integration/osparc_image_specs.py | 6 +- .../src/service_integration/settings.py | 9 +- .../src/service_integration/versioning.py | 22 +- .../tests/test_command_compose.py | 6 +- .../tests/test_compose_spec_model.py | 2 +- .../tests/test_oci_image_spec.py | 2 +- .../tests/test_osparc_config.py | 13 +- .../tests/test_osparc_image_specs.py | 4 +- .../tests/test_osparc_runtime_specs.py | 10 +- .../service-library/requirements/_aiohttp.txt | 6 +- .../service-library/requirements/_base.txt | 24 +- .../service-library/requirements/_fastapi.txt | 19 +- .../service-library/requirements/_test.txt | 25 +- .../service-library/requirements/_tools.txt | 10 - .../servicelib/aiohttp/application_setup.py | 6 +- .../src/servicelib/aiohttp/docker_utils.py | 16 +- .../aiohttp/long_running_tasks/_server.py | 18 +- .../aiohttp/long_running_tasks/client.py | 4 +- .../servicelib/aiohttp/requests_validation.py | 19 +- .../src/servicelib/background_task.py | 4 +- .../deferred_tasks/_redis_task_tracker.py | 10 +- .../src/servicelib/docker_utils.py | 58 +- .../src/servicelib/error_codes.py | 11 +- .../src/servicelib/fastapi/docker_utils.py | 20 +- .../src/servicelib/fastapi/errors.py | 4 +- .../servicelib/fastapi/http_client_thin.py | 4 +- .../fastapi/long_running_tasks/_client.py | 19 +- .../long_running_tasks/_context_manager.py | 2 +- .../fastapi/long_running_tasks/_server.py | 2 +- .../src/servicelib/file_utils.py | 4 +- .../servicelib/long_running_tasks/_errors.py | 6 +- .../servicelib/long_running_tasks/_models.py | 8 +- .../servicelib/long_running_tasks/_task.py | 6 +- .../src/servicelib/progress_bar.py | 4 +- .../src/servicelib/rabbitmq/_errors.py | 8 +- .../src/servicelib/rabbitmq/_models.py | 7 +- .../rabbitmq/rpc_interfaces/catalog/errors.py | 5 +- .../rpc_interfaces/catalog/services.py | 26 +- .../dynamic_scheduler/errors.py | 4 +- .../dynamic_scheduler/services.py | 10 +- .../efs_guardian/efs_manager.py | 4 +- .../resource_usage_tracker/errors.py | 4 +- .../resource_usage_tracker/pricing_plans.py | 18 +- .../resource_usage_tracker/pricing_units.py | 10 +- .../resource_usage_tracker/service_runs.py | 12 +- .../service-library/src/servicelib/redis.py | 4 +- .../src/servicelib/utils_meta.py | 4 +- .../src/servicelib/utils_secrets.py | 4 +- .../aiohttp/long_running_tasks/conftest.py | 6 +- .../test_long_running_tasks.py | 10 +- ...st_long_running_tasks_with_task_context.py | 6 +- .../tests/aiohttp/test_docker_utils.py | 9 +- .../tests/aiohttp/test_requests_validation.py | 78 +- .../tests/deferred_tasks/example_app.py | 4 +- .../test__redis_task_tracker.py | 5 +- .../deferred_tasks/test_deferred_tasks.py | 20 +- .../test_long_running_tasks.py | 24 +- ...test_long_running_tasks_context_manager.py | 15 +- .../tests/fastapi/test_docker_utils.py | 11 +- .../tests/fastapi/test_exceptions_utils.py | 6 +- .../tests/fastapi/test_http_client_thin.py | 17 +- .../tests/fastapi/test_openapi.py | 2 +- .../tests/fastapi/test_rabbitmq.py | 2 +- .../rabbitmq/test_rabbitmq_connection.py | 6 +- .../tests/rabbitmq/test_rabbitmq_rpc.py | 5 +- .../tests/rabbitmq/test_rabbitmq_utils.py | 6 +- .../tests/test_archiving_utils.py | 5 +- .../settings-library/requirements/_base.in | 6 +- .../settings-library/requirements/_base.txt | 14 +- .../settings-library/requirements/_test.txt | 10 +- .../settings-library/requirements/_tools.txt | 10 - packages/settings-library/requirements/ci.txt | 2 +- .../settings-library/requirements/dev.txt | 2 +- .../src/settings_library/base.py | 116 +- .../src/settings_library/basic_types.py | 19 +- .../src/settings_library/catalog.py | 3 +- .../src/settings_library/comp_services.py | 11 +- .../src/settings_library/director_v2.py | 3 +- .../src/settings_library/docker_registry.py | 12 +- .../src/settings_library/ec2.py | 18 +- .../src/settings_library/efs.py | 4 +- .../src/settings_library/email.py | 27 +- .../src/settings_library/node_ports.py | 20 +- .../src/settings_library/postgres.py | 62 +- .../src/settings_library/prometheus.py | 22 +- .../src/settings_library/r_clone.py | 2 +- .../src/settings_library/rabbit.py | 17 +- .../src/settings_library/redis.py | 26 +- .../resource_usage_tracker.py | 3 +- .../src/settings_library/s3.py | 18 +- .../src/settings_library/ssm.py | 17 +- .../src/settings_library/storage.py | 3 +- .../src/settings_library/tracing.py | 8 +- .../src/settings_library/twilio.py | 22 +- .../src/settings_library/utils_cli.py | 55 +- .../src/settings_library/utils_encoders.py | 2 +- .../src/settings_library/utils_service.py | 31 +- .../src/settings_library/webserver.py | 4 +- packages/settings-library/tests/conftest.py | 8 +- .../tests/test__models_examples.py | 2 +- .../tests/test__pydantic_settings.py | 110 +- packages/settings-library/tests/test_base.py | 73 +- .../tests/test_base_w_postgres.py | 102 +- packages/settings-library/tests/test_email.py | 2 +- .../settings-library/tests/test_postgres.py | 17 +- .../settings-library/tests/test_twilio.py | 2 +- .../settings-library/tests/test_utils_cli.py | 13 +- .../tests/test_utils_logging.py | 20 +- .../tests/test_utils_service.py | 12 +- packages/simcore-sdk/requirements/_base.txt | 26 +- packages/simcore-sdk/requirements/_test.txt | 28 +- packages/simcore-sdk/requirements/_tools.txt | 10 - .../src/simcore_sdk/node_data/data_manager.py | 6 +- .../node_ports_common/_filemanager.py | 18 +- .../node_ports_common/aws_s3_cli.py | 5 +- .../node_ports_common/file_io_utils.py | 4 +- .../node_ports_common/filemanager.py | 8 +- .../simcore_sdk/node_ports_common/r_clone.py | 11 +- .../node_ports_common/r_clone_utils.py | 11 +- .../node_ports_common/storage_client.py | 10 +- .../src/simcore_sdk/node_ports_v2/links.py | 17 +- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 31 +- .../src/simcore_sdk/node_ports_v2/port.py | 57 +- .../node_ports_v2/port_validation.py | 18 +- .../node_ports_v2/ports_mapping.py | 25 +- .../node_ports_v2/serialization_v2.py | 2 +- .../tests/helpers/utils_port_v2.py | 2 +- .../simcore-sdk/tests/integration/conftest.py | 2 +- .../test_node_ports_common_filemanager.py | 1398 ++++++++--------- .../test_node_ports_v2_nodeports2.py | 122 +- .../tests/unit/test_node_data_data_manager.py | 2 +- .../test_node_ports_common_file_io_utils.py | 21 +- .../tests/unit/test_node_ports_v2_links.py | 2 +- .../unit/test_node_ports_v2_nodeports_v2.py | 59 + .../tests/unit/test_node_ports_v2_port.py | 7 +- .../unit/test_node_ports_v2_port_mapping.py | 50 +- .../test_node_ports_v2_port_validation.py | 13 +- .../tests/unit/test_storage_client.py | 32 +- .../requirements/requirements.txt | 28 +- tests/public-api/requirements/_base.txt | 19 +- tests/swarm-deploy/requirements/_test.txt | 170 +- tests/swarm-deploy/requirements/_tools.txt | 10 - 286 files changed, 3271 insertions(+), 2646 deletions(-) create mode 100644 packages/common-library/src/common_library/pydantic_type_adapters.py diff --git a/ci/helpers/requirements.txt b/ci/helpers/requirements.txt index 55a3ab4e163..daf26ed5c0b 100644 --- a/ci/helpers/requirements.txt +++ b/ci/helpers/requirements.txt @@ -1,17 +1,19 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements.in +# uv pip compile requirements.in -o requirements.txt aiohttp==3.9.5 + # via + # -c ../../requirements/constraints.txt + # -r requirements.in aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via starlette -async-timeout==4.0.3 - # via aiohttp attrs==23.2.0 # via aiohttp -exceptiongroup==1.2.1 - # via anyio -fastapi==0.99.1 +fastapi==0.115.0 + # via -r requirements.in frozenlist==1.4.1 # via # aiohttp @@ -24,16 +26,22 @@ multidict==6.0.5 # via # aiohttp # yarl -pydantic==1.10.15 - # via fastapi +pydantic==2.9.2 + # via + # -c ../../requirements/constraints.txt + # fastapi +pydantic-core==2.23.4 + # via pydantic sniffio==1.3.1 # via anyio -starlette==0.27.0 - # via fastapi +starlette==0.38.6 + # via + # -c ../../requirements/constraints.txt + # fastapi typing-extensions==4.11.0 # via - # anyio # fastapi # pydantic + # pydantic-core yarl==1.9.4 # via aiohttp diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 50f6f157311..53832fe75c5 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -32,6 +32,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # fast-depends @@ -105,7 +107,7 @@ orjson==3.10.7 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in pamqp==3.3.0 # via aiormq -pydantic==1.10.17 +pydantic==2.9.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -120,6 +122,20 @@ pydantic==1.10.17 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.3 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.7.2 @@ -128,6 +144,8 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -198,6 +216,7 @@ typing-extensions==4.12.2 # aiodebug # faststream # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt index 213e7ccb561..1a9b4a7afa2 100644 --- a/packages/aws-library/requirements/_test.txt +++ b/packages/aws-library/requirements/_test.txt @@ -1,3 +1,7 @@ +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto appdirs==1.4.4 @@ -151,11 +155,15 @@ py-partiql-parser==0.5.5 # via moto pycparser==2.22 # via cffi -pydantic==1.10.17 +pydantic==2.9.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.3 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via moto pytest==8.3.2 @@ -193,7 +201,9 @@ python-dateutil==2.9.0.post0 # faker # moto python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt @@ -268,6 +278,7 @@ typing-extensions==4.12.2 # flexparser # pint # pydantic + # pydantic-core # types-aioboto3 # types-aiobotocore urllib3==2.2.2 diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt index 7fcd69a010f..f39ca4e3672 100644 --- a/packages/aws-library/requirements/ci.txt +++ b/packages/aws-library/requirements/ci.txt @@ -11,8 +11,9 @@ --requirement _test.txt # installs this repo's packages +simcore-common-library @ ../common-library +simcore-models-library @ ../models-library/ pytest-simcore @ ../pytest-simcore -simcore-models-library @ ../models-library simcore-service-library @ ../service-library/ simcore-settings-library @ ../settings-library/ diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt index f8956725407..34cc644b370 100644 --- a/packages/aws-library/requirements/dev.txt +++ b/packages/aws-library/requirements/dev.txt @@ -12,8 +12,9 @@ --requirement _tools.txt # installs this repo's packages ---editable ../pytest-simcore +--editable ../common-library/ --editable ../models-library/ +--editable ../pytest-simcore/ --editable ../service-library/ --editable ../settings-library/ diff --git a/packages/aws-library/src/aws_library/ec2/__init__.py b/packages/aws-library/src/aws_library/ec2/__init__.py index 02fcf10b00e..112c70861b2 100644 --- a/packages/aws-library/src/aws_library/ec2/__init__.py +++ b/packages/aws-library/src/aws_library/ec2/__init__.py @@ -1,6 +1,10 @@ from ._client import SimcoreEC2API from ._errors import EC2AccessError, EC2NotConnectedError, EC2RuntimeError from ._models import ( + AWS_TAG_KEY_MAX_LENGTH, + AWS_TAG_KEY_MIN_LENGTH, + AWS_TAG_VALUE_MAX_LENGTH, + AWS_TAG_VALUE_MIN_LENGTH, AWSTagKey, AWSTagValue, EC2InstanceBootSpecific, @@ -14,6 +18,10 @@ __all__: tuple[str, ...] = ( "AWSTagKey", "AWSTagValue", + "AWS_TAG_KEY_MIN_LENGTH", + "AWS_TAG_KEY_MAX_LENGTH", + "AWS_TAG_VALUE_MIN_LENGTH", + "AWS_TAG_VALUE_MAX_LENGTH", "EC2AccessError", "EC2InstanceBootSpecific", "EC2InstanceConfig", diff --git a/packages/aws-library/src/aws_library/ec2/_errors.py b/packages/aws-library/src/aws_library/ec2/_errors.py index c39047db00d..4fb0e611ed2 100644 --- a/packages/aws-library/src/aws_library/ec2/_errors.py +++ b/packages/aws-library/src/aws_library/ec2/_errors.py @@ -1,12 +1,9 @@ # pylint: disable=too-many-ancestors -from typing import Any - -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class EC2BaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + pass class EC2RuntimeError(EC2BaseError, RuntimeError): diff --git a/packages/aws-library/src/aws_library/ec2/_models.py b/packages/aws-library/src/aws_library/ec2/_models.py index fed1f3ea46a..ad15a74f0eb 100644 --- a/packages/aws-library/src/aws_library/ec2/_models.py +++ b/packages/aws-library/src/aws_library/ec2/_models.py @@ -2,18 +2,19 @@ import re import tempfile from dataclasses import dataclass -from typing import Any, ClassVar, TypeAlias +from typing import Annotated, Final, TypeAlias import sh # type: ignore[import-untyped] from models_library.docker import DockerGenericTag from pydantic import ( BaseModel, ByteSize, - ConstrainedStr, + ConfigDict, Field, NonNegativeFloat, NonNegativeInt, - validator, + StringConstraints, + field_validator, ) from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -33,26 +34,26 @@ def __gt__(self, other: "Resources") -> bool: return self.cpus > other.cpus or self.ram > other.ram def __add__(self, other: "Resources") -> "Resources": - return Resources.construct( + return Resources.model_construct( **{ key: a + b for (key, a), b in zip( - self.dict().items(), other.dict().values(), strict=True + self.model_dump().items(), other.model_dump().values(), strict=True ) } ) def __sub__(self, other: "Resources") -> "Resources": - return Resources.construct( + return Resources.model_construct( **{ key: a - b for (key, a), b in zip( - self.dict().items(), other.dict().values(), strict=True + self.model_dump().items(), other.model_dump().values(), strict=True ) } ) - @validator("cpus", pre=True) + @field_validator("cpus", mode="before") @classmethod def _floor_cpus_to_0(cls, v: float) -> float: return max(v, 0) @@ -67,19 +68,31 @@ class EC2InstanceType: InstancePrivateDNSName: TypeAlias = str -class AWSTagKey(ConstrainedStr): +AWS_TAG_KEY_MIN_LENGTH: Final[int] = 1 +AWS_TAG_KEY_MAX_LENGTH: Final[int] = 128 +AWSTagKey: TypeAlias = Annotated[ # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions] - regex = re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$") - min_length = 1 - max_length = 128 - - -class AWSTagValue(ConstrainedStr): + str, + StringConstraints( + min_length=AWS_TAG_KEY_MIN_LENGTH, + max_length=AWS_TAG_KEY_MAX_LENGTH, + pattern=re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$"), + ), +] + + +AWS_TAG_VALUE_MIN_LENGTH: Final[int] = 0 +AWS_TAG_VALUE_MAX_LENGTH: Final[int] = 256 +AWSTagValue: TypeAlias = Annotated[ # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions] # quotes []{} were added as it allows to json encode. it seems to be accepted as a value - regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$") - min_length = 0 - max_length = 256 + str, + StringConstraints( + min_length=0, + max_length=256, + pattern=r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$", + ), +] EC2Tags: TypeAlias = dict[AWSTagKey, AWSTagValue] @@ -148,8 +161,23 @@ class EC2InstanceBootSpecific(BaseModel): default=0, description="number of buffer EC2s to keep (defaults to 0)" ) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + @field_validator("custom_boot_scripts") + @classmethod + def validate_bash_calls(cls, v): + try: + with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file: + temp_file.writelines(v) + temp_file.flush() + # NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes + sh.bash("-n", temp_file.name) + except sh.ErrorReturnCode as exc: + msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}" + raise ValueError(msg) from exc + + return v + + model_config = ConfigDict( + json_schema_extra={ "examples": [ { # just AMI @@ -205,18 +233,4 @@ class Config: }, ] } - - @validator("custom_boot_scripts") - @classmethod - def validate_bash_calls(cls, v): - try: - with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file: - temp_file.writelines(v) - temp_file.flush() - # NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes - sh.bash("-n", temp_file.name) - except sh.ErrorReturnCode as exc: - msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}" - raise ValueError(msg) from exc - - return v + ) diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py index 4ddb2bfb9c2..816208c22c8 100644 --- a/packages/aws-library/src/aws_library/s3/_client.py +++ b/packages/aws-library/src/aws_library/s3/_client.py @@ -13,9 +13,10 @@ from boto3.s3.transfer import TransferConfig from botocore import exceptions as botocore_exc from botocore.client import Config +from common_library.pydantic_type_adapters import AnyUrlLegacyAdapter from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart from models_library.basic_types import SHA256Str -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.logging_utils import log_catch, log_context from servicelib.utils import limited_gather from settings_library.s3 import S3Settings @@ -44,6 +45,9 @@ _AWS_MAX_ITEMS_PER_PAGE: Final[int] = 1000 +ListAnyUrlTypeAdapter: Final[TypeAdapter[list[AnyUrl]]] = TypeAdapter(list[AnyUrl]) + + class UploadedBytesTransferredCallback(Protocol): def __call__(self, bytes_transferred: int, *, file_name: str) -> None: ... @@ -251,7 +255,7 @@ async def create_single_presigned_download_link( bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int, - ) -> AnyUrl: + ) -> str: # NOTE: ensure the bucket/object exists, this will raise if not await self._client.head_bucket(Bucket=bucket) await self._client.head_object(Bucket=bucket, Key=object_key) @@ -260,13 +264,12 @@ async def create_single_presigned_download_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - url: AnyUrl = parse_obj_as(AnyUrl, generated_link) - return url + return f"{AnyUrlLegacyAdapter.validate_python(generated_link)}" @s3_exception_handler(_logger) async def create_single_presigned_upload_link( self, *, bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int - ) -> AnyUrl: + ) -> str: # NOTE: ensure the bucket/object exists, this will raise if not await self._client.head_bucket(Bucket=bucket) generated_link = await self._client.generate_presigned_url( @@ -274,8 +277,7 @@ async def create_single_presigned_upload_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - url: AnyUrl = parse_obj_as(AnyUrl, generated_link) - return url + return f"{AnyUrlLegacyAdapter.validate_python(generated_link)}" @s3_exception_handler(_logger) async def create_multipart_upload_links( @@ -298,8 +300,7 @@ async def create_multipart_upload_links( # compute the number of links, based on the announced file size num_upload_links, chunk_size = compute_num_file_chunks(file_size) # now create the links - upload_links = parse_obj_as( - list[AnyUrl], + upload_links = ListAnyUrlTypeAdapter.validate_python( await asyncio.gather( *( self._client.generate_presigned_url( @@ -473,7 +474,6 @@ def is_multipart(file_size: ByteSize) -> bool: @staticmethod def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl: - url: AnyUrl = parse_obj_as( - AnyUrl, f"s3://{bucket}/{urllib.parse.quote(object_key)}" + return AnyUrlLegacyAdapter.validate_python( + f"s3://{bucket}/{urllib.parse.quote(object_key)}" ) - return url diff --git a/packages/aws-library/src/aws_library/s3/_constants.py b/packages/aws-library/src/aws_library/s3/_constants.py index 05f2b3dc6d6..a94cd555f43 100644 --- a/packages/aws-library/src/aws_library/s3/_constants.py +++ b/packages/aws-library/src/aws_library/s3/_constants.py @@ -1,10 +1,14 @@ from typing import Final -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter # NOTE: AWS S3 upload limits https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html -MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "100MiB") -MULTIPART_COPY_THRESHOLD: Final[ByteSize] = parse_obj_as(ByteSize, "100MiB") +MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = TypeAdapter( + ByteSize +).validate_python("100MiB") +MULTIPART_COPY_THRESHOLD: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( + "100MiB" +) -PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "5GiB") -S3_MAX_FILE_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "5TiB") +PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5GiB") +S3_MAX_FILE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5TiB") diff --git a/packages/aws-library/src/aws_library/s3/_errors.py b/packages/aws-library/src/aws_library/s3/_errors.py index f297b04b64d..d14105dbd30 100644 --- a/packages/aws-library/src/aws_library/s3/_errors.py +++ b/packages/aws-library/src/aws_library/s3/_errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class S3RuntimeError(PydanticErrorMixin, RuntimeError): +class S3RuntimeError(OsparcErrorMixin, RuntimeError): msg_template: str = "S3 client unexpected error" @@ -10,25 +10,25 @@ class S3NotConnectedError(S3RuntimeError): class S3AccessError(S3RuntimeError): - code = "s3_access.error" + code = "s3_access.error" # type: ignore[assignment] msg_template: str = "Unexpected error while accessing S3 backend" class S3BucketInvalidError(S3AccessError): - code = "s3_bucket.invalid_error" + code = "s3_bucket.invalid_error" # type: ignore[assignment] msg_template: str = "The bucket '{bucket}' is invalid" class S3KeyNotFoundError(S3AccessError): - code = "s3_key.not_found_error" + code = "s3_key.not_found_error" # type: ignore[assignment] msg_template: str = "The file {key} in {bucket} was not found" class S3UploadNotFoundError(S3AccessError): - code = "s3_upload.not_found_error" + code = "s3_upload.not_found_error" # type: ignore[assignment] msg_template: str = "The upload for {key} in {bucket} was not found" class S3DestinationNotEmptyError(S3AccessError): - code = "s3_destination.not_empty_error" + code = "s3_destination.not_empty_error" # type: ignore[assignment] msg_template: str = "The destination {dst_prefix} is not empty" diff --git a/packages/aws-library/src/aws_library/s3/_utils.py b/packages/aws-library/src/aws_library/s3/_utils.py index 00a1bcc59bb..96ad59f57d3 100644 --- a/packages/aws-library/src/aws_library/s3/_utils.py +++ b/packages/aws-library/src/aws_library/s3/_utils.py @@ -1,13 +1,13 @@ from typing import Final -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter _MULTIPART_MAX_NUMBER_OF_PARTS: Final[int] = 10000 # this is artifically defined, if possible we keep a maximum number of requests for parallel # uploading. If that is not possible then we create as many upload part as the max part size allows _MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE: Final[list[ByteSize]] = [ - parse_obj_as(ByteSize, x) + TypeAdapter(ByteSize).validate_python(x) for x in [ "10Mib", "50Mib", diff --git a/packages/aws-library/src/aws_library/ssm/_errors.py b/packages/aws-library/src/aws_library/ssm/_errors.py index 32300d08d29..5d3ea16b6c6 100644 --- a/packages/aws-library/src/aws_library/ssm/_errors.py +++ b/packages/aws-library/src/aws_library/ssm/_errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class SSMRuntimeError(PydanticErrorMixin, RuntimeError): +class SSMRuntimeError(OsparcErrorMixin, RuntimeError): msg_template: str = "SSM client unexpected error" diff --git a/packages/aws-library/tests/test_ec2_models.py b/packages/aws-library/tests/test_ec2_models.py index f7c114932be..ed232ad0043 100644 --- a/packages/aws-library/tests/test_ec2_models.py +++ b/packages/aws-library/tests/test_ec2_models.py @@ -6,7 +6,7 @@ import pytest from aws_library.ec2._models import AWSTagKey, AWSTagValue, EC2InstanceData, Resources from faker import Faker -from pydantic import ByteSize, ValidationError, parse_obj_as +from pydantic import ByteSize, TypeAdapter, ValidationError @pytest.mark.parametrize( @@ -88,9 +88,9 @@ def test_resources_gt_operator(a: Resources, b: Resources, a_greater_than_b: boo Resources(cpus=1, ram=ByteSize(34)), ), ( - Resources(cpus=0.1, ram=ByteSize(-1)), + Resources(cpus=0.1, ram=ByteSize(1)), Resources(cpus=1, ram=ByteSize(34)), - Resources(cpus=1.1, ram=ByteSize(33)), + Resources(cpus=1.1, ram=ByteSize(35)), ), ], ) @@ -108,14 +108,14 @@ def test_resources_create_as_empty(): "a,b,result", [ ( - Resources(cpus=0, ram=ByteSize(0)), - Resources(cpus=1, ram=ByteSize(34)), - Resources.construct(cpus=-1, ram=ByteSize(-34)), + Resources(cpus=0, ram=ByteSize(34)), + Resources(cpus=1, ram=ByteSize(0)), + Resources.model_construct(cpus=-1, ram=ByteSize(34)), ), ( - Resources(cpus=0.1, ram=ByteSize(-1)), - Resources(cpus=1, ram=ByteSize(34)), - Resources.construct(cpus=-0.9, ram=ByteSize(-35)), + Resources(cpus=0.1, ram=ByteSize(34)), + Resources(cpus=1, ram=ByteSize(1)), + Resources.model_construct(cpus=-0.9, ram=ByteSize(33)), ), ], ) @@ -129,10 +129,10 @@ def test_resources_sub(a: Resources, b: Resources, result: Resources): def test_aws_tag_key_invalid(ec2_tag_key: str): # for a key it raises with pytest.raises(ValidationError): - parse_obj_as(AWSTagKey, ec2_tag_key) + TypeAdapter(AWSTagKey).validate_python(ec2_tag_key) # for a value it does not - parse_obj_as(AWSTagValue, ec2_tag_key) + TypeAdapter(AWSTagValue).validate_python(ec2_tag_key) def test_ec2_instance_data_hashable(faker: Faker): diff --git a/packages/aws-library/tests/test_s3_client.py b/packages/aws-library/tests/test_s3_client.py index 93ee29fe5b0..af98fd6523f 100644 --- a/packages/aws-library/tests/test_s3_client.py +++ b/packages/aws-library/tests/test_s3_client.py @@ -14,7 +14,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Final import botocore.exceptions import pytest @@ -32,7 +32,7 @@ from models_library.api_schemas_storage import S3BucketName, UploadedPart from models_library.basic_types import SHA256Str from moto.server import ThreadedMotoServer -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_benchmark.plugin import BenchmarkFixture from pytest_simcore.helpers.logging_tools import log_context from pytest_simcore.helpers.parametrizations import ( @@ -49,6 +49,8 @@ from types_aiobotocore_s3 import S3Client from types_aiobotocore_s3.literals import BucketLocationConstraintType +_BYTE_SIZE_ADAPTER: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize) + @pytest.fixture async def simcore_s3_api( @@ -67,7 +69,9 @@ async def simcore_s3_api( @pytest.fixture def bucket_name(faker: Faker) -> S3BucketName: # NOTE: no faker here as we need some specific namings - return parse_obj_as(S3BucketName, faker.pystr().replace("_", "-").lower()) + return TypeAdapter(S3BucketName).validate_python( + faker.pystr().replace("_", "-").lower() + ) @pytest.fixture @@ -89,7 +93,9 @@ async def with_s3_bucket( @pytest.fixture def non_existing_s3_bucket(faker: Faker) -> S3BucketName: - return parse_obj_as(S3BucketName, faker.pystr().replace("_", "-").lower()) + return TypeAdapter(S3BucketName).validate_python( + faker.pystr().replace("_", "-").lower() + ) @pytest.fixture @@ -107,7 +113,7 @@ async def _( file, MultiPartUploadLinks( upload_id="fake", - chunk_size=parse_obj_as(ByteSize, file.stat().st_size), + chunk_size=_BYTE_SIZE_ADAPTER.validate_python(file.stat().st_size), urls=[presigned_url], ), ) @@ -131,7 +137,7 @@ async def with_uploaded_file_on_s3( s3_client: S3Client, with_s3_bucket: S3BucketName, ) -> AsyncIterator[UploadedFile]: - test_file = create_file_of_size(parse_obj_as(ByteSize, "10Kib")) + test_file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("10Kib")) await s3_client.upload_file( Filename=f"{test_file}", Bucket=with_s3_bucket, @@ -200,7 +206,7 @@ async def _uploader( object_key=object_key, file_size=ByteSize(file.stat().st_size), expiration_secs=default_expiration_time_seconds, - sha256_checksum=parse_obj_as(SHA256Str, faker.sha256()), + sha256_checksum=TypeAdapter(SHA256Str).validate_python(faker.sha256()), ) assert upload_links @@ -586,7 +592,7 @@ async def test_undelete_file( assert file_metadata.size == with_uploaded_file_on_s3.local_path.stat().st_size # upload another file on top of the existing one - new_file = create_file_of_size(parse_obj_as(ByteSize, "5Kib")) + new_file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("5Kib")) await s3_client.upload_file( Filename=f"{new_file}", Bucket=with_s3_bucket, @@ -688,7 +694,7 @@ async def test_create_single_presigned_download_link( object_key=with_uploaded_file_on_s3.s3_key, expiration_secs=default_expiration_time_seconds, ) - assert isinstance(download_url, AnyUrl) + assert download_url dest_file = tmp_path / faker.file_name() async with ClientSession() as session: @@ -738,10 +744,10 @@ async def test_create_single_presigned_upload_link( create_file_of_size: Callable[[ByteSize], Path], default_expiration_time_seconds: int, upload_to_presigned_link: Callable[ - [Path, AnyUrl, S3BucketName, S3ObjectKey], Awaitable[None] + [Path, str, S3BucketName, S3ObjectKey], Awaitable[None] ], ): - file = create_file_of_size(parse_obj_as(ByteSize, "1Mib")) + file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib")) s3_object_key = file.name presigned_url = await simcore_s3_api.create_single_presigned_upload_link( bucket=with_s3_bucket, @@ -769,7 +775,7 @@ async def test_create_single_presigned_upload_link_with_non_existing_bucket_rais create_file_of_size: Callable[[ByteSize], Path], default_expiration_time_seconds: int, ): - file = create_file_of_size(parse_obj_as(ByteSize, "1Mib")) + file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib")) s3_object_key = file.name with pytest.raises(S3BucketInvalidError): await simcore_s3_api.create_single_presigned_upload_link( @@ -863,7 +869,7 @@ async def test_create_multipart_presigned_upload_link_invalid_raises( object_key=faker.pystr(), file_size=ByteSize(file.stat().st_size), expiration_secs=default_expiration_time_seconds, - sha256_checksum=parse_obj_as(SHA256Str, faker.sha256()), + sha256_checksum=TypeAdapter(SHA256Str).validate_python(faker.sha256()), ) # completing with invalid bucket @@ -1076,7 +1082,7 @@ async def test_copy_file_invalid_raises( create_file_of_size: Callable[[ByteSize], Path], faker: Faker, ): - file = create_file_of_size(parse_obj_as(ByteSize, "1MiB")) + file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1MiB")) uploaded_file = await upload_file(file) dst_object_key = faker.file_name() # NOTE: since aioboto3 13.1.0 this raises S3KeyNotFoundError instead of S3BucketInvalidError @@ -1101,9 +1107,9 @@ async def test_copy_file_invalid_raises( "directory_size, min_file_size, max_file_size", [ ( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "1B"), - parse_obj_as(ByteSize, "10Kib"), + _BYTE_SIZE_ADAPTER.validate_python("1Mib"), + _BYTE_SIZE_ADAPTER.validate_python("1B"), + _BYTE_SIZE_ADAPTER.validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1127,9 +1133,9 @@ async def test_get_directory_metadata( "directory_size, min_file_size, max_file_size", [ ( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "1B"), - parse_obj_as(ByteSize, "10Kib"), + _BYTE_SIZE_ADAPTER.validate_python("1Mib"), + _BYTE_SIZE_ADAPTER.validate_python("1B"), + _BYTE_SIZE_ADAPTER.validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1159,9 +1165,9 @@ async def test_get_directory_metadata_raises( "directory_size, min_file_size, max_file_size", [ ( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "1B"), - parse_obj_as(ByteSize, "10Kib"), + _BYTE_SIZE_ADAPTER.validate_python("1Mib"), + _BYTE_SIZE_ADAPTER.validate_python("1B"), + _BYTE_SIZE_ADAPTER.validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1195,9 +1201,9 @@ async def test_delete_file_recursively( "directory_size, min_file_size, max_file_size", [ ( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "1B"), - parse_obj_as(ByteSize, "10Kib"), + _BYTE_SIZE_ADAPTER.validate_python("1Mib"), + _BYTE_SIZE_ADAPTER.validate_python("1B"), + _BYTE_SIZE_ADAPTER.validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1233,9 +1239,9 @@ async def test_delete_file_recursively_raises( "directory_size, min_file_size, max_file_size", [ ( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "1B"), - parse_obj_as(ByteSize, "10Kib"), + _BYTE_SIZE_ADAPTER.validate_python("1Mib"), + _BYTE_SIZE_ADAPTER.validate_python("1B"), + _BYTE_SIZE_ADAPTER.validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1298,7 +1304,7 @@ def test_compute_s3_url( bucket: S3BucketName, object_key: S3ObjectKey, expected_s3_url: AnyUrl ): assert ( - SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key) + str(SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key)) == expected_s3_url ) @@ -1333,14 +1339,14 @@ def run_async_test(*args, **kwargs) -> None: "directory_size, min_file_size, max_file_size", [ ( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "1B"), - parse_obj_as(ByteSize, "10Kib"), + _BYTE_SIZE_ADAPTER.validate_python("1Mib"), + _BYTE_SIZE_ADAPTER.validate_python("1B"), + _BYTE_SIZE_ADAPTER.validate_python("10Kib"), ), ( - parse_obj_as(ByteSize, "500Mib"), - parse_obj_as(ByteSize, "10Mib"), - parse_obj_as(ByteSize, "50Mib"), + _BYTE_SIZE_ADAPTER.validate_python("500Mib"), + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + _BYTE_SIZE_ADAPTER.validate_python("50Mib"), ), ], ids=byte_size_ids, diff --git a/packages/aws-library/tests/test_s3_utils.py b/packages/aws-library/tests/test_s3_utils.py index 5354da8bc66..bf021978585 100644 --- a/packages/aws-library/tests/test_s3_utils.py +++ b/packages/aws-library/tests/test_s3_utils.py @@ -4,29 +4,73 @@ # pylint: disable=unused-variable +from typing import Final + import pytest from aws_library.s3._utils import ( _MULTIPART_MAX_NUMBER_OF_PARTS, _MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE, compute_num_file_chunks, ) -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.parametrizations import byte_size_ids +_BYTE_SIZE_ADAPTER: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize) + @pytest.mark.parametrize( "file_size, expected_num_chunks, expected_chunk_size", [ - (parse_obj_as(ByteSize, "5Mib"), 1, parse_obj_as(ByteSize, "10Mib")), - (parse_obj_as(ByteSize, "10Mib"), 1, parse_obj_as(ByteSize, "10Mib")), - (parse_obj_as(ByteSize, "20Mib"), 2, parse_obj_as(ByteSize, "10Mib")), - (parse_obj_as(ByteSize, "50Mib"), 5, parse_obj_as(ByteSize, "10Mib")), - (parse_obj_as(ByteSize, "150Mib"), 15, parse_obj_as(ByteSize, "10Mib")), - (parse_obj_as(ByteSize, "550Mib"), 55, parse_obj_as(ByteSize, "10Mib")), - (parse_obj_as(ByteSize, "560Gib"), 5735, parse_obj_as(ByteSize, "100Mib")), - (parse_obj_as(ByteSize, "5Tib"), 8739, parse_obj_as(ByteSize, "600Mib")), - (parse_obj_as(ByteSize, "15Tib"), 7680, parse_obj_as(ByteSize, "2Gib")), - (parse_obj_as(ByteSize, "9431773844"), 900, parse_obj_as(ByteSize, "10Mib")), + ( + _BYTE_SIZE_ADAPTER.validate_python("5Mib"), + 1, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + 1, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("20Mib"), + 2, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("50Mib"), + 5, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("150Mib"), + 15, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("550Mib"), + 55, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("560Gib"), + 5735, + _BYTE_SIZE_ADAPTER.validate_python("100Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("5Tib"), + 8739, + _BYTE_SIZE_ADAPTER.validate_python("600Mib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("15Tib"), + 7680, + _BYTE_SIZE_ADAPTER.validate_python("2Gib"), + ), + ( + _BYTE_SIZE_ADAPTER.validate_python("9431773844"), + 900, + _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + ), ], ids=byte_size_ids, ) @@ -39,8 +83,7 @@ def test_compute_num_file_chunks( def test_enormous_file_size_raises_value_error(): - enormous_file_size = parse_obj_as( - ByteSize, + enormous_file_size = _BYTE_SIZE_ADAPTER.validate_python( ( max(_MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE) * _MULTIPART_MAX_NUMBER_OF_PARTS diff --git a/packages/common-library/requirements/_test.in b/packages/common-library/requirements/_test.in index 4454d79d36a..1fe37ac0151 100644 --- a/packages/common-library/requirements/_test.in +++ b/packages/common-library/requirements/_test.in @@ -10,6 +10,7 @@ coverage faker +pydantic-settings pytest pytest-asyncio pytest-cov diff --git a/packages/common-library/requirements/_test.txt b/packages/common-library/requirements/_test.txt index 2354abd790d..89b9a19eca6 100644 --- a/packages/common-library/requirements/_test.txt +++ b/packages/common-library/requirements/_test.txt @@ -1,8 +1,12 @@ +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -faker==30.1.0 +faker==30.3.0 # via -r requirements/_test.in icdiff==2.0.7 # via pytest-icdiff @@ -16,6 +20,17 @@ pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff +pydantic==2.9.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # pydantic-settings +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic +pydantic-settings==2.5.2 + # via -r requirements/_test.in pytest==8.3.3 # via # -r requirements/_test.in @@ -44,7 +59,9 @@ pytest-sugar==1.0.0 python-dateutil==2.9.0.post0 # via faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -r requirements/_test.in + # pydantic-settings six==1.16.0 # via python-dateutil termcolor==2.5.0 @@ -53,3 +70,5 @@ typing-extensions==4.12.2 # via # -c requirements/_base.txt # faker + # pydantic + # pydantic-core diff --git a/packages/common-library/requirements/_tools.txt b/packages/common-library/requirements/_tools.txt index a333bb822ae..b5f85d4efcc 100644 --- a/packages/common-library/requirements/_tools.txt +++ b/packages/common-library/requirements/_tools.txt @@ -1,6 +1,6 @@ astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt build==1.2.2.post1 # via pip-tools diff --git a/packages/common-library/src/common_library/errors_classes.py b/packages/common-library/src/common_library/errors_classes.py index 1438536d550..99ed586e744 100644 --- a/packages/common-library/src/common_library/errors_classes.py +++ b/packages/common-library/src/common_library/errors_classes.py @@ -1,3 +1,4 @@ +from typing import Any from pydantic.errors import PydanticErrorMixin @@ -15,8 +16,8 @@ def __new__(cls, *_args, **_kwargs): cls.code = cls._get_full_class_name() return super().__new__(cls) - def __init__(self, *_args, **kwargs) -> None: - self.__dict__ = kwargs + def __init__(self, **ctx: Any) -> None: + self.__dict__ = ctx super().__init__(message=self._build_message(), code=self.code) # type: ignore[arg-type] def __str__(self) -> str: diff --git a/packages/common-library/src/common_library/pydantic_networks_extension.py b/packages/common-library/src/common_library/pydantic_networks_extension.py index b53a2bfc8ae..79c5da906b1 100644 --- a/packages/common-library/src/common_library/pydantic_networks_extension.py +++ b/packages/common-library/src/common_library/pydantic_networks_extension.py @@ -1,5 +1,25 @@ from typing import Annotated, TypeAlias -from pydantic import AfterValidator, AnyHttpUrl +from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, HttpUrl +from pydantic_core import Url -AnyHttpUrlLegacy: TypeAlias = Annotated[str, AnyHttpUrl, AfterValidator(lambda u: u.rstrip("/"))] + +def _strip_last_slash(url: Url) -> str: + return f"{url}".rstrip("/") + + +AnyUrlLegacy: TypeAlias = Annotated[ + AnyUrl, + AfterValidator(_strip_last_slash), +] + +AnyHttpUrlLegacy: TypeAlias = Annotated[ + AnyHttpUrl, + AfterValidator(_strip_last_slash), +] + + +HttpUrlLegacy: TypeAlias = Annotated[ + HttpUrl, + AfterValidator(_strip_last_slash), +] diff --git a/packages/common-library/src/common_library/pydantic_type_adapters.py b/packages/common-library/src/common_library/pydantic_type_adapters.py new file mode 100644 index 00000000000..883100c4717 --- /dev/null +++ b/packages/common-library/src/common_library/pydantic_type_adapters.py @@ -0,0 +1,8 @@ +from typing import Final + +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy, AnyUrlLegacy +from pydantic import TypeAdapter + +AnyUrlLegacyAdapter: Final[TypeAdapter[AnyUrlLegacy]] = TypeAdapter(AnyUrlLegacy) + +AnyHttpUrlLegacyAdapter: Final[TypeAdapter[AnyHttpUrlLegacy]] = TypeAdapter(AnyHttpUrlLegacy) diff --git a/packages/common-library/src/common_library/serialization.py b/packages/common-library/src/common_library/serialization.py index 510bdf6a469..a2178cc3ae6 100644 --- a/packages/common-library/src/common_library/serialization.py +++ b/packages/common-library/src/common_library/serialization.py @@ -1,3 +1,4 @@ +from datetime import timedelta from typing import Any from common_library.pydantic_fields_extension import get_type @@ -15,6 +16,9 @@ def model_dump_with_secrets( field_data = data[field_name] + if isinstance(field_data, timedelta): + data[field_name] = field_data.total_seconds() + if isinstance(field_data, SecretStr): if show_secrets: data[field_name] = field_data.get_secret_value() diff --git a/packages/common-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py index ae0ed8c1e3d..63674fbd3b4 100644 --- a/packages/common-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -38,8 +38,7 @@ class B12(B1, ValueError): def test_error_codes_and_msg_template(): class MyBaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) # Do not forget this for base exceptions! + pass class MyValueError(MyBaseError, ValueError): msg_template = "Wrong value {value}" diff --git a/packages/common-library/tests/test_pydantic_fields_extension.py b/packages/common-library/tests/test_pydantic_fields_extension.py index 50ff5443c41..9f5aa1ae2fc 100644 --- a/packages/common-library/tests/test_pydantic_fields_extension.py +++ b/packages/common-library/tests/test_pydantic_fields_extension.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Any, Callable, Literal import pytest from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable @@ -68,5 +68,5 @@ class MyModel(BaseModel): (is_nullable, False, "e"), ], ) -def test_field_fn(fn, expected, name): +def test_field_fn(fn: Callable[[Any], Any], expected: Any, name: str): assert expected == fn(MyModel.model_fields[name]) diff --git a/packages/common-library/tests/test_pydantic_networks_extension.py b/packages/common-library/tests/test_pydantic_networks_extension.py index 3390f7c2acf..6ab50a42a2b 100644 --- a/packages/common-library/tests/test_pydantic_networks_extension.py +++ b/packages/common-library/tests/test_pydantic_networks_extension.py @@ -1,20 +1,39 @@ +import pytest from common_library.pydantic_networks_extension import AnyHttpUrlLegacy -from pydantic import AnyHttpUrl, TypeAdapter +from pydantic import AnyHttpUrl, BaseModel, TypeAdapter, ValidationError from pydantic_core import Url +class A(BaseModel): + url: AnyHttpUrlLegacy + + def test_any_http_url(): url = TypeAdapter(AnyHttpUrl).validate_python( "http://backgroud.testserver.io", ) assert isinstance(url, Url) - assert f"{url}" == "http://backgroud.testserver.io/" # NOTE: trailing '/' added in Pydantic v2 + assert ( + f"{url}" == "http://backgroud.testserver.io/" + ) # trailing slash added (in Pydantic v2) + def test_any_http_url_legacy(): url = TypeAdapter(AnyHttpUrlLegacy).validate_python( - "http://backgroud.testserver.io", + "http://backgroud.testserver.io", ) assert isinstance(url, str) - assert url == "http://backgroud.testserver.io" + assert url == "http://backgroud.testserver.io" # no trailing slash was added + + +def test_valid_any_http_url_legacy_field(): + a = A(url="http://backgroud.testserver.io") # type: ignore + + assert a.url == "http://backgroud.testserver.io" # no trailing slash was added + + +def test_not_valid_any_http_url_legacy_field(): + with pytest.raises(ValidationError): + A(url="htttttp://backgroud.testserver.io") # type: ignore diff --git a/packages/common-library/tests/test_serialization.py b/packages/common-library/tests/test_serialization.py index d53db58809c..d897ff5ec5d 100644 --- a/packages/common-library/tests/test_serialization.py +++ b/packages/common-library/tests/test_serialization.py @@ -1,5 +1,3 @@ -from typing import Final - import pytest from common_library.serialization import model_dump_with_secrets from pydantic import BaseModel, SecretStr @@ -10,9 +8,6 @@ class Credentials(BaseModel): PASSWORD: SecretStr | None = None -ME: Final[Credentials] = Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42")) - - @pytest.mark.parametrize( "expected,show_secrets", [ @@ -27,4 +22,4 @@ class Credentials(BaseModel): ], ) def test_model_dump_with_secrets(expected: dict, show_secrets: bool): - assert expected == model_dump_with_secrets(ME, show_secrets=show_secrets) + assert expected == model_dump_with_secrets(Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42")), show_secrets=show_secrets) diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt index 327a04c5678..81807f5ffe6 100644 --- a/packages/dask-task-models-library/requirements/_base.txt +++ b/packages/dask-task-models-library/requirements/_base.txt @@ -1,3 +1,5 @@ +annotated-types==0.7.0 + # via pydantic arrow==1.3.0 # via -r requirements/../../../packages/models-library/requirements/_base.in attrs==24.2.0 @@ -65,7 +67,7 @@ partd==1.4.2 # via dask psutil==6.0.0 # via distributed -pydantic==1.10.17 +pydantic==2.9.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -73,10 +75,22 @@ pydantic==1.10.17 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.3 + # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -118,6 +132,7 @@ types-python-dateutil==2.9.0.20240821 typing-extensions==4.12.2 # via # pydantic + # pydantic-core # typer urllib3==2.2.2 # via diff --git a/packages/dask-task-models-library/requirements/_test.txt b/packages/dask-task-models-library/requirements/_test.txt index aa7e91d4d23..521d13265d9 100644 --- a/packages/dask-task-models-library/requirements/_test.txt +++ b/packages/dask-task-models-library/requirements/_test.txt @@ -4,8 +4,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in flexcache==0.3 @@ -67,10 +65,6 @@ six==1.16.0 # python-dateutil termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # pytest typing-extensions==4.12.2 # via # -c requirements/_base.txt diff --git a/packages/dask-task-models-library/requirements/_tools.txt b/packages/dask-task-models-library/requirements/_tools.txt index b76f8083592..b9ee0a3c96d 100644 --- a/packages/dask-task-models-library/requirements/_tools.txt +++ b/packages/dask-task-models-library/requirements/_tools.txt @@ -70,22 +70,12 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/docker.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/docker.py index 4e9d36df3fb..b4fa976b665 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/docker.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/docker.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Extra, SecretStr +from pydantic import BaseModel, ConfigDict, SecretStr class DockerBasicAuth(BaseModel): @@ -6,9 +6,9 @@ class DockerBasicAuth(BaseModel): username: str password: SecretStr - class Config: - extra = Extra.forbid - schema_extra = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { "server_address": "docker.io", @@ -16,4 +16,5 @@ class Config: "password": "123456", } ] - } + }, + ) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py index f4060531f7f..f0a6813ba15 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py @@ -1,20 +1,20 @@ """ Dask task exceptions """ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class TaskValueError(PydanticErrorMixin, ValueError): - code = "task.value_error" +class TaskValueError(OsparcErrorMixin, ValueError): + code = "task.value_error" # type: ignore[assignment] -class TaskCancelledError(PydanticErrorMixin, RuntimeError): - code = "task.cancelled_error" +class TaskCancelledError(OsparcErrorMixin, RuntimeError): + code = "task.cancelled_error" # type: ignore[assignment] msg_template = "The task was cancelled" -class ServiceRuntimeError(PydanticErrorMixin, RuntimeError): - code = "service.runtime_error" +class ServiceRuntimeError(OsparcErrorMixin, RuntimeError): + code = "service.runtime_error" # type: ignore[assignment] msg_template = ( "The service {service_key}:{service_version}" " running in container {container_id} failed with code" diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/events.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/events.py index 1455c00cbff..a27bb027e94 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/events.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/events.py @@ -1,10 +1,10 @@ import logging from abc import ABC, abstractmethod -from typing import Any, ClassVar, TypeAlias +from typing import TypeAlias import dask.typing from distributed.worker import get_worker -from pydantic import BaseModel, Extra, validator +from pydantic import BaseModel, ConfigDict, field_validator from .protocol import TaskOwner @@ -19,8 +19,7 @@ class BaseTaskEvent(BaseModel, ABC): def topic_name() -> str: raise NotImplementedError - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") def _dask_key_to_dask_task_id(key: dask.typing.Key) -> str: @@ -51,8 +50,8 @@ def from_dask_worker( task_owner=task_owner, ) - class Config(BaseTaskEvent.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "job_id": "simcore/services/comp/sleeper:1.1.0:projectid_ec7e595a-63ee-46a1-a04a-901b11b649f8:nodeid_39467d89-b659-4914-9359-c40b1b6d1d6d:uuid_5ee5c655-450d-4711-a3ec-32ffe16bc580", @@ -78,8 +77,9 @@ class Config(BaseTaskEvent.Config): }, ] } + ) - @validator("progress", always=True) + @field_validator("progress") @classmethod def ensure_between_0_1(cls, v): if 0 <= v <= 1: @@ -112,8 +112,8 @@ def from_dask_worker( task_owner=task_owner, ) - class Config(BaseTaskEvent.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "job_id": "simcore/services/comp/sleeper:1.1.0:projectid_ec7e595a-63ee-46a1-a04a-901b11b649f8:nodeid_39467d89-b659-4914-9359-c40b1b6d1d6d:uuid_5ee5c655-450d-4711-a3ec-32ffe16bc580", @@ -129,3 +129,4 @@ class Config(BaseTaskEvent.Config): }, ] } + ) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py index 887397d4227..0bb95130723 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py @@ -1,7 +1,7 @@ import json from contextlib import suppress from pathlib import Path -from typing import Any, ClassVar, TypeAlias, Union +from typing import Any, TypeAlias, Union from models_library.basic_regex import MIME_TYPE_RE from models_library.generics import DictModel @@ -9,7 +9,7 @@ from pydantic import ( AnyUrl, BaseModel, - Extra, + ConfigDict, Field, StrictBool, StrictFloat, @@ -23,9 +23,9 @@ class PortSchema(BaseModel): required: bool - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ { "required": True, @@ -34,15 +34,16 @@ class Config: "required": False, }, ] - } + }, + ) class FilePortSchema(PortSchema): mapping: str | None = None url: AnyUrl - class Config(PortSchema.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "mapping": "some_filename.txt", @@ -55,6 +56,7 @@ class Config(PortSchema.Config): }, ] } + ) class FileUrl(BaseModel): @@ -64,12 +66,12 @@ class FileUrl(BaseModel): description="Local file relpath name (if given), otherwise it takes the url filename", ) file_mime_type: str | None = Field( - default=None, description="the file MIME type", regex=MIME_TYPE_RE + default=None, description="the file MIME type", pattern=MIME_TYPE_RE ) - class Config: - extra = Extra.forbid - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ "examples": [ {"url": "https://some_file_url", "file_mime_type": "application/json"}, { @@ -78,7 +80,8 @@ class Config: "file_mime_type": "application/json", }, ] - } + }, + ) PortValue: TypeAlias = Union[ @@ -94,8 +97,8 @@ class Config: class TaskInputData(DictModel[ServicePortKey, PortValue]): - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "boolean_input": False, @@ -106,6 +109,7 @@ class Config: }, ] } + ) PortSchemaValue: TypeAlias = Union[PortSchema, FilePortSchema] @@ -118,8 +122,8 @@ class TaskOutputDataSchema(DictModel[ServicePortKey, PortSchemaValue]): # does not work well in that case. For that reason, the schema is # sent as a json-schema instead of with a dynamically-created model class # - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "boolean_output": {"required": False}, @@ -138,6 +142,7 @@ class Config: }, ] } + ) class TaskOutputData(DictModel[ServicePortKey, PortValue]): @@ -170,10 +175,10 @@ def from_task_output( msg = f"Could not locate '{output_key}' in {output_data_file}" raise ValueError(msg) - return cls.parse_obj(data) + return cls.model_validate(data) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "boolean_output": False, @@ -184,3 +189,4 @@ class Config: }, ] } + ) diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/protocol.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/protocol.py index 00f89d96d94..fd6acf554e0 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/protocol.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/protocol.py @@ -1,4 +1,4 @@ -from typing import Any, ClassVar, Protocol, TypeAlias +from typing import Any, Protocol, TypeAlias from models_library.basic_types import EnvVarKey from models_library.docker import DockerLabelKey @@ -6,7 +6,7 @@ from models_library.projects_nodes_io import NodeID from models_library.services_resources import BootMode from models_library.users import UserID -from pydantic import AnyUrl, BaseModel, root_validator +from pydantic import AnyUrl, BaseModel, ConfigDict, model_validator from settings_library.s3 import S3Settings from .docker import DockerBasicAuth @@ -32,7 +32,7 @@ class TaskOwner(BaseModel): def has_parent(self) -> bool: return bool(self.parent_node_id and self.parent_project_id) - @root_validator + @model_validator(mode="before") @classmethod def check_parent_valid(cls, values: dict[str, Any]) -> dict[str, Any]: parent_project_id = values.get("parent_project_id") @@ -44,8 +44,8 @@ def check_parent_valid(cls, values: dict[str, Any]) -> dict[str, Any]: raise ValueError(msg) return values - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "user_id": 32, @@ -63,6 +63,7 @@ class Config: }, ] } + ) class ContainerTaskParameters(BaseModel): @@ -76,24 +77,23 @@ class ContainerTaskParameters(BaseModel): boot_mode: BootMode task_owner: TaskOwner - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "image": "ubuntu", "tag": "latest", - "input_data": TaskInputData.Config.schema_extra["examples"][0], - "output_data_keys": TaskOutputDataSchema.Config.schema_extra[ - "examples" - ][0], + "input_data": TaskInputData.model_config["json_schema_extra"]["examples"][0], # type: ignore[index] + "output_data_keys": TaskOutputDataSchema.model_config["json_schema_extra"]["examples"][0], # type: ignore[index] "command": ["sleep 10", "echo hello"], "envs": {"MYENV": "is an env"}, "labels": {"io.simcore.thelabel": "is amazing"}, "boot_mode": BootMode.CPU.value, - "task_owner": TaskOwner.Config.schema_extra["examples"][0], + "task_owner": TaskOwner.model_config["json_schema_extra"]["examples"][0], # type: ignore[index] }, ] } + ) class ContainerRemoteFct(Protocol): diff --git a/packages/dask-task-models-library/tests/container_tasks/test_docker.py b/packages/dask-task-models-library/tests/container_tasks/test_docker.py index 307fe175547..4eb5bc74980 100644 --- a/packages/dask-task-models-library/tests/container_tasks/test_docker.py +++ b/packages/dask-task-models-library/tests/container_tasks/test_docker.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize("model_cls", [(DockerBasicAuth)]) def test_docker_models_examples(model_cls): - examples = model_cls.Config.schema_extra["examples"] + examples = model_cls.model_config["json_schema_extra"]["examples"] for index, example in enumerate(examples): print(f"{index:-^10}:\n", example) diff --git a/packages/dask-task-models-library/tests/container_tasks/test_events.py b/packages/dask-task-models-library/tests/container_tasks/test_events.py index 16a308e11e0..1aa4139720d 100644 --- a/packages/dask-task-models-library/tests/container_tasks/test_events.py +++ b/packages/dask-task-models-library/tests/container_tasks/test_events.py @@ -26,7 +26,7 @@ def test_task_event_abstract(): @pytest.mark.parametrize("model_cls", [TaskProgressEvent, TaskLogEvent]) def test_events_models_examples(model_cls): - examples = model_cls.Config.schema_extra["examples"] + examples = model_cls.model_config["json_schema_extra"]["examples"] for index, example in enumerate(examples): print(f"{index:-^10}:\n", example) @@ -51,7 +51,7 @@ def mocked_dask_worker_job_id(mocker: MockerFixture, job_id: str) -> str: return job_id -@pytest.fixture(params=TaskOwner.Config.schema_extra["examples"]) +@pytest.fixture(params=TaskOwner.model_config["json_schema_extra"]["examples"]) def task_owner(request: pytest.FixtureRequest) -> TaskOwner: return TaskOwner(**request.param) diff --git a/packages/dask-task-models-library/tests/container_tasks/test_io.py b/packages/dask-task-models-library/tests/container_tasks/test_io.py index 14527d92391..db6357d930c 100644 --- a/packages/dask-task-models-library/tests/container_tasks/test_io.py +++ b/packages/dask-task-models-library/tests/container_tasks/test_io.py @@ -30,7 +30,7 @@ def test_io_models_examples(model_cls, model_cls_examples): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) - model_instance = model_cls.parse_obj(example) + model_instance = model_cls.model_validate(example) assert model_instance, f"Failed with {name}" print(name, ":", model_instance) @@ -69,9 +69,11 @@ def _create_fake_outputs( def test_create_task_output_from_task_with_optional_fields_as_required( tmp_path: Path, optional_fields_set: bool, faker: Faker ): - for schema_example in TaskOutputDataSchema.Config.schema_extra["examples"]: + for schema_example in TaskOutputDataSchema.model_config["json_schema_extra"][ + "examples" + ]: - task_output_schema = TaskOutputDataSchema.parse_obj(schema_example) + task_output_schema = TaskOutputDataSchema.model_validate(schema_example) outputs_file_name = _create_fake_outputs( task_output_schema, tmp_path, optional_fields_set, faker ) @@ -92,7 +94,7 @@ def test_create_task_output_from_task_with_optional_fields_as_required( def test_create_task_output_from_task_throws_when_there_are_missing_files( tmp_path: Path, faker: Faker ): - task_output_schema = TaskOutputDataSchema.parse_obj( + task_output_schema = TaskOutputDataSchema.model_validate( { "required_file_output": { "required": True, @@ -113,7 +115,7 @@ def test_create_task_output_from_task_throws_when_there_are_missing_files( def test_create_task_output_from_task_does_not_throw_when_there_are_optional_missing_files( tmp_path: Path, faker: Faker ): - task_output_schema = TaskOutputDataSchema.parse_obj( + task_output_schema = TaskOutputDataSchema.model_validate( { "optional_file_output": { "required": False, @@ -134,7 +136,7 @@ def test_create_task_output_from_task_does_not_throw_when_there_are_optional_mis def test_create_task_output_from_task_throws_when_there_are_entries( tmp_path: Path, faker: Faker ): - task_output_schema = TaskOutputDataSchema.parse_obj( + task_output_schema = TaskOutputDataSchema.model_validate( { "some_output": { "required": True, @@ -153,7 +155,7 @@ def test_create_task_output_from_task_throws_when_there_are_entries( def test_create_task_output_from_task_does_not_throw_when_there_are_optional_entries( tmp_path: Path, faker: Faker ): - task_output_schema = TaskOutputDataSchema.parse_obj( + task_output_schema = TaskOutputDataSchema.model_validate( { "some_output": { "required": False, @@ -182,6 +184,6 @@ def test_objects_are_compatible_with_dask_requirements(model_cls, model_cls_exam for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) - model_instance = model_cls.parse_obj(example) + model_instance = model_cls.model_validate(example) reloaded_instance = loads(dumps(model_instance)) assert reloaded_instance == model_instance diff --git a/packages/dask-task-models-library/tests/container_tasks/test_protocol.py b/packages/dask-task-models-library/tests/container_tasks/test_protocol.py index d17202adabd..3c70924a043 100644 --- a/packages/dask-task-models-library/tests/container_tasks/test_protocol.py +++ b/packages/dask-task-models-library/tests/container_tasks/test_protocol.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize("model_cls", [TaskOwner, ContainerTaskParameters]) def test_events_models_examples(model_cls): - examples = model_cls.Config.schema_extra["examples"] + examples = model_cls.model_config["json_schema_extra"]["examples"] for index, example in enumerate(examples): print(f"{index:-^10}:\n", example) @@ -19,7 +19,9 @@ def test_events_models_examples(model_cls): def test_task_owner_parent_valid(faker: Faker): - invalid_task_owner_example = TaskOwner.Config.schema_extra["examples"][0] + invalid_task_owner_example = TaskOwner.model_config["json_schema_extra"][ + "examples" + ][0] invalid_task_owner_example["parent_project_id"] = faker.uuid4() assert invalid_task_owner_example["parent_node_id"] is None with pytest.raises(ValidationError, match=r".+ are None or both are set!"): diff --git a/packages/models-library/requirements/_base.txt b/packages/models-library/requirements/_base.txt index 03e1b39d677..f07b0ddd44b 100644 --- a/packages/models-library/requirements/_base.txt +++ b/packages/models-library/requirements/_base.txt @@ -20,13 +20,13 @@ orjson==3.10.7 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in -pydantic==2.9.1 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in # pydantic-extra-types # pydantic-settings -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via -r requirements/_base.in diff --git a/packages/models-library/scripts/validate-pg-projects.py b/packages/models-library/scripts/validate-pg-projects.py index 978e32cfc6f..648b6846876 100644 --- a/packages/models-library/scripts/validate-pg-projects.py +++ b/packages/models-library/scripts/validate-pg-projects.py @@ -59,12 +59,12 @@ def validate_csv_exported_pg_project( pid = row.get("uuid", index + 1) try: - model = ProjectFromCsv.parse_obj(row) + model = ProjectFromCsv.model_validate(row) if verbose > 1: typer.secho(f"{pid} OK", fg=typer.colors.GREEN) if verbose > 2: - typer.echo(model.json(indent=2)) + typer.echo(model.model_dump_json(indent=2)) except ValidationError as err: failed.append(pid) typer.secho( diff --git a/packages/models-library/src/models_library/api_schemas_catalog/__init__.py b/packages/models-library/src/models_library/api_schemas_catalog/__init__.py index 84d761729a4..2e8c8f75a24 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/__init__.py @@ -1,7 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -CATALOG_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as(RPCNamespace, "catalog") +CATALOG_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter(RPCNamespace).validate_python( + "catalog" +) diff --git a/packages/models-library/src/models_library/api_schemas_clusters_keeper/__init__.py b/packages/models-library/src/models_library/api_schemas_clusters_keeper/__init__.py index b6570d01c89..79be28f2021 100644 --- a/packages/models-library/src/models_library/api_schemas_clusters_keeper/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_clusters_keeper/__init__.py @@ -1,9 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -CLUSTERS_KEEPER_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as( - RPCNamespace, "clusters-keeper" -) +CLUSTERS_KEEPER_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter( + RPCNamespace +).validate_python("clusters-keeper") diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py index f0958695e15..d103a3ea8c5 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py @@ -33,7 +33,7 @@ class CommonServiceDetails(BaseModel): class ServiceDetails(CommonServiceDetails): - basepath: Path = Field( + basepath: Path | None = Field( default=None, description="predefined path where the dynamic service should be served. If empty, the service shall use the root endpoint.", alias="service_basepath", @@ -68,7 +68,7 @@ class RunningDynamicServiceDetails(ServiceDetails): internal_port: PortInt = Field( ..., description="the service swarm internal port", alias="service_port" ) - published_port: PortInt = Field( + published_port: PortInt | None = Field( default=None, description="the service swarm published port if any", deprecated=True, diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/__init__.py b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/__init__.py index 5631d38e5f9..70a4f1247ba 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_scheduler/__init__.py @@ -1,9 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -DYNAMIC_SCHEDULER_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as( - RPCNamespace, "dynamic-scheduler" -) +DYNAMIC_SCHEDULER_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter( + RPCNamespace +).validate_python("dynamic-scheduler") diff --git a/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py b/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py index 50793febaf9..f47a9a3f8d3 100644 --- a/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_efs_guardian/__init__.py @@ -1,9 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -EFS_GUARDIAN_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as( - RPCNamespace, "efs-guardian" -) +EFS_GUARDIAN_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter( + RPCNamespace +).validate_python("efs-guardian") diff --git a/packages/models-library/src/models_library/api_schemas_payments/__init__.py b/packages/models-library/src/models_library/api_schemas_payments/__init__.py index 30d68367ded..73928d6ccd7 100644 --- a/packages/models-library/src/models_library/api_schemas_payments/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_payments/__init__.py @@ -1,7 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -PAYMENTS_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as(RPCNamespace, "payments") +PAYMENTS_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter(RPCNamespace).validate_python( + "payments" +) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/__init__.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/__init__.py index 295897e5b1d..d32b474edf6 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/__init__.py @@ -1,9 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -RESOURCE_USAGE_TRACKER_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as( - RPCNamespace, "resource-usage-tracker" -) +RESOURCE_USAGE_TRACKER_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter( + RPCNamespace +).validate_python("resource-usage-tracker") diff --git a/packages/models-library/src/models_library/api_schemas_storage.py b/packages/models-library/src/models_library/api_schemas_storage.py index fac13d42d40..bd0185a9173 100644 --- a/packages/models-library/src/models_library/api_schemas_storage.py +++ b/packages/models-library/src/models_library/api_schemas_storage.py @@ -37,10 +37,11 @@ ETag: TypeAlias = str - S3BucketName: TypeAlias = Annotated[str, StringConstraints(pattern=S3_BUCKET_NAME_RE)] -DatCoreDatasetName: TypeAlias = Annotated[str, StringConstraints(pattern=DATCORE_DATASET_NAME_RE)] +DatCoreDatasetName: TypeAlias = Annotated[ + str, StringConstraints(pattern=DATCORE_DATASET_NAME_RE) +] # / diff --git a/packages/models-library/src/models_library/api_schemas_webserver/__init__.py b/packages/models-library/src/models_library/api_schemas_webserver/__init__.py index f30e0f0790d..c95f68ab78c 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/__init__.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/__init__.py @@ -1,7 +1,9 @@ from typing import Final -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq_basic_types import RPCNamespace -WEBSERVER_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as(RPCNamespace, "webserver") +WEBSERVER_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter( + RPCNamespace +).validate_python("webserver") diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index cda166e0d13..02fabd46f7a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -103,7 +103,7 @@ class NodeGet(OutputSchema): "service_basepath": "/x/E1O2E-LAH", "service_state": "pending", "service_message": "no suitable node (insufficient resources on 1 node)", - "user_id": 123, + "user_id": "123", } } ) diff --git a/packages/models-library/src/models_library/basic_regex.py b/packages/models-library/src/models_library/basic_regex.py index 51441fe39e6..b65c0fd1fe1 100644 --- a/packages/models-library/src/models_library/basic_regex.py +++ b/packages/models-library/src/models_library/basic_regex.py @@ -49,7 +49,9 @@ SIMCORE_S3_DIRECTORY_ID_RE = rf"^({UUID_RE_BASE})\/({UUID_RE_BASE})\/(.+)\/$" # S3 - AWS bucket names [https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html] -S3_BUCKET_NAME_RE = r"(?!(^xn--|-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$" +S3_BUCKET_NAME_RE = re.compile( + r"^(?!xn--)[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$(? "DockerLabelKey": # NOTE: https://docs.docker.com/engine/reference/commandline/tag/#description -DockerGenericTag: TypeAlias = Annotated[str, StringConstraints(pattern=DOCKER_GENERIC_TAG_KEY_RE)] +DockerGenericTag: TypeAlias = Annotated[ + str, StringConstraints(pattern=DOCKER_GENERIC_TAG_KEY_RE) +] class DockerPlacementConstraint(ConstrainedStr): @@ -139,7 +141,7 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: """returns a dictionary of strings as required by docker""" return { to_simcore_runtime_docker_label_key(k): f"{v}" - for k, v in sorted(self.dict().items()) + for k, v in sorted(self.model_dump().items()) } @classmethod diff --git a/packages/models-library/src/models_library/function_services_catalog/_utils.py b/packages/models-library/src/models_library/function_services_catalog/_utils.py index 4cd1275b5e0..a58a524d094 100644 --- a/packages/models-library/src/models_library/function_services_catalog/_utils.py +++ b/packages/models-library/src/models_library/function_services_catalog/_utils.py @@ -14,10 +14,10 @@ "email": "unknown@osparc.io", "affiliation": "unknown", } -EN = Author.parse_obj(AUTHORS.get("EN", _DEFAULT)) -OM = Author.parse_obj(AUTHORS.get("OM", _DEFAULT)) -PC = Author.parse_obj(AUTHORS.get("PC", _DEFAULT)) -WVG = Author.parse_obj(AUTHORS.get("WVG", _DEFAULT)) +EN = Author.model_validate(AUTHORS.get("EN", _DEFAULT)) +OM = Author.model_validate(AUTHORS.get("OM", _DEFAULT)) +PC = Author.model_validate(AUTHORS.get("PC", _DEFAULT)) +WVG = Author.model_validate(AUTHORS.get("WVG", _DEFAULT)) def create_fake_thumbnail_url(label: str) -> str: diff --git a/packages/models-library/src/models_library/function_services_catalog/services/demo_units.py b/packages/models-library/src/models_library/function_services_catalog/services/demo_units.py index 298ac02c82b..44bd30e0899 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/demo_units.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/demo_units.py @@ -15,7 +15,7 @@ # If this assumption cannot be guaranteed anymore the test must be updated. # -META = ServiceMetaDataPublished.parse_obj( +META = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/data-iterator/demo-units", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/file_picker.py b/packages/models-library/src/models_library/function_services_catalog/services/file_picker.py index 0e0554842fb..2245a8ba3ff 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/file_picker.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/file_picker.py @@ -8,7 +8,7 @@ from .._key_labels import FUNCTION_SERVICE_KEY_PREFIX from .._utils import OM, FunctionServices -META: Final = ServiceMetaDataPublished.parse_obj( +META: Final = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/file-picker", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py b/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py index 662cbf327cf..d59e37735e8 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py @@ -12,7 +12,7 @@ def create_metadata( ) -> ServiceMetaDataPublished: prefix = prefix or type_name LABEL = f"{type_name.capitalize()} iterator" - return ServiceMetaDataPublished.parse_obj( + return ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/data-iterator/{prefix}-range", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py b/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py index f0199389885..a2be976c651 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py @@ -13,7 +13,7 @@ LIST_NUMBERS_SCHEMA: dict[str, Any] = schema_of(list[float], title="list[number]") -META = ServiceMetaDataPublished.parse_obj( +META = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/data-iterator/sensitivity", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/nodes_group.py b/packages/models-library/src/models_library/function_services_catalog/services/nodes_group.py index bfde87e52c3..40adb28f342 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/nodes_group.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/nodes_group.py @@ -7,7 +7,7 @@ # NOTE: DO not mistake with simcore/services/frontend/nodes-group/macros/ # which needs to be redefined. # -META = ServiceMetaDataPublished.parse_obj( +META = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/nodes-group", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/parameters.py b/packages/models-library/src/models_library/function_services_catalog/services/parameters.py index e0e25b6ee11..d62a4a88dfb 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/parameters.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/parameters.py @@ -12,7 +12,7 @@ def _create_metadata(type_name: str) -> ServiceMetaDataPublished: This is a parametrized node (or param-node in short) """ - meta = ServiceMetaDataPublished.parse_obj( + meta = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/parameter/{type_name}", @@ -45,7 +45,7 @@ def _create_metadata(type_name: str) -> ServiceMetaDataPublished: META_BOOL: Final = _create_metadata(type_name="boolean") META_INT: Final = _create_metadata(type_name="integer") META_STR: Final = _create_metadata(type_name="string") -META_ARRAY: Final = ServiceMetaDataPublished.parse_obj( +META_ARRAY: Final = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/parameter/array", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/probes.py b/packages/models-library/src/models_library/function_services_catalog/services/probes.py index e736efb2fb1..4c710a90ade 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/probes.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/probes.py @@ -7,7 +7,7 @@ def _create_metadata(type_name: str) -> ServiceMetaDataPublished: - obj: ServiceMetaDataPublished = ServiceMetaDataPublished.parse_obj( + obj: ServiceMetaDataPublished = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/iterator-consumer/probe/{type_name}", @@ -38,7 +38,7 @@ def _create_metadata(type_name: str) -> ServiceMetaDataPublished: META_BOOL: Final = _create_metadata("boolean") META_INT: Final = _create_metadata("integer") META_STR: Final = _create_metadata("string") -META_ARRAY: Final = ServiceMetaDataPublished.parse_obj( +META_ARRAY: Final = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/iterator-consumer/probe/array", @@ -67,7 +67,7 @@ def _create_metadata(type_name: str) -> ServiceMetaDataPublished: } ) -META_FILE: Final = ServiceMetaDataPublished.parse_obj( +META_FILE: Final = ServiceMetaDataPublished.model_validate( { "integration-version": LATEST_INTEGRATION_VERSION, "key": f"{FUNCTION_SERVICE_KEY_PREFIX}/iterator-consumer/probe/file", diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 440bfe915bb..120e54d899d 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -1,7 +1,6 @@ """ Models a study's project document """ -from copy import deepcopy from datetime import datetime from enum import Enum from typing import Any, Final, TypeAlias @@ -172,16 +171,7 @@ class Project(BaseProjectModel): alias="workspaceId", ) - def _patch_json_schema_extra(self, schema: dict) -> None: - # Patch to allow jsonschema nullable - # SEE https://github.com/samuelcolvin/pydantic/issues/990#issuecomment-645961530 - state_pydantic_schema = deepcopy(schema["properties"]["state"]) - schema["properties"]["state"] = { - "anyOf": [{"type": "null"}, state_pydantic_schema] - } - model_config = ConfigDict( title="osparc-simcore project", extra="forbid", - json_schema_extra=_patch_json_schema_extra, # type: ignore[typeddict-item] ) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index b074dd4dec6..3a6ea052313 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -113,6 +113,16 @@ class NodeState(BaseModel): ) +def _patch_json_schema_extra(schema: dict) -> None: + # NOTE: exporting without this trick does not make runHash as nullable. + # It is a Pydantic issue see https://github.com/samuelcolvin/pydantic/issues/1270 + for prop_name in ["parent", "runHash"]: + if prop_name in schema.get("properties", {}): + prop = deepcopy(schema["properties"][prop_name]) + prop["nullable"] = True + schema["properties"][prop_name] = prop + + class Node(BaseModel): key: ServiceKey = Field( ..., @@ -234,16 +244,7 @@ def convert_from_enum(cls, v): return NodeState(currentStatus=running_state_value) return v - def _patch_json_schema_extra(self, schema: dict) -> None: - # NOTE: exporting without this trick does not make runHash as nullable. - # It is a Pydantic issue see https://github.com/samuelcolvin/pydantic/issues/1270 - for prop_name in ["parent", "runHash"]: - if prop_name in schema.get("properties", {}): - prop = deepcopy(schema["properties"][prop_name]) - prop["nullable"] = True - schema["properties"][prop_name] = prop - model_config = ConfigDict( extra="forbid", - json_schema_extra=_patch_json_schema_extra, # type: ignore[typeddict-item] + json_schema_extra=_patch_json_schema_extra, ) diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 876c2f71744..3a79b6acf00 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -6,7 +6,6 @@ - Link to another port: PortLink """ -import re from pathlib import Path from typing import Annotated, TypeAlias from uuid import UUID @@ -15,6 +14,7 @@ from pydantic import ( AnyUrl, BaseModel, + BeforeValidator, ConfigDict, Field, StringConstraints, @@ -40,8 +40,9 @@ LocationName = str -class SimcoreS3FileID(ConstrainedStr): - pattern: re.Pattern[str] | None = re.compile(SIMCORE_S3_FILE_ID_RE) +SimcoreS3FileID: TypeAlias = Annotated[ + str, StringConstraints(pattern=SIMCORE_S3_FILE_ID_RE) +] class SimcoreS3DirectoryID(ConstrainedStr): @@ -50,7 +51,7 @@ class SimcoreS3DirectoryID(ConstrainedStr): `{project_id}/{node_id}/simcore-dir-name/` """ - pattern: re.Pattern[str] | None = re.compile(SIMCORE_S3_DIRECTORY_ID_RE) + pattern: str = SIMCORE_S3_DIRECTORY_ID_RE @staticmethod def _get_parent(s3_object: str, *, parent_index: int) -> str: @@ -87,9 +88,7 @@ def from_simcore_s3_object(cls, s3_object: str) -> "SimcoreS3DirectoryID": return TypeAdapter(cls).validate_python(f"{parent_path}/") -class DatCoreFileID(ConstrainedStr): - regex: re.Pattern[str] | None = re.compile(DATCORE_FILE_ID_RE) - +DatCoreFileID: TypeAlias = Annotated[str, StringConstraints(pattern=DATCORE_FILE_ID_RE)] StorageFileID: TypeAlias = SimcoreS3FileID | DatCoreFileID @@ -123,7 +122,9 @@ class PortLink(BaseModel): class DownloadLink(BaseModel): """I/O port type to hold a generic download link to a file (e.g. S3 pre-signed link, etc)""" - download_link: Annotated[str, AnyUrl] = Field(..., alias="downloadLink") + download_link: Annotated[ + str, BeforeValidator(lambda x: str(TypeAdapter(AnyUrl).validate_python(x))) + ] = Field(..., alias="downloadLink") label: str | None = Field(default=None, description="Display name") model_config = ConfigDict( extra="forbid", @@ -145,11 +146,13 @@ class BaseFileLink(BaseModel): store: LocationID = Field( ..., description="The store identifier: 0 for simcore S3, 1 for datcore", + validate_default=True, ) path: StorageFileID = Field( ..., description="The path to the file in the storage provider domain", + union_mode="left_to_right", ) label: str | None = Field( @@ -170,6 +173,8 @@ def legacy_enforce_str_to_int(cls, v): return int(v) return v + model_config = ConfigDict(populate_by_name=True) + class SimCoreFileLink(BaseFileLink): """I/O port type to hold a link to a file in simcore S3 storage""" diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 13ecda316ae..11e0ad55796 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -46,7 +46,7 @@ def routing_key(self) -> str | None: """ def body(self) -> bytes: - return self.json().encode() + return self.model_dump_json().encode() class ProjectMessageBase(BaseModel): diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index d8e3b9990b3..63b4e4948b0 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -1,8 +1,9 @@ from typing import Annotated, Final, Generic, TypeAlias, TypeVar +from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter from pydantic import ( - AnyHttpUrl, BaseModel, + BeforeValidator, ConfigDict, Field, NonNegativeInt, @@ -20,7 +21,9 @@ MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE: Final[int] = 50 -PageLimitInt: TypeAlias = Annotated[int, Field(ge=1, lt=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE)] +PageLimitInt: TypeAlias = Annotated[ + int, Field(ge=1, lt=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE) +] DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = TypeAdapter( PageLimitInt @@ -92,7 +95,14 @@ class PageRefs(BaseModel, Generic[RefT]): model_config = ConfigDict(extra="forbid") -class PageLinks(PageRefs[Annotated[str, AnyHttpUrl]]): +class PageLinks( + PageRefs[ + Annotated[ + str, + BeforeValidator(lambda x: str(AnyHttpUrlLegacyAdapter.validate_python(x))), + ] + ] +): ... diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index 8d901d50f30..1adfc5625c6 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -1,9 +1,8 @@ from math import ceil from typing import Any, Protocol, TypedDict, Union, runtime_checkable -from pydantic import parse_obj_as +from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter -from .basic_types import AnyHttpUrl from .rest_pagination import PageLinks, PageMetaInfoLimitOffset # NOTE: In this repo we use two type of URL-like data structures: @@ -39,7 +38,9 @@ def _replace_query(url: _URLType, query: dict[str, Any]) -> str: new_url = url.update_query(query) else: new_url = url.replace_query_params(**query) - return f"{new_url}" + + new_url_str = f"{new_url}" + return f"{AnyHttpUrlLegacyAdapter.validate_python(new_url_str)}" class PageDict(TypedDict): @@ -61,7 +62,7 @@ def paginate_data( Usage: obj: PageDict = paginate_data( ... ) - model = Page[MyModelItem].parse_obj(obj) + model = Page[MyModelItem].model_validate(obj) raises ValidationError """ @@ -72,37 +73,21 @@ def paginate_data( total=total, count=len(chunk), limit=limit, offset=offset ), _links=PageLinks( - self=( - parse_obj_as( - AnyHttpUrl, - _replace_query(request_url, {"offset": offset, "limit": limit}), - ) - ), - first=parse_obj_as( - AnyHttpUrl, _replace_query(request_url, {"offset": 0, "limit": limit}) - ), - prev=parse_obj_as( - AnyHttpUrl, - _replace_query( - request_url, {"offset": max(offset - limit, 0), "limit": limit} - ), + self=_replace_query(request_url, {"offset": offset, "limit": limit}), + first=_replace_query(request_url, {"offset": 0, "limit": limit}), + prev=_replace_query( + request_url, {"offset": max(offset - limit, 0), "limit": limit} ) if offset > 0 else None, - next=parse_obj_as( - AnyHttpUrl, - _replace_query( - request_url, - {"offset": min(offset + limit, last_page * limit), "limit": limit}, - ), + next=_replace_query( + request_url, + {"offset": min(offset + limit, last_page * limit), "limit": limit}, ) if offset < (last_page * limit) else None, - last=parse_obj_as( - AnyHttpUrl, - _replace_query( - request_url, {"offset": last_page * limit, "limit": limit} - ), + last=_replace_query( + request_url, {"offset": last_page * limit, "limit": limit} ), ), data=chunk, diff --git a/packages/models-library/src/models_library/rpc_pagination.py b/packages/models-library/src/models_library/rpc_pagination.py index 0ec454cc9fd..92470b30d67 100644 --- a/packages/models-library/src/models_library/rpc_pagination.py +++ b/packages/models-library/src/models_library/rpc_pagination.py @@ -31,7 +31,7 @@ class PageRefsParams(PageRefs[PageQueryParameters]): @classmethod def create(cls, total: int, limit: int, offset: int) -> "PageRefsParams": last_page = ceil(total / limit) - 1 - return cls.parse_obj( + return cls.model_validate( { "self": {"offset": offset, "limit": limit}, "first": {"offset": 0, "limit": limit}, diff --git a/packages/models-library/src/models_library/service_settings_labels.py b/packages/models-library/src/models_library/service_settings_labels.py index 8f954737231..851b1880cc3 100644 --- a/packages/models-library/src/models_library/service_settings_labels.py +++ b/packages/models-library/src/models_library/service_settings_labels.py @@ -12,11 +12,11 @@ Field, Json, PrivateAttr, + TypeAdapter, ValidationError, ValidationInfo, field_validator, model_validator, - parse_obj_as, ) from .callbacks_mapping import CallbacksMapping @@ -43,8 +43,7 @@ class ContainerSpec(BaseModel): max_length=2, ) - model_config = ConfigDict( - **_BaseConfig, + model_config = _BaseConfig | ConfigDict( json_schema_extra={ "examples": [ {"Command": ["executable"]}, @@ -102,8 +101,7 @@ def ensure_backwards_compatible_setting_type(cls, v): return "Resources" return v - model_config = ConfigDict( - **_BaseConfig, + model_config = _BaseConfig | ConfigDict( populate_by_name=True, json_schema_extra={ "examples": [ @@ -203,7 +201,7 @@ def validate_volume_limits(cls, v, info: ValidationInfo) -> str | None: for path_str, size_str in v.items(): # checks that format is correct try: - parse_obj_as(ByteSize, size_str) + TypeAdapter(ByteSize).validate_python(size_str) except ValidationError as e: msg = f"Provided size='{size_str}' contains invalid charactes: {e!s}" raise ValueError(msg) from e @@ -221,8 +219,7 @@ def validate_volume_limits(cls, v, info: ValidationInfo) -> str | None: output: str | None = v return output - model_config = ConfigDict( - **_BaseConfig, + model_config = _BaseConfig | ConfigDict( json_schema_extra={ "examples": [ { diff --git a/packages/models-library/src/models_library/service_settings_nat_rule.py b/packages/models-library/src/models_library/service_settings_nat_rule.py index ee937c81254..1f50b62f503 100644 --- a/packages/models-library/src/models_library/service_settings_nat_rule.py +++ b/packages/models-library/src/models_library/service_settings_nat_rule.py @@ -1,21 +1,14 @@ from collections.abc import Generator from typing import Final -from pydantic import ( - BaseModel, - ConfigDict, - Field, - ValidationInfo, - field_validator, - parse_obj_as, -) +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationInfo, field_validator from .basic_types import PortInt from .osparc_variable_identifier import OsparcVariableIdentifier, raise_if_unresolved # Cloudflare DNS server address DEFAULT_DNS_SERVER_ADDRESS: Final[str] = "1.1.1.1" # NOSONAR -DEFAULT_DNS_SERVER_PORT: Final[PortInt] = parse_obj_as(PortInt, 53) +DEFAULT_DNS_SERVER_PORT: Final[PortInt] = TypeAdapter(PortInt).validate_python(53) class _PortRange(BaseModel): diff --git a/packages/models-library/src/models_library/services_io.py b/packages/models-library/src/models_library/services_io.py index 49264f19799..6f1acd3d494 100644 --- a/packages/models-library/src/models_library/services_io.py +++ b/packages/models-library/src/models_library/services_io.py @@ -211,7 +211,7 @@ class ServiceInput(BaseServiceIOModel): def from_json_schema(cls, port_schema: dict[str, Any]) -> "ServiceInput": """Creates input port model from a json-schema""" data = cls._from_json_schema_base_implementation(port_schema) - return cls.parse_obj(data) + return cls.model_validate(data) class ServiceOutput(BaseServiceIOModel): @@ -258,4 +258,4 @@ class ServiceOutput(BaseServiceIOModel): def from_json_schema(cls, port_schema: dict[str, Any]) -> "ServiceOutput": """Creates output port model from a json-schema""" data = cls._from_json_schema_base_implementation(port_schema) - return cls.parse_obj(data) + return cls.model_validate(data) diff --git a/packages/models-library/src/models_library/utils/json_serialization.py b/packages/models-library/src/models_library/utils/json_serialization.py index 69ffb00572d..a2fee1295f1 100644 --- a/packages/models-library/src/models_library/utils/json_serialization.py +++ b/packages/models-library/src/models_library/utils/json_serialization.py @@ -56,8 +56,8 @@ def decimal_encoder(dec_value: Decimal) -> int | float: """ if dec_value.as_tuple().exponent >= 0: # type: ignore[operator] return int(dec_value) - else: - return float(dec_value) + + return float(dec_value) ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { @@ -95,7 +95,8 @@ def pydantic_encoder(obj: Any) -> Any: if isinstance(obj, BaseModel): return obj.model_dump() - elif is_dataclass(obj): + + if is_dataclass(obj): return asdict(obj) # type: ignore[call-overload] # Check the class type and its superclasses for a matching encoder diff --git a/packages/models-library/src/models_library/utils/nodes.py b/packages/models-library/src/models_library/utils/nodes.py index 1def98ec507..dd791677d19 100644 --- a/packages/models-library/src/models_library/utils/nodes.py +++ b/packages/models-library/src/models_library/utils/nodes.py @@ -5,7 +5,7 @@ from copy import deepcopy from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter from ..projects import Project from ..projects_nodes_io import NodeID, PortLink, UUIDStr @@ -20,7 +20,7 @@ def project_node_io_payload_cb( async def node_io_payload_cb(node_id: NodeID) -> dict[str, Any]: node_io_payload: dict[str, Any] = {"inputs": None, "outputs": None} - node = project.workbench.get(UUIDStr(node_id)) + node = project.workbench.get(TypeAdapter(UUIDStr).validate_python(node_id)) if node: node_io_payload = {"inputs": node.inputs, "outputs": node.outputs} @@ -58,7 +58,7 @@ async def compute_node_hash( # ensure we do not get pydantic types for hashing here, only jsoneable stuff if isinstance(payload, BaseModel): - payload = payload.dict(by_alias=True, exclude_unset=True) + payload = payload.model_dump(by_alias=True, exclude_unset=True) # remove the payload if it is null and it was resolved if payload is not None: diff --git a/packages/models-library/tests/test__models_examples.py b/packages/models-library/tests/test__models_examples.py index 12809db713b..2345b5451f1 100644 --- a/packages/models-library/tests/test__models_examples.py +++ b/packages/models-library/tests/test__models_examples.py @@ -14,6 +14,6 @@ def test_all_models_library_models_config_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): - assert model_cls.parse_obj( + assert model_cls.model_validate( example_data ), f"Failed {example_name} : {json.dumps(example_data)}" diff --git a/packages/models-library/tests/test__pydantic_models.py b/packages/models-library/tests/test__pydantic_models.py index 548d34f6569..645dc1ffe21 100644 --- a/packages/models-library/tests/test__pydantic_models.py +++ b/packages/models-library/tests/test__pydantic_models.py @@ -50,7 +50,7 @@ class ArgumentAnnotation(BaseModel): "items": {"type": "integer"}, } - assert x_annotation.dict() == { + assert x_annotation.model_dump() == { "name": "x", "data_schema": { "title": "schema[x]", @@ -64,7 +64,7 @@ class ArgumentAnnotation(BaseModel): # # the constructor would expect a raw string but we produced a nested dict with pytest.raises(ValidationError) as exc_info: - ArgumentAnnotation(**x_annotation.dict()) + ArgumentAnnotation(**x_annotation.model_dump()) assert exc_info.value.errors()[0] == { "input": {"items": {"type": "integer"}, "title": "schema[x]", "type": "array"}, @@ -147,7 +147,7 @@ class Func(BaseModel): assert model.input == {"w": 42, "z": False} assert model.output == "some/path/or/string" - # (undefined) json string vs SimCoreFileLink.dict() ------------ + # (undefined) json string vs SimCoreFileLink.model_dump() ------------ MINIMAL = 2 # <--- index of the example with the minimum required fields assert SimCoreFileLink in get_args(OutputTypes) example = SimCoreFileLink.model_validate( diff --git a/packages/models-library/tests/test__pydantic_models_and_enums.py b/packages/models-library/tests/test__pydantic_models_and_enums.py index 51f0226ee80..00c67c32c9b 100644 --- a/packages/models-library/tests/test__pydantic_models_and_enums.py +++ b/packages/models-library/tests/test__pydantic_models_and_enums.py @@ -2,7 +2,7 @@ import pytest from models_library.utils.enums import are_equivalent_enums, enum_to_dict -from pydantic import BaseModel, ValidationError, parse_obj_as +from pydantic import BaseModel, TypeAdapter, ValidationError # @@ -76,16 +76,16 @@ class Model(BaseModel): def test_parsing_enums_in_pydantic(): - model = parse_obj_as(Model, {"color": Color1.RED}) + model = TypeAdapter(Model).validate_python({"color": Color1.RED}) assert model.color == Color1.RED # Can parse from STRING - model = parse_obj_as(Model, {"color": "RED"}) + model = TypeAdapter(Model).validate_python({"color": "RED"}) assert model.color == Color1.RED # Can **NOT** parse from equilalent enum with pytest.raises(ValidationError): - parse_obj_as(Model, {"color": Color2.RED}) + TypeAdapter(Model).validate_python({"color": Color2.RED}) class ModelStrAndEnum(BaseModel): @@ -95,30 +95,32 @@ class ModelStrAndEnum(BaseModel): def test_parsing_strenum_in_pydantic(): assert are_equivalent_enums(Color1, ColorStrAndEnum1) - model = parse_obj_as(ModelStrAndEnum, {"color": ColorStrAndEnum1.RED}) + model = TypeAdapter(ModelStrAndEnum).validate_python( + {"color": ColorStrAndEnum1.RED} + ) assert model.color == ColorStrAndEnum1.RED # Can parse from string - model = parse_obj_as(ModelStrAndEnum, {"color": "RED"}) + model = TypeAdapter(ModelStrAndEnum).validate_python({"color": "RED"}) assert model.color == ColorStrAndEnum1.RED # **CAN** parse other equivalent str-enum # Using str-enums allow you to parse from equivalent enums! - parse_obj_as(ModelStrAndEnum, {"color": ColorStrAndEnum2.RED}) + TypeAdapter(ModelStrAndEnum).validate_python({"color": ColorStrAndEnum2.RED}) def test_parsing_str_and_enum_in_pydantic(): - # Can still NOT parse equilalent enum(-only) + # Can still NOT parse equivalent enum(-only) # with pytest.raises(ValidationError): - # parse_obj_as(ModelStrAndEnum, {"color": Color1.RED}) + # TypeAdapter(ModelStrAndEnum).validate_python({"color": Color1.RED}) # And the opposite? NO!!! with pytest.raises(ValidationError): - parse_obj_as(Color1, {"color": ColorStrAndEnum1.RED}) + TypeAdapter(Color1).validate_python({"color": ColorStrAndEnum1.RED}) with pytest.raises(ValidationError): - parse_obj_as(Color1, {"color": ColorStrAndEnum2.RED}) + TypeAdapter(Color1).validate_python({"color": ColorStrAndEnum2.RED}) # CONCLUSION: we need a validator to pre-process inputs ! # SEE models_library.utils.common_validators diff --git a/packages/models-library/tests/test_api_schemas_catalog.py b/packages/models-library/tests/test_api_schemas_catalog.py index 0c815d7bd0c..721f27481e2 100644 --- a/packages/models-library/tests/test_api_schemas_catalog.py +++ b/packages/models-library/tests/test_api_schemas_catalog.py @@ -9,7 +9,7 @@ def test_service_port_with_file(): - io = ServiceInput.parse_obj( + io = ServiceInput.model_validate( { "displayOrder": 1, "label": "Input files", @@ -21,7 +21,7 @@ def test_service_port_with_file(): } ) - port = ServicePortGet.from_service_io("input", "input_1", io).dict( + port = ServicePortGet.from_service_io("input", "input_1", io).model_dump( exclude_unset=True ) @@ -39,7 +39,7 @@ def test_service_port_with_file(): def test_service_port_with_boolean(): - io = ServiceInput.parse_obj( + io = ServiceInput.model_validate( { "displayOrder": 3, "label": "Same title and description is more usual than you might think", @@ -49,7 +49,7 @@ def test_service_port_with_boolean(): } ) - port = ServicePortGet.from_service_io("input", "input_1", io).dict( + port = ServicePortGet.from_service_io("input", "input_1", io).model_dump( exclude_unset=True ) diff --git a/packages/models-library/tests/test_api_schemas_webserver_projects.py b/packages/models-library/tests/test_api_schemas_webserver_projects.py index b8e4fcbdc47..295e9ee2304 100644 --- a/packages/models-library/tests/test_api_schemas_webserver_projects.py +++ b/packages/models-library/tests/test_api_schemas_webserver_projects.py @@ -14,7 +14,7 @@ ) from models_library.generics import Envelope from models_library.rest_pagination import Page -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.simcore_webserver_projects_rest_api import ( CREATE_FROM_SERVICE, CREATE_FROM_TEMPLATE, @@ -34,12 +34,12 @@ ids=lambda c: c.name, ) def test_create_project_schemas(api_call: HttpApiCallCapture): - request_payload = ProjectCreateNew.parse_obj(api_call.request_payload) + request_payload = ProjectCreateNew.model_validate(api_call.request_payload) assert request_payload - response_body = parse_obj_as( - Envelope[ProjectGet] | Envelope[TaskProjectGet], api_call.response_body - ) + response_body = TypeAdapter( + Envelope[ProjectGet] | Envelope[TaskProjectGet] + ).validate_python(api_call.response_body) assert response_body @@ -51,7 +51,9 @@ def test_create_project_schemas(api_call: HttpApiCallCapture): def test_list_project_schemas(api_call: HttpApiCallCapture): assert api_call.request_payload is None - response_body = parse_obj_as(Page[ProjectListItem], api_call.response_body) + response_body = TypeAdapter(Page[ProjectListItem]).validate_python( + api_call.response_body + ) assert response_body @@ -64,7 +66,9 @@ def test_get_project_schemas(api_call: HttpApiCallCapture): # NOTE: that response_body here is the exported values # and therefore ProjectGet has to be implemented in such a way that # can also parse exported values! (e.g. Json does not allow that, or ocassionaly exclude_none) - response_body = parse_obj_as(Envelope[ProjectGet], api_call.response_body) + response_body = TypeAdapter(Envelope[ProjectGet]).validate_python( + api_call.response_body + ) assert response_body @@ -74,8 +78,12 @@ def test_get_project_schemas(api_call: HttpApiCallCapture): ids=lambda c: c.name, ) def test_replace_project_schemas(api_call: HttpApiCallCapture): - request_payload = parse_obj_as(ProjectReplace, api_call.request_payload) + request_payload = TypeAdapter(ProjectReplace).validate_python( + api_call.request_payload + ) assert request_payload - response_body = parse_obj_as(Envelope[ProjectGet], api_call.response_body) + response_body = TypeAdapter(Envelope[ProjectGet]).validate_python( + api_call.response_body + ) assert response_body diff --git a/packages/models-library/tests/test_callbacks_mapping.py b/packages/models-library/tests/test_callbacks_mapping.py index e1c0df003c6..e39db6367ad 100644 --- a/packages/models-library/tests/test_callbacks_mapping.py +++ b/packages/models-library/tests/test_callbacks_mapping.py @@ -6,7 +6,7 @@ TIMEOUT_MIN, CallbacksMapping, ) -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError def _format_with_timeout(timeout: float) -> dict[str, Any]: @@ -20,8 +20,10 @@ def test_inactivity_time_out_is_max_capped(): INACTIVITY_TIMEOUT_CAP - 1, INACTIVITY_TIMEOUT_CAP, ]: - parse_obj_as(CallbacksMapping, _format_with_timeout(in_bounds)) + TypeAdapter(CallbacksMapping).validate_python(_format_with_timeout(in_bounds)) for out_of_bounds in [INACTIVITY_TIMEOUT_CAP + 1, TIMEOUT_MIN - 1]: with pytest.raises(ValidationError): - parse_obj_as(CallbacksMapping, _format_with_timeout(out_of_bounds)) + TypeAdapter(CallbacksMapping).validate_python( + _format_with_timeout(out_of_bounds) + ) diff --git a/packages/models-library/tests/test_docker.py b/packages/models-library/tests/test_docker.py index 55cb9419bbc..dd5fed89951 100644 --- a/packages/models-library/tests/test_docker.py +++ b/packages/models-library/tests/test_docker.py @@ -13,7 +13,7 @@ DockerLabelKey, StandardSimcoreDockerLabels, ) -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError _faker = Faker() @@ -40,11 +40,11 @@ def test_docker_label_key(label_key: str, valid: bool): # NOTE: https://docs.docker.com/config/labels-custom-metadata/#key-format-recommendations if valid: - instance = parse_obj_as(DockerLabelKey, label_key) + instance = TypeAdapter(DockerLabelKey).validate_python(label_key) assert instance else: with pytest.raises(ValidationError): - parse_obj_as(DockerLabelKey, label_key) + TypeAdapter(DockerLabelKey).validate_python(label_key) @pytest.mark.parametrize( @@ -94,11 +94,11 @@ def test_docker_label_key(label_key: str, valid: bool): ) def test_docker_generic_tag(image_name: str, valid: bool): if valid: - instance = parse_obj_as(DockerGenericTag, image_name) + instance = TypeAdapter(DockerGenericTag).validate_python(image_name) assert instance else: with pytest.raises(ValidationError): - parse_obj_as(DockerGenericTag, image_name) + TypeAdapter(DockerGenericTag).validate_python(image_name) @pytest.mark.parametrize( @@ -107,7 +107,9 @@ def test_docker_generic_tag(image_name: str, valid: bool): ids=str, ) def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]): - simcore_service_docker_label_keys = StandardSimcoreDockerLabels.parse_obj(obj_data) + simcore_service_docker_label_keys = StandardSimcoreDockerLabels.model_validate( + obj_data + ) exported_dict = simcore_service_docker_label_keys.to_simcore_runtime_docker_labels() assert all( isinstance(v, str) for v in exported_dict.values() @@ -115,8 +117,8 @@ def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]): assert all( key.startswith(_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX) for key in exported_dict ) - re_imported_docker_label_keys = parse_obj_as( - StandardSimcoreDockerLabels, exported_dict - ) + re_imported_docker_label_keys = TypeAdapter( + StandardSimcoreDockerLabels + ).validate_python(exported_dict) assert re_imported_docker_label_keys assert simcore_service_docker_label_keys == re_imported_docker_label_keys diff --git a/packages/models-library/tests/test_generics.py b/packages/models-library/tests/test_generics.py index b778cd4a490..f94436f1214 100644 --- a/packages/models-library/tests/test_generics.py +++ b/packages/models-library/tests/test_generics.py @@ -20,7 +20,7 @@ def test_dict_base_model(): "another key": "a string value", "yet another key": Path("some_path"), } - some_instance = DictModel[str, Any].parse_obj(some_dict) + some_instance = DictModel[str, Any].model_validate(some_dict) assert some_instance # test some typical dict methods @@ -78,10 +78,10 @@ def test_enveloped_data_builtin(builtin_type: type, builtin_value: Any): assert envelope == Envelope[builtin_type].from_data(builtin_value) # exports - assert envelope.dict(exclude_unset=True, exclude_none=True) == { + assert envelope.model_dump(exclude_unset=True, exclude_none=True) == { "data": builtin_value } - assert envelope.dict() == {"data": builtin_value, "error": None} + assert envelope.model_dump() == {"data": builtin_value, "error": None} def test_enveloped_data_model(): @@ -92,7 +92,9 @@ class User(BaseModel): enveloped = Envelope[User](data={"idr": 3}) assert isinstance(enveloped.data, User) - assert enveloped.dict(exclude_unset=True, exclude_none=True) == {"data": {"idr": 3}} + assert enveloped.model_dump(exclude_unset=True, exclude_none=True) == { + "data": {"idr": 3} + } def test_enveloped_data_dict(): diff --git a/packages/models-library/tests/test_osparc_variable_identifier.py b/packages/models-library/tests/test_osparc_variable_identifier.py index 18b48c299bd..cb23b19f60a 100644 --- a/packages/models-library/tests/test_osparc_variable_identifier.py +++ b/packages/models-library/tests/test_osparc_variable_identifier.py @@ -10,7 +10,7 @@ raise_if_unresolved_osparc_variable_identifier_found, replace_osparc_variable_identifier, ) -from pydantic import BaseModel, ValidationError, parse_obj_as +from pydantic import BaseModel, TypeAdapter, ValidationError VALID_IDENTIFIERS: list[str] = [ "$OSPARC_VARIABLE_One121_", @@ -41,6 +41,11 @@ ] +_OSPARC_VARIABLE_IDENTIFIER_ADAPTER: TypeAdapter[ + OsparcVariableIdentifier +] = TypeAdapter(OsparcVariableIdentifier) + + @pytest.fixture(params=VALID_IDENTIFIERS) def osparc_variable_identifier_str(request: pytest.FixtureRequest) -> str: return request.param @@ -50,13 +55,15 @@ def osparc_variable_identifier_str(request: pytest.FixtureRequest) -> str: def identifier( osparc_variable_identifier_str: str, ) -> OsparcVariableIdentifier: - return parse_obj_as(OsparcVariableIdentifier, osparc_variable_identifier_str) + return _OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python( + osparc_variable_identifier_str + ) @pytest.mark.parametrize("invalid_var_name", INVALID_IDENTIFIERS) def test_osparc_variable_identifier_does_not_validate(invalid_var_name: str): with pytest.raises(ValidationError): - parse_obj_as(OsparcVariableIdentifier, invalid_var_name) + _OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python(invalid_var_name) def test_raise_if_unresolved(identifier: OsparcVariableIdentifier): @@ -76,13 +83,19 @@ class Example(BaseModel): @pytest.mark.parametrize( "object_template", [ - parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_1"), - [parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_1")], - (parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_1"),), - {parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_1")}, - {"test": parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_1")}, + _OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python("$OSPARC_VARIABLE_1"), + [_OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python("$OSPARC_VARIABLE_1")], + (_OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python("$OSPARC_VARIABLE_1"),), + {_OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python("$OSPARC_VARIABLE_1")}, + { + "test": _OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python( + "$OSPARC_VARIABLE_1" + ) + }, Example( - nested_objects=parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_1") + nested_objects=_OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python( + "$OSPARC_VARIABLE_1" + ) ), ], ) @@ -147,6 +160,8 @@ def test_osparc_variable_name_and_default_value( expected_osparc_variable_name: str, expected_default_value: str | None, ): - osparc_variable_identifer = parse_obj_as(OsparcVariableIdentifier, str_identifier) + osparc_variable_identifer = _OSPARC_VARIABLE_IDENTIFIER_ADAPTER.validate_python( + str_identifier + ) assert osparc_variable_identifer.name == expected_osparc_variable_name assert osparc_variable_identifer.default_value == expected_default_value diff --git a/packages/models-library/tests/test_project_networks.py b/packages/models-library/tests/test_project_networks.py index c91f0503a8e..a929ac2a0aa 100644 --- a/packages/models-library/tests/test_project_networks.py +++ b/packages/models-library/tests/test_project_networks.py @@ -7,7 +7,7 @@ DockerNetworkName, NetworksWithAliases, ) -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError @pytest.mark.parametrize( @@ -19,7 +19,7 @@ ], ) def test_networks_with_aliases_ok(valid_example: dict) -> None: - assert NetworksWithAliases.parse_obj(valid_example) + assert NetworksWithAliases.model_validate(valid_example) @pytest.mark.parametrize( @@ -39,26 +39,26 @@ def test_networks_with_aliases_ok(valid_example: dict) -> None: ) def test_networks_with_aliases_fail(invalid_example: dict) -> None: with pytest.raises(ValidationError): - assert NetworksWithAliases.parse_obj(invalid_example) + assert NetworksWithAliases.model_validate(invalid_example) @pytest.mark.parametrize("network_name", ["a", "ok", "a_", "A_", "a1", "a-"]) def test_projects_networks_validation(network_name: str) -> None: - assert parse_obj_as(DockerNetworkName, network_name) == network_name - assert parse_obj_as(DockerNetworkAlias, network_name) == network_name + assert TypeAdapter(DockerNetworkName).validate_python(network_name) == network_name + assert TypeAdapter(DockerNetworkAlias).validate_python(network_name) == network_name @pytest.mark.parametrize("network_name", ["", "1", "-", "_"]) def test_projects_networks_validation_fails(network_name: str) -> None: with pytest.raises(ValidationError): - parse_obj_as(DockerNetworkName, network_name) + TypeAdapter(DockerNetworkName).validate_python(network_name) with pytest.raises(ValidationError): - parse_obj_as(DockerNetworkAlias, network_name) + TypeAdapter(DockerNetworkAlias).validate_python(network_name) def test_class_constructors_fail() -> None: with pytest.raises(ValidationError): - NetworksWithAliases.parse_obj( + NetworksWithAliases.model_validate( { "ok-netowrk_naeme": { UUID( diff --git a/packages/models-library/tests/test_project_nodes.py b/packages/models-library/tests/test_project_nodes.py index 2edefd1533d..96f427a19cb 100644 --- a/packages/models-library/tests/test_project_nodes.py +++ b/packages/models-library/tests/test_project_nodes.py @@ -31,7 +31,7 @@ def test_create_minimal_node(minimal_node_data_sample: dict[str, Any]): assert node.parent is None assert node.progress is None - assert node.dict(exclude_unset=True) == minimal_node_data_sample + assert node.model_dump(exclude_unset=True) == minimal_node_data_sample def test_create_minimal_node_with_new_data_type( @@ -69,4 +69,4 @@ def test_backwards_compatibility_node_data(minimal_node_data_sample: dict[str, A assert node.state.modified is True assert node.state.dependencies == set() - assert node.dict(exclude_unset=True) != old_node_data + assert node.model_dump(exclude_unset=True) != old_node_data diff --git a/packages/models-library/tests/test_project_nodes_io.py b/packages/models-library/tests/test_project_nodes_io.py index aac9568eccb..9a191c7d674 100644 --- a/packages/models-library/tests/test_project_nodes_io.py +++ b/packages/models-library/tests/test_project_nodes_io.py @@ -12,7 +12,7 @@ SimCoreFileLink, SimcoreS3DirectoryID, ) -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError @pytest.fixture() @@ -120,11 +120,13 @@ def test_store_discriminator(): def test_simcore_s3_directory_id(): # the only allowed path is the following - result = parse_obj_as(SimcoreS3DirectoryID, f"{UUID_0}/{UUID_0}/ok-simcore-dir/") + result = TypeAdapter(SimcoreS3DirectoryID).validate_python( + f"{UUID_0}/{UUID_0}/ok-simcore-dir/" + ) assert result == f"{UUID_0}/{UUID_0}/ok-simcore-dir/" # re-parsing must work the same thing works - assert parse_obj_as(SimcoreS3DirectoryID, result) + assert TypeAdapter(SimcoreS3DirectoryID).validate_python(result) # all below are not allowed for invalid_path in ( @@ -132,10 +134,12 @@ def test_simcore_s3_directory_id(): f"{UUID_0}/{UUID_0}/a-dir/a-file", ): with pytest.raises(ValidationError): - parse_obj_as(SimcoreS3DirectoryID, invalid_path) + TypeAdapter(SimcoreS3DirectoryID).validate_python(invalid_path) with pytest.raises(ValidationError, match="Not allowed subdirectory found in"): - parse_obj_as(SimcoreS3DirectoryID, f"{UUID_0}/{UUID_0}/a-dir/a-subdir/") + TypeAdapter(SimcoreS3DirectoryID).validate_python( + f"{UUID_0}/{UUID_0}/a-dir/a-subdir/" + ) @pytest.mark.parametrize( diff --git a/packages/models-library/tests/test_projects.py b/packages/models-library/tests/test_projects.py index 8b646345c2d..5cbb0e13573 100644 --- a/packages/models-library/tests/test_projects.py +++ b/packages/models-library/tests/test_projects.py @@ -28,7 +28,7 @@ def minimal_project(faker: Faker) -> dict[str, Any]: def test_project_minimal_model(minimal_project: dict[str, Any]): - project = Project.parse_obj(minimal_project) + project = Project.model_validate(minimal_project) assert project assert project.thumbnail is None @@ -37,7 +37,7 @@ def test_project_minimal_model(minimal_project: dict[str, Any]): def test_project_with_thumbnail_as_empty_string(minimal_project: dict[str, Any]): thumbnail_empty_string = deepcopy(minimal_project) thumbnail_empty_string.update({"thumbnail": ""}) - project = Project.parse_obj(thumbnail_empty_string) + project = Project.model_validate(thumbnail_empty_string) assert project assert project.thumbnail is None diff --git a/packages/models-library/tests/test_projects_state.py b/packages/models-library/tests/test_projects_state.py index 2895d71f3a1..236d65a5538 100644 --- a/packages/models-library/tests/test_projects_state.py +++ b/packages/models-library/tests/test_projects_state.py @@ -5,7 +5,7 @@ def test_project_locked_with_missing_owner_raises(): with pytest.raises(ValueError): ProjectLocked(value=True, status=ProjectStatus.OPENED) - ProjectLocked.parse_obj({"value": False, "status": ProjectStatus.OPENED}) + ProjectLocked.model_validate({"value": False, "status": ProjectStatus.OPENED}) @pytest.mark.parametrize( @@ -19,4 +19,4 @@ def test_project_locked_with_missing_owner_raises(): ) def test_project_locked_with_allowed_values(lock: bool, status: ProjectStatus): with pytest.raises(ValueError): - ProjectLocked.parse_obj({"value": lock, "status": status}) + ProjectLocked.model_validate({"value": lock, "status": status}) diff --git a/packages/models-library/tests/test_rest_pagination_utils.py b/packages/models-library/tests/test_rest_pagination_utils.py index f9887a1bf71..acaf6bc9d5c 100644 --- a/packages/models-library/tests/test_rest_pagination_utils.py +++ b/packages/models-library/tests/test_rest_pagination_utils.py @@ -41,7 +41,7 @@ def test_paginating_data(base_url): ) assert data_obj - model_instance = Page[int].parse_obj(data_obj) + model_instance = Page[int].model_validate(data_obj) assert model_instance assert model_instance.meta == PageMetaInfoLimitOffset( total=total_number_of_items, count=len(data_chunk), limit=limit, offset=offset @@ -75,7 +75,7 @@ def test_paginating_data(base_url): offset += len(data_chunk) assert model_instance.links.next is not None - data_obj: PageDict = paginate_data( + data_obj: PageDict = paginate_data( # type: ignore[no-redef] data_chunk, request_url=URL(model_instance.links.next), total=total_number_of_items, @@ -83,7 +83,7 @@ def test_paginating_data(base_url): offset=offset, ) - model_instance = Page[int].parse_obj(data_obj) + model_instance = Page[int].model_validate(data_obj) assert model_instance assert model_instance.meta == PageMetaInfoLimitOffset( total=total_number_of_items, @@ -127,7 +127,7 @@ def test_paginating_data(base_url): assert offset == last_chunk_offset assert model_instance.links.next is not None - data_obj: PageDict = paginate_data( + data_obj: PageDict = paginate_data( # type: ignore[no-redef] data_chunk, request_url=URL(model_instance.links.next), total=total_number_of_items, @@ -136,7 +136,7 @@ def test_paginating_data(base_url): ) assert data_obj - model_instance = Page[int].parse_obj(data_obj) + model_instance = Page[int].model_validate(data_obj) assert model_instance assert model_instance.meta == PageMetaInfoLimitOffset( diff --git a/packages/models-library/tests/test_service_settings_labels.py b/packages/models-library/tests/test_service_settings_labels.py index f9c096804cc..287e3d5614b 100644 --- a/packages/models-library/tests/test_service_settings_labels.py +++ b/packages/models-library/tests/test_service_settings_labels.py @@ -31,7 +31,7 @@ ) from models_library.services_resources import DEFAULT_SINGLE_SERVICE_NAME from models_library.utils.string_substitution import TextTemplate -from pydantic import BaseModel, TypeAdapter, ValidationError, parse_obj_as +from pydantic import BaseModel, TypeAdapter, ValidationError from pydantic.json import pydantic_encoder @@ -69,12 +69,12 @@ def test_simcore_service_labels(example: dict, items: int, uses_dynamic_sidecar: simcore_service_labels = SimcoreServiceLabels.model_validate(example) assert simcore_service_labels - assert len(simcore_service_labels.dict(exclude_unset=True)) == items + assert len(simcore_service_labels.model_dump(exclude_unset=True)) == items assert simcore_service_labels.needs_dynamic_sidecar == uses_dynamic_sidecar def test_service_settings(): - simcore_settings_settings_label = SimcoreServiceSettingsLabel.parse_obj( + simcore_settings_settings_label = SimcoreServiceSettingsLabel.model_validate( SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"] ) assert simcore_settings_settings_label @@ -126,7 +126,7 @@ def test_path_mappings_json_encoding(): path_mappings = PathMappingsLabel.model_validate(example) print(path_mappings) assert ( - PathMappingsLabel.parse_raw(path_mappings.model_dump_json()) + PathMappingsLabel.model_validate_json(path_mappings.model_dump_json()) == path_mappings ) @@ -262,7 +262,7 @@ def test_container_outgoing_permit_list_and_container_allow_internet_with_compos "simcore.service.container-http-entrypoint": container_name_1, } - instance = DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + instance = DynamicSidecarServiceLabels.model_validate_json(json.dumps(dict_data)) assert ( instance.containers_allowed_outgoing_permit_list[container_name_1][0] == expected_host_permit_list_policy @@ -291,7 +291,7 @@ def test_container_outgoing_permit_list_and_container_allow_internet_without_com ) }, ): - assert DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + assert TypeAdapter(DynamicSidecarServiceLabels).validate_json(json.dumps(dict_data)) def test_container_allow_internet_no_compose_spec_not_ok(): @@ -299,7 +299,7 @@ def test_container_allow_internet_no_compose_spec_not_ok(): "simcore.service.containers-allowed-outgoing-internet": json.dumps(["hoho"]), } with pytest.raises(ValidationError) as exec_info: - assert DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + assert DynamicSidecarServiceLabels.model_validate_json(json.dumps(dict_data)) assert "Expected only 1 entry 'container' not '{'hoho'}" in f"{exec_info.value}" @@ -312,7 +312,7 @@ def test_container_allow_internet_compose_spec_not_ok(): "simcore.service.containers-allowed-outgoing-internet": json.dumps(["hoho"]), } with pytest.raises(ValidationError) as exec_info: - assert DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + assert DynamicSidecarServiceLabels.model_validate_json(json.dumps(dict_data)) assert f"container='hoho' not found in {compose_spec=}" in f"{exec_info.value}" @@ -331,7 +331,7 @@ def test_container_outgoing_permit_list_no_compose_spec_not_ok(): ), } with pytest.raises(ValidationError) as exec_info: - assert DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + assert DynamicSidecarServiceLabels.model_validate_json(json.dumps(dict_data)) assert ( f"Expected only one entry '{DEFAULT_SINGLE_SERVICE_NAME}' not 'container_name'" in f"{exec_info.value}" @@ -355,7 +355,7 @@ def test_container_outgoing_permit_list_compose_spec_not_ok(): "simcore.service.compose-spec": json.dumps(compose_spec), } with pytest.raises(ValidationError) as exec_info: - assert DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + assert DynamicSidecarServiceLabels.model_validate_json(json.dumps(dict_data)) assert ( f"Trying to permit list container='container_name' which was not found in {compose_spec=}" in f"{exec_info.value}" @@ -378,7 +378,7 @@ def test_not_allowed_in_both_permit_list_and_outgoing_internet(): } with pytest.raises(ValidationError) as exec_info: - DynamicSidecarServiceLabels.parse_raw(json.dumps(dict_data)) + DynamicSidecarServiceLabels.model_validate_json(json.dumps(dict_data)) assert ( f"Not allowed common_containers={{'{container_name}'}} detected" @@ -520,30 +520,27 @@ def test_can_parse_labels_with_osparc_identifiers( vendor_environments: dict[str, Any], service_labels: dict[str, str] ): # can load OSPARC_VARIABLE_ identifiers!! - service_meta = SimcoreServiceLabels.parse_obj(service_labels) + service_meta = SimcoreServiceLabels.model_validate(service_labels) assert service_meta.containers_allowed_outgoing_permit_list nat_rule: NATRule = service_meta.containers_allowed_outgoing_permit_list[ "s4l-core" ][0] - assert nat_rule.hostname == parse_obj_as( - OsparcVariableIdentifier, + assert nat_rule.hostname == TypeAdapter(OsparcVariableIdentifier).validate_python( "${OSPARC_VARIABLE_VENDOR_SECRET_LICENSE_SERVER_HOSTNAME}", ) assert nat_rule.tcp_ports == [ - parse_obj_as( - OsparcVariableIdentifier, + TypeAdapter(OsparcVariableIdentifier).validate_python( "$OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_1", ), - parse_obj_as( - OsparcVariableIdentifier, + TypeAdapter(OsparcVariableIdentifier).validate_python( "$OSPARC_VARIABLE_VENDOR_SECRET_TCP_PORTS_2", ), 3, ] service_meta = replace_osparc_variable_identifier(service_meta, vendor_environments) - service_meta_str = service_meta.json() + service_meta_str = service_meta.model_dump_json() not_replaced_vars = {"OSPARC_VARIABLE_OS_TYPE_LINUX"} @@ -552,7 +549,7 @@ def test_can_parse_labels_with_osparc_identifiers( continue assert osparc_variable_name not in service_meta_str - service_meta_str = service_meta.json( + service_meta_str = service_meta.model_dump_json( include={"containers_allowed_outgoing_permit_list"} ) @@ -568,7 +565,7 @@ def test_resolving_some_service_labels_at_load_time( vendor_environments: dict[str, Any], service_labels: dict[str, str] ): print(json.dumps(service_labels, indent=1)) - service_meta = SimcoreServiceLabels.parse_obj(service_labels) + service_meta = SimcoreServiceLabels.model_validate(service_labels) # NOTE: replacing all OsparcVariableIdentifier instances nested inside objects # this also does a partial replacement if there is no entry inside the vendor_environments @@ -593,7 +590,7 @@ def test_resolving_some_service_labels_at_load_time( # NOTE: that this model needs all values to be resolved before parsing them # otherwise it might fail!! The question is whether these values can be resolved at this point # NOTE: vendor values are in the database and therefore are available at this point - labels = SimcoreServiceLabels.parse_obj(service_labels) + labels = SimcoreServiceLabels.model_validate(service_labels) print("After", labels.model_dump_json(indent=1)) formatted_json = service_meta.model_dump_json(indent=1) @@ -613,4 +610,4 @@ def test_user_preferences_path_is_part_of_exiting_volume(): ), } with pytest.raises(ValidationError, match="user_preferences_path=/tmp/outputs"): - assert DynamicSidecarServiceLabels.parse_raw(json.dumps(labels_data)) + assert DynamicSidecarServiceLabels.model_validate_json(json.dumps(labels_data)) diff --git a/packages/models-library/tests/test_service_settings_nat_rule.py b/packages/models-library/tests/test_service_settings_nat_rule.py index 66319e9435c..c6f9f05497c 100644 --- a/packages/models-library/tests/test_service_settings_nat_rule.py +++ b/packages/models-library/tests/test_service_settings_nat_rule.py @@ -9,7 +9,7 @@ replace_osparc_variable_identifier, ) from models_library.service_settings_nat_rule import NATRule -from pydantic import parse_obj_as +from pydantic import TypeAdapter SUPPORTED_TEMPLATES: set[str] = { "$OSPARC_VARIABLE_%s", @@ -79,7 +79,7 @@ def _all_combinations_from_dict(data: dict[Any, Any]) -> list[dict[Any, Any]]: def test_nat_rule_with_osparc_variable_identifier( nat_rule_dict: dict[str, Any], osparc_variables: dict[str, Any] ): - nat_rule = parse_obj_as(NATRule, nat_rule_dict) + nat_rule = TypeAdapter(NATRule).validate_python(nat_rule_dict) with pytest.raises(UnresolvedOsparcVariableIdentifierError): list(nat_rule.iter_tcp_ports()) @@ -87,7 +87,7 @@ def test_nat_rule_with_osparc_variable_identifier( # NOTE: values are mostly replaced in place unless it's used as first level replace_osparc_variable_identifier(nat_rule, osparc_variables) - nat_rule_str = nat_rule.json() + nat_rule_str = nat_rule.model_dump_json() for osparc_variable_name in osparc_variables: assert osparc_variable_name not in nat_rule_str @@ -108,7 +108,9 @@ def test_nat_rule_with_osparc_variable_identifier( ], ) def test_______(replace_with_value: Any): - a_var = parse_obj_as(OsparcVariableIdentifier, "$OSPARC_VARIABLE_some_var") + a_var = TypeAdapter(OsparcVariableIdentifier).validate_python( + "$OSPARC_VARIABLE_some_var" + ) assert isinstance(a_var, OsparcVariableIdentifier) replaced_var = replace_osparc_variable_identifier( @@ -151,7 +153,7 @@ def test_replace_an_instance_of_osparc_variable_identifier( except TypeError: formatted_template = var_template - a_var = parse_obj_as(OsparcVariableIdentifier, formatted_template) + a_var = TypeAdapter(OsparcVariableIdentifier).validate_python(formatted_template) assert isinstance(a_var, OsparcVariableIdentifier) replace_with_identifier_default = identifier_has_default and replace_with_default diff --git a/packages/models-library/tests/test_services_io.py b/packages/models-library/tests/test_services_io.py index acfb02a05b1..e056647665f 100644 --- a/packages/models-library/tests/test_services_io.py +++ b/packages/models-library/tests/test_services_io.py @@ -15,7 +15,7 @@ def test_service_port_units(tests_data_dir: Path): data = yaml.safe_load((tests_data_dir / "metadata-sleeper-2.0.2.yaml").read_text()) print(ServiceMetaDataPublished.schema_json(indent=2)) - service_meta = ServiceMetaDataPublished.parse_obj(data) + service_meta = ServiceMetaDataPublished.model_validate(data) assert service_meta.inputs for input_nameid, input_meta in service_meta.inputs.items(): diff --git a/packages/models-library/tests/test_sidecar_volumes.py b/packages/models-library/tests/test_sidecar_volumes.py index e9c54554288..402899726bc 100644 --- a/packages/models-library/tests/test_sidecar_volumes.py +++ b/packages/models-library/tests/test_sidecar_volumes.py @@ -14,4 +14,4 @@ def test_volume_state_equality_does_not_use_last_changed(status: VolumeStatus): # at the moment of the creation of the object. assert VolumeState(status=status) == VolumeState(status=status) schema_property_count = len(VolumeState.schema()["properties"]) - assert len(VolumeState(status=status).dict()) == schema_property_count + assert len(VolumeState(status=status).model_dump()) == schema_property_count diff --git a/packages/models-library/tests/test_user_preferences.py b/packages/models-library/tests/test_user_preferences.py index 272e73cf6e5..edac734f0c7 100644 --- a/packages/models-library/tests/test_user_preferences.py +++ b/packages/models-library/tests/test_user_preferences.py @@ -15,20 +15,24 @@ _AutoRegisterMeta, _BaseUserPreferenceModel, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter _SERVICE_KEY_AND_VERSION_SAMPLES: list[tuple[ServiceKey, ServiceVersion]] = [ ( - parse_obj_as(ServiceKey, "simcore/services/comp/something-1231"), - parse_obj_as(ServiceVersion, "0.0.1"), + TypeAdapter(ServiceKey).validate_python("simcore/services/comp/something-1231"), + TypeAdapter(ServiceVersion).validate_python("0.0.1"), ), ( - parse_obj_as(ServiceKey, "simcore/services/dynamic/something-1231"), - parse_obj_as(ServiceVersion, "0.0.1"), + TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/something-1231" + ), + TypeAdapter(ServiceVersion).validate_python("0.0.1"), ), ( - parse_obj_as(ServiceKey, "simcore/services/frontend/something-1231"), - parse_obj_as(ServiceVersion, "0.0.1"), + TypeAdapter(ServiceKey).validate_python( + "simcore/services/frontend/something-1231" + ), + TypeAdapter(ServiceVersion).validate_python("0.0.1"), ), ] @@ -54,7 +58,7 @@ def test_base_user_preference_model(value: Any, preference_type: PreferenceType) base_data = _get_base_user_preferences_data( preference_type=preference_type, value=value ) - assert parse_obj_as(_BaseUserPreferenceModel, base_data) + assert TypeAdapter(_BaseUserPreferenceModel).validate_python(base_data) def test_frontend_preferences(value: Any): @@ -64,7 +68,7 @@ def test_frontend_preferences(value: Any): base_data.update({"preference_identifier": "pref-name"}) # check serialization - frontend_preference = parse_obj_as(FrontendUserPreference, base_data) + frontend_preference = TypeAdapter(FrontendUserPreference).validate_python(base_data) assert set(frontend_preference.to_db().keys()) == {"value"} @@ -80,7 +84,7 @@ def test_user_service_preferences(value: Any, mock_file_path: Path): "file_path": mock_file_path, } ) - instance = parse_obj_as(UserServiceUserPreference, base_data) + instance = TypeAdapter(UserServiceUserPreference).validate_python(base_data) assert set(instance.to_db().keys()) == { "value", "service_key", @@ -96,7 +100,7 @@ def unregister_defined_classes() -> Iterator[None]: def test__frontend__user_preference(value: Any, unregister_defined_classes: None): - pref1 = FrontendUserPreference.parse_obj( + pref1 = FrontendUserPreference.model_validate( {"preference_identifier": "pref_id", "value": value} ) assert isinstance(pref1, FrontendUserPreference) @@ -112,7 +116,7 @@ def test__user_service__user_preference( mock_file_path: Path, unregister_defined_classes: None, ): - pref1 = UserServiceUserPreference.parse_obj( + pref1 = UserServiceUserPreference.model_validate( { "value": value, "service_key": service_key, @@ -123,8 +127,8 @@ def test__user_service__user_preference( # NOTE: these will be stored as bytes, # check bytes serialization/deserialization - pref1_as_bytes = pref1.json().encode() - new_instance = UserServiceUserPreference.parse_raw(pref1_as_bytes) + pref1_as_bytes = pref1.model_dump_json().encode() + new_instance = UserServiceUserPreference.model_validate_json(pref1_as_bytes) assert new_instance == pref1 diff --git a/packages/models-library/tests/test_utils_common_validators.py b/packages/models-library/tests/test_utils_common_validators.py index d4c7cb5409f..db9df708b0f 100644 --- a/packages/models-library/tests/test_utils_common_validators.py +++ b/packages/models-library/tests/test_utils_common_validators.py @@ -48,8 +48,8 @@ class Model(BaseModel): empty_str_to_none_pre_validator ) - model = Model.parse_obj({"nullable_message": None}) - assert model == Model.parse_obj({"nullable_message": ""}) + model = Model.model_validate({"nullable_message": None}) + assert model == Model.model_validate({"nullable_message": ""}) def test_none_to_empty_str_pre_validator(): @@ -60,8 +60,8 @@ class Model(BaseModel): none_to_empty_str_pre_validator ) - model = Model.parse_obj({"message": ""}) - assert model == Model.parse_obj({"message": None}) + model = Model.model_validate({"message": ""}) + assert model == Model.model_validate({"message": None}) def test_null_or_none_str_to_none_validator(): @@ -72,20 +72,20 @@ class Model(BaseModel): null_or_none_str_to_none_validator ) - model = Model.parse_obj({"message": "none"}) - assert model == Model.parse_obj({"message": None}) + model = Model.model_validate({"message": "none"}) + assert model == Model.model_validate({"message": None}) - model = Model.parse_obj({"message": "null"}) - assert model == Model.parse_obj({"message": None}) + model = Model.model_validate({"message": "null"}) + assert model == Model.model_validate({"message": None}) - model = Model.parse_obj({"message": "NoNe"}) - assert model == Model.parse_obj({"message": None}) + model = Model.model_validate({"message": "NoNe"}) + assert model == Model.model_validate({"message": None}) - model = Model.parse_obj({"message": "NuLl"}) - assert model == Model.parse_obj({"message": None}) + model = Model.model_validate({"message": "NuLl"}) + assert model == Model.model_validate({"message": None}) - model = Model.parse_obj({"message": None}) - assert model == Model.parse_obj({"message": None}) + model = Model.model_validate({"message": None}) + assert model == Model.model_validate({"message": None}) - model = Model.parse_obj({"message": ""}) - assert model == Model.parse_obj({"message": ""}) + model = Model.model_validate({"message": ""}) + assert model == Model.model_validate({"message": ""}) diff --git a/packages/models-library/tests/test_utils_pydantic_tools_extension.py b/packages/models-library/tests/test_utils_pydantic_tools_extension.py index 34cbb528dbb..0bf8abdbb41 100644 --- a/packages/models-library/tests/test_utils_pydantic_tools_extension.py +++ b/packages/models-library/tests/test_utils_pydantic_tools_extension.py @@ -39,8 +39,8 @@ def test_schema(): def test_only_required(): model = MyModel(a=1, b=2) - assert model.dict() == {"a": 1, "b": 2, "c": 42, "d": None, "e": None} - assert model.dict(exclude_unset=True) == {"a": 1, "b": 2} + assert model.model_dump() == {"a": 1, "b": 2, "c": 42, "d": None, "e": None} + assert model.model_dump(exclude_unset=True) == {"a": 1, "b": 2} def test_parse_obj_or_none(): diff --git a/packages/models-library/tests/test_utils_service_io.py b/packages/models-library/tests/test_utils_service_io.py index 51e4324e2be..7ef8d4070a0 100644 --- a/packages/models-library/tests/test_utils_service_io.py +++ b/packages/models-library/tests/test_utils_service_io.py @@ -17,7 +17,7 @@ from models_library.services import ServiceInput, ServiceOutput, ServicePortKey from models_library.utils.json_schema import jsonschema_validate_schema from models_library.utils.services_io import get_service_io_json_schema -from pydantic import parse_obj_as +from pydantic import TypeAdapter example_inputs_labels = [ e for e in ServiceInput.model_config["json_schema_extra"]["examples"] if e["label"] @@ -32,11 +32,11 @@ def service_port(request: pytest.FixtureRequest) -> ServiceInput | ServiceOutput try: index = example_inputs_labels.index(request.param) example = ServiceInput.model_config["json_schema_extra"]["examples"][index] - return ServiceInput.parse_obj(example) + return ServiceInput.model_validate(example) except ValueError: index = example_outputs_labels.index(request.param) example = ServiceOutput.model_config["json_schema_extra"]["examples"][index] - return ServiceOutput.parse_obj(example) + return ServiceOutput.model_validate(example) def test_get_schema_from_port(service_port: ServiceInput | ServiceOutput): @@ -73,8 +73,12 @@ def test_against_service_metadata_configs(metadata_path: Path): meta = json.loads(metadata_path.read_text()) - inputs = parse_obj_as(dict[ServicePortKey, ServiceInput], meta["inputs"]) - outputs = parse_obj_as(dict[ServicePortKey, ServiceOutput], meta["outputs"]) + inputs = TypeAdapter(dict[ServicePortKey, ServiceInput]).validate_python( + meta["inputs"] + ) + outputs = TypeAdapter(dict[ServicePortKey, ServiceOutput]).validate_python( + meta["outputs"] + ) for port in itertools.chain(inputs.values(), outputs.values()): schema = get_service_io_json_schema(port) diff --git a/packages/models-library/tests/test_utils_specs_substitution.py b/packages/models-library/tests/test_utils_specs_substitution.py index 0670e56e271..c523271bd2a 100644 --- a/packages/models-library/tests/test_utils_specs_substitution.py +++ b/packages/models-library/tests/test_utils_specs_substitution.py @@ -12,7 +12,7 @@ SpecsSubstitutionsResolver, SubstitutionValue, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter @pytest.fixture() @@ -49,7 +49,7 @@ def available_osparc_variables( "SERVICE_VERSION": service_version, "DISPLAY": "True", } - return parse_obj_as(dict[str, SubstitutionValue], environs) + return TypeAdapter(dict[str, SubstitutionValue]).validate_python(environs) @pytest.mark.parametrize( diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt index b7e4320e9e0..a4df8b512d1 100644 --- a/packages/notifications-library/requirements/_base.txt +++ b/packages/notifications-library/requirements/_base.txt @@ -4,6 +4,8 @@ aiosmtplib==3.0.2 # via -r requirements/_base.in alembic==1.13.2 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic arrow==1.3.0 # via -r requirements/../../../packages/models-library/requirements/_base.in async-timeout==4.0.3 @@ -63,7 +65,7 @@ orjson==3.10.7 # -r requirements/../../../packages/models-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==1.10.17 +pydantic==2.9.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -72,10 +74,22 @@ pydantic==1.10.17 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.3 + # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings referencing==0.35.1 # via # jsonschema @@ -108,6 +122,7 @@ typing-extensions==4.12.2 # via # alembic # pydantic + # pydantic-core # typer yarl==1.9.4 # via -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/packages/notifications-library/requirements/_test.txt b/packages/notifications-library/requirements/_test.txt index 48dd95db3fc..ab645dfb576 100644 --- a/packages/notifications-library/requirements/_test.txt +++ b/packages/notifications-library/requirements/_test.txt @@ -10,8 +10,6 @@ coverage==7.6.1 # pytest-cov docker==7.1.0 # via -r requirements/_test.in -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in greenlet==3.0.3 @@ -68,7 +66,9 @@ python-dateutil==2.9.0.post0 # -c requirements/_base.txt # faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt @@ -90,11 +90,6 @@ tenacity==9.0.0 # via -r requirements/_test.in termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # mypy - # pytest types-aiofiles==24.1.0.20240626 # via -r requirements/_test.in typing-extensions==4.12.2 diff --git a/packages/notifications-library/requirements/_tools.txt b/packages/notifications-library/requirements/_tools.txt index 71884cfdaf4..8204f34a33c 100644 --- a/packages/notifications-library/requirements/_tools.txt +++ b/packages/notifications-library/requirements/_tools.txt @@ -71,22 +71,12 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/notifications-library/tests/email/test_email_events.py b/packages/notifications-library/tests/email/test_email_events.py index 5e3786ab234..995da5faf4e 100644 --- a/packages/notifications-library/tests/email/test_email_events.py +++ b/packages/notifications-library/tests/email/test_email_events.py @@ -66,8 +66,8 @@ def ipinfo(faker: Faker) -> dict[str, Any]: @pytest.fixture def request_form(faker: Faker) -> dict[str, Any]: return AccountRequestInfo( - **AccountRequestInfo.Config.schema_extra["example"] - ).dict() + **AccountRequestInfo.model_config["json_schema_extra"]["example"] + ).model_dump() @pytest.fixture diff --git a/packages/notifications-library/tests/with_db/conftest.py b/packages/notifications-library/tests/with_db/conftest.py index bdd3d0f3d09..750f3cc24a4 100644 --- a/packages/notifications-library/tests/with_db/conftest.py +++ b/packages/notifications-library/tests/with_db/conftest.py @@ -14,7 +14,7 @@ from models_library.products import ProductName from models_library.users import GroupID, UserID from notifications_library._templates import get_default_named_templates -from pydantic import validate_arguments +from pydantic import validate_call from simcore_postgres_database.models.jinja2_templates import jinja2_templates from simcore_postgres_database.models.payments_transactions import payments_transactions from simcore_postgres_database.models.products import products @@ -165,7 +165,7 @@ def set_template_to_product( sqlalchemy_async_engine: AsyncEngine, product: dict[str, Any] ): # NOTE: needs all fixture products in db - @validate_arguments + @validate_call async def _(template_name: IDStr, product_name: ProductName) -> None: async with sqlalchemy_async_engine.begin() as conn: await conn.execute( @@ -179,7 +179,7 @@ async def _(template_name: IDStr, product_name: ProductName) -> None: @pytest.fixture def unset_template_to_product(sqlalchemy_async_engine: AsyncEngine): - @validate_arguments + @validate_call async def _(template_name: IDStr, product_name: ProductName) -> None: async with sqlalchemy_async_engine.begin() as conn: await conn.execute( diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index 48679f44663..0294edf9114 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -3,6 +3,7 @@ # --constraint ../../../requirements/constraints.txt --constraint ./constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in alembic pydantic diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index aaf19732f53..5cb99144fd9 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -1,5 +1,7 @@ alembic==1.13.2 # via -r requirements/_base.in +annotated-types==0.7.0 + # via pydantic async-timeout==4.0.3 # via asyncpg asyncpg==0.29.0 @@ -18,10 +20,12 @@ multidict==6.0.5 # via yarl psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==1.10.17 +pydantic==2.9.1 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in +pydantic-core==2.23.3 + # via pydantic sqlalchemy==1.4.53 # via # -c requirements/../../../requirements/constraints.txt @@ -31,5 +35,6 @@ typing-extensions==4.12.2 # via # alembic # pydantic + # pydantic-core yarl==1.9.4 # via -r requirements/_base.in diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt index 245a367c69b..8bd80b78b95 100644 --- a/packages/postgres-database/requirements/_test.txt +++ b/packages/postgres-database/requirements/_test.txt @@ -10,8 +10,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in greenlet==3.0.3 @@ -70,11 +68,6 @@ sqlalchemy==1.4.53 # aiopg sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -tomli==2.0.1 - # via - # coverage - # mypy - # pytest types-docker==7.1.0.20240821 # via -r requirements/_test.in types-psycopg2==2.9.21.20240819 diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt index 10d7162ab81..9247bc4b1a9 100644 --- a/packages/postgres-database/requirements/_tools.txt +++ b/packages/postgres-database/requirements/_tools.txt @@ -70,22 +70,12 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/postgres-database/requirements/ci.txt b/packages/postgres-database/requirements/ci.txt index b901feff8e4..8df1aecf670 100644 --- a/packages/postgres-database/requirements/ci.txt +++ b/packages/postgres-database/requirements/ci.txt @@ -12,7 +12,8 @@ --requirement _test.txt # installs this repo's packages -pytest-simcore @ ../../packages/pytest-simcore/ +simcore-common-library @ ../common-library/ +pytest-simcore @ ../pytest-simcore/ # current module simcore-postgres-database @ . diff --git a/packages/postgres-database/requirements/dev.txt b/packages/postgres-database/requirements/dev.txt index 8136f1a48b5..095f8383b2a 100644 --- a/packages/postgres-database/requirements/dev.txt +++ b/packages/postgres-database/requirements/dev.txt @@ -13,7 +13,9 @@ --requirement _tools.txt # installs this repo's packages ---editable ../../packages/pytest-simcore/ +--editable ../common-library/ +--editable ../pytest-simcore/ + # current module --editable . diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_folders.py b/packages/postgres-database/src/simcore_postgres_database/utils_folders.py index e0f59cdcfd2..6dcca321a73 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_folders.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_folders.py @@ -5,21 +5,22 @@ from datetime import datetime from enum import Enum from functools import reduce -from typing import Any, ClassVar, Final, TypeAlias, cast +from typing import Annotated, Any, ClassVar, Final, TypeAlias, cast import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy +from common_library.errors_classes import OsparcErrorMixin from pydantic import ( BaseModel, - ConstrainedStr, + ConfigDict, Field, NonNegativeInt, PositiveInt, + StringConstraints, + TypeAdapter, ValidationError, - parse_obj_as, ) -from pydantic.errors import PydanticErrorMixin from simcore_postgres_database.utils_ordering import OrderByDict from sqlalchemy import Column, func from sqlalchemy.dialects import postgresql @@ -63,8 +64,8 @@ """ -class FoldersError(PydanticErrorMixin, RuntimeError): - pass +class FoldersError(OsparcErrorMixin, RuntimeError): + ... class InvalidFolderNameError(FoldersError): @@ -294,13 +295,17 @@ def _get_filter_for_enabled_permissions( ### -class FolderName(ConstrainedStr): - regex = re.compile( - r'^(?!.*[<>:"/\\|?*\]])(?!.*\b(?:LPT9|COM1|LPT1|COM2|LPT3|LPT4|CON|COM5|COM3|COM4|AUX|PRN|LPT2|LPT5|COM6|LPT7|NUL|COM8|LPT6|COM9|COM7|LPT8)\b).+$', - re.IGNORECASE, - ) - min_length = 1 - max_length = 255 +FolderName: TypeAlias = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=255, + pattern=re.compile( + r'^(?!.*[<>:"/\\|?*\]])(?!.*\b(?:LPT9|COM1|LPT1|COM2|LPT3|LPT4|CON|COM5|COM3|COM4|AUX|PRN|LPT2|LPT5|COM6|LPT7|NUL|COM8|LPT6|COM9|COM7|LPT8)\b).+$', + re.IGNORECASE, + ), + ), +] class FolderEntry(BaseModel): @@ -313,9 +318,7 @@ class FolderEntry(BaseModel): modified: datetime = Field(alias="access_modified") my_access_rights: _FolderPermissions access_rights: dict[_GroupID, _FolderPermissions] - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class _ResolvedAccessRights(BaseModel): @@ -327,9 +330,7 @@ class _ResolvedAccessRights(BaseModel): write: bool delete: bool level: int - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) async def _get_resolved_access_rights( @@ -529,7 +530,7 @@ async def folder_create( RootFolderRequiresAtLeastOnePrimaryGroupError """ try: - parse_obj_as(FolderName, name) + TypeAdapter(FolderName).validate_python(name) except ValidationError as exc: raise InvalidFolderNameError(name=name, reason=f"{exc}") from exc diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 39749b7fdbf..c8aa9962d43 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -5,8 +5,8 @@ import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import ResultProxy, RowProxy -from pydantic import BaseModel -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin +from pydantic import BaseModel, ConfigDict from sqlalchemy.dialects.postgresql import insert as pg_insert from .errors import ForeignKeyViolation @@ -18,7 +18,7 @@ # -class BaseProjectsMetadataError(PydanticErrorMixin, RuntimeError): +class BaseProjectsMetadataError(OsparcErrorMixin, RuntimeError): msg_template: str = "Project metadata unexpected error" @@ -53,10 +53,7 @@ class ProjectMetadata(BaseModel): parent_node_id: uuid.UUID | None root_parent_project_uuid: uuid.UUID | None root_parent_node_id: uuid.UUID | None - - class Config: - frozen = True - orm_mode = True + model_config = ConfigDict(frozen=True, from_attributes=True) # diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 09cb8a561f4..cb47141b1ab 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -5,8 +5,8 @@ import sqlalchemy from aiopg.sa.connection import SAConnection -from pydantic import BaseModel, Field -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin +from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.dialects.postgresql import insert as pg_insert from .errors import ForeignKeyViolation, UniqueViolation @@ -17,7 +17,7 @@ # # Errors # -class BaseProjectNodesError(PydanticErrorMixin, RuntimeError): +class BaseProjectNodesError(OsparcErrorMixin, RuntimeError): msg_template: str = "Project nodes unexpected error" @@ -43,18 +43,16 @@ class ProjectNodeCreate(BaseModel): @classmethod def get_field_names(cls, *, exclude: set[str]) -> set[str]: - return {name for name in cls.__fields__ if name not in exclude} + return {name for name in cls.model_fields.keys() if name not in exclude} - class Config: - frozen = True + model_config = ConfigDict(frozen=True) class ProjectNode(ProjectNodeCreate): created: datetime.datetime modified: datetime.datetime - class Config(ProjectNodeCreate.Config): - orm_mode = True + model_config = ConfigDict(from_attributes=True) @dataclass(frozen=True, kw_only=True) @@ -85,7 +83,7 @@ async def add( [ { "project_uuid": f"{self.project_uuid}", - **node.dict(), + **node.model_dump(), } for node in nodes ] diff --git a/packages/postgres-database/tests/test_utils_projects_nodes.py b/packages/postgres-database/tests/test_utils_projects_nodes.py index a20083608dd..50d2af96911 100644 --- a/packages/postgres-database/tests/test_utils_projects_nodes.py +++ b/packages/postgres-database/tests/test_utils_projects_nodes.py @@ -412,9 +412,9 @@ async def test_get_project_id_from_node_id_raises_if_multiple_projects_with_same assert len(project1_nodes) == 1 project2_nodes = await project2_repo.add(connection, nodes=[shared_node]) assert len(project2_nodes) == 1 - assert project1_nodes[0].dict( + assert project1_nodes[0].model_dump( include=ProjectNodeCreate.get_field_names(exclude={"created", "modified"}) - ) == project2_nodes[0].dict( + ) == project2_nodes[0].model_dump( include=ProjectNodeCreate.get_field_names(exclude={"created", "modified"}) ) with pytest.raises(ProjectNodesNonUniqueNodeFoundError): diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py b/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py index 48fb2d1283e..e6afeac8e7b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py @@ -32,7 +32,7 @@ async def s3_client(s3_settings: S3Settings) -> typing.AsyncIterator[S3Client]: config=Config(signature_version="s3v4"), ) assert isinstance(session_client, ClientCreatorContext) - client = typing.cast(S3Client, await exit_stack.enter_async_context(session_client)) + client = typing.cast(S3Client, await exit_stack.enter_async_context(session_client)) # type: ignore[arg-type] yield client diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py index 077fb25d51a..74f007973c5 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -11,7 +11,7 @@ from faker import Faker from models_library.utils.fastapi_encoders import jsonable_encoder from moto.server import ThreadedMotoServer -from pydantic import AnyHttpUrl, SecretStr, parse_obj_as +from pydantic import SecretStr from pytest_mock.plugin import MockerFixture from settings_library.basic_types import IDStr from settings_library.ec2 import EC2Settings @@ -75,7 +75,7 @@ def mocked_ec2_server_envs( mocked_ec2_server_settings: EC2Settings, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - changed_envs: EnvVarsDict = mocked_ec2_server_settings.dict() + changed_envs: EnvVarsDict = mocked_ec2_server_settings.model_dump() return setenvs_from_dict(monkeypatch, {**changed_envs}) @@ -101,10 +101,7 @@ def mocked_ssm_server_settings( ) -> SSMSettings: return SSMSettings( SSM_ACCESS_KEY_ID=SecretStr("xxx"), - SSM_ENDPOINT=parse_obj_as( - AnyHttpUrl, - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - ), + SSM_ENDPOINT=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # type: ignore[arg-type] # pylint: disable=protected-access # noqa: SLF001 SSM_SECRET_ACCESS_KEY=SecretStr("xxx"), ) @@ -124,10 +121,7 @@ def mocked_s3_server_settings( ) -> S3Settings: return S3Settings( S3_ACCESS_KEY=IDStr("xxx"), - S3_ENDPOINT=parse_obj_as( - AnyHttpUrl, - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - ), + S3_ENDPOINT=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # type: ignore[arg-type] # pylint: disable=protected-access # noqa: SLF001 S3_SECRET_KEY=IDStr("xxx"), S3_BUCKET_NAME=IDStr(f"pytest{faker.pystr().lower()}"), S3_REGION=IDStr("us-east-1"), @@ -139,5 +133,7 @@ def mocked_s3_server_envs( mocked_s3_server_settings: S3Settings, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - changed_envs: EnvVarsDict = mocked_s3_server_settings.dict(exclude_unset=True) + changed_envs: EnvVarsDict = mocked_s3_server_settings.model_dump( + mode="json", exclude_unset=True + ) return setenvs_from_dict(monkeypatch, {**changed_envs}) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index 5780937a2c0..91cd5e2d428 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -106,9 +106,9 @@ def external_registry_settings( if external_envfile_dict: config = { field: external_envfile_dict.get(field, None) - for field in RegistrySettings.__fields__ + for field in RegistrySettings.model_fields } - return RegistrySettings.parse_obj(config) + return RegistrySettings.model_validate(config) return None diff --git a/packages/pytest-simcore/src/pytest_simcore/faker_payments_data.py b/packages/pytest-simcore/src/pytest_simcore/faker_payments_data.py index 9d675c45e11..3f4058b72e9 100644 --- a/packages/pytest-simcore/src/pytest_simcore/faker_payments_data.py +++ b/packages/pytest-simcore/src/pytest_simcore/faker_payments_data.py @@ -24,7 +24,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import EmailStr, HttpUrl, parse_obj_as +from pydantic import EmailStr, HttpUrl, TypeAdapter from simcore_postgres_database.models.payments_transactions import ( PaymentTransactionState, ) @@ -34,27 +34,27 @@ @pytest.fixture def wallet_id(faker: Faker) -> WalletID: - return parse_obj_as(WalletID, faker.pyint()) + return TypeAdapter(WalletID).validate_python(faker.pyint()) @pytest.fixture def wallet_name(faker: Faker) -> IDStr: - return parse_obj_as(IDStr, f"wallet-{faker.word()}") + return TypeAdapter(IDStr).validate_python(f"wallet-{faker.word()}") @pytest.fixture -def invoice_url(faker: Faker) -> HttpUrl: - return parse_obj_as(HttpUrl, faker.image_url()) +def invoice_url(faker: Faker) -> str: + return faker.image_url() @pytest.fixture -def invoice_pdf_url(faker: Faker) -> HttpUrl: - return parse_obj_as(HttpUrl, faker.image_url()) +def invoice_pdf_url(faker: Faker) -> str: + return faker.image_url() @pytest.fixture def stripe_invoice_id(faker: Faker) -> StripeInvoiceID: - return parse_obj_as(StripeInvoiceID, f"in_{faker.word()}") + return TypeAdapter(StripeInvoiceID).validate_python(f"in_{faker.word()}") @pytest.fixture diff --git a/packages/pytest-simcore/src/pytest_simcore/faker_products_data.py b/packages/pytest-simcore/src/pytest_simcore/faker_products_data.py index f82636b6633..e55c1e489f0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/faker_products_data.py +++ b/packages/pytest-simcore/src/pytest_simcore/faker_products_data.py @@ -14,7 +14,7 @@ import pytest from faker import Faker from models_library.products import ProductName, StripePriceID, StripeTaxRateID -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from .helpers.faker_factories import random_product @@ -51,8 +51,7 @@ def product_name() -> ProductName: def support_email( request: pytest.FixtureRequest, product_name: ProductName ) -> EmailStr: - return parse_obj_as( - EmailStr, + return TypeAdapter(EmailStr).validate_python( request.config.getoption("--faker-support-email", default=None) or f"support@{product_name}.info", ) @@ -60,8 +59,7 @@ def support_email( @pytest.fixture def bcc_email(request: pytest.FixtureRequest, product_name: ProductName) -> EmailStr: - return parse_obj_as( - EmailStr, + return TypeAdapter(EmailStr).validate_python( request.config.getoption("--faker-bcc-email", default=None) or f"finance@{product_name}-department.info", ) diff --git a/packages/pytest-simcore/src/pytest_simcore/faker_projects_data.py b/packages/pytest-simcore/src/pytest_simcore/faker_projects_data.py index 643ffee8859..f2d0eee8105 100644 --- a/packages/pytest-simcore/src/pytest_simcore/faker_projects_data.py +++ b/packages/pytest-simcore/src/pytest_simcore/faker_projects_data.py @@ -13,7 +13,7 @@ from faker import Faker from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from pydantic import parse_obj_as +from pydantic import TypeAdapter _MESSAGE = ( "If set, it overrides the fake value of `{}` fixture." @@ -34,12 +34,11 @@ def pytest_addoption(parser: pytest.Parser): @pytest.fixture def project_id(faker: Faker, request: pytest.FixtureRequest) -> ProjectID: - return parse_obj_as( - ProjectID, + return TypeAdapter(ProjectID).validate_python( request.config.getoption("--faker-project-id", default=None) or faker.uuid4(), ) @pytest.fixture def node_id(faker: Faker) -> NodeID: - return parse_obj_as(NodeID, faker.uuid4()) + return TypeAdapter(NodeID).validate_python(faker.uuid4()) diff --git a/packages/pytest-simcore/src/pytest_simcore/faker_users_data.py b/packages/pytest-simcore/src/pytest_simcore/faker_users_data.py index 6ba011db47c..4e59b6db93a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/faker_users_data.py +++ b/packages/pytest-simcore/src/pytest_simcore/faker_users_data.py @@ -14,7 +14,7 @@ from faker import Faker from models_library.basic_types import IDStr from models_library.users import UserID -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from .helpers.faker_factories import DEFAULT_TEST_PASSWORD, random_user @@ -61,8 +61,7 @@ def pytest_addoption(parser: pytest.Parser): @pytest.fixture def user_id(faker: Faker, request: pytest.FixtureRequest) -> UserID: - return parse_obj_as( - UserID, + return TypeAdapter(UserID).validate_python( request.config.getoption("--faker-user-id", default=None) or faker.pyint(), ) @@ -74,8 +73,7 @@ def is_external_user_email(request: pytest.FixtureRequest) -> bool: @pytest.fixture def user_email(faker: Faker, request: pytest.FixtureRequest) -> EmailStr: - return parse_obj_as( - EmailStr, + return TypeAdapter(EmailStr).validate_python( request.config.getoption(_FAKE_USER_EMAIL_OPTION, default=None) or faker.email(), ) @@ -93,7 +91,7 @@ def user_last_name(faker: Faker) -> str: @pytest.fixture def user_name(user_email: str) -> IDStr: - return parse_obj_as(IDStr, user_email.split("@")[0]) + return TypeAdapter(IDStr).validate_python(user_email.split("@")[0]) @pytest.fixture diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index bc415ab3161..f51a5d8211b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -209,7 +209,7 @@ def random_product( registration_email_template: str | None = None, fake: Faker = DEFAULT_FAKER, **overrides, -): +) -> dict[str, Any]: """ Foreign keys are: diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_openapi.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_openapi.py index fd5afaa183a..177b1330e36 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_openapi.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_openapi.py @@ -5,7 +5,7 @@ import httpx import jsonref -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError from settings_library.catalog import CatalogSettings from settings_library.director_v2 import DirectorV2Settings from settings_library.storage import StorageSettings @@ -87,7 +87,7 @@ def _get_params( raise VerbNotInPathError(msg) if (params := verb_spec.get("parameters")) is None: continue - all_params += parse_obj_as(list[CapturedParameter], params) + all_params += TypeAdapter(list[CapturedParameter]).validate_python(params) return set(all_params) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py index 89783d0591c..25f2abc8cd0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py @@ -1,17 +1,15 @@ from typing import Literal -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field from .httpx_calls_capture_errors import OpenApiSpecError class CapturedParameterSchema(BaseModel): - title: str | None - type_: Literal["str", "int", "float", "bool"] | None = Field( - None, alias="type", optional=True - ) + title: str | None = None + type_: Literal["str", "int", "float", "bool"] | None = Field(None, alias="type") pattern: str | None - format_: Literal["uuid"] | None = Field(None, alias="format", optional=True) + format_: Literal["uuid"] | None = Field(None, alias="format") exclusiveMinimum: bool | None minimum: int | None anyOf: list["CapturedParameterSchema"] | None @@ -22,7 +20,7 @@ class Config: validate_always = True allow_population_by_field_name = True - @validator("type_", pre=True) + @field_validator("type_", mode="before") @classmethod def preprocess_type_(cls, val): if val == "string": @@ -33,7 +31,7 @@ def preprocess_type_(cls, val): val = "bool" return val - @root_validator(pre=False) + @model_validator(mode="after") @classmethod def check_compatibility(cls, values): type_ = values.get("type_") @@ -100,10 +98,7 @@ class CapturedParameter(BaseModel): response_value: str | None = ( None # attribute for storing the params value in a concrete response ) - - class Config: - validate_always = True - allow_population_by_field_name = True + model_config = ConfigDict(validate_default=True, populate_by_name=True) def __hash__(self): return hash( diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_client_base_dev.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_client_base_dev.py index d9b5bb64437..9a36d4cc020 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_client_base_dev.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_client_base_dev.py @@ -6,7 +6,7 @@ from fastapi.encoders import jsonable_encoder from httpx._types import URLTypes from jsonschema import ValidationError -from pydantic import parse_file_as +from pydantic import TypeAdapter from .httpx_calls_capture_errors import CaptureProcessingError from .httpx_calls_capture_models import HttpApiCallCaptureModel, get_captured_model @@ -14,6 +14,11 @@ _logger = logging.getLogger(__name__) +_HTTP_API_CALL_CAPTURE_MODEL_ADAPTER: TypeAdapter[ + list[HttpApiCallCaptureModel] +] = TypeAdapter(list[HttpApiCallCaptureModel]) + + class AsyncClientCaptureWrapper(httpx.AsyncClient): """ Adds captures mechanism @@ -41,8 +46,11 @@ async def request(self, method: str, url: URLTypes, **kwargs): or self._capture_file.read_text().strip() == "" ): self._capture_file.write_text("[]") - serialized_captures: list[HttpApiCallCaptureModel] = parse_file_as( - list[HttpApiCallCaptureModel], self._capture_file + + serialized_captures: list[ + HttpApiCallCaptureModel + ] = _HTTP_API_CALL_CAPTURE_MODEL_ADAPTER.validate_json( + self._capture_file.read_text() ) serialized_captures.append(capture) self._capture_file.write_text( diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/parametrizations.py b/packages/pytest-simcore/src/pytest_simcore/helpers/parametrizations.py index ed6381f5611..6eae044643b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/parametrizations.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/parametrizations.py @@ -1,6 +1,6 @@ import pytest from _pytest.mark.structures import ParameterSet -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter def byte_size_ids(val) -> str | None: @@ -10,4 +10,4 @@ def byte_size_ids(val) -> str | None: def parametrized_file_size(size_str: str) -> ParameterSet: - return pytest.param(parse_obj_as(ByteSize, size_str), id=size_str) + return pytest.param(TypeAdapter(ByteSize).validate_python(size_str), id=size_str) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/s3.py b/packages/pytest-simcore/src/pytest_simcore/helpers/s3.py index 5d7e721a832..2f0a03b575d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/s3.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/s3.py @@ -8,14 +8,14 @@ from aiohttp import ClientSession from aws_library.s3 import MultiPartUploadLinks from models_library.api_schemas_storage import ETag, FileUploadSchema, UploadedPart -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status from servicelib.utils import limited_as_completed, logged_gather from types_aiobotocore_s3 import S3Client from .logging_tools import log_context -_SENDER_CHUNK_SIZE: Final[int] = parse_obj_as(ByteSize, "16Mib") +_SENDER_CHUNK_SIZE: Final[int] = TypeAdapter(ByteSize).validate_python("16Mib") async def _file_sender( @@ -51,7 +51,7 @@ async def upload_file_part( f"--> uploading {this_file_chunk_size=} of {file=}, [{part_index+1}/{num_parts}]..." ) response = await session.put( - upload_url, + str(upload_url), data=_file_sender( file, offset=file_offset, diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index b6687e22239..a190fa6900e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -78,9 +78,9 @@ async def create_project( project_nodes={ NodeID(node_id): ProjectNodeCreate( node_id=NodeID(node_id), - required_resources=ServiceResourcesDictHelpers.Config.schema_extra[ - "examples" - ][0], + required_resources=ServiceResourcesDictHelpers.model_config[ + "json_schema_extra" + ]["examples"][0], ) for node_id in project_data.get("workbench", {}) }, diff --git a/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py b/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py index b6c0a5aad3b..6bc71929eb3 100644 --- a/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py @@ -14,7 +14,7 @@ import requests import requests.exceptions from docker.errors import APIError -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from tenacity import retry from tenacity.after import after_log from tenacity.retry import retry_if_exception_type @@ -56,7 +56,7 @@ def _wait_until_httpbin_is_responsive(): _wait_until_httpbin_is_responsive() - yield parse_obj_as(HttpUrl, base_url) + yield TypeAdapter(HttpUrl).validate_python(base_url) finally: with suppress(APIError): diff --git a/packages/pytest-simcore/src/pytest_simcore/httpx_calls_capture.py b/packages/pytest-simcore/src/pytest_simcore/httpx_calls_capture.py index 5c8df1ff6c5..d8cd056c115 100644 --- a/packages/pytest-simcore/src/pytest_simcore/httpx_calls_capture.py +++ b/packages/pytest-simcore/src/pytest_simcore/httpx_calls_capture.py @@ -38,7 +38,7 @@ import pytest import respx import yaml -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers.docker import get_service_published_port from pytest_simcore.helpers.host import get_localhost_ip @@ -213,9 +213,9 @@ def _( assert capture_path.suffix == ".json" if services_mocks_enabled: - captures: list[HttpApiCallCaptureModel] = parse_obj_as( - list[HttpApiCallCaptureModel], json.loads(capture_path.read_text()) - ) + captures: list[HttpApiCallCaptureModel] = TypeAdapter( + list[HttpApiCallCaptureModel] + ).validate_python(json.loads(capture_path.read_text())) if len(side_effects_callbacks) > 0: assert len(side_effects_callbacks) == len(captures) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index 46cee6fbeeb..38b9d2bdf8d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -31,5 +31,5 @@ def minio_s3_settings_envs( minio_s3_settings: S3Settings, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - changed_envs: EnvVarsDict = minio_s3_settings.dict(exclude_unset=True) + changed_envs: EnvVarsDict = minio_s3_settings.model_dump(exclude_unset=True) return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py index 297e9a9ab13..04d285a601e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py @@ -51,7 +51,7 @@ def walk_model_examples_in_package(package: ModuleType) -> Iterator[ModelExample def iter_model_examples_in_module(module: object) -> Iterator[ModelExample]: - """Iterates on all examples defined as BaseModelClass.Config.schema_extra["example"] + """Iterates on all examples defined as BaseModelClass.model_config["json_schema_extra"]["example"] Usage: @@ -64,7 +64,7 @@ def test_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): print(example_name, ":", json.dumps(example_data)) - assert model_cls.parse_obj(example_data) + assert model_cls.model_validate(example_data) """ def _is_model_cls(obj) -> bool: diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py index 47188400e79..240e0100648 100644 --- a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -56,7 +56,7 @@ def rabbit_env_vars_dict( async def rabbit_settings(rabbit_env_vars_dict: EnvVarsDict) -> RabbitSettings: """Returns the settings of a rabbit service that is up and responsive""" - settings = RabbitSettings.parse_obj(rabbit_env_vars_dict) + settings = RabbitSettings.model_validate(rabbit_env_vars_dict) await wait_till_rabbit_responsive(settings.dsn) return settings diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index dffe3883c61..429783e7061 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -28,7 +28,7 @@ from models_library.projects_pipeline import ComputationTask from models_library.projects_state import RunningState from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status from yarl import URL @@ -107,8 +107,8 @@ def create_computation_cb(url, **kwargs) -> CallbackResult: "62237c33-8d6c-4709-aa92-c3cf693dd6d2", ], } - returned_computation = ComputationTask.parse_obj( - ComputationTask.Config.schema_extra["examples"][0] + returned_computation = ComputationTask.model_validate( + ComputationTask.model_config["json_schema_extra"]["examples"][0] ).copy( update={ "id": f"{kwargs['json']['project_id']}", @@ -131,8 +131,8 @@ def get_computation_cb(url, **kwargs) -> CallbackResult: state = RunningState.NOT_STARTED pipeline: dict[str, list[str]] = FULL_PROJECT_PIPELINE_ADJACENCY node_states = FULL_PROJECT_NODE_STATES - returned_computation = ComputationTask.parse_obj( - ComputationTask.Config.schema_extra["examples"][0] + returned_computation = ComputationTask.model_validate( + ComputationTask.model_config["json_schema_extra"]["examples"][0] ).copy( update={ "id": Path(url.path).name, @@ -154,11 +154,11 @@ def get_computation_cb(url, **kwargs) -> CallbackResult: def create_cluster_cb(url, **kwargs) -> CallbackResult: assert "json" in kwargs, f"missing body in call to {url}" assert url.query.get("user_id") - random_cluster = Cluster.parse_obj( - random.choice(Cluster.Config.schema_extra["examples"]) + random_cluster = Cluster.model_validate( + random.choice(Cluster.model_config["json_schema_extra"]["examples"]) ) return CallbackResult( - status=201, payload=json.loads(random_cluster.json(by_alias=True)) + status=201, payload=json.loads(random_cluster.model_dump_json(by_alias=True)) ) @@ -169,9 +169,11 @@ def list_clusters_cb(url, **kwargs) -> CallbackResult: body=json.dumps( [ json.loads( - Cluster.parse_obj( - random.choice(Cluster.Config.schema_extra["examples"]) - ).json(by_alias=True) + Cluster.model_validate( + random.choice( + Cluster.model_config["json_schema_extra"]["examples"] + ) + ).model_dump_json(by_alias=True) ) for _ in range(3) ] @@ -185,12 +187,14 @@ def get_cluster_cb(url, **kwargs) -> CallbackResult: return CallbackResult( status=200, payload=json.loads( - Cluster.parse_obj( + Cluster.model_validate( { - **random.choice(Cluster.Config.schema_extra["examples"]), + **random.choice( + Cluster.model_config["json_schema_extra"]["examples"] + ), **{"id": cluster_id}, } - ).json(by_alias=True) + ).model_dump_json(by_alias=True) ), ) @@ -214,12 +218,14 @@ def patch_cluster_cb(url, **kwargs) -> CallbackResult: return CallbackResult( status=200, payload=json.loads( - Cluster.parse_obj( + Cluster.model_validate( { - **random.choice(Cluster.Config.schema_extra["examples"]), + **random.choice( + Cluster.model_config["json_schema_extra"]["examples"] + ), **{"id": cluster_id}, } - ).json(by_alias=True) + ).model_dump_json(by_alias=True) ), ) @@ -366,11 +372,13 @@ def get_upload_link_cb(url: URL, **kwargs) -> CallbackResult: if file_size := kwargs["params"].get("file_size") is not None: assert file_size upload_schema = FileUploadSchema( - chunk_size=parse_obj_as(ByteSize, "5GiB"), - urls=[parse_obj_as(AnyUrl, f"{scheme[link_type]}://{file_id}")], + chunk_size=TypeAdapter(ByteSize).validate_python("5GiB"), + urls=[ + TypeAdapter(AnyUrl).validate_python(f"{scheme[link_type]}://{file_id}") + ], links=FileUploadLinks( - abort_upload=parse_obj_as(AnyUrl, f"{url}:abort"), - complete_upload=parse_obj_as(AnyUrl, f"{url}:complete"), + abort_upload=TypeAdapter(AnyUrl).validate_python(f"{url}:abort"), + complete_upload=TypeAdapter(AnyUrl).validate_python(f"{url}:complete"), ), ) return CallbackResult( @@ -379,7 +387,7 @@ def get_upload_link_cb(url: URL, **kwargs) -> CallbackResult: ) # version 1 returns a presigned link presigned_link = PresignedLink( - link=parse_obj_as(AnyUrl, f"{scheme[link_type]}://{file_id}") + link=TypeAdapter(AnyUrl).validate_python(f"{scheme[link_type]}://{file_id}") ) return CallbackResult( status=status.HTTP_200_OK, @@ -436,7 +444,9 @@ async def storage_v0_service_mock( aioresponses_mocker.get( get_file_metadata_pattern, status=status.HTTP_200_OK, - payload={"data": FileMetaDataGet.Config.schema_extra["examples"][0]}, + payload={ + "data": FileMetaDataGet.model_config["json_schema_extra"]["examples"][0] + }, repeat=True, ) aioresponses_mocker.get( @@ -465,8 +475,9 @@ def generate_future_link(url, **kwargs): (parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", "") ) - payload: FileUploadCompleteResponse = parse_obj_as( - FileUploadCompleteResponse, + payload: FileUploadCompleteResponse = TypeAdapter( + FileUploadCompleteResponse + ).validate_python( { "links": { "state": stripped_url + ":complete/futures/" + str(faker.uuid4()) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 9628d1058c9..e2f7654d3d0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -10,7 +10,7 @@ import tenacity from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_mock import MockerFixture from servicelib.minio_utils import ServiceRetryPolicyUponInitialization from yarl import URL @@ -82,6 +82,8 @@ def create_simcore_file_id() -> Callable[[ProjectID, NodeID, str], SimcoreS3File def _creator( project_id: ProjectID, node_id: NodeID, file_name: str ) -> SimcoreS3FileID: - return parse_obj_as(SimcoreS3FileID, f"{project_id}/{node_id}/{file_name}") + return TypeAdapter(SimcoreS3FileID).validate_python( + f"{project_id}/{node_id}/{file_name}" + ) return _creator diff --git a/packages/service-integration/requirements/_base.txt b/packages/service-integration/requirements/_base.txt index 904f97c614f..bbd97171063 100644 --- a/packages/service-integration/requirements/_base.txt +++ b/packages/service-integration/requirements/_base.txt @@ -1,3 +1,5 @@ +annotated-types==0.7.0 + # via pydantic arrow==1.3.0 # via # -r requirements/../../../packages/models-library/requirements/_base.in @@ -31,8 +33,6 @@ docker==7.1.0 # via -r requirements/_base.in email-validator==2.2.0 # via pydantic -exceptiongroup==1.2.2 - # via pytest idna==3.7 # via # email-validator @@ -68,17 +68,27 @@ packaging==24.1 # via pytest pluggy==1.5.0 # via pytest -pydantic==1.10.17 +pydantic==2.9.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via -r requirements/../../../packages/models-library/requirements/_base.in pygments==2.18.0 # via rich pytest==8.3.2 # via -r requirements/_base.in python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings python-slugify==8.0.4 # via cookiecutter pyyaml==6.0.2 @@ -109,8 +119,6 @@ six==1.16.0 # via python-dateutil text-unidecode==1.3 # via python-slugify -tomli==2.0.1 - # via pytest typer==0.12.4 # via -r requirements/_base.in types-python-dateutil==2.9.0.20240821 @@ -118,6 +126,7 @@ types-python-dateutil==2.9.0.20240821 typing-extensions==4.12.2 # via # pydantic + # pydantic-core # typer urllib3==2.2.2 # via diff --git a/packages/service-integration/requirements/_test.txt b/packages/service-integration/requirements/_test.txt index 925a176d40c..fa704698091 100644 --- a/packages/service-integration/requirements/_test.txt +++ b/packages/service-integration/requirements/_test.txt @@ -6,10 +6,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.2 - # via - # -c requirements/_base.txt - # pytest iniconfig==2.0.0 # via # -c requirements/_base.txt @@ -48,11 +44,6 @@ rpds-py==0.20.0 # referencing termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # -c requirements/_base.txt - # coverage - # pytest types-docker==7.1.0.20240821 # via -r requirements/_test.in types-jsonschema==4.23.0.20240813 diff --git a/packages/service-integration/requirements/_tools.txt b/packages/service-integration/requirements/_tools.txt index f76d26f846f..354746f70e4 100644 --- a/packages/service-integration/requirements/_tools.txt +++ b/packages/service-integration/requirements/_tools.txt @@ -69,22 +69,11 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_base.txt - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py b/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py index a390a469a41..a0a5f295402 100644 --- a/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py +++ b/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py @@ -5,9 +5,10 @@ # type:ignore from enum import Enum -from typing import Any +from typing import Any, TypeAlias -from pydantic import BaseModel, ConstrainedInt, Extra, Field, conint, constr +from pydantic import BaseModel, ConfigDict, Field, RootModel, StringConstraints +from typing_extensions import Annotated # MODIFICATIONS ------------------------------------------------------------------------- # @@ -19,17 +20,14 @@ # UserWarning: format of 'subnet_ip_address' not understood for 'string' - using default # port number range -class PortInt(ConstrainedInt): - gt = 0 - lt = 65535 +PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)] # ---------------------------------------------------------------------------------------- class Configuration(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") source: str | None = None target: str | None = None @@ -39,8 +37,7 @@ class Config: class CredentialSpec(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") config: str | None = None file: str | None = None @@ -54,31 +51,29 @@ class Condition(Enum): class DependsOn(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") condition: Condition class Extend(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") service: str file: str | None = None class Logging(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") driver: str | None = None - options: dict[constr(regex=r"^.+$"), str | float | None] | None = None + options: dict[ + Annotated[str, StringConstraints(pattern=r"^.+$")], str | float | None + ] | None = None class Port(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") mode: str | None = None host_ip: str | None = None @@ -96,8 +91,7 @@ class PullPolicy(Enum): class Secret1(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") source: str | None = None target: str | None = None @@ -107,38 +101,33 @@ class Config: class Ulimit(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") hard: int soft: int class Bind(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") propagation: str | None = None create_host_path: bool | None = None class Volume2(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") nocopy: bool | None = None class Tmpfs(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") - size: conint(ge=0) | str | None = None + size: Annotated[int, Field(ge=0)] | str | None = None class Volume1(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") type: str source: str | None = None @@ -151,8 +140,7 @@ class Config: class Healthcheck(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") disable: bool | None = None interval: str | None = None @@ -168,8 +156,7 @@ class Order(Enum): class RollbackConfig(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") parallelism: int | None = None delay: str | None = None @@ -185,8 +172,7 @@ class Order1(Enum): class UpdateConfig(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") parallelism: int | None = None delay: str | None = None @@ -197,16 +183,14 @@ class Config: class Limits(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") cpus: float | str | None = None memory: str | None = None class RestartPolicy(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") condition: str | None = None delay: str | None = None @@ -215,15 +199,13 @@ class Config: class Preference(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") spread: str | None = None class Placement(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") constraints: list[str] | None = None preferences: list[Preference] | None = None @@ -231,53 +213,49 @@ class Config: class DiscreteResourceSpec(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") kind: str | None = None value: float | None = None class GenericResource(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") discrete_resource_spec: DiscreteResourceSpec | None = None -class GenericResources(BaseModel): - __root__: list[GenericResource] +class GenericResources(RootModel): + root: list[GenericResource] class ConfigItem(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") subnet: str | None = None ip_range: str | None = None gateway: str | None = None - aux_addresses: dict[constr(regex=r"^.+$"), str] | None = None + aux_addresses: dict[ + Annotated[str, StringConstraints(pattern=r"^.+$")], str + ] | None = None class Ipam(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") driver: str | None = None config: list[ConfigItem] | None = None - options: dict[constr(regex=r"^.+$"), str] | None = None + options: dict[Annotated[str, StringConstraints(pattern=r"^.+$")], str] | None = None class External(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") name: str | None = None class External1(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") name: str | None = None @@ -290,37 +268,39 @@ class External3(BaseModel): name: str | None = None -class ListOfStrings(BaseModel): - __root__: list[str] +class ListOfStrings(RootModel): + root: list[str] -class ListOrDict(BaseModel): - __root__: (dict[constr(regex=r".+"), str | float | bool | None] | list[str]) +class ListOrDict(RootModel): + root: ( + dict[ + Annotated[str, StringConstraints(pattern=r".+")], str | float | bool | None + ] + | list[str] + ) class BlkioLimit(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") path: str | None = None rate: int | str | None = None class BlkioWeight(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") path: str | None = None weight: int | None = None -class Constraints(BaseModel): - __root__: Any +class Constraints(RootModel): + root: Any = None class BuildItem(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") context: str | None = None dockerfile: str | None = None @@ -335,8 +315,7 @@ class Config: class BlkioConfig(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") device_read_bps: list[BlkioLimit] | None = None device_read_iops: list[BlkioLimit] | None = None @@ -347,8 +326,7 @@ class Config: class Network1(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") aliases: ListOfStrings | None = None ipv4_address: str | None = None @@ -358,8 +336,7 @@ class Config: class Device(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") capabilities: ListOfStrings | None = None count: str | int | None = None @@ -368,17 +345,18 @@ class Config: options: ListOrDict | None = None -class Devices(BaseModel): - __root__: list[Device] +class Devices(RootModel): + root: list[Device] class Network(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") name: str | None = None driver: str | None = None - driver_opts: dict[constr(regex=r"^.+$"), str | float] | None = None + driver_opts: dict[ + Annotated[str, StringConstraints(pattern=r"^.+$")], str | float + ] | None = None ipam: Ipam | None = None external: External | None = None internal: bool | None = None @@ -388,32 +366,33 @@ class Config: class Volume(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") name: str | None = None driver: str | None = None - driver_opts: dict[constr(regex=r"^.+$"), str | float] | None = None + driver_opts: dict[ + Annotated[str, StringConstraints(pattern=r"^.+$")], str | float + ] | None = None external: External1 | None = None labels: ListOrDict | None = None class Secret(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") name: str | None = None file: str | None = None external: External2 | None = None labels: ListOrDict | None = None driver: str | None = None - driver_opts: dict[constr(regex=r"^.+$"), str | float] | None = None + driver_opts: dict[ + Annotated[str, StringConstraints(pattern=r"^.+$")], str | float + ] | None = None template_driver: str | None = None class ComposeSpecConfig(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") name: str | None = None file: str | None = None @@ -422,13 +401,12 @@ class Config: template_driver: str | None = None -class StringOrList(BaseModel): - __root__: str | ListOfStrings +class StringOrList(RootModel): + root: str | ListOfStrings class Reservations(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") cpus: float | str | None = None memory: str | None = None @@ -437,16 +415,14 @@ class Config: class Resources(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") limits: Limits | None = None reservations: Reservations | None = None class Deployment(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") mode: str | None = None endpoint_mode: str | None = None @@ -460,8 +436,7 @@ class Config: class Service(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") deploy: Deployment | None = None build: str | BuildItem | None = None @@ -472,8 +447,8 @@ class Config: command: str | list[str] | None = None configs: list[str | Configuration] | None = None container_name: str | None = None - cpu_count: conint(ge=0) | None = None - cpu_percent: conint(ge=0, le=100) | None = None + cpu_count: Annotated[int, Field(ge=0)] | None = None + cpu_percent: Annotated[int, Field(ge=0, le=100)] | None = None cpu_shares: float | str | None = None cpu_quota: float | str | None = None cpu_period: float | str | None = None @@ -483,7 +458,10 @@ class Config: cpuset: str | None = None credential_spec: CredentialSpec | None = None depends_on: None | ( - ListOfStrings | dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), DependsOn] + ListOfStrings + | dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], DependsOn + ] ) = None device_cgroup_rules: ListOfStrings | None = None devices: list[str] | None = None @@ -515,10 +493,14 @@ class Config: memswap_limit: float | str | None = None network_mode: str | None = None networks: None | ( - ListOfStrings | dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), Network1 | None] + ListOfStrings + | dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], + Network1 | None, + ] ) = None oom_kill_disable: bool | None = None - oom_score_adj: conint(ge=-1000, le=1000) | None = None + oom_score_adj: Annotated[int, Field(ge=-1000, le=1000)] | None = None pid: str | None = None pids_limit: float | str | None = None platform: str | None = None @@ -540,7 +522,9 @@ class Config: storage_opt: dict[str, Any] | None = None tmpfs: StringOrList | None = None tty: bool | None = None - ulimits: dict[constr(regex=r"^[a-z]+$"), int | Ulimit] | None = None + ulimits: dict[ + Annotated[str, StringConstraints(pattern=r"^[a-z]+$")], int | Ulimit + ] | None = None user: str | None = None userns_mode: str | None = None volumes: list[str | Volume1] | None = None @@ -553,15 +537,27 @@ class ComposeSpecification(BaseModel): The Compose file is a YAML file defining a multi-containers based application. """ - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") version: str | None = Field( None, description="Version of the Compose specification used. Tools not implementing required version MUST reject the configuration file.", ) - services: dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), Service] | None = None - networks: dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), Network] | None = None - volumes: dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), Volume] | None = None - secrets: dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), Secret] | None = None - configs: None | (dict[constr(regex=r"^[a-zA-Z0-9._-]+$"), ComposeSpecConfig]) = None + services: dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], Service + ] | None = None + networks: dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], Network + ] | None = None + volumes: dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], Volume + ] | None = None + secrets: dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], Secret + ] | None = None + configs: None | ( + dict[ + Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9._-]+$")], + ComposeSpecConfig, + ] + ) = None diff --git a/packages/service-integration/src/service_integration/cli/__init__.py b/packages/service-integration/src/service_integration/cli/__init__.py index 7a1c058957e..a146de5735d 100644 --- a/packages/service-integration/src/service_integration/cli/__init__.py +++ b/packages/service-integration/src/service_integration/cli/__init__.py @@ -62,7 +62,7 @@ def main( overrides["COMPOSE_VERSION"] = compose_version # save states - ctx.settings = AppSettings.parse_obj(overrides) # type: ignore[attr-defined] # pylint:disable=no-member + ctx.settings = AppSettings.model_validate(overrides) # type: ignore[attr-defined] # pylint:disable=no-member # diff --git a/packages/service-integration/src/service_integration/cli/_compose_spec.py b/packages/service-integration/src/service_integration/cli/_compose_spec.py index 117a4afa5ef..f6d9b16be9e 100644 --- a/packages/service-integration/src/service_integration/cli/_compose_spec.py +++ b/packages/service-integration/src/service_integration/cli/_compose_spec.py @@ -196,7 +196,7 @@ def create_compose( for n, config_name in enumerate(configs_kwargs_map): nth_compose_spec = create_docker_compose_image_spec( settings, **configs_kwargs_map[config_name] - ).dict(exclude_unset=True) + ).model_dump(exclude_unset=True) if n == 0: compose_spec_dict = nth_compose_spec diff --git a/packages/service-integration/src/service_integration/cli/_config.py b/packages/service-integration/src/service_integration/cli/_config.py index 2f41dcb6f72..4437907efa0 100644 --- a/packages/service-integration/src/service_integration/cli/_config.py +++ b/packages/service-integration/src/service_integration/cli/_config.py @@ -25,7 +25,7 @@ def _get_labels_or_raise(build_labels) -> dict[str, str]: return dict(item.strip().split("=") for item in build_labels) if isinstance(build_labels, dict): return build_labels - if labels__root__ := build_labels.__root__: + if labels__root__ := build_labels.root: assert isinstance(labels__root__, dict) # nosec return labels__root__ raise InvalidLabelsError(build_labels=build_labels) @@ -39,7 +39,7 @@ def _create_config_from_compose_spec( ): rich.print(f"Creating osparc config files from {compose_spec_path}") - compose_spec = ComposeSpecification.parse_obj( + compose_spec = ComposeSpecification.model_validate( yaml.safe_load(compose_spec_path.read_text()) ) @@ -56,7 +56,7 @@ def _save(service_name: str, filename: Path, model: BaseModel): rich.print(f"Creating {output_path} ...", end="") with output_path.open("wt") as fh: - data = json.loads(model.json(by_alias=True, exclude_none=True)) + data = json.loads(model.model_dump_json(by_alias=True, exclude_none=True)) yaml.safe_dump(data, fh, sort_keys=False) rich.print("DONE") diff --git a/packages/service-integration/src/service_integration/errors.py b/packages/service-integration/src/service_integration/errors.py index 8d216b7d918..65521d36371 100644 --- a/packages/service-integration/src/service_integration/errors.py +++ b/packages/service-integration/src/service_integration/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class ServiceIntegrationError(PydanticErrorMixin, RuntimeError): +class ServiceIntegrationError(OsparcErrorMixin, RuntimeError): pass @@ -13,5 +13,5 @@ class UndefinedOciImageSpecError(ServiceIntegrationError): ... -class InvalidLabelsError(PydanticErrorMixin, ValueError): +class InvalidLabelsError(OsparcErrorMixin, ValueError): template_msg = "Invalid build labels {build_labels}" diff --git a/packages/service-integration/src/service_integration/oci_image_spec.py b/packages/service-integration/src/service_integration/oci_image_spec.py index e07a5e4cafc..3b9e45b46ab 100644 --- a/packages/service-integration/src/service_integration/oci_image_spec.py +++ b/packages/service-integration/src/service_integration/oci_image_spec.py @@ -11,8 +11,7 @@ from models_library.basic_types import SHA1Str, VersionStr from models_library.utils.labels_annotations import from_labels, to_labels -from pydantic import BaseModel, Field -from pydantic.config import Extra +from pydantic import BaseModel, ConfigDict, Field from pydantic.networks import AnyUrl # @@ -100,22 +99,20 @@ class OciImageSpecAnnotations(BaseModel): None, description="Digest of the image this image is based on (string)", ) - - class Config: - alias_generator = _underscore_as_dot - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict( + alias_generator=_underscore_as_dot, populate_by_name=True, extra="forbid" + ) @classmethod def from_labels_annotations( cls, labels: dict[str, str] ) -> "OciImageSpecAnnotations": data = from_labels(labels, prefix_key=OCI_LABEL_PREFIX, trim_key_head=False) - return cls.parse_obj(data) + return cls.model_validate(data) def to_labels_annotations(self) -> dict[str, str]: labels: dict[str, str] = to_labels( - self.dict(exclude_unset=True, by_alias=True, exclude_none=True), + self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True), prefix_key=OCI_LABEL_PREFIX, ) return labels @@ -131,30 +128,30 @@ class LabelSchemaAnnotations(BaseModel): build_date: datetime vcs_ref: str vcs_url: AnyUrl - - class Config: - alias_generator = lambda field_name: field_name.replace("_", "-") - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict( + alias_generator=lambda field_name: field_name.replace("_", "-"), + populate_by_name=True, + extra="forbid", + ) @classmethod def create_from_env(cls) -> "LabelSchemaAnnotations": data = {} - for field_name in cls.__fields__: + for field_name in cls.model_fields: if value := os.environ.get(field_name.upper()): data[field_name] = value - return cls.parse_obj(data) + return cls.model_validate(data) def to_oci_data(self) -> dict[str, Any]: """Collects data that be converted to OCI labels. WARNING: label-schema has be deprecated in favor of OCI image specs """ - convertable_data = self.dict( + convertable_data = self.model_dump( include=set(_TO_OCI.keys()), exclude_unset=True, exclude_none=True ) assert set(convertable_data.keys()).issubset( # nosec - set(self.__fields__.keys()) + set(self.model_fields.keys()) ) # nosec return {_TO_OCI[key]: value for key, value in convertable_data.items()} diff --git a/packages/service-integration/src/service_integration/osparc_config.py b/packages/service-integration/src/service_integration/osparc_config.py index 1a340729e41..9382b98b447 100644 --- a/packages/service-integration/src/service_integration/osparc_config.py +++ b/packages/service-integration/src/service_integration/osparc_config.py @@ -36,9 +36,14 @@ from_labels, to_labels, ) -from pydantic import NonNegativeInt, ValidationError -from pydantic.class_validators import root_validator, validator -from pydantic.config import Extra +from pydantic import ( + ConfigDict, + NonNegativeInt, + ValidationError, + ValidationInfo, + field_validator, + model_validator, +) from pydantic.fields import Field from pydantic.main import BaseModel @@ -67,7 +72,7 @@ class DockerComposeOverwriteConfig(ComposeSpecification): def create_default( cls, service_name: str | None = None ) -> "DockerComposeOverwriteConfig": - model: "DockerComposeOverwriteConfig" = cls.parse_obj( + model: "DockerComposeOverwriteConfig" = cls.model_validate( { "services": { service_name: { @@ -84,7 +89,7 @@ def create_default( def from_yaml(cls, path: Path) -> "DockerComposeOverwriteConfig": with path.open() as fh: data = yaml_safe_load(fh) - model: "DockerComposeOverwriteConfig" = cls.parse_obj(data) + model: "DockerComposeOverwriteConfig" = cls.model_validate(data) return model @@ -101,11 +106,11 @@ class MetadataConfig(ServiceMetaDataPublished): exclude=True, ) - @validator("contact") + @field_validator("contact") @classmethod - def _check_contact_in_authors(cls, v, values): + def _check_contact_in_authors(cls, v, info: ValidationInfo): """catalog service relies on contact and author to define access rights""" - authors_emails = {author.email for author in values["authors"]} + authors_emails = {author.email for author in info.data["authors"]} if v not in authors_emails: msg = "Contact {v} must be registered as an author" raise ValueError(msg) @@ -115,7 +120,7 @@ def _check_contact_in_authors(cls, v, values): def from_yaml(cls, path: Path) -> "MetadataConfig": with path.open() as fh: data = yaml_safe_load(fh) - model: "MetadataConfig" = cls.parse_obj(data) + model: "MetadataConfig" = cls.model_validate(data) return model @classmethod @@ -123,12 +128,12 @@ def from_labels_annotations(cls, labels: dict[str, str]) -> "MetadataConfig": data = from_labels( labels, prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False ) - model: "MetadataConfig" = cls.parse_obj(data) + model: "MetadataConfig" = cls.model_validate(data) return model def to_labels_annotations(self) -> dict[str, str]: labels: dict[str, str] = to_labels( - self.dict(exclude_unset=True, by_alias=True, exclude_none=True), + self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True), prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False, ) @@ -175,7 +180,7 @@ class SettingsItem(BaseModel): description="The value of the service setting (shall follow Docker REST API scheme for services", ) - @validator("type_", pre=True) + @field_validator("type_", mode="before") @classmethod def ensure_backwards_compatible_setting_type(cls, v): if v == "resources": @@ -183,18 +188,16 @@ def ensure_backwards_compatible_setting_type(cls, v): return "Resources" return v - @validator("value", pre=True) + @field_validator("value", mode="before") @classmethod - def check_value_against_custom_types(cls, v, values): - if (type_ := values.get("type_")) and type_ == "ContainerSpec": - ContainerSpec.parse_obj(v) + def check_value_against_custom_types(cls, v, info: ValidationInfo): + if (type_ := info.data.get("type_")) and type_ == "ContainerSpec": + ContainerSpec.model_validate(v) return v class ValidatingDynamicSidecarServiceLabels(DynamicSidecarServiceLabels): - class Config: - extra = Extra.allow - allow_population_by_field_name = True + model_config = ConfigDict(extra="allow", populate_by_name=True) def _underscore_as_minus(field_name: str) -> str: @@ -225,13 +228,13 @@ class RuntimeConfig(BaseModel): settings: list[SettingsItem] = Field(default_factory=list) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def ensure_compatibility(cls, v): # NOTE: if changes are applied to `DynamicSidecarServiceLabels` # these are also validated when ooil runs. try: - ValidatingDynamicSidecarServiceLabels.parse_obj(v) + ValidatingDynamicSidecarServiceLabels.model_validate(v) except ValidationError: _logger.exception( "Could not validate %s via %s", @@ -242,25 +245,26 @@ def ensure_compatibility(cls, v): return v - class Config: - alias_generator = _underscore_as_minus - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict( + alias_generator=_underscore_as_minus, + populate_by_name=True, + extra="forbid", + ) @classmethod def from_yaml(cls, path: Path) -> "RuntimeConfig": with path.open() as fh: data = yaml_safe_load(fh) - return cls.parse_obj(data) + return cls.model_validate(data) @classmethod def from_labels_annotations(cls, labels: dict[str, str]) -> "RuntimeConfig": data = from_labels(labels, prefix_key=OSPARC_LABEL_PREFIXES[1]) - return cls.parse_obj(data) + return cls.model_validate(data) def to_labels_annotations(self) -> dict[str, str]: labels: dict[str, str] = to_labels( - self.dict(exclude_unset=True, by_alias=True, exclude_none=True), + self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True), prefix_key=OSPARC_LABEL_PREFIXES[1], ) return labels diff --git a/packages/service-integration/src/service_integration/osparc_image_specs.py b/packages/service-integration/src/service_integration/osparc_image_specs.py index df97e7c18b1..7f6dec6ca15 100644 --- a/packages/service-integration/src/service_integration/osparc_image_specs.py +++ b/packages/service-integration/src/service_integration/osparc_image_specs.py @@ -41,9 +41,9 @@ def create_image_spec( docker_compose_overwrite_cfg.services[service_name].build.labels = labels - overwrite_options = docker_compose_overwrite_cfg.services[service_name].build.dict( - exclude_none=True - ) + overwrite_options = docker_compose_overwrite_cfg.services[ + service_name + ].build.model_dump(exclude_none=True, serialize_as_any=True) build_spec = BuildItem(**overwrite_options) return ComposeSpecification( diff --git a/packages/service-integration/src/service_integration/settings.py b/packages/service-integration/src/service_integration/settings.py index 70c971c8db9..f8b977cc9a4 100644 --- a/packages/service-integration/src/service_integration/settings.py +++ b/packages/service-integration/src/service_integration/settings.py @@ -1,4 +1,5 @@ -from pydantic import BaseModel, BaseSettings, Field, SecretStr +from pydantic import BaseModel, Field, SecretStr +from pydantic_settings import BaseSettings, SettingsConfigDict class Registry(BaseModel): @@ -26,9 +27,9 @@ class AppSettings(BaseSettings): COMPOSE_VERSION: str = Field( "3.7", description="version of the docker-compose spec" ) - - class Config: - env_file_encoding = "utf-8" + model_config = SettingsConfigDict( + env_file_encoding="utf-8", + ) # TODO: load from ~/.osparc/service-integration.json or env file # TODO: add access to secrets diff --git a/packages/service-integration/src/service_integration/versioning.py b/packages/service-integration/src/service_integration/versioning.py index 3ed56868e50..0d7685a818f 100644 --- a/packages/service-integration/src/service_integration/versioning.py +++ b/packages/service-integration/src/service_integration/versioning.py @@ -1,15 +1,13 @@ -import re from datetime import datetime -from re import Pattern -from typing import Any, ClassVar +from typing import Annotated, TypeAlias from models_library.basic_regex import SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS from packaging.version import Version -from pydantic import BaseModel, ConstrainedStr, Field +from pydantic import BaseModel, ConfigDict, Field, StringConstraints - -class SemanticVersionStr(ConstrainedStr): - regex: Pattern[str] | None = re.compile(SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS) +SemanticVersionStr: TypeAlias = Annotated[ + str, StringConstraints(pattern=SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS) +] def bump_version_string(current_version: str, bump: str) -> str: @@ -52,8 +50,8 @@ class ExecutableVersionInfo(BaseModel): version: SemanticVersionStr released: datetime - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "display_name": "SEMCAD X", "display_version": "Matterhorn Student Edition 1", @@ -63,6 +61,7 @@ class Config: "released": "2021-11-19T14:58:45.900979", } } + ) class ServiceVersionInfo(BaseModel): @@ -72,11 +71,12 @@ class ServiceVersionInfo(BaseModel): ) released: datetime = Field(..., description="Publication/release date") - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "version": "1.0.0", # e.g. first time released as an osparc "integration_version": "2.1.0", "released": "2021-11-19T14:58:45.900979", } } + ) diff --git a/packages/service-integration/tests/test_command_compose.py b/packages/service-integration/tests/test_command_compose.py index 371d8a9dbdc..50f8b5b67b4 100644 --- a/packages/service-integration/tests/test_command_compose.py +++ b/packages/service-integration/tests/test_command_compose.py @@ -39,7 +39,7 @@ def test_make_docker_compose_meta( assert target_compose_specs.exists() # valid compose specs - compose_cfg = ComposeSpecification.parse_obj( + compose_cfg = ComposeSpecification.model_validate( yaml.safe_load(target_compose_specs.read_text()) ) assert compose_cfg.services @@ -48,8 +48,8 @@ def test_make_docker_compose_meta( compose_labels = compose_cfg.services[metadata_cfg.service_name()].build.labels assert compose_labels - assert isinstance(compose_labels.__root__, dict) + assert isinstance(compose_labels.root, dict) assert ( - MetadataConfig.from_labels_annotations(compose_labels.__root__) == metadata_cfg + MetadataConfig.from_labels_annotations(compose_labels.root) == metadata_cfg ) diff --git a/packages/service-integration/tests/test_compose_spec_model.py b/packages/service-integration/tests/test_compose_spec_model.py index 63cd0924c99..416dfbb8eef 100644 --- a/packages/service-integration/tests/test_compose_spec_model.py +++ b/packages/service-integration/tests/test_compose_spec_model.py @@ -9,7 +9,7 @@ def test_autogenerated_compose_spec_model(tests_data_dir: Path): docker_compose_path = tests_data_dir / "docker-compose-meta.yml" # tests if parses valid file - compose_spec = ComposeSpecification.parse_obj( + compose_spec = ComposeSpecification.model_validate( yaml.safe_load(docker_compose_path.read_text()) ) diff --git a/packages/service-integration/tests/test_oci_image_spec.py b/packages/service-integration/tests/test_oci_image_spec.py index ef2bd8b47d9..641594c9966 100644 --- a/packages/service-integration/tests/test_oci_image_spec.py +++ b/packages/service-integration/tests/test_oci_image_spec.py @@ -18,7 +18,7 @@ def test_label_schema_to_oci_conversion(monkeypatch): lsa = LabelSchemaAnnotations.create_from_env() - OciImageSpecAnnotations.parse_obj(lsa.to_oci_data()) + OciImageSpecAnnotations.model_validate(lsa.to_oci_data()) def test_create_annotations_from_metadata(tests_data_dir: Path): diff --git a/packages/service-integration/tests/test_osparc_config.py b/packages/service-integration/tests/test_osparc_config.py index e993bc25392..9a5a8bd7a81 100644 --- a/packages/service-integration/tests/test_osparc_config.py +++ b/packages/service-integration/tests/test_osparc_config.py @@ -52,8 +52,8 @@ def test_load_from_labels( runtime_cfg = RuntimeConfig.from_labels_annotations(labels) assert runtime_cfg.callbacks_mapping is not None - print(meta_cfg.json(exclude_unset=True, indent=2)) - print(runtime_cfg.json(exclude_unset=True, indent=2)) + print(meta_cfg.model_dump_json(exclude_unset=True, indent=2)) + print(runtime_cfg.model_dump_json(exclude_unset=True, indent=2)) # create yamls from config for model in (runtime_cfg, meta_cfg): @@ -62,7 +62,7 @@ def test_load_from_labels( ) with open(config_path, "w") as fh: data = json.loads( - model.json(exclude_unset=True, by_alias=True, exclude_none=True) + model.model_dump_json(exclude_unset=True, by_alias=True, exclude_none=True) ) yaml.safe_dump(data, fh, sort_keys=False) @@ -72,7 +72,8 @@ def test_load_from_labels( @pytest.mark.parametrize( - "example_data", SimcoreServiceSettingLabelEntry.Config.schema_extra["examples"] + "example_data", + SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"], ) def test_settings_item_in_sync_with_service_settings_label( example_data: dict[str, Any] @@ -81,7 +82,7 @@ def test_settings_item_in_sync_with_service_settings_label( # First we parse with SimcoreServiceSettingLabelEntry since it also supports backwards compatibility # and will upgrade old version - example_model = SimcoreServiceSettingLabelEntry.parse_obj(example_data) + example_model = SimcoreServiceSettingLabelEntry.model_validate(example_data) # SettingsItem is exclusively for NEW labels, so it should not support backwards compatibility new_model = SettingsItem( @@ -91,4 +92,4 @@ def test_settings_item_in_sync_with_service_settings_label( ) # check back - SimcoreServiceSettingLabelEntry.parse_obj(new_model.dict(by_alias=True)) + SimcoreServiceSettingLabelEntry.model_validate(new_model.model_dump(by_alias=True)) diff --git a/packages/service-integration/tests/test_osparc_image_specs.py b/packages/service-integration/tests/test_osparc_image_specs.py index b482bc85a4c..6bec87425ad 100644 --- a/packages/service-integration/tests/test_osparc_image_specs.py +++ b/packages/service-integration/tests/test_osparc_image_specs.py @@ -58,8 +58,8 @@ def test_create_image_spec_impl(tests_data_dir: Path, settings: AppSettings): assert build_spec assert isinstance(build_spec, BaseModel) - print(build_spec.json(exclude_unset=True, indent=2)) - print(yaml.safe_dump(compose_spec.dict(exclude_unset=True), sort_keys=False)) + print(build_spec.model_dump_json(exclude_unset=True, indent=2)) + print(yaml.safe_dump(compose_spec.model_dump(exclude_unset=True), sort_keys=False)) def test_image_digest_is_not_a_label_annotation(tests_data_dir: Path): diff --git a/packages/service-integration/tests/test_osparc_runtime_specs.py b/packages/service-integration/tests/test_osparc_runtime_specs.py index 74d63e15e5b..153c85d27c4 100644 --- a/packages/service-integration/tests/test_osparc_runtime_specs.py +++ b/packages/service-integration/tests/test_osparc_runtime_specs.py @@ -17,8 +17,8 @@ def test_create_runtime_spec_impl(tests_data_dir: Path): osparc_spec: dict = yaml.safe_load((tests_data_dir / "runtime.yml").read_text()) - pm_spec1 = PathMappingsLabel.parse_obj(osparc_spec["paths-mapping"]) - pm_spec2 = PathMappingsLabel.parse_obj( + pm_spec1 = PathMappingsLabel.model_validate(osparc_spec["paths-mapping"]) + pm_spec2 = PathMappingsLabel.model_validate( { "outputs_path": "/outputs", "inputs_path": "/inputs", @@ -58,12 +58,12 @@ def test_create_runtime_spec_impl(tests_data_dir: Path): # FIXME: ensure all sources are different! (e.g. a/b/c and z/c have the same name!) - print(Service(volumes=volumes).json(exclude_unset=True, indent=2)) + print(Service(volumes=volumes).model_dump_json(exclude_unset=True, indent=2)) # TODO: _auto_map_to_service(osparc_spec["settings"]) data = {} for obj in osparc_spec["settings"]: - item = SettingsItem.parse_obj(obj) + item = SettingsItem.model_validate(obj) if item.name == "resources": # https://docs.docker.com/compose/compose-file/compose-file-v3/#resources @@ -87,7 +87,7 @@ def test_create_runtime_spec_impl(tests_data_dir: Path): else: raise AssertionError(item) - print(Service(**data).json(exclude_unset=True, indent=2)) + print(Service(**data).model_dump_json(exclude_unset=True, indent=2)) def test_compatibility(): diff --git a/packages/service-library/requirements/_aiohttp.txt b/packages/service-library/requirements/_aiohttp.txt index cebbcc2dd9f..f97b5b35ee8 100644 --- a/packages/service-library/requirements/_aiohttp.txt +++ b/packages/service-library/requirements/_aiohttp.txt @@ -14,9 +14,7 @@ aiosignal==1.3.1 aiozipkin==1.1.1 # via -r requirements/_aiohttp.in async-timeout==4.0.3 - # via - # aiohttp - # aiopg + # via aiopg attrs==24.2.0 # via # -r requirements/_aiohttp.in @@ -35,8 +33,6 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -greenlet==3.0.3 - # via sqlalchemy idna==3.7 # via # requests diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index 69daeedb073..6d0447e7c0e 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -18,6 +18,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # fast-depends @@ -26,10 +28,6 @@ arrow==1.3.0 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/_base.in -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==24.2.0 # via # aiohttp @@ -41,8 +39,6 @@ dnspython==2.6.1 # via email-validator email-validator==2.2.0 # via pydantic -exceptiongroup==1.2.2 - # via anyio fast-depends==2.4.8 # via faststream faststream==0.5.18 @@ -76,7 +72,7 @@ orjson==3.10.7 # -r requirements/../../../packages/models-library/requirements/_base.in pamqp==3.3.0 # via aiormq -pydantic==1.10.17 +pydantic==2.9.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -85,12 +81,24 @@ pydantic==1.10.17 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.7.2 # via -r requirements/_base.in python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -137,9 +145,9 @@ types-python-dateutil==2.9.0.20240821 typing-extensions==4.12.2 # via # aiodebug - # anyio # faststream # pydantic + # pydantic-core # typer yarl==1.9.4 # via diff --git a/packages/service-library/requirements/_fastapi.txt b/packages/service-library/requirements/_fastapi.txt index 9a07c682d49..abaaac8e111 100644 --- a/packages/service-library/requirements/_fastapi.txt +++ b/packages/service-library/requirements/_fastapi.txt @@ -1,3 +1,5 @@ +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # httpx @@ -11,13 +13,8 @@ certifi==2024.7.4 # httpx click==8.1.7 # via uvicorn -exceptiongroup==1.2.2 - # via anyio -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/_fastapi.in # prometheus-fastapi-instrumentator h11==0.14.0 @@ -42,18 +39,19 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/_fastapi.in -pydantic==1.10.17 +pydantic==2.9.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../requirements/constraints.txt # fastapi +pydantic-core==2.23.4 + # via pydantic sniffio==1.3.1 # via # anyio # httpx -starlette==0.27.0 +starlette==0.38.5 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -61,9 +59,8 @@ starlette==0.27.0 # fastapi typing-extensions==4.12.2 # via - # anyio # fastapi # pydantic - # uvicorn + # pydantic-core uvicorn==0.30.6 # via -r requirements/_fastapi.in diff --git a/packages/service-library/requirements/_test.txt b/packages/service-library/requirements/_test.txt index 789fdc81902..38282969f74 100644 --- a/packages/service-library/requirements/_test.txt +++ b/packages/service-library/requirements/_test.txt @@ -21,11 +21,6 @@ anyio==4.4.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -async-timeout==4.0.3 - # via - # -c requirements/_aiohttp.txt - # -c requirements/_base.txt - # aiohttp attrs==24.2.0 # via # -c requirements/_aiohttp.txt @@ -52,12 +47,6 @@ coverage==7.6.1 # pytest-cov docker==7.1.0 # via -r requirements/_test.in -exceptiongroup==1.2.2 - # via - # -c requirements/_base.txt - # -c requirements/_fastapi.txt - # anyio - # pytest execnet==2.1.1 # via pytest-xdist faker==27.0.0 @@ -70,10 +59,6 @@ frozenlist==1.4.1 # -c requirements/_base.txt # aiohttp # aiosignal -greenlet==3.0.3 - # via - # -c requirements/_aiohttp.txt - # sqlalchemy h11==0.14.0 # via # -c requirements/_fastapi.txt @@ -197,7 +182,9 @@ python-dateutil==2.9.0.post0 # -c requirements/_base.txt # faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt @@ -250,11 +237,6 @@ sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # mypy - # pytest types-aiofiles==24.1.0.20240626 # via -r requirements/_test.in types-psycopg2==2.9.21.20240819 @@ -265,7 +247,6 @@ typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_fastapi.txt - # anyio # mypy # sqlalchemy2-stubs urllib3==2.2.2 diff --git a/packages/service-library/requirements/_tools.txt b/packages/service-library/requirements/_tools.txt index 4695266d9c8..b54db6d8f5c 100644 --- a/packages/service-library/requirements/_tools.txt +++ b/packages/service-library/requirements/_tools.txt @@ -72,22 +72,12 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/service-library/src/servicelib/aiohttp/application_setup.py b/packages/service-library/src/servicelib/aiohttp/application_setup.py index 4fae3acc09f..4da40aa0182 100644 --- a/packages/service-library/src/servicelib/aiohttp/application_setup.py +++ b/packages/service-library/src/servicelib/aiohttp/application_setup.py @@ -8,7 +8,7 @@ import arrow from aiohttp import web -from pydantic import parse_obj_as +from pydantic import TypeAdapter from .application_keys import APP_CONFIG_KEY, APP_SETTINGS_KEY @@ -94,7 +94,9 @@ def _is_addon_enabled_from_config( for part in parts: if section and part == "enabled": # if section exists, no need to explicitly enable it - return parse_obj_as(bool, searched_config.get(part, True)) + return TypeAdapter(bool).validate_python( + searched_config.get(part, True) + ) searched_config = searched_config[part] except KeyError as ee: diff --git a/packages/service-library/src/servicelib/aiohttp/docker_utils.py b/packages/service-library/src/servicelib/aiohttp/docker_utils.py index 636b3492616..8e9393e1e69 100644 --- a/packages/service-library/src/servicelib/aiohttp/docker_utils.py +++ b/packages/service-library/src/servicelib/aiohttp/docker_utils.py @@ -2,7 +2,7 @@ import aiohttp from models_library.docker import DockerGenericTag -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError from settings_library.docker_registry import RegistrySettings from yarl import URL @@ -68,9 +68,9 @@ async def retrieve_image_layer_information( # if the image has multiple architectures json_response = await response.json() try: - multi_arch_manifests = parse_obj_as( - DockerImageMultiArchManifestsV2, json_response - ) + multi_arch_manifests = TypeAdapter( + DockerImageMultiArchManifestsV2 + ).validate_python(json_response) # find the correct platform digest = "" for manifest in multi_arch_manifests.manifests: @@ -89,8 +89,12 @@ async def retrieve_image_layer_information( response.raise_for_status() assert response.status == status.HTTP_200_OK # nosec json_response = await response.json() - return parse_obj_as(DockerImageManifestsV2, json_response) + return TypeAdapter(DockerImageManifestsV2).validate_python( + json_response + ) except ValidationError: - return parse_obj_as(DockerImageManifestsV2, json_response) + return TypeAdapter(DockerImageManifestsV2).validate_python( + json_response + ) return None diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index df81371cbb8..bce905e9cd6 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -5,8 +5,9 @@ from typing import Any from aiohttp import web +from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter from models_library.utils.json_serialization import json_dumps -from pydantic import AnyHttpUrl, PositiveFloat +from pydantic import PositiveFloat from ...aiohttp import status from ...long_running_tasks._models import TaskGet @@ -67,17 +68,14 @@ async def start_long_running_task( ip_addr, port = request_.transport.get_extra_info( "sockname" ) # https://docs.python.org/3/library/asyncio-protocol.html#asyncio.BaseTransport.get_extra_info - status_url = AnyHttpUrl( - url=f"http://{ip_addr}:{port}{request_.app.router['get_task_status'].url_for(task_id=task_id)}", - scheme="http", + status_url = AnyHttpUrlLegacyAdapter.validate_python( + f"http://{ip_addr}:{port}{request_.app.router['get_task_status'].url_for(task_id=task_id)}" # NOSONAR ) - result_url = AnyHttpUrl( - url=f"http://{ip_addr}:{port}{request_.app.router['get_task_result'].url_for(task_id=task_id)}", - scheme="http", + result_url = AnyHttpUrlLegacyAdapter.validate_python( + f"http://{ip_addr}:{port}{request_.app.router['get_task_result'].url_for(task_id=task_id)}" # NOSONAR ) - abort_url = AnyHttpUrl( - url=f"http://{ip_addr}:{port}{request_.app.router['cancel_and_delete_task'].url_for(task_id=task_id)}", - scheme="http", + abort_url = AnyHttpUrlLegacyAdapter.validate_python( + f"http://{ip_addr}:{port}{request_.app.router['cancel_and_delete_task'].url_for(task_id=task_id)}" # NOSONAR ) task_get = TaskGet( task_id=task_id, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py index c99cb1ce671..04071d5d07c 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/client.py @@ -35,7 +35,7 @@ async def _start(session: ClientSession, url: URL, json: RequestBody | None) -> data, error = unwrap_envelope(await response.json()) assert not error # nosec assert data is not None # nosec - return TaskGet.parse_obj(data) + return TaskGet.model_validate(data) @retry(**_DEFAULT_AIOHTTP_RETRY_POLICY) @@ -57,7 +57,7 @@ async def _wait_for_completion( data, error = unwrap_envelope(await response.json()) assert not error # nosec assert data is not None # nosec - task_status = TaskStatus.parse_obj(data) + task_status = TaskStatus.model_validate(data) yield task_status.task_progress if not task_status.done: await asyncio.sleep( diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index 2fd5d0e41f0..b70f0e821e4 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -14,7 +14,7 @@ from aiohttp import web from models_library.utils.json_serialization import json_dumps -from pydantic import BaseModel, Extra, ValidationError, parse_obj_as +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from ..mimetype_constants import MIMETYPE_APPLICATION_JSON from . import status @@ -31,8 +31,9 @@ class RequestParams(BaseModel): class StrictRequestParams(BaseModel): """Use a base class for context, path and query parameters""" - class Config: - extra = Extra.forbid # strict + model_config = ConfigDict( + extra="forbid", + ) @contextmanager @@ -139,7 +140,7 @@ def parse_request_path_parameters_as( use_error_v1=use_enveloped_error_v1, ): data = dict(request.match_info) - return parameters_schema_cls.parse_obj(data) + return parameters_schema_cls.model_validate(data) def parse_request_query_parameters_as( @@ -168,8 +169,8 @@ def parse_request_query_parameters_as( ): data = dict(request.query) if hasattr(parameters_schema_cls, "parse_obj"): - return parameters_schema_cls.parse_obj(data) - model: ModelClass = parse_obj_as(parameters_schema_cls, data) + return parameters_schema_cls.model_validate(data) + model: ModelClass = TypeAdapter(parameters_schema_cls).validate_python(data) return model @@ -185,7 +186,7 @@ def parse_request_headers_as( use_error_v1=use_enveloped_error_v1, ): data = dict(request.headers) - return parameters_schema_cls.parse_obj(data) + return parameters_schema_cls.model_validate(data) async def parse_request_body_as( @@ -224,7 +225,7 @@ async def parse_request_body_as( # NOTE: model_schema can be 'list[T]' or 'dict[T]' which raise TypeError # with issubclass(model_schema, BaseModel) assert issubclass(model_schema_cls, BaseModel) # nosec - return model_schema_cls.parse_obj(body) # type: ignore [return-value] + return model_schema_cls.model_validate(body) # type: ignore [return-value] # used for model_schema like 'list[T]' or 'dict[T]' - return parse_obj_as(model_schema_cls, body) + return TypeAdapter(model_schema_cls).validate_python(body) # type: ignore[no-any-return] diff --git a/packages/service-library/src/servicelib/background_task.py b/packages/service-library/src/servicelib/background_task.py index e7a4c665c49..b1eba9bc54b 100644 --- a/packages/service-library/src/servicelib/background_task.py +++ b/packages/service-library/src/servicelib/background_task.py @@ -5,7 +5,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from typing import Final -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying from tenacity.stop import stop_after_attempt @@ -21,7 +21,7 @@ _MAX_TASK_CANCELLATION_ATTEMPTS: Final[int] = 3 -class PeriodicTaskCancellationError(PydanticErrorMixin, Exception): +class PeriodicTaskCancellationError(OsparcErrorMixin, Exception): msg_template: str = "Could not cancel task '{task_name}'" diff --git a/packages/service-library/src/servicelib/deferred_tasks/_redis_task_tracker.py b/packages/service-library/src/servicelib/deferred_tasks/_redis_task_tracker.py index 69762108e71..382cb6c9f04 100644 --- a/packages/service-library/src/servicelib/deferred_tasks/_redis_task_tracker.py +++ b/packages/service-library/src/servicelib/deferred_tasks/_redis_task_tracker.py @@ -33,13 +33,19 @@ async def get_new_unique_identifier(self) -> TaskUID: async def _get_raw(self, redis_key: str) -> TaskScheduleModel | None: found_data = await self.redis_client_sdk.redis.get(redis_key) - return None if found_data is None else TaskScheduleModel.parse_raw(found_data) + return ( + None + if found_data is None + else TaskScheduleModel.model_validate_json(found_data) + ) async def get(self, task_uid: TaskUID) -> TaskScheduleModel | None: return await self._get_raw(_get_key(task_uid)) async def save(self, task_uid: TaskUID, task_schedule: TaskScheduleModel) -> None: - await self.redis_client_sdk.redis.set(_get_key(task_uid), task_schedule.json()) + await self.redis_client_sdk.redis.set( + _get_key(task_uid), task_schedule.model_dump_json() + ) async def remove(self, task_uid: TaskUID) -> None: await self.redis_client_sdk.redis.delete(_get_key(task_uid)) diff --git a/packages/service-library/src/servicelib/docker_utils.py b/packages/service-library/src/servicelib/docker_utils.py index 3b3159d5916..df976c623af 100644 --- a/packages/service-library/src/servicelib/docker_utils.py +++ b/packages/service-library/src/servicelib/docker_utils.py @@ -11,7 +11,7 @@ from models_library.docker import DockerGenericTag from models_library.generated_models.docker_rest_api import ProgressDetail from models_library.utils.change_case import snake_to_camel -from pydantic import BaseModel, ByteSize, ValidationError, parse_obj_as +from pydantic import BaseModel, ByteSize, ConfigDict, TypeAdapter, ValidationError from settings_library.docker_registry import RegistrySettings from yarl import URL @@ -39,11 +39,11 @@ class DockerLayerSizeV2(BaseModel): media_type: str size: ByteSize digest: str - - class Config: - frozen = True - alias_generator = snake_to_camel - allow_population_by_field_name = True + model_config = ConfigDict( + frozen=True, + alias_generator=snake_to_camel, + populate_by_name=True, + ) class DockerImageManifestsV2(BaseModel): @@ -51,39 +51,41 @@ class DockerImageManifestsV2(BaseModel): media_type: str config: DockerLayerSizeV2 layers: list[DockerLayerSizeV2] - - class Config: - keep_untouched = (cached_property,) - frozen = True - alias_generator = snake_to_camel - allow_population_by_field_name = True + model_config = ConfigDict( + ignored_types=(cached_property,), + frozen=True, + alias_generator=snake_to_camel, + populate_by_name=True, + ) @cached_property def layers_total_size(self) -> ByteSize: - return parse_obj_as(ByteSize, sum(layer.size for layer in self.layers)) + return TypeAdapter(ByteSize).validate_python( + sum(layer.size for layer in self.layers) + ) class DockerImageMultiArchManifestsV2(BaseModel): schema_version: Literal[2] media_type: Literal["application/vnd.oci.image.index.v1+json"] manifests: list[dict[str, Any]] - - class Config: - frozen = True - alias_generator = snake_to_camel - allow_population_by_field_name = True + model_config = ConfigDict( + frozen=True, + alias_generator=snake_to_camel, + populate_by_name=True, + ) class _DockerPullImage(BaseModel): status: str - id: str | None - progress_detail: ProgressDetail | None - progress: str | None - - class Config: - frozen = True - alias_generator = snake_to_camel - allow_population_by_field_name = True + id: str | None = None + progress_detail: ProgressDetail | None = None + progress: str | None = None + model_config = ConfigDict( + frozen=True, + alias_generator=snake_to_camel, + populate_by_name=True, + ) DOCKER_HUB_HOST: Final[str] = "registry-1.docker.io" @@ -241,7 +243,9 @@ async def pull_image( image, stream=True, auth=registry_auth ): try: - parsed_progress = parse_obj_as(_DockerPullImage, pull_progress) + parsed_progress = TypeAdapter(_DockerPullImage).validate_python( + pull_progress + ) except ValidationError: _logger.exception( "Unexpected error while validating '%s'. " diff --git a/packages/service-library/src/servicelib/error_codes.py b/packages/service-library/src/servicelib/error_codes.py index 2803e3627ab..06cd14ac8bc 100644 --- a/packages/service-library/src/servicelib/error_codes.py +++ b/packages/service-library/src/servicelib/error_codes.py @@ -9,10 +9,9 @@ import re -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated -from pydantic.tools import parse_obj_as -from pydantic.types import constr +from pydantic import StringConstraints, TypeAdapter _LABEL = "OEC:{}" _PATTERN = r"OEC:\d+" @@ -20,11 +19,13 @@ if TYPE_CHECKING: ErrorCodeStr = str else: - ErrorCodeStr = constr(strip_whitespace=True, regex=_PATTERN) + ErrorCodeStr = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=_PATTERN) + ] def create_error_code(exception: BaseException) -> ErrorCodeStr: - return parse_obj_as(ErrorCodeStr, _LABEL.format(id(exception))) + return TypeAdapter(ErrorCodeStr).validate_python(_LABEL.format(id(exception))) def parse_error_code(obj) -> set[ErrorCodeStr]: diff --git a/packages/service-library/src/servicelib/fastapi/docker_utils.py b/packages/service-library/src/servicelib/fastapi/docker_utils.py index 1c71c190a47..c7db4c1cf2d 100644 --- a/packages/service-library/src/servicelib/fastapi/docker_utils.py +++ b/packages/service-library/src/servicelib/fastapi/docker_utils.py @@ -5,7 +5,7 @@ import httpx from models_library.basic_types import IDStr from models_library.docker import DockerGenericTag -from pydantic import ByteSize, ValidationError, parse_obj_as +from pydantic import ByteSize, TypeAdapter, ValidationError from settings_library.docker_registry import RegistrySettings from yarl import URL @@ -72,9 +72,9 @@ async def retrieve_image_layer_information( # if the image has multiple architectures json_response = response.json() try: - multi_arch_manifests = parse_obj_as( - DockerImageMultiArchManifestsV2, json_response - ) + multi_arch_manifests = TypeAdapter( + DockerImageMultiArchManifestsV2 + ).validate_python(json_response) # find the correct platform digest = "" for manifest in multi_arch_manifests.manifests: @@ -93,14 +93,20 @@ async def retrieve_image_layer_information( response.raise_for_status() assert response.status_code == status.HTTP_200_OK # nosec json_response = response.json() - return parse_obj_as(DockerImageManifestsV2, json_response) + return TypeAdapter(DockerImageManifestsV2).validate_python( + json_response + ) except ValidationError: - return parse_obj_as(DockerImageManifestsV2, json_response) + return TypeAdapter(DockerImageManifestsV2).validate_python( + json_response + ) return None -_DEFAULT_MIN_IMAGE_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "200MiB") +_DEFAULT_MIN_IMAGE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( + "200MiB" +) async def pull_images( diff --git a/packages/service-library/src/servicelib/fastapi/errors.py b/packages/service-library/src/servicelib/fastapi/errors.py index 9eebef84637..139ed573fbe 100644 --- a/packages/service-library/src/servicelib/fastapi/errors.py +++ b/packages/service-library/src/servicelib/fastapi/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class ApplicationRuntimeError(PydanticErrorMixin, RuntimeError): +class ApplicationRuntimeError(OsparcErrorMixin, RuntimeError): pass diff --git a/packages/service-library/src/servicelib/fastapi/http_client_thin.py b/packages/service-library/src/servicelib/fastapi/http_client_thin.py index e00e0d636a2..c113321a488 100644 --- a/packages/service-library/src/servicelib/fastapi/http_client_thin.py +++ b/packages/service-library/src/servicelib/fastapi/http_client_thin.py @@ -7,7 +7,7 @@ from httpx import AsyncClient, ConnectError, HTTPError, PoolTimeout, Response from httpx._types import TimeoutTypes, URLTypes -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin from tenacity import RetryCallState from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -30,7 +30,7 @@ """ -class BaseClientError(PydanticErrorMixin, Exception): +class BaseClientError(OsparcErrorMixin, Exception): """Used as based for all the raised errors""" msg_template: str = "{message}" diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index a2dda66735a..36458031ff8 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -4,9 +4,10 @@ import warnings from typing import Any, Awaitable, Callable, Final +from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter from fastapi import FastAPI, status from httpx import AsyncClient, HTTPError -from pydantic import AnyHttpUrl, PositiveFloat, parse_obj_as +from pydantic import PositiveFloat from tenacity import RetryCallState from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -23,6 +24,7 @@ DEFAULT_HTTP_REQUESTS_TIMEOUT: Final[PositiveFloat] = 15 + logger = logging.getLogger(__name__) @@ -113,7 +115,7 @@ class Client: status, result and/or cancel of a long running task. """ - def __init__(self, app: FastAPI, async_client: AsyncClient, base_url: AnyHttpUrl): + def __init__(self, app: FastAPI, async_client: AsyncClient, base_url: str): """ `app`: used byt the `Client` to recover the `ClientConfiguration` `async_client`: an AsyncClient instance used by `Client` @@ -128,12 +130,9 @@ def _client_configuration(self) -> ClientConfiguration: output: ClientConfiguration = self.app.state.long_running_client_configuration return output - def _get_url(self, path: str) -> AnyHttpUrl: - output: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, - f"{self._base_url}{self._client_configuration.router_prefix}{path}", - ) - return output + def _get_url(self, path: str) -> str: + url = f"{self._base_url}{self._client_configuration.router_prefix}{path}" + return f"{AnyHttpUrlLegacyAdapter.validate_python(url)}" @retry_on_http_errors async def get_task_status( @@ -152,7 +151,7 @@ async def get_task_status( body=result.text, ) - return TaskStatus.parse_obj(result.json()) + return TaskStatus.model_validate(result.json()) @retry_on_http_errors async def get_task_result( @@ -171,7 +170,7 @@ async def get_task_result( body=result.text, ) - task_result = TaskResult.parse_obj(result.json()) + task_result = TaskResult.model_validate(result.json()) if task_result.error is not None: raise TaskClientResultError(message=task_result.error) return task_result.result diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py index 7cb61f29140..2c001525173 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py @@ -96,7 +96,7 @@ async def periodic_task_result( async def _status_update() -> TaskStatus: task_status: TaskStatus = await client.get_task_status(task_id) - logger.debug("Task status %s", task_status.json()) + logger.debug("Task status %s", task_status.model_dump_json()) await progress_manager.update( task_id=task_id, message=task_status.task_progress.message, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index c5d7429f01a..e8306b6d187 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -50,4 +50,4 @@ async def on_shutdown() -> None: # add error handlers # NOTE: Exception handler can not be added during the on_startup script, otherwise not working correctly - app.add_exception_handler(BaseLongRunningError, base_long_running_error_handler) + app.add_exception_handler(BaseLongRunningError, base_long_running_error_handler) # type: ignore[arg-type] diff --git a/packages/service-library/src/servicelib/file_utils.py b/packages/service-library/src/servicelib/file_utils.py index c90468cba2a..a52854c26e7 100644 --- a/packages/service-library/src/servicelib/file_utils.py +++ b/packages/service-library/src/servicelib/file_utils.py @@ -10,9 +10,9 @@ # https://docs.python.org/3/library/os.html#os.remove from aiofiles.os import remove from aiofiles.os import wrap as sync_to_async -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter -CHUNK_4KB: Final[ByteSize] = parse_obj_as(ByteSize, "4kb") # 4K blocks +CHUNK_4KB: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("4kb") # 4K blocks class AsyncStream(Protocol): diff --git a/packages/service-library/src/servicelib/long_running_tasks/_errors.py b/packages/service-library/src/servicelib/long_running_tasks/_errors.py index 73722f746ac..44dc03157f2 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_errors.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_errors.py @@ -1,10 +1,10 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class BaseLongRunningError(PydanticErrorMixin, Exception): +class BaseLongRunningError(OsparcErrorMixin, Exception): """base exception for this module""" - code: str = "long_running_task.base_long_running_error" + code: str = "long_running_task.base_long_running_error" # type: ignore[assignment] class TaskAlreadyRunningError(BaseLongRunningError): diff --git a/packages/service-library/src/servicelib/long_running_tasks/_models.py b/packages/service-library/src/servicelib/long_running_tasks/_models.py index b211ca29fdc..fc240160b81 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_models.py @@ -15,7 +15,7 @@ TaskResult, TaskStatus, ) -from pydantic import BaseModel, Field, PositiveFloat +from pydantic import BaseModel, ConfigDict, Field, PositiveFloat TaskName: TypeAlias = str @@ -46,9 +46,9 @@ class TrackedTask(BaseModel): "polled by the client who created it" ), ) - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict( + arbitrary_types_allowed=True, + ) class ClientConfiguration(BaseModel): diff --git a/packages/service-library/src/servicelib/long_running_tasks/_task.py b/packages/service-library/src/servicelib/long_running_tasks/_task.py index 88960cb6327..641e78a96a8 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_task.py @@ -123,7 +123,9 @@ async def _stale_tasks_monitor_worker(self) -> None: logger.warning( "Removing stale task '%s' with status '%s'", task_id, - self.get_task_status(task_id, with_task_context=None).json(), + self.get_task_status( + task_id, with_task_context=None + ).model_dump_json(), ) await self.remove_task( task_id, with_task_context=None, reraise_errors=False @@ -210,7 +212,7 @@ def get_task_status( task = tracked_task.task done = task.done() - return TaskStatus.parse_obj( + return TaskStatus.model_validate( { "task_progress": tracked_task.task_progress, "done": done, diff --git a/packages/service-library/src/servicelib/progress_bar.py b/packages/service-library/src/servicelib/progress_bar.py index 782f89ba550..bf70c0c3e88 100644 --- a/packages/service-library/src/servicelib/progress_bar.py +++ b/packages/service-library/src/servicelib/progress_bar.py @@ -10,7 +10,7 @@ ProgressStructuredMessage, ProgressUnit, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter from .logging_utils import log_catch @@ -95,7 +95,7 @@ async def main_fct(): def __post_init__(self) -> None: if self.progress_unit is not None: - parse_obj_as(ProgressUnit, self.progress_unit) # type: ignore[arg-type] # mypy does not like Literal with parse_obj_as + TypeAdapter(ProgressUnit).validate_python(self.progress_unit) self._continuous_value_lock = asyncio.Lock() self.num_steps = max(1, self.num_steps) if self.step_weights: diff --git a/packages/service-library/src/servicelib/rabbitmq/_errors.py b/packages/service-library/src/servicelib/rabbitmq/_errors.py index 0e3efbf3a11..c105c2b8ff3 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/_errors.py @@ -1,21 +1,21 @@ from typing import Final -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin _ERROR_PREFIX: Final[str] = "rabbitmq_error" -class BaseRPCError(PydanticErrorMixin, RuntimeError): +class BaseRPCError(OsparcErrorMixin, RuntimeError): ... class RPCNotInitializedError(BaseRPCError): - code = f"{_ERROR_PREFIX}.not_started" + code = f"{_ERROR_PREFIX}.not_started" # type: ignore[assignment] msg_template = "Please check that the RabbitMQ RPC backend was initialized!" class RemoteMethodNotRegisteredError(BaseRPCError): - code = f"{_ERROR_PREFIX}.remote_not_registered" + code = f"{_ERROR_PREFIX}.remote_not_registered" # type: ignore[assignment] msg_template = ( "Could not find a remote method named: '{method_name}'. " "Message from remote server was returned: {incoming_message}. " diff --git a/packages/service-library/src/servicelib/rabbitmq/_models.py b/packages/service-library/src/servicelib/rabbitmq/_models.py index 565447072fa..e48e4bb13aa 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_models.py +++ b/packages/service-library/src/servicelib/rabbitmq/_models.py @@ -2,12 +2,13 @@ from collections.abc import Awaitable, Callable from typing import Any, Protocol +from models_library.basic_types import ConstrainedStr from models_library.rabbitmq_basic_types import ( REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS, RPCMethodName, RPCNamespace, ) -from pydantic import ConstrainedStr, parse_obj_as +from pydantic import TypeAdapter MessageHandler = Callable[[Any], Awaitable[bool]] @@ -23,11 +24,11 @@ def routing_key(self) -> str | None: class RPCNamespacedMethodName(ConstrainedStr): min_length: int = 1 max_length: int = 255 - regex: re.Pattern[str] | None = re.compile(REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS) + pattern: str = REGEX_RABBIT_QUEUE_ALLOWED_SYMBOLS @classmethod def from_namespace_and_method( cls, namespace: RPCNamespace, method_name: RPCMethodName ) -> "RPCNamespacedMethodName": namespaced_method_name = f"{namespace}.{method_name}" - return parse_obj_as(cls, namespaced_method_name) + return TypeAdapter(cls).validate_python(namespaced_method_name) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py index 5837beddcae..d278bb350ba 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py @@ -1,11 +1,8 @@ -from typing import Any - from common_library.errors_classes import OsparcErrorMixin class CatalogApiBaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + pass class CatalogItemNotFoundError(CatalogApiBaseError): diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index cc67413aefe..83efa8f1d0e 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -16,7 +16,7 @@ ) from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import NonNegativeInt, parse_obj_as, validate_arguments +from pydantic import NonNegativeInt, TypeAdapter, validate_call from servicelib.logging_utils import log_decorator from servicelib.rabbitmq._constants import RPC_REQUEST_DEFAULT_TIMEOUT_S @@ -40,7 +40,7 @@ async def list_services_paginated( # pylint: disable=too-many-arguments CatalogForbiddenError: no access-rights to list services """ - @validate_arguments() + @validate_call() async def _call( product_name: ProductName, user_id: UserID, @@ -49,7 +49,7 @@ async def _call( ): return await rpc_client.request( CATALOG_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "list_services_paginated"), + TypeAdapter(RPCMethodName).validate_python("list_services_paginated"), product_name=product_name, user_id=user_id, limit=limit, @@ -60,7 +60,9 @@ async def _call( result = await _call( product_name=product_name, user_id=user_id, limit=limit, offset=offset ) - assert parse_obj_as(PageRpc[ServiceGetV2], result) is not None # nosec + assert ( + TypeAdapter(PageRpc[ServiceGetV2]).validate_python(result) is not None + ) # nosec return cast(PageRpc[ServiceGetV2], result) @@ -80,7 +82,7 @@ async def get_service( CatalogForbiddenError: not access rights to read this service """ - @validate_arguments() + @validate_call() async def _call( product_name: ProductName, user_id: UserID, @@ -89,7 +91,7 @@ async def _call( ) -> Any: return await rpc_client.request( CATALOG_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_service"), + TypeAdapter(RPCMethodName).validate_python("get_service"), product_name=product_name, user_id=user_id, service_key=service_key, @@ -103,7 +105,7 @@ async def _call( service_key=service_key, service_version=service_version, ) - assert parse_obj_as(ServiceGetV2, result) is not None # nosec + assert TypeAdapter(ServiceGetV2).validate_python(result) is not None # nosec return cast(ServiceGetV2, result) @@ -125,7 +127,7 @@ async def update_service( CatalogForbiddenError: not access rights to read this service """ - @validate_arguments() + @validate_call() async def _call( product_name: ProductName, user_id: UserID, @@ -135,7 +137,7 @@ async def _call( ): return await rpc_client.request( CATALOG_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "update_service"), + TypeAdapter(RPCMethodName).validate_python("update_service"), product_name=product_name, user_id=user_id, service_key=service_key, @@ -150,7 +152,7 @@ async def _call( service_version=service_version, update=update, ) - assert parse_obj_as(ServiceGetV2, result) is not None # nosec + assert TypeAdapter(ServiceGetV2).validate_python(result) is not None # nosec return cast(ServiceGetV2, result) @@ -170,7 +172,7 @@ async def check_for_service( CatalogForbiddenError: not access rights to read this service """ - @validate_arguments() + @validate_call() async def _call( product_name: ProductName, user_id: UserID, @@ -179,7 +181,7 @@ async def _call( ): return await rpc_client.request( CATALOG_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "check_for_service"), + TypeAdapter(RPCMethodName).validate_python("check_for_service"), product_name=product_name, user_id=user_id, service_key=service_key, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/errors.py index 5e104db333c..6d7bf2a722c 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class BaseDynamicSchedulerRPCError(PydanticErrorMixin, Exception): +class BaseDynamicSchedulerRPCError(OsparcErrorMixin, Exception): ... diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py index 9da2dad425e..3dcc9ed502f 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py @@ -10,7 +10,7 @@ from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RabbitMQRPCClient @@ -26,6 +26,8 @@ DEFAULT_LEGACY_WB_TO_DV2_HTTP_REQUESTS_TIMEOUT_S * 2 ) +_RPC_METHOD_NAME_ADAPTER: TypeAdapter[RPCMethodName] = TypeAdapter(RPCMethodName) + @log_decorator(_logger, level=logging.DEBUG) async def get_service_status( @@ -33,7 +35,7 @@ async def get_service_status( ) -> NodeGetIdle | DynamicServiceGet | NodeGet: result = await rabbitmq_rpc_client.request( DYNAMIC_SCHEDULER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_service_status"), + _RPC_METHOD_NAME_ADAPTER.validate_python("get_service_status"), node_id=node_id, timeout_s=_RPC_DEFAULT_TIMEOUT_S, ) @@ -49,7 +51,7 @@ async def run_dynamic_service( ) -> DynamicServiceGet | NodeGet: result = await rabbitmq_rpc_client.request( DYNAMIC_SCHEDULER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "run_dynamic_service"), + _RPC_METHOD_NAME_ADAPTER.validate_python("run_dynamic_service"), dynamic_service_start=dynamic_service_start, timeout_s=_RPC_DEFAULT_TIMEOUT_S, ) @@ -66,7 +68,7 @@ async def stop_dynamic_service( ) -> None: result = await rabbitmq_rpc_client.request( DYNAMIC_SCHEDULER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "stop_dynamic_service"), + _RPC_METHOD_NAME_ADAPTER.validate_python("stop_dynamic_service"), dynamic_service_stop=dynamic_service_stop, timeout_s=timeout_s, ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py index 592959eb08c..ec05906b1ef 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/efs_guardian/efs_manager.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from ....logging_utils import log_decorator from ....rabbitmq import RabbitMQRPCClient @@ -27,7 +27,7 @@ async def create_project_specific_data_dir( ) -> Path: output: Path = await rabbitmq_rpc_client.request( EFS_GUARDIAN_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "create_project_specific_data_dir"), + TypeAdapter(RPCMethodName).validate_python("create_project_specific_data_dir"), project_id=project_id, node_id=node_id, storage_directory_name=storage_directory_name, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py index 44549841802..f9c1a24f406 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class ResourceUsageTrackerRuntimeError(PydanticErrorMixin, RuntimeError): +class ResourceUsageTrackerRuntimeError(OsparcErrorMixin, RuntimeError): msg_template: str = "Resource-usage-tracker unexpected error" diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_plans.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_plans.py index a7dc4b5d404..218cd139fb4 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_plans.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_plans.py @@ -16,7 +16,7 @@ PricingPlanUpdate, ) from models_library.services import ServiceKey, ServiceVersion -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from ....logging_utils import log_decorator from ....rabbitmq import RabbitMQRPCClient @@ -26,6 +26,8 @@ _DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 20 +_RPC_METHOD_NAME_ADAPTER: TypeAdapter[RPCMethodName] = TypeAdapter(RPCMethodName) + @log_decorator(_logger, level=logging.DEBUG) async def get_pricing_plan( @@ -36,7 +38,7 @@ async def get_pricing_plan( ) -> PricingPlanGet: result: PricingPlanGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_pricing_plan"), + _RPC_METHOD_NAME_ADAPTER.validate_python("get_pricing_plan"), product_name=product_name, pricing_plan_id=pricing_plan_id, timeout_s=_DEFAULT_TIMEOUT_S, @@ -53,7 +55,7 @@ async def list_pricing_plans( ) -> list[PricingPlanGet]: result: PricingPlanGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "list_pricing_plans"), + _RPC_METHOD_NAME_ADAPTER.validate_python("list_pricing_plans"), product_name=product_name, timeout_s=_DEFAULT_TIMEOUT_S, ) @@ -69,7 +71,7 @@ async def create_pricing_plan( ) -> PricingPlanGet: result: PricingPlanGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "create_pricing_plan"), + _RPC_METHOD_NAME_ADAPTER.validate_python("create_pricing_plan"), data=data, timeout_s=_DEFAULT_TIMEOUT_S, ) @@ -86,7 +88,7 @@ async def update_pricing_plan( ) -> PricingPlanGet: result: PricingPlanGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "update_pricing_plan"), + _RPC_METHOD_NAME_ADAPTER.validate_python("update_pricing_plan"), product_name=product_name, data=data, timeout_s=_DEFAULT_TIMEOUT_S, @@ -104,8 +106,8 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ) -> list[PricingPlanToServiceGet]: result: PricingPlanGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as( - RPCMethodName, "list_connected_services_to_pricing_plan_by_pricing_plan" + _RPC_METHOD_NAME_ADAPTER.validate_python( + "list_connected_services_to_pricing_plan_by_pricing_plan" ), product_name=product_name, pricing_plan_id=pricing_plan_id, @@ -126,7 +128,7 @@ async def connect_service_to_pricing_plan( ) -> PricingPlanToServiceGet: result: PricingPlanGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "connect_service_to_pricing_plan"), + _RPC_METHOD_NAME_ADAPTER.validate_python("connect_service_to_pricing_plan"), product_name=product_name, pricing_plan_id=pricing_plan_id, service_key=service_key, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_units.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_units.py index cec80e7186a..afa5611a92d 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_units.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/pricing_units.py @@ -15,7 +15,7 @@ PricingUnitWithCostCreate, PricingUnitWithCostUpdate, ) -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from ....logging_utils import log_decorator from ....rabbitmq import RabbitMQRPCClient @@ -25,6 +25,8 @@ _DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 20 +_RPC_METHOD_NAME_ADAPTER: TypeAdapter[RPCMethodName] = TypeAdapter(RPCMethodName) + @log_decorator(_logger, level=logging.DEBUG) async def get_pricing_unit( @@ -36,7 +38,7 @@ async def get_pricing_unit( ) -> PricingUnitGet: result: PricingUnitGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_pricing_unit"), + _RPC_METHOD_NAME_ADAPTER.validate_python("get_pricing_unit"), product_name=product_name, pricing_plan_id=pricing_plan_id, pricing_unit_id=pricing_unit_id, @@ -55,7 +57,7 @@ async def create_pricing_unit( ) -> PricingUnitGet: result: PricingUnitGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "create_pricing_unit"), + _RPC_METHOD_NAME_ADAPTER.validate_python("create_pricing_unit"), product_name=product_name, data=data, timeout_s=_DEFAULT_TIMEOUT_S, @@ -73,7 +75,7 @@ async def update_pricing_unit( ) -> PricingUnitGet: result: PricingUnitGet = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "update_pricing_unit"), + _RPC_METHOD_NAME_ADAPTER.validate_python("update_pricing_unit"), product_name=product_name, data=data, timeout_s=_DEFAULT_TIMEOUT_S, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py index e826363897a..ad7b2fd908b 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py @@ -18,7 +18,7 @@ from models_library.rest_ordering import OrderBy from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import AnyUrl, NonNegativeInt, parse_obj_as +from pydantic import AnyUrl, NonNegativeInt, TypeAdapter from ....logging_utils import log_decorator from ....rabbitmq import RabbitMQRPCClient @@ -28,6 +28,8 @@ _DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 20 +_RPC_METHOD_NAME_ADAPTER: TypeAdapter[RPCMethodName] = TypeAdapter(RPCMethodName) + @log_decorator(_logger, level=logging.DEBUG) async def get_service_run_page( @@ -44,7 +46,7 @@ async def get_service_run_page( ) -> ServiceRunPage: result = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_service_run_page"), + _RPC_METHOD_NAME_ADAPTER.validate_python("get_service_run_page"), user_id=user_id, product_name=product_name, limit=limit, @@ -74,7 +76,9 @@ async def get_osparc_credits_aggregated_usages_page( ) -> OsparcCreditsAggregatedUsagesPage: result = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_osparc_credits_aggregated_usages_page"), + _RPC_METHOD_NAME_ADAPTER.validate_python( + "get_osparc_credits_aggregated_usages_page" + ), user_id=user_id, product_name=product_name, limit=limit, @@ -102,7 +106,7 @@ async def export_service_runs( ) -> AnyUrl: result: AnyUrl = await rabbitmq_rpc_client.request( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "export_service_runs"), + _RPC_METHOD_NAME_ADAPTER.validate_python("export_service_runs"), user_id=user_id, product_name=product_name, wallet_id=wallet_id, diff --git a/packages/service-library/src/servicelib/redis.py b/packages/service-library/src/servicelib/redis.py index 03847ae0b04..fce89d7790e 100644 --- a/packages/service-library/src/servicelib/redis.py +++ b/packages/service-library/src/servicelib/redis.py @@ -10,8 +10,8 @@ import redis.asyncio as aioredis import redis.exceptions +from common_library.errors_classes import OsparcErrorMixin from pydantic import NonNegativeFloat, NonNegativeInt -from pydantic.errors import PydanticErrorMixin from redis.asyncio.lock import Lock from redis.asyncio.retry import Retry from redis.backoff import ExponentialBackoff @@ -36,7 +36,7 @@ _logger = logging.getLogger(__name__) -class BaseRedisError(PydanticErrorMixin, RuntimeError): +class BaseRedisError(OsparcErrorMixin, RuntimeError): ... diff --git a/packages/service-library/src/servicelib/utils_meta.py b/packages/service-library/src/servicelib/utils_meta.py index 46fa78dd83e..6ee48fd4d56 100644 --- a/packages/service-library/src/servicelib/utils_meta.py +++ b/packages/service-library/src/servicelib/utils_meta.py @@ -6,7 +6,7 @@ from models_library.basic_types import VersionStr from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter class PackageInfo: @@ -40,7 +40,7 @@ def version(self) -> Version: @property def __version__(self) -> VersionStr: - return parse_obj_as(VersionStr, self._distribution.version) + return TypeAdapter(VersionStr).validate_python(self._distribution.version) @property def api_prefix_path_tag(self) -> str: diff --git a/packages/service-library/src/servicelib/utils_secrets.py b/packages/service-library/src/servicelib/utils_secrets.py index 7b74a491080..67c440ce044 100644 --- a/packages/service-library/src/servicelib/utils_secrets.py +++ b/packages/service-library/src/servicelib/utils_secrets.py @@ -2,7 +2,7 @@ import string from typing import Final -from pydantic import StrictInt, validate_arguments +from pydantic import StrictInt, validate_call MIN_PASSWORD_LENGTH = 30 _SAFE_SYMBOLS = "!$%*+,-.:=?@^_~" # avoid issues with parsing, espapes etc @@ -48,7 +48,7 @@ def are_secrets_equal(got: str, expected: str) -> bool: return secrets.compare_digest(got.encode("utf8"), expected.encode("utf8")) -@validate_arguments +@validate_call def secure_randint(start: StrictInt, end: StrictInt) -> int: """Generate a random integer between start (inclusive) and end (exclusive).""" if start >= end: diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py index 68d06d15278..987a68a4036 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py @@ -9,7 +9,7 @@ from aiohttp import web from aiohttp.test_utils import TestClient from faker import Faker -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.long_running_tasks.server import TaskId @@ -92,7 +92,7 @@ async def _caller(client: TestClient, **query_kwargs) -> TaskId: data, error = await assert_status(resp, status.HTTP_202_ACCEPTED) assert data assert not error - task_get = parse_obj_as(long_running_tasks.server.TaskGet, data) + task_get = TypeAdapter(long_running_tasks.server.TaskGet).validate_python(data) return task_get.task_id return _caller @@ -122,7 +122,7 @@ async def _waiter( data, error = await assert_status(result, status.HTTP_200_OK) assert data assert not error - task_status = long_running_tasks.server.TaskStatus.parse_obj(data) + task_status = long_running_tasks.server.TaskStatus.model_validate(data) assert task_status assert task_status.done diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index afd9e8f4fde..7907f092c24 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -18,7 +18,7 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.long_running_tasks.server import TaskGet, TaskId @@ -75,12 +75,12 @@ async def test_workflow( data, error = await assert_status(result, status.HTTP_200_OK) assert data assert not error - task_status = long_running_tasks.server.TaskStatus.parse_obj(data) + task_status = long_running_tasks.server.TaskStatus.model_validate(data) assert task_status progress_updates.append( (task_status.task_progress.message, task_status.task_progress.percent) ) - print(f"<-- received task status: {task_status.json(indent=2)}") + print(f"<-- received task status: {task_status.model_dump_json(indent=2)}") assert task_status.done, "task incomplete" print( f"-- waiting for task status completed successfully: {json.dumps(attempt.retry_state.retry_object.statistics, indent=2)}" @@ -216,7 +216,7 @@ async def test_list_tasks( result = await client.get(f"{list_url}") data, error = await assert_status(result, status.HTTP_200_OK) assert not error - list_of_tasks = parse_obj_as(list[TaskGet], data) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(data) assert len(list_of_tasks) == NUM_TASKS # the task name is properly formatted @@ -235,5 +235,5 @@ async def test_list_tasks( result = await client.get(f"{list_url}") data, error = await assert_status(result, status.HTTP_200_OK) assert not error - list_of_tasks = parse_obj_as(list[TaskGet], data) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(data) assert len(list_of_tasks) == NUM_TASKS - (task_index + 1) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index 941ae31359d..5671eda108f 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -18,7 +18,7 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient -from pydantic import create_model, parse_obj_as +from pydantic import TypeAdapter, create_model from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import long_running_tasks, status from servicelib.aiohttp.long_running_tasks._server import ( @@ -108,7 +108,7 @@ async def test_list_tasks( result = await client_with_task_context.get(f"{list_url}") data, error = await assert_status(result, status.HTTP_200_OK) assert not error - list_of_tasks = parse_obj_as(list[TaskGet], data) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(data) assert len(list_of_tasks) == 0 # the list should be full if we pass the expected context @@ -117,7 +117,7 @@ async def test_list_tasks( ) data, error = await assert_status(result, status.HTTP_200_OK) assert not error - list_of_tasks = parse_obj_as(list[TaskGet], data) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(data) assert len(list_of_tasks) == 1 diff --git a/packages/service-library/tests/aiohttp/test_docker_utils.py b/packages/service-library/tests/aiohttp/test_docker_utils.py index 890ffdc588b..bcd2129abd2 100644 --- a/packages/service-library/tests/aiohttp/test_docker_utils.py +++ b/packages/service-library/tests/aiohttp/test_docker_utils.py @@ -11,7 +11,7 @@ from faker import Faker from models_library.docker import DockerGenericTag from models_library.progress_bar import ProgressReport -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from servicelib import progress_bar from servicelib.aiohttp.docker_utils import retrieve_image_layer_information @@ -42,8 +42,7 @@ async def test_retrieve_image_layer_information( if "sha256" in service_tag: image_name = f"{service_repo}@{service_tag}" await remove_images_from_host([image_name]) - docker_image = parse_obj_as( - DockerGenericTag, + docker_image = TypeAdapter(DockerGenericTag).validate_python( f"{registry_settings.REGISTRY_URL}/{osparc_service['image']['name']}:{osparc_service['image']['tag']}", ) layer_information = await retrieve_image_layer_information( @@ -97,13 +96,13 @@ def _assert_progress_report_values( # check first progress assert mocked_progress_cb.call_args_list[0].args[0].dict( exclude={"message"} - ) == ProgressReport(actual_value=0, total=total, unit="Byte").dict( + ) == ProgressReport(actual_value=0, total=total, unit="Byte").model_dump( exclude={"message"} ) # check last progress assert mocked_progress_cb.call_args_list[-1].args[0].dict( exclude={"message"} - ) == ProgressReport(actual_value=total, total=total, unit="Byte").dict( + ) == ProgressReport(actual_value=total, total=total, unit="Byte").model_dump( exclude={"message"} ) diff --git a/packages/service-library/tests/aiohttp/test_requests_validation.py b/packages/service-library/tests/aiohttp/test_requests_validation.py index 08e2f07bfbe..4e1b4f4e2e7 100644 --- a/packages/service-library/tests/aiohttp/test_requests_validation.py +++ b/packages/service-library/tests/aiohttp/test_requests_validation.py @@ -11,7 +11,7 @@ from aiohttp.test_utils import TestClient from faker import Faker from models_library.utils.json_serialization import json_dumps -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -41,9 +41,9 @@ def create_fake(cls, faker: Faker): class MyRequestPathParams(BaseModel): project_uuid: UUID - - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra="forbid", + ) @classmethod def create_fake(cls, faker: Faker): @@ -55,7 +55,7 @@ class MyRequestQueryParams(BaseModel): label: str def as_params(self, **kwargs) -> dict[str, str]: - data = self.dict(**kwargs) + data = self.model_dump(**kwargs) return {k: f"{v}" for k, v in data.items()} @classmethod @@ -66,9 +66,9 @@ def create_fake(cls, faker: Faker): class MyRequestHeadersParams(BaseModel): user_agent: str = Field(alias="X-Simcore-User-Agent") optional_header: str | None = Field(default=None, alias="X-Simcore-Optional-Header") - - class Config: - allow_population_by_field_name = False + model_config = ConfigDict( + populate_by_name=False, + ) @classmethod def create_fake(cls, faker: Faker): @@ -111,7 +111,9 @@ def client(event_loop, aiohttp_client: Callable, faker: Faker) -> TestClient: async def _handler(request: web.Request) -> web.Response: # --------- UNDER TEST ------- # NOTE: app context does NOT need to be validated everytime! - context = MyRequestContext.parse_obj({**dict(request.app), **dict(request)}) + context = MyRequestContext.model_validate( + {**dict(request.app), **dict(request)} + ) path_params = parse_request_path_parameters_as( MyRequestPathParams, request, use_enveloped_error_v1=False @@ -129,11 +131,11 @@ async def _handler(request: web.Request) -> web.Response: return web.json_response( { - "parameters": path_params.dict(), - "queries": query_params.dict(), - "body": body.dict(), - "context": context.dict(), - "headers": headers_params.dict(), + "parameters": path_params.model_dump(), + "queries": query_params.model_dump(), + "body": body.model_dump(), + "context": context.model_dump(), + "headers": headers_params.model_dump(), }, dumps=json_dumps, ) @@ -194,21 +196,21 @@ async def test_parse_request_as( r = await client.get( f"/projects/{path_params.project_uuid}", params=query_params.as_params(), - json=body.dict(), - headers=headers_params.dict(by_alias=True), + json=body.model_dump(), + headers=headers_params.model_dump(by_alias=True), ) assert r.status == status.HTTP_200_OK, f"{await r.text()}" got = await r.json() - assert got["parameters"] == jsonable_encoder(path_params.dict()) - assert got["queries"] == jsonable_encoder(query_params.dict()) - assert got["body"] == body.dict() + assert got["parameters"] == jsonable_encoder(path_params.model_dump()) + assert got["queries"] == jsonable_encoder(query_params.model_dump()) + assert got["body"] == body.model_dump() assert got["context"] == { "secret": client.app[APP_SECRET_KEY], "user_id": 42, } - assert got["headers"] == jsonable_encoder(headers_params.dict()) + assert got["headers"] == jsonable_encoder(headers_params.model_dump()) async def test_parse_request_with_invalid_path_params( @@ -221,8 +223,8 @@ async def test_parse_request_with_invalid_path_params( r = await client.get( "/projects/invalid-uuid", params=query_params.as_params(), - json=body.dict(), - headers=headers_params.dict(by_alias=True), + json=body.model_dump(), + headers=headers_params.model_dump(by_alias=True), ) assert r.status == status.HTTP_422_UNPROCESSABLE_ENTITY, f"{await r.text()}" @@ -234,8 +236,8 @@ async def test_parse_request_with_invalid_path_params( "details": [ { "loc": "project_uuid", - "msg": "value is not a valid uuid", - "type": "type_error.uuid", + "msg": "Input should be a valid UUID, invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "type": "uuid_parsing", } ], } @@ -252,8 +254,8 @@ async def test_parse_request_with_invalid_query_params( r = await client.get( f"/projects/{path_params.project_uuid}", params={}, - json=body.dict(), - headers=headers_params.dict(by_alias=True), + json=body.model_dump(), + headers=headers_params.model_dump(by_alias=True), ) assert r.status == status.HTTP_422_UNPROCESSABLE_ENTITY, f"{await r.text()}" @@ -265,8 +267,8 @@ async def test_parse_request_with_invalid_query_params( "details": [ { "loc": "label", - "msg": "field required", - "type": "value_error.missing", + "msg": "Field required", + "type": "missing", } ], } @@ -284,7 +286,7 @@ async def test_parse_request_with_invalid_body( f"/projects/{path_params.project_uuid}", params=query_params.as_params(), json={"invalid": "body"}, - headers=headers_params.dict(by_alias=True), + headers=headers_params.model_dump(by_alias=True), ) assert r.status == status.HTTP_422_UNPROCESSABLE_ENTITY, f"{await r.text()}" @@ -298,13 +300,13 @@ async def test_parse_request_with_invalid_body( "details": [ { "loc": "x", - "msg": "field required", - "type": "value_error.missing", + "msg": "Field required", + "type": "missing", }, { "loc": "z", - "msg": "field required", - "type": "value_error.missing", + "msg": "Field required", + "type": "missing", }, ], } @@ -322,7 +324,7 @@ async def test_parse_request_with_invalid_json_body( f"/projects/{path_params.project_uuid}", params=query_params.as_params(), data=b"[ 1 2, 3 'broken-json' ]", - headers=headers_params.dict(by_alias=True), + headers=headers_params.model_dump(by_alias=True), ) body = await r.text() @@ -340,8 +342,8 @@ async def test_parse_request_with_invalid_headers_params( r = await client.get( f"/projects/{path_params.project_uuid}", params=query_params.as_params(), - json=body.dict(), - headers=headers_params.dict(), # we pass the wrong names + json=body.model_dump(), + headers=headers_params.model_dump(), # we pass the wrong names ) assert r.status == status.HTTP_422_UNPROCESSABLE_ENTITY, f"{await r.text()}" @@ -353,8 +355,8 @@ async def test_parse_request_with_invalid_headers_params( "details": [ { "loc": "X-Simcore-User-Agent", - "msg": "field required", - "type": "value_error.missing", + "msg": "Field required", + "type": "missing", } ], } diff --git a/packages/service-library/tests/deferred_tasks/example_app.py b/packages/service-library/tests/deferred_tasks/example_app.py index 75850fddc2e..1962d0b4232 100644 --- a/packages/service-library/tests/deferred_tasks/example_app.py +++ b/packages/service-library/tests/deferred_tasks/example_app.py @@ -107,8 +107,8 @@ async def _commands_handler( ) -> Any: """Handles all commands send by remote party""" if command == "init-context": - context.redis_settings = RedisSettings.parse_raw(payload["redis"]) - context.rabbit_settings = RabbitSettings.parse_raw(payload["rabbit"]) + context.redis_settings = RedisSettings.model_validate_json(payload["redis"]) + context.rabbit_settings = RabbitSettings.model_validate_json(payload["rabbit"]) # using the same db as the deferred tasks with different keys context.in_memory_lists = InMemoryLists(context.redis_settings, port) diff --git a/packages/service-library/tests/deferred_tasks/test__redis_task_tracker.py b/packages/service-library/tests/deferred_tasks/test__redis_task_tracker.py index 3ec3bde01ed..366759e22d3 100644 --- a/packages/service-library/tests/deferred_tasks/test__redis_task_tracker.py +++ b/packages/service-library/tests/deferred_tasks/test__redis_task_tracker.py @@ -5,7 +5,7 @@ from datetime import timedelta import pytest -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.deferred_tasks._models import TaskUID from servicelib.deferred_tasks._redis_task_tracker import RedisTaskTracker from servicelib.deferred_tasks._task_schedule import TaskScheduleModel, TaskState @@ -19,8 +19,7 @@ @pytest.fixture def task_schedule() -> TaskScheduleModel: - return parse_obj_as( - TaskScheduleModel, + return TypeAdapter(TaskScheduleModel).validate_python( { "timeout": timedelta(seconds=1), "execution_attempts": 1, diff --git a/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py b/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py index 0ea55a62eee..9ea22f87ed1 100644 --- a/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py +++ b/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py @@ -16,6 +16,8 @@ import psutil import pytest from aiohttp.test_utils import unused_port +from common_library.serialization import model_dump_with_secrets +from models_library.utils.json_serialization import json_dumps from pydantic import NonNegativeFloat, NonNegativeInt from pytest_mock import MockerFixture from servicelib import redis as servicelib_redis @@ -24,7 +26,6 @@ from servicelib.sequences_utils import partition_gen from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings -from settings_library.utils_encoders import create_json_encoder_wo_secrets from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -123,7 +124,6 @@ async def _tcp_command( def _get_serialization_options() -> dict[str, Any]: return { - "encoder": create_json_encoder_wo_secrets(RabbitSettings), "exclude_defaults": True, "exclude_none": True, "exclude_unset": True, @@ -158,8 +158,20 @@ async def start(self) -> None: response = await _tcp_command( "init-context", { - "rabbit": self.rabbit_service.json(**_get_serialization_options()), - "redis": self.redis_service.json(**_get_serialization_options()), + "rabbit": json_dumps( + model_dump_with_secrets( + self.rabbit_service, + show_secrets=True, + **_get_serialization_options(), + ) + ), + "redis": json_dumps( + model_dump_with_secrets( + self.redis_service, + show_secrets=True, + **_get_serialization_options(), + ) + ), "max-workers": self.max_workers, }, port=self.remote_process.port, diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index bd55b44d498..52527f138d9 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -18,7 +18,7 @@ from asgi_lifespan import LifespanManager from fastapi import APIRouter, Depends, FastAPI, status from httpx import AsyncClient -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.fastapi import long_running_tasks from servicelib.long_running_tasks._models import TaskGet, TaskId from servicelib.long_running_tasks._task import TaskContext @@ -94,7 +94,9 @@ async def _caller(app: FastAPI, client: AsyncClient, **query_kwargs) -> TaskId: ) resp = await client.post(f"{url}") assert resp.status_code == status.HTTP_202_ACCEPTED - task_id = parse_obj_as(long_running_tasks.server.TaskId, resp.json()) + task_id = TypeAdapter(long_running_tasks.server.TaskId).validate_python( + resp.json() + ) return task_id return _caller @@ -122,7 +124,7 @@ async def _waiter( with attempt: result = await client.get(f"{status_url}") assert result.status_code == status.HTTP_200_OK - task_status = long_running_tasks.server.TaskStatus.parse_obj( + task_status = long_running_tasks.server.TaskStatus.model_validate( result.json() ) assert task_status @@ -149,12 +151,14 @@ async def test_workflow( with attempt: result = await client.get(f"{status_url}") assert result.status_code == status.HTTP_200_OK - task_status = long_running_tasks.server.TaskStatus.parse_obj(result.json()) + task_status = long_running_tasks.server.TaskStatus.model_validate( + result.json() + ) assert task_status progress_updates.append( (task_status.task_progress.message, task_status.task_progress.percent) ) - print(f"<-- received task status: {task_status.json(indent=2)}") + print(f"<-- received task status: {task_status.model_dump_json(indent=2)}") assert task_status.done, "task incomplete" print( f"-- waiting for task status completed successfully: {json.dumps(attempt.retry_state.retry_object.statistics, indent=2)}" @@ -179,7 +183,7 @@ async def test_workflow( result = await client.get(f"{result_url}") # NOTE: this is DIFFERENT than with aiohttp where we return the real result assert result.status_code == status.HTTP_200_OK - task_result = long_running_tasks.server.TaskResult.parse_obj(result.json()) + task_result = long_running_tasks.server.TaskResult.model_validate(result.json()) assert not task_result.error assert task_result.result == [f"{x}" for x in range(10)] # getting the result again should raise a 404 @@ -218,7 +222,7 @@ async def test_failing_task_returns_error( result_url = app.url_path_for("get_task_result", task_id=task_id) result = await client.get(f"{result_url}") assert result.status_code == status.HTTP_200_OK - task_result = long_running_tasks.server.TaskResult.parse_obj(result.json()) + task_result = long_running_tasks.server.TaskResult.model_validate(result.json()) assert not task_result.result assert task_result.error @@ -274,7 +278,7 @@ async def test_list_tasks_empty_list(app: FastAPI, client: AsyncClient): list_url = app.url_path_for("list_tasks") result = await client.get(f"{list_url}") assert result.status_code == status.HTTP_200_OK - list_of_tasks = parse_obj_as(list[TaskGet], result.json()) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(result.json()) assert list_of_tasks == [] @@ -296,7 +300,7 @@ async def test_list_tasks( list_url = app.url_path_for("list_tasks") result = await client.get(f"{list_url}") assert result.status_code == status.HTTP_200_OK - list_of_tasks = parse_obj_as(list[TaskGet], result.json()) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(result.json()) assert len(list_of_tasks) == NUM_TASKS # now wait for them to finish @@ -311,5 +315,5 @@ async def test_list_tasks( # the list shall go down one by one result = await client.get(f"{list_url}") assert result.status_code == status.HTTP_200_OK - list_of_tasks = parse_obj_as(list[TaskGet], result.json()) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(result.json()) assert len(list_of_tasks) == NUM_TASKS - (task_index + 1) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 9f15184b052..0dc440bf33e 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -4,11 +4,12 @@ import asyncio from typing import AsyncIterable, Final +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy import pytest from asgi_lifespan import LifespanManager from fastapi import APIRouter, Depends, FastAPI, status from httpx import AsyncClient -from pydantic import AnyHttpUrl, PositiveFloat, parse_obj_as +from pydantic import PositiveFloat, TypeAdapter from servicelib.fastapi.long_running_tasks._context_manager import _ProgressManager from servicelib.fastapi.long_running_tasks.client import ( Client, @@ -90,7 +91,7 @@ async def bg_task_app( @pytest.fixture def mock_task_id() -> TaskId: - return parse_obj_as(TaskId, "fake_task_id") + return TypeAdapter(TaskId).validate_python("fake_task_id") async def test_task_result( @@ -100,7 +101,7 @@ async def test_task_result( assert result.status_code == status.HTTP_200_OK, result.text task_id = result.json() - url = parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io") client = Client(app=bg_task_app, async_client=async_client, base_url=url) async with periodic_task_result( client, @@ -120,7 +121,7 @@ async def test_task_result_times_out( assert result.status_code == status.HTTP_200_OK, result.text task_id = result.json() - url = parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io") client = Client(app=bg_task_app, async_client=async_client, base_url=url) timeout = TASK_SLEEP_INTERVAL / 10 with pytest.raises(TaskClientTimeoutError) as exec_info: @@ -146,7 +147,7 @@ async def test_task_result_task_result_is_an_error( assert result.status_code == status.HTTP_200_OK, result.text task_id = result.json() - url = parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io") client = Client(app=bg_task_app, async_client=async_client, base_url=url) with pytest.raises(TaskClientResultError) as exec_info: async with periodic_task_result( @@ -185,13 +186,13 @@ async def progress_update( assert received == ("", None) for _ in range(repeat): - await progress_updater.update(mock_task_id, percent=ProgressPercent(0.0)) + await progress_updater.update(mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(0.0)) assert counter == 2 assert received == ("", 0.0) for _ in range(repeat): await progress_updater.update( - mock_task_id, percent=ProgressPercent(1.0), message="done" + mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(1.0), message="done" ) assert counter == 3 assert received == ("done", 1.0) diff --git a/packages/service-library/tests/fastapi/test_docker_utils.py b/packages/service-library/tests/fastapi/test_docker_utils.py index 4db0db99bd0..f6d78066c97 100644 --- a/packages/service-library/tests/fastapi/test_docker_utils.py +++ b/packages/service-library/tests/fastapi/test_docker_utils.py @@ -12,7 +12,7 @@ from faker import Faker from models_library.docker import DockerGenericTag from models_library.progress_bar import ProgressReport -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from servicelib import progress_bar from servicelib.docker_utils import pull_image @@ -46,8 +46,7 @@ async def test_retrieve_image_layer_information( if "sha256" in service_tag: image_name = f"{service_repo}@{service_tag}" await remove_images_from_host([image_name]) - docker_image = parse_obj_as( - DockerGenericTag, + docker_image = TypeAdapter(DockerGenericTag).validate_python( f"{registry_settings.REGISTRY_URL}/{osparc_service['image']['name']}:{osparc_service['image']['tag']}", ) layer_information = await retrieve_image_layer_information( @@ -103,13 +102,13 @@ def _assert_progress_report_values( # check first progress assert mocked_progress_cb.call_args_list[0].args[0].dict( exclude={"message"} - ) == ProgressReport(actual_value=0, total=total, unit="Byte").dict( + ) == ProgressReport(actual_value=0, total=total, unit="Byte").model_dump( exclude={"message"} ) # check last progress assert mocked_progress_cb.call_args_list[-1].args[0].dict( exclude={"message"} - ) == ProgressReport(actual_value=total, total=total, unit="Byte").dict( + ) == ProgressReport(actual_value=total, total=total, unit="Byte").model_dump( exclude={"message"} ) @@ -202,7 +201,7 @@ async def test_pull_image_without_layer_information( assert layer_information print(f"{image=} has {layer_information.layers_total_size=}") - fake_number_of_steps = parse_obj_as(ByteSize, "200MiB") + fake_number_of_steps = TypeAdapter(ByteSize).validate_python("200MiB") assert fake_number_of_steps > layer_information.layers_total_size async with progress_bar.ProgressBarData( num_steps=fake_number_of_steps, diff --git a/packages/service-library/tests/fastapi/test_exceptions_utils.py b/packages/service-library/tests/fastapi/test_exceptions_utils.py index 845043f3405..cfe7fbde0e8 100644 --- a/packages/service-library/tests/fastapi/test_exceptions_utils.py +++ b/packages/service-library/tests/fastapi/test_exceptions_utils.py @@ -10,7 +10,7 @@ from fastapi import FastAPI, HTTPException from httpx import AsyncClient from models_library.api_schemas__common.errors import DefaultApiError -from pydantic import parse_raw_as +from pydantic import TypeAdapter from servicelib.fastapi.exceptions_utils import ( handle_errors_as_500, http_exception_as_json_response, @@ -66,7 +66,7 @@ async def test_http_errors_respond_with_error_model( response = await client.post(f"/error/{code}") assert response.status_code == code - error = parse_raw_as(DefaultApiError, response.text) + error = TypeAdapter(DefaultApiError).validate_json(response.text) assert error.detail == f"test {code}" assert error.name @@ -79,4 +79,4 @@ async def test_non_http_error_handling( response = await client.post(f"/raise/{code}") print(response) - error = parse_raw_as(DefaultApiError, response.text) + error = TypeAdapter(DefaultApiError).validate_json(response.text) diff --git a/packages/service-library/tests/fastapi/test_http_client_thin.py b/packages/service-library/tests/fastapi/test_http_client_thin.py index f98de720c33..7bd96b25eee 100644 --- a/packages/service-library/tests/fastapi/test_http_client_thin.py +++ b/packages/service-library/tests/fastapi/test_http_client_thin.py @@ -3,6 +3,7 @@ import logging from collections.abc import AsyncIterable, Iterable from typing import Final +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy import arrow import pytest @@ -15,7 +16,7 @@ TransportError, codes, ) -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from respx import MockRouter from servicelib.fastapi.http_client_thin import ( BaseThinClient, @@ -76,12 +77,14 @@ async def thick_client(request_timeout: int) -> AsyncIterable[FakeThickClient]: @pytest.fixture -def test_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://missing-host:1111") +def test_url() -> str: + url =TypeAdapter(AnyHttpUrlLegacy).validate_python("http://missing-host:1111") + return f"{url}" async def test_connection_error( - thick_client: FakeThickClient, test_url: AnyHttpUrl + thick_client: FakeThickClient, + test_url: str, ) -> None: with pytest.raises(ClientHttpError) as exe_info: await thick_client.get_provided_url(test_url) @@ -92,7 +95,7 @@ async def test_connection_error( async def test_retry_on_errors( request_timeout: int, - test_url: AnyHttpUrl, + test_url: str, caplog_info_level: pytest.LogCaptureFixture, ) -> None: client = FakeThickClient(total_retry_interval=request_timeout) @@ -108,7 +111,7 @@ async def test_retry_on_errors_by_error_type( error_class: type[RequestError], caplog_info_level: pytest.LogCaptureFixture, request_timeout: int, - test_url: AnyHttpUrl, + test_url: str, ) -> None: class ATestClient(BaseThinClient): # pylint: disable=no-self-use @@ -177,7 +180,7 @@ async def public_method_no_annotation(self): async def test_expect_state_decorator( - test_url: AnyHttpUrl, + test_url: str, respx_mock: MockRouter, request_timeout: int, ) -> None: diff --git a/packages/service-library/tests/fastapi/test_openapi.py b/packages/service-library/tests/fastapi/test_openapi.py index 0edd438c73a..54f7e017799 100644 --- a/packages/service-library/tests/fastapi/test_openapi.py +++ b/packages/service-library/tests/fastapi/test_openapi.py @@ -44,7 +44,7 @@ def test_exclusive_min_openapi_issue(app: FastAPI): def test_overriding_openapi_method(app: FastAPI): assert not hasattr(app, "_original_openapi") - assert app.openapi.__doc__ is None + #assert app.openapi.__doc__ is None # PC why was this set to check that it is none? it's coming from the base fastapi applicaiton and now they provide some docs override_fastapi_openapi_method(app) diff --git a/packages/service-library/tests/fastapi/test_rabbitmq.py b/packages/service-library/tests/fastapi/test_rabbitmq.py index 9c94cfa0766..b41a94097f2 100644 --- a/packages/service-library/tests/fastapi/test_rabbitmq.py +++ b/packages/service-library/tests/fastapi/test_rabbitmq.py @@ -132,6 +132,6 @@ async def test_post_message( f"--> checking for message in rabbit exchange {rabbit_message.channel_name}, {attempt.retry_state.retry_object.statistics}" ) mocked_message_handler.assert_called_once_with( - rabbit_message.json().encode() + rabbit_message.model_dump_json().encode() ) print("... message received") diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py b/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py index 3019b07d6ab..ba7576e3027 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py @@ -86,13 +86,13 @@ async def test_rabbit_client_with_paused_container( await rabbit_client.publish(exchange_name, message) -def _get_rabbitmq_api_params(rabbit_service: RabbitSettings) -> dict[str, str]: +def _get_rabbitmq_api_params(rabbit_service: RabbitSettings) -> dict[str, Any]: return { "scheme": "http", - "user": rabbit_service.RABBIT_USER, + "username": rabbit_service.RABBIT_USER, "password": rabbit_service.RABBIT_PASSWORD.get_secret_value(), "host": rabbit_service.RABBIT_HOST, - "port": "15672", + "port": 15672, } diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc.py b/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc.py index e192afc611e..46588de6e87 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc.py @@ -350,11 +350,12 @@ async def _a_handler() -> None: pass if expect_fail: - with pytest.raises(ValidationError) as exec_info: + with pytest.raises( + ValidationError, match="String should have at most 255 characters" + ): await rpc_server.register_handler( RPCNamespace("a"), RPCMethodName(handler_name), _a_handler ) - assert "ensure this value has at most 255 characters" in f"{exec_info.value}" else: await rpc_server.register_handler( RPCNamespace("a"), RPCMethodName(handler_name), _a_handler diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq_utils.py b/packages/service-library/tests/rabbitmq/test_rabbitmq_utils.py index b07f8e8cb8d..2615a92ac56 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq_utils.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq_utils.py @@ -26,18 +26,18 @@ def test_rpc_namespace_sorts_elements(): def test_rpc_namespace_too_long(): with pytest.raises(ValidationError) as exec_info: RPCNamespace.from_entries({f"test{i}": f"test{i}" for i in range(20)}) - assert "ensure this value has at most 252 characters" in f"{exec_info.value}" + assert "String should have at most 252 characters" in f"{exec_info.value}" @pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() # no rabbitmq instance running def test_rpc_namespace_too_short(): with pytest.raises(ValidationError) as exec_info: RPCNamespace.from_entries({}) - assert "ensure this value has at least 1 characters" in f"{exec_info.value}" + assert "String should have at least 1 character" in f"{exec_info.value}" @pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() # no rabbitmq instance running def test_rpc_namespace_invalid_symbols(): with pytest.raises(ValidationError) as exec_info: RPCNamespace.from_entries({"test": "@"}) - assert "string does not match regex" in f"{exec_info.value}" + assert "String should match pattern" in f"{exec_info.value}" diff --git a/packages/service-library/tests/test_archiving_utils.py b/packages/service-library/tests/test_archiving_utils.py index f6886ea509a..bb6f2b486c4 100644 --- a/packages/service-library/tests/test_archiving_utils.py +++ b/packages/service-library/tests/test_archiving_utils.py @@ -18,7 +18,7 @@ import pytest from faker import Faker -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_benchmark.plugin import BenchmarkFixture from servicelib import archiving_utils from servicelib.archiving_utils import ArchiveError, archive_dir, unarchive_dir @@ -566,7 +566,8 @@ async def _archive_dir_performance( @pytest.mark.skip(reason="manual testing") @pytest.mark.parametrize( - "compress, file_size, num_files", [(False, parse_obj_as(ByteSize, "1Mib"), 10000)] + "compress, file_size, num_files", + [(False, TypeAdapter(ByteSize).validate_python("1Mib"), 10000)], ) def test_archive_dir_performance( benchmark: BenchmarkFixture, diff --git a/packages/settings-library/requirements/_base.in b/packages/settings-library/requirements/_base.in index 9672f593a7f..91f4dd23b04 100644 --- a/packages/settings-library/requirements/_base.in +++ b/packages/settings-library/requirements/_base.in @@ -2,12 +2,10 @@ # Specifies third-party dependencies for 'settings-library' # --constraint ../../../requirements/constraints.txt - -# intra-repo required dependencies --requirement ../../../packages/common-library/requirements/_base.in -pydantic>=1.9 - +pydantic +pydantic-settings # extra rich diff --git a/packages/settings-library/requirements/_base.txt b/packages/settings-library/requirements/_base.txt index 900c4fea2aa..fc4a246dc95 100644 --- a/packages/settings-library/requirements/_base.txt +++ b/packages/settings-library/requirements/_base.txt @@ -1,15 +1,26 @@ +annotated-types==0.7.0 + # via pydantic click==8.1.7 # via typer markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -pydantic==1.10.17 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-settings==2.5.2 + # via -r requirements/_base.in pygments==2.18.0 # via rich +python-dotenv==1.0.1 + # via pydantic-settings rich==13.7.1 # via # -r requirements/_base.in @@ -21,4 +32,5 @@ typer==0.12.4 typing-extensions==4.12.2 # via # pydantic + # pydantic-core # typer diff --git a/packages/settings-library/requirements/_test.txt b/packages/settings-library/requirements/_test.txt index 1ca7d43dd3c..56bf15d9c2d 100644 --- a/packages/settings-library/requirements/_test.txt +++ b/packages/settings-library/requirements/_test.txt @@ -2,8 +2,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in iniconfig==2.0.0 @@ -34,12 +32,10 @@ pytest-sugar==1.0.0 python-dateutil==2.9.0.post0 # via faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in six==1.16.0 # via python-dateutil termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # pytest diff --git a/packages/settings-library/requirements/_tools.txt b/packages/settings-library/requirements/_tools.txt index a75c5397d80..d14257822b0 100644 --- a/packages/settings-library/requirements/_tools.txt +++ b/packages/settings-library/requirements/_tools.txt @@ -67,21 +67,11 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/settings-library/requirements/ci.txt b/packages/settings-library/requirements/ci.txt index d950945b44b..aeacf04c2e5 100644 --- a/packages/settings-library/requirements/ci.txt +++ b/packages/settings-library/requirements/ci.txt @@ -11,8 +11,8 @@ --requirement _test.txt # installs this repo's packages -pytest-simcore @ ../pytest-simcore simcore-common-library @ ../common-library/ +pytest-simcore @ ../pytest-simcore # current module simcore-settings-library @ . diff --git a/packages/settings-library/requirements/dev.txt b/packages/settings-library/requirements/dev.txt index 2168d7f36e1..de2adb4ecbb 100644 --- a/packages/settings-library/requirements/dev.txt +++ b/packages/settings-library/requirements/dev.txt @@ -1,4 +1,4 @@ -# Shortcut to install all packages needed to develop 'models-library' +# Shortcut to install all packages needed to develop 'settings-library' # # - As ci.txt but with current and repo packages in develop (edit) mode # diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index 296b453e26c..c3f0e103e7a 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -1,19 +1,12 @@ import logging -from collections.abc import Sequence from functools import cached_property -from typing import Final, get_args, get_origin - -from pydantic import ( - BaseConfig, - BaseSettings, - ConfigError, - Extra, - ValidationError, - validator, -) -from pydantic.error_wrappers import ErrorList, ErrorWrapper -from pydantic.fields import ModelField, Undefined -from pydantic.typing import is_literal_type +from typing import Any, Final, get_origin + +from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable +from pydantic import ValidationInfo, field_validator +from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined, ValidationError +from pydantic_settings import BaseSettings, SettingsConfigDict _logger = logging.getLogger(__name__) @@ -22,41 +15,33 @@ ] = "%s auto_default_from_env unresolved, defaulting to None" -class DefaultFromEnvFactoryError(ValidationError): - ... +class DefaultFromEnvFactoryError(ValueError): + def __init__(self, errors): + super().__init__() + self.errors = errors -def create_settings_from_env(field: ModelField): +def _create_settings_from_env(field_name: str, info: FieldInfo): # NOTE: Cannot pass only field.type_ because @prepare_field (when this function is called) # this value is still not resolved (field.type_ at that moment has a weak_ref). # Therefore we keep the entire 'field' but MUST be treated here as read-only def _default_factory(): """Creates default from sub-settings or None (if nullable)""" - field_settings_cls = field.type_ + field_settings_cls = get_type(info) try: return field_settings_cls() except ValidationError as err: - if field.allow_none: + if is_nullable(info): # e.g. Optional[PostgresSettings] would warn if defaults to None _logger.warning( _DEFAULTS_TO_NONE_MSG, - field.name, + field_name, ) return None - - def _prepend_field_name(ee: ErrorList): - if isinstance(ee, ErrorWrapper): - return ErrorWrapper(ee.exc, (field.name, *ee.loc_tuple())) - assert isinstance(ee, Sequence) # nosec - return [_prepend_field_name(e) for e in ee] - - raise DefaultFromEnvFactoryError( - errors=_prepend_field_name(err.raw_errors), - model=err.model, - # FIXME: model = shall be the parent settings?? but I dont find how retrieve it from the field - ) from err + _logger.warning("Validation errors=%s", err.errors()) + raise DefaultFromEnvFactoryError(errors=err.errors()) from err return _default_factory @@ -70,40 +55,46 @@ class BaseCustomSettings(BaseSettings): SEE tests for details. """ - @validator("*", pre=True) + @field_validator("*", mode="before") @classmethod - def parse_none(cls, v, field: ModelField): + def _parse_none(cls, v, info: ValidationInfo): # WARNING: In nullable fields, envs equal to null or none are parsed as None !! - if field.allow_none and isinstance(v, str) and v.lower() in ("null", "none"): + if ( + info.field_name + and is_nullable(cls.model_fields[info.field_name]) + and isinstance(v, str) + and v.lower() in ("none",) + ): return None return v - class Config(BaseConfig): - case_sensitive = True # All must be capitalized - extra = Extra.forbid - allow_mutation = False - frozen = True - validate_all = True - keep_untouched = (cached_property,) - - @classmethod - def prepare_field(cls, field: ModelField) -> None: - super().prepare_field(field) + model_config = SettingsConfigDict( + case_sensitive=True, # All must be capitalized + extra="forbid", + frozen=True, + validate_default=True, + ignored_types=(cached_property,), + env_parse_none_str="null", + ) - auto_default_from_env = field.field_info.extra.get( - "auto_default_from_env", False + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any): + super().__pydantic_init_subclass__(**kwargs) + + for name, field in cls.model_fields.items(): + auto_default_from_env = ( + field.json_schema_extra is not None + and field.json_schema_extra.get( # type: ignore[union-attr] + "auto_default_from_env", False + ) ) - - field_type = field.type_ - if args := get_args(field_type): - field_type = next(a for a in args if a != type(None)) + field_type = get_type(field) # Avoids issubclass raising TypeError. SEE test_issubclass_type_error_with_pydantic_models is_not_composed = ( get_origin(field_type) is None ) # is not composed as dict[str, Any] or Generic[Base] - # avoid literals raising TypeError - is_not_literal = is_literal_type(field.type_) is False + is_not_literal = not is_literal(field) if ( is_not_literal @@ -111,25 +102,26 @@ def prepare_field(cls, field: ModelField) -> None: and issubclass(field_type, BaseCustomSettings) ): if auto_default_from_env: - assert field.field_info.default is Undefined - assert field.field_info.default_factory is None + assert field.default is PydanticUndefined + assert field.default_factory is None # Transform it into something like `Field(default_factory=create_settings_from_env(field))` - field.default_factory = create_settings_from_env(field) + field.default_factory = _create_settings_from_env(name, field) field.default = None - field.required = False # has a default now elif ( is_not_literal and is_not_composed and issubclass(field_type, BaseSettings) ): - msg = f"{cls}.{field.name} of type {field_type} must inherit from BaseCustomSettings" - raise ConfigError(msg) + msg = f"{cls}.{name} of type {field_type} must inherit from BaseCustomSettings" + raise ValueError(msg) elif auto_default_from_env: - msg = f"auto_default_from_env=True can only be used in BaseCustomSettings subclassesbut field {cls}.{field.name} is {field_type} " - raise ConfigError(msg) + msg = f"auto_default_from_env=True can only be used in BaseCustomSettings subclasses but field {cls}.{name} is {field_type} " + raise ValueError(msg) + + cls.model_rebuild(force=True) @classmethod def create_from_envs(cls, **overrides): diff --git a/packages/settings-library/src/settings_library/basic_types.py b/packages/settings-library/src/settings_library/basic_types.py index 277832669e1..d912b5d4e39 100644 --- a/packages/settings-library/src/settings_library/basic_types.py +++ b/packages/settings-library/src/settings_library/basic_types.py @@ -3,21 +3,17 @@ # This is a minor evil to avoid the maintenance burden that creates # an extra dependency to a larger models_library (intra-repo library) -import re from enum import Enum +from typing import Annotated, TypeAlias -from pydantic import ConstrainedInt, ConstrainedStr - +from pydantic import Field, StringConstraints # port number range -class PortInt(ConstrainedInt): - gt = 0 - lt = 65535 +PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)] # e.g. 'v5' -class VersionTag(ConstrainedStr): - regex = re.compile(r"^v\d$") +VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")] class LogLevel(str, Enum): @@ -55,7 +51,6 @@ class BuildTargetEnum(str, Enum): # non-empty bounded string used as identifier # e.g. "123" or "name_123" or "fa327c73-52d8-462a-9267-84eeaf0f90e3" but NOT "" -class IDStr(ConstrainedStr): - strip_whitespace = True - min_length = 1 - max_length = 50 +IDStr: TypeAlias = Annotated[ + str, StringConstraints(strip_whitespace=True, min_length=1, max_length=50) +] diff --git a/packages/settings-library/src/settings_library/catalog.py b/packages/settings-library/src/settings_library/catalog.py index e5f44f29269..17c71237e81 100644 --- a/packages/settings-library/src/settings_library/catalog.py +++ b/packages/settings-library/src/settings_library/catalog.py @@ -1,6 +1,5 @@ from functools import cached_property -from pydantic import parse_obj_as from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -13,7 +12,7 @@ class CatalogSettings(BaseCustomSettings, MixinServiceSettings): CATALOG_HOST: str = "catalog" CATALOG_PORT: PortInt = DEFAULT_FASTAPI_PORT - CATALOG_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + CATALOG_VTAG: VersionTag = "v0" @cached_property def api_base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/comp_services.py b/packages/settings-library/src/settings_library/comp_services.py index e3cb628f7b7..71901e61624 100644 --- a/packages/settings-library/src/settings_library/comp_services.py +++ b/packages/settings-library/src/settings_library/comp_services.py @@ -1,5 +1,4 @@ -from pydantic import ByteSize, NonNegativeInt, validator -from pydantic.tools import parse_raw_as +from pydantic import ByteSize, NonNegativeInt, TypeAdapter, field_validator from settings_library.base import BaseCustomSettings from ._constants import GB @@ -10,19 +9,19 @@ class ComputationalServices(BaseCustomSettings): DEFAULT_MAX_NANO_CPUS: NonNegativeInt = _DEFAULT_MAX_NANO_CPUS_VALUE - DEFAULT_MAX_MEMORY: ByteSize = parse_raw_as( - ByteSize, f"{_DEFAULT_MAX_MEMORY_VALUE}" + DEFAULT_MAX_MEMORY: ByteSize = TypeAdapter(ByteSize).validate_python( + f"{_DEFAULT_MAX_MEMORY_VALUE}" ) DEFAULT_RUNTIME_TIMEOUT: NonNegativeInt = 0 - @validator("DEFAULT_MAX_NANO_CPUS", pre=True) + @field_validator("DEFAULT_MAX_NANO_CPUS", mode="before") @classmethod def _set_default_cpus_if_negative(cls, v): if v is None or v == "" or int(v) <= 0: v = _DEFAULT_MAX_NANO_CPUS_VALUE return v - @validator("DEFAULT_MAX_MEMORY", pre=True) + @field_validator("DEFAULT_MAX_MEMORY", mode="before") @classmethod def _set_default_memory_if_negative(cls, v): if v is None or v == "" or int(v) <= 0: diff --git a/packages/settings-library/src/settings_library/director_v2.py b/packages/settings-library/src/settings_library/director_v2.py index 78c5edd78c6..baf32956c8e 100644 --- a/packages/settings-library/src/settings_library/director_v2.py +++ b/packages/settings-library/src/settings_library/director_v2.py @@ -1,6 +1,5 @@ from functools import cached_property -from pydantic import parse_obj_as from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -13,7 +12,7 @@ class DirectorV2Settings(BaseCustomSettings, MixinServiceSettings): DIRECTOR_V2_HOST: str = "director-v2" DIRECTOR_V2_PORT: PortInt = DEFAULT_FASTAPI_PORT - DIRECTOR_V2_VTAG: VersionTag = parse_obj_as(VersionTag, "v2") + DIRECTOR_V2_VTAG: VersionTag = "v2" @cached_property def api_base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/docker_registry.py b/packages/settings-library/src/settings_library/docker_registry.py index bb365cb9785..e899ce45718 100644 --- a/packages/settings-library/src/settings_library/docker_registry.py +++ b/packages/settings-library/src/settings_library/docker_registry.py @@ -1,7 +1,8 @@ from functools import cached_property -from typing import Any, ClassVar +from typing import Any -from pydantic import Field, SecretStr, validator +from pydantic import Field, SecretStr, field_validator +from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings @@ -23,7 +24,7 @@ class RegistrySettings(BaseCustomSettings): ) REGISTRY_SSL: bool = Field(..., description="access to registry through ssl") - @validator("REGISTRY_PATH", pre=True) + @field_validator("REGISTRY_PATH", mode="before") @classmethod def _escape_none_string(cls, v) -> Any | None: return None if v == "None" else v @@ -36,8 +37,8 @@ def resolved_registry_url(self) -> str: def api_url(self) -> str: return f"{self.REGISTRY_URL}/v2" - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + json_schema_extra={ "examples": [ { "REGISTRY_AUTH": "True", @@ -48,3 +49,4 @@ class Config(BaseCustomSettings.Config): } ], } + ) diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py index 2cd7cf0b9a6..22d2d9af9ee 100644 --- a/packages/settings-library/src/settings_library/ec2.py +++ b/packages/settings-library/src/settings_library/ec2.py @@ -1,20 +1,23 @@ -from typing import Any, ClassVar +from typing import Annotated -from pydantic import Field +from pydantic import AnyHttpUrl, BeforeValidator, Field, TypeAdapter +from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings +ANY_HTTP_URL_ADAPTER: TypeAdapter = TypeAdapter(AnyHttpUrl) + class EC2Settings(BaseCustomSettings): EC2_ACCESS_KEY_ID: str - EC2_ENDPOINT: str | None = Field( - default=None, description="do not define if using standard AWS" - ) + EC2_ENDPOINT: Annotated[ + str, BeforeValidator(lambda x: str(ANY_HTTP_URL_ADAPTER.validate_python(x))) + ] | None = Field(default=None, description="do not define if using standard AWS") EC2_REGION_NAME: str = "us-east-1" EC2_SECRET_ACCESS_KEY: str - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + json_schema_extra={ "examples": [ { "EC2_ACCESS_KEY_ID": "my_access_key_id", @@ -24,3 +27,4 @@ class Config(BaseCustomSettings.Config): } ], } + ) diff --git a/packages/settings-library/src/settings_library/efs.py b/packages/settings-library/src/settings_library/efs.py index d09b8abb20f..34c48f9dca6 100644 --- a/packages/settings-library/src/settings_library/efs.py +++ b/packages/settings-library/src/settings_library/efs.py @@ -8,7 +8,7 @@ class AwsEfsSettings(BaseCustomSettings): EFS_DNS_NAME: str = Field( description="AWS Elastic File System DNS name", - example="fs-xxx.efs.us-east-1.amazonaws.com", + examples=["fs-xxx.efs.us-east-1.amazonaws.com"], ) EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str EFS_MOUNTED_PATH: Path = Field( @@ -16,7 +16,7 @@ class AwsEfsSettings(BaseCustomSettings): ) EFS_ONLY_ENABLED_FOR_USERIDS: list[int] = Field( description="This is temporary solution so we can enable it for specific users for testing purpose", - example=[1], + examples=[[1]], ) diff --git a/packages/settings-library/src/settings_library/email.py b/packages/settings-library/src/settings_library/email.py index b15bf209405..fe5f8448b34 100644 --- a/packages/settings-library/src/settings_library/email.py +++ b/packages/settings-library/src/settings_library/email.py @@ -1,6 +1,7 @@ from enum import Enum +from typing import Self -from pydantic import root_validator +from pydantic import model_validator from pydantic.fields import Field from pydantic.types import SecretStr @@ -31,25 +32,23 @@ class SMTPSettings(BaseCustomSettings): SMTP_USERNAME: str | None = Field(None, min_length=1) SMTP_PASSWORD: SecretStr | None = Field(None, min_length=1) - @root_validator - @classmethod - def _both_credentials_must_be_set(cls, values): - username = values.get("SMTP_USERNAME") - password = values.get("SMTP_PASSWORD") + @model_validator(mode="after") + def _both_credentials_must_be_set(self) -> Self: + username = self.SMTP_USERNAME + password = self.SMTP_PASSWORD if username is None and password or username and password is None: msg = f"Please provide both {username=} and {password=} not just one" raise ValueError(msg) - return values + return self - @root_validator - @classmethod - def _enabled_tls_required_authentication(cls, values): - smtp_protocol = values.get("SMTP_PROTOCOL") + @model_validator(mode="after") + def _enabled_tls_required_authentication(self) -> Self: + smtp_protocol = self.SMTP_PROTOCOL - username = values.get("SMTP_USERNAME") - password = values.get("SMTP_PASSWORD") + username = self.SMTP_USERNAME + password = self.SMTP_PASSWORD tls_enabled = smtp_protocol == EmailProtocol.TLS starttls_enabled = smtp_protocol == EmailProtocol.STARTTLS @@ -57,7 +56,7 @@ def _enabled_tls_required_authentication(cls, values): if (tls_enabled or starttls_enabled) and not (username or password): msg = "when using SMTP_PROTOCOL other than UNENCRYPTED username and password are required" raise ValueError(msg) - return values + return self @property def has_credentials(self) -> bool: diff --git a/packages/settings-library/src/settings_library/node_ports.py b/packages/settings-library/src/settings_library/node_ports.py index 2a5d12f1bd7..522fcdd0991 100644 --- a/packages/settings-library/src/settings_library/node_ports.py +++ b/packages/settings-library/src/settings_library/node_ports.py @@ -1,7 +1,7 @@ from datetime import timedelta from typing import Final -from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, root_validator +from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator from .base import BaseCustomSettings from .postgres import PostgresSettings @@ -11,8 +11,8 @@ class StorageAuthSettings(StorageSettings): - STORAGE_USERNAME: str | None - STORAGE_PASSWORD: SecretStr | None + STORAGE_USERNAME: str | None = None + STORAGE_PASSWORD: SecretStr | None = None STORAGE_SECURE: bool = False @property @@ -21,11 +21,11 @@ def auth_required(self) -> bool: # for details see https://github.com/ITISFoundation/osparc-issues/issues/1264 return self.STORAGE_USERNAME is not None and self.STORAGE_PASSWORD is not None - @root_validator + @model_validator(mode="after") @classmethod def _validate_auth_fields(cls, values): - username = values["STORAGE_USERNAME"] - password = values["STORAGE_PASSWORD"] + username = values.STORAGE_USERNAME + password = values.STORAGE_PASSWORD if (username is None) != (password is None): msg = f"Both {username=} and {password=} must be either set or unset!" raise ValueError(msg) @@ -33,9 +33,13 @@ def _validate_auth_fields(cls, values): class NodePortsSettings(BaseCustomSettings): - NODE_PORTS_STORAGE_AUTH: StorageAuthSettings = Field(auto_default_from_env=True) + NODE_PORTS_STORAGE_AUTH: StorageAuthSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - POSTGRES_SETTINGS: PostgresSettings = Field(auto_default_from_env=True) + POSTGRES_SETTINGS: PostgresSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) NODE_PORTS_MULTIPART_UPLOAD_COMPLETION_TIMEOUT_S: NonNegativeInt = int( timedelta(minutes=5).total_seconds() diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index f8335bbeed2..ed7377877be 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -1,8 +1,15 @@ import urllib.parse from functools import cached_property -from typing import Any, ClassVar -from pydantic import Field, PostgresDsn, SecretStr, validator +from pydantic import ( + AliasChoices, + Field, + PostgresDsn, + SecretStr, + ValidationInfo, + field_validator, +) +from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings from .basic_types import PortInt @@ -11,7 +18,7 @@ class PostgresSettings(BaseCustomSettings): # entrypoint POSTGRES_HOST: str - POSTGRES_PORT: PortInt = PortInt(5432) + POSTGRES_PORT: PortInt = 5432 # auth POSTGRES_USER: str @@ -31,43 +38,47 @@ class PostgresSettings(BaseCustomSettings): POSTGRES_CLIENT_NAME: str | None = Field( default=None, description="Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)", - env=[ + validation_alias=AliasChoices( "POSTGRES_CLIENT_NAME", # This is useful when running inside a docker container, then the hostname is set each client gets a different name "HOST", "HOSTNAME", - ], + ), ) - @validator("POSTGRES_MAXSIZE") + @field_validator("POSTGRES_MAXSIZE") @classmethod - def _check_size(cls, v, values): - if not (values["POSTGRES_MINSIZE"] <= v): - msg = f"assert POSTGRES_MINSIZE={values['POSTGRES_MINSIZE']} <= POSTGRES_MAXSIZE={v}" + def _check_size(cls, v, info: ValidationInfo): + if info.data["POSTGRES_MINSIZE"] > v: + msg = f"assert POSTGRES_MINSIZE={info.data['POSTGRES_MINSIZE']} <= POSTGRES_MAXSIZE={v}" raise ValueError(msg) return v @cached_property def dsn(self) -> str: - dsn: str = PostgresDsn.build( - scheme="postgresql", - user=self.POSTGRES_USER, - password=self.POSTGRES_PASSWORD.get_secret_value(), - host=self.POSTGRES_HOST, - port=f"{self.POSTGRES_PORT}", - path=f"/{self.POSTGRES_DB}", + dsn: str = str( + PostgresDsn.build( # pylint: disable=no-member + scheme="postgresql", + username=self.POSTGRES_USER, + password=self.POSTGRES_PASSWORD.get_secret_value(), + host=self.POSTGRES_HOST, + port=self.POSTGRES_PORT, + path=f"{self.POSTGRES_DB}", + ) ) return dsn @cached_property def dsn_with_async_sqlalchemy(self) -> str: - dsn: str = PostgresDsn.build( - scheme="postgresql+asyncpg", - user=self.POSTGRES_USER, - password=self.POSTGRES_PASSWORD.get_secret_value(), - host=self.POSTGRES_HOST, - port=f"{self.POSTGRES_PORT}", - path=f"/{self.POSTGRES_DB}", + dsn: str = str( + PostgresDsn.build( # pylint: disable=no-member + scheme="postgresql+asyncpg", + username=self.POSTGRES_USER, + password=self.POSTGRES_PASSWORD.get_secret_value(), + host=self.POSTGRES_HOST, + port=self.POSTGRES_PORT, + path=f"{self.POSTGRES_DB}", + ) ) return dsn @@ -81,8 +92,8 @@ def dsn_with_query(self) -> str: ) return dsn - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + json_schema_extra={ "examples": [ # minimal required { @@ -94,3 +105,4 @@ class Config(BaseCustomSettings.Config): } ], } + ) diff --git a/packages/settings-library/src/settings_library/prometheus.py b/packages/settings-library/src/settings_library/prometheus.py index 065c7e930f0..9c40293d463 100644 --- a/packages/settings-library/src/settings_library/prometheus.py +++ b/packages/settings-library/src/settings_library/prometheus.py @@ -9,7 +9,7 @@ class PrometheusSettings(BaseCustomSettings, MixinServiceSettings): PROMETHEUS_URL: AnyUrl - PROMETHEUS_VTAG: VersionTag = VersionTag("v1") + PROMETHEUS_VTAG: VersionTag = "v1" PROMETHEUS_USERNAME: str | None = None PROMETHEUS_PASSWORD: SecretStr | None = None @@ -24,14 +24,16 @@ def origin(self) -> str: @cached_property def api_url(self) -> str: assert self.PROMETHEUS_URL.host # nosec - prometheus_url: str = AnyUrl.build( - scheme=self.PROMETHEUS_URL.scheme, - user=self.PROMETHEUS_USERNAME, - password=self.PROMETHEUS_PASSWORD.get_secret_value() - if self.PROMETHEUS_PASSWORD - else None, - host=self.PROMETHEUS_URL.host, - port=self.PROMETHEUS_URL.port, - path=self.PROMETHEUS_URL.path, + prometheus_url: str = str( + AnyUrl.build( + scheme=self.PROMETHEUS_URL.scheme, + username=self.PROMETHEUS_USERNAME, + password=self.PROMETHEUS_PASSWORD.get_secret_value() + if self.PROMETHEUS_PASSWORD + else None, + host=self.PROMETHEUS_URL.host, + port=self.PROMETHEUS_URL.port, + path=self.PROMETHEUS_URL.path, + ) ) return prometheus_url diff --git a/packages/settings-library/src/settings_library/r_clone.py b/packages/settings-library/src/settings_library/r_clone.py index ff04d509bef..c4288466928 100644 --- a/packages/settings-library/src/settings_library/r_clone.py +++ b/packages/settings-library/src/settings_library/r_clone.py @@ -13,7 +13,7 @@ class S3Provider(StrEnum): class RCloneSettings(BaseCustomSettings): - R_CLONE_S3: S3Settings = Field(auto_default_from_env=True) + R_CLONE_S3: S3Settings = Field(json_schema_extra={"auto_default_from_env": True}) R_CLONE_PROVIDER: S3Provider # SEE https://rclone.org/docs/#transfers-n diff --git a/packages/settings-library/src/settings_library/rabbit.py b/packages/settings-library/src/settings_library/rabbit.py index 19c6af0b656..e2cc2e271ce 100644 --- a/packages/settings-library/src/settings_library/rabbit.py +++ b/packages/settings-library/src/settings_library/rabbit.py @@ -1,6 +1,5 @@ from functools import cached_property -from pydantic import parse_obj_as from pydantic.networks import AnyUrl from pydantic.types import SecretStr @@ -15,7 +14,7 @@ class RabbitDsn(AnyUrl): class RabbitSettings(BaseCustomSettings): # host RABBIT_HOST: str - RABBIT_PORT: PortInt = parse_obj_as(PortInt, 5672) + RABBIT_PORT: PortInt = 5672 RABBIT_SECURE: bool # auth @@ -24,11 +23,13 @@ class RabbitSettings(BaseCustomSettings): @cached_property def dsn(self) -> str: - rabbit_dsn: str = RabbitDsn.build( - scheme="amqps" if self.RABBIT_SECURE else "amqp", - user=self.RABBIT_USER, - password=self.RABBIT_PASSWORD.get_secret_value(), - host=self.RABBIT_HOST, - port=f"{self.RABBIT_PORT}", + rabbit_dsn: str = str( + RabbitDsn.build( + scheme="amqps" if self.RABBIT_SECURE else "amqp", + username=self.RABBIT_USER, + password=self.RABBIT_PASSWORD.get_secret_value(), + host=self.RABBIT_HOST, + port=self.RABBIT_PORT, + ) ) return rabbit_dsn diff --git a/packages/settings-library/src/settings_library/redis.py b/packages/settings-library/src/settings_library/redis.py index ecccad69c10..6f9d7ad9d38 100644 --- a/packages/settings-library/src/settings_library/redis.py +++ b/packages/settings-library/src/settings_library/redis.py @@ -1,6 +1,6 @@ from enum import Enum -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.networks import RedisDsn from pydantic.types import SecretStr @@ -22,20 +22,22 @@ class RedisDatabase(int, Enum): class RedisSettings(BaseCustomSettings): # host REDIS_HOST: str = "redis" - REDIS_PORT: PortInt = parse_obj_as(PortInt, 6789) + REDIS_PORT: PortInt = TypeAdapter(PortInt).validate_python(6789) # auth REDIS_USER: str | None = None REDIS_PASSWORD: SecretStr | None = None - def build_redis_dsn(self, db_index: RedisDatabase): - return RedisDsn.build( - scheme="redis", - user=self.REDIS_USER or None, - password=( - self.REDIS_PASSWORD.get_secret_value() if self.REDIS_PASSWORD else None - ), - host=self.REDIS_HOST, - port=f"{self.REDIS_PORT}", - path=f"/{db_index}", + def build_redis_dsn(self, db_index: RedisDatabase) -> str: + return str( + RedisDsn.build( # pylint: disable=no-member + scheme="redis", + username=self.REDIS_USER or None, + password=( + self.REDIS_PASSWORD.get_secret_value() if self.REDIS_PASSWORD else None + ), + host=self.REDIS_HOST, + port=self.REDIS_PORT, + path=f"/{db_index}", + ) ) diff --git a/packages/settings-library/src/settings_library/resource_usage_tracker.py b/packages/settings-library/src/settings_library/resource_usage_tracker.py index dc696fab76c..d0df8f093ad 100644 --- a/packages/settings-library/src/settings_library/resource_usage_tracker.py +++ b/packages/settings-library/src/settings_library/resource_usage_tracker.py @@ -1,7 +1,6 @@ from datetime import timedelta from functools import cached_property -from pydantic import parse_obj_as from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -16,7 +15,7 @@ class ResourceUsageTrackerSettings(BaseCustomSettings, MixinServiceSettings): RESOURCE_USAGE_TRACKER_HOST: str = "resource-usage-tracker" RESOURCE_USAGE_TRACKER_PORT: PortInt = DEFAULT_FASTAPI_PORT - RESOURCE_USAGE_TRACKER_VTAG: VersionTag = parse_obj_as(VersionTag, "v1") + RESOURCE_USAGE_TRACKER_VTAG: VersionTag = "v1" @cached_property def api_base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/s3.py b/packages/settings-library/src/settings_library/s3.py index cef1bf11be5..95268b41920 100644 --- a/packages/settings-library/src/settings_library/s3.py +++ b/packages/settings-library/src/settings_library/s3.py @@ -1,22 +1,25 @@ -from typing import Any, ClassVar +from typing import Annotated -from pydantic import AnyHttpUrl, Field +from pydantic import AnyHttpUrl, BeforeValidator, Field, TypeAdapter +from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings from .basic_types import IDStr +ANY_HTTP_URL_ADAPTER: TypeAdapter = TypeAdapter(AnyHttpUrl) + class S3Settings(BaseCustomSettings): S3_ACCESS_KEY: IDStr S3_BUCKET_NAME: IDStr - S3_ENDPOINT: AnyHttpUrl | None = Field( - default=None, description="do not define if using standard AWS" - ) + S3_ENDPOINT: Annotated[ + str, BeforeValidator(lambda x: str(ANY_HTTP_URL_ADAPTER.validate_python(x))) + ] | None = Field(default=None, description="do not define if using standard AWS") S3_REGION: IDStr S3_SECRET_KEY: IDStr - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + json_schema_extra={ "examples": [ { # non AWS use-case @@ -35,3 +38,4 @@ class Config(BaseCustomSettings.Config): }, ], } + ) diff --git a/packages/settings-library/src/settings_library/ssm.py b/packages/settings-library/src/settings_library/ssm.py index 32b965fa123..73bee409464 100644 --- a/packages/settings-library/src/settings_library/ssm.py +++ b/packages/settings-library/src/settings_library/ssm.py @@ -1,20 +1,22 @@ -from typing import Any, ClassVar +from typing import Annotated -from pydantic import AnyHttpUrl, Field, SecretStr +from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter +from pydantic import BeforeValidator, Field, SecretStr +from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings class SSMSettings(BaseCustomSettings): SSM_ACCESS_KEY_ID: SecretStr - SSM_ENDPOINT: AnyHttpUrl | None = Field( - default=None, description="do not define if using standard AWS" - ) + SSM_ENDPOINT: Annotated[ + str, BeforeValidator(lambda x: str(AnyHttpUrlLegacyAdapter.validate_python(x))) + ] | None = Field(default=None, description="do not define if using standard AWS") SSM_REGION_NAME: str = "us-east-1" SSM_SECRET_ACCESS_KEY: SecretStr - class Config(BaseCustomSettings.Config): - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + json_schema_extra={ "examples": [ { "SSM_ACCESS_KEY_ID": "my_access_key_id", @@ -24,3 +26,4 @@ class Config(BaseCustomSettings.Config): } ], } + ) diff --git a/packages/settings-library/src/settings_library/storage.py b/packages/settings-library/src/settings_library/storage.py index 92ec0301257..00ef1987037 100644 --- a/packages/settings-library/src/settings_library/storage.py +++ b/packages/settings-library/src/settings_library/storage.py @@ -1,6 +1,5 @@ from functools import cached_property -from pydantic import parse_obj_as from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -13,7 +12,7 @@ class StorageSettings(BaseCustomSettings, MixinServiceSettings): STORAGE_HOST: str = "storage" STORAGE_PORT: PortInt = DEFAULT_AIOHTTP_PORT - STORAGE_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + STORAGE_VTAG: VersionTag = "v0" @cached_property def base_url(self) -> str: diff --git a/packages/settings-library/src/settings_library/tracing.py b/packages/settings-library/src/settings_library/tracing.py index 28a11cbbf6a..85b7abae5ff 100644 --- a/packages/settings-library/src/settings_library/tracing.py +++ b/packages/settings-library/src/settings_library/tracing.py @@ -1,4 +1,4 @@ -from pydantic import AnyUrl, Field, parse_obj_as +from pydantic import AliasChoices, AnyUrl, Field, TypeAdapter from .base import BaseCustomSettings @@ -7,15 +7,15 @@ class TracingSettings(BaseCustomSettings): TRACING_ZIPKIN_ENDPOINT: AnyUrl = Field( - default=parse_obj_as(AnyUrl, "http://jaeger:9411"), + default=TypeAdapter(AnyUrl).validate_python("http://jaeger:9411"), # NOSONAR description="Zipkin compatible endpoint", ) TRACING_THRIFT_COMPACT_ENDPOINT: AnyUrl = Field( - default=parse_obj_as(AnyUrl, "http://jaeger:5775"), + default=TypeAdapter(AnyUrl).validate_python("http://jaeger:5775"), # NOSONAR description="accept zipkin.thrift over compact thrift protocol (deprecated, used by legacy clients only)", ) TRACING_CLIENT_NAME: str = Field( default=UNDEFINED_CLIENT_NAME, description="Name of the application connecting the tracing service", - env=["HOST", "HOSTNAME", "TRACING_CLIENT_NAME"], + validation_alias=AliasChoices("HOST", "HOSTNAME", "TRACING_CLIENT_NAME"), ) diff --git a/packages/settings-library/src/settings_library/twilio.py b/packages/settings-library/src/settings_library/twilio.py index eb4ec0c707a..b63e35caf61 100644 --- a/packages/settings-library/src/settings_library/twilio.py +++ b/packages/settings-library/src/settings_library/twilio.py @@ -6,29 +6,25 @@ """ -import re -from re import Pattern +from typing import Annotated, TypeAlias -from pydantic import ConstrainedStr, Field, parse_obj_as +from pydantic import BeforeValidator, Field, StringConstraints, TypeAdapter from .base import BaseCustomSettings - -class CountryCodeStr(ConstrainedStr): - # Based on https://countrycode.org/ - strip_whitespace: bool = True - regex: Pattern[str] | None = re.compile(r"^\d{1,4}") - - class Config: - frozen = True +# Based on https://countrycode.org/ +CountryCodeStr: TypeAlias = Annotated[ + str, + BeforeValidator(str), + StringConstraints(strip_whitespace=True, pattern=r"^\d{1,4}"), +] class TwilioSettings(BaseCustomSettings): TWILIO_ACCOUNT_SID: str = Field(..., description="Twilio account String Identifier") TWILIO_AUTH_TOKEN: str = Field(..., description="API tokens") TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT: list[CountryCodeStr] = Field( - default=parse_obj_as( - list[CountryCodeStr], + default=TypeAdapter(list[CountryCodeStr]).validate_python( [ "41", ], diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index 79d0e1ac145..2c1ab37116d 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -1,3 +1,4 @@ +import json import logging import os from collections.abc import Callable @@ -6,12 +7,12 @@ import rich import typer +from common_library.serialization import model_dump_with_secrets from pydantic import ValidationError -from pydantic.env_settings import BaseSettings +from pydantic_settings import BaseSettings from ._constants import HEADER_STR from .base import BaseCustomSettings -from .utils_encoders import create_json_encoder_wo_secrets def print_as_envfile( @@ -24,14 +25,15 @@ def print_as_envfile( ): exclude_unset = pydantic_export_options.get("exclude_unset", False) - for field in settings_obj.__fields__.values(): - auto_default_from_env = field.field_info.extra.get( - "auto_default_from_env", False + for name, field in settings_obj.model_fields.items(): + auto_default_from_env = ( + field.json_schema_extra is not None + and field.json_schema_extra.get("auto_default_from_env", False) ) - value = getattr(settings_obj, field.name) + value = getattr(settings_obj, name) - if exclude_unset and field.name not in settings_obj.__fields_set__: + if exclude_unset and name not in settings_obj.model_fields_set: if not auto_default_from_env: continue if value is None: @@ -39,10 +41,14 @@ def print_as_envfile( if isinstance(value, BaseSettings): if compact: - value = f"'{value.json(**pydantic_export_options)}'" # flat + value = json.dumps( + model_dump_with_secrets( + value, show_secrets=show_secrets, **pydantic_export_options + ) + ) # flat else: if verbose: - typer.echo(f"\n# --- {field.name} --- ") + typer.echo(f"\n# --- {name} --- ") print_as_envfile( value, compact=False, @@ -54,17 +60,22 @@ def print_as_envfile( elif show_secrets and hasattr(value, "get_secret_value"): value = value.get_secret_value() - if verbose: - field_info = field.field_info - if field_info.description: - typer.echo(f"# {field_info.description}") + if verbose and field.description: + typer.echo(f"# {field.description}") - typer.echo(f"{field.name}={value}") + typer.echo(f"{name}={value}") -def print_as_json(settings_obj, *, compact=False, **pydantic_export_options): +def print_as_json( + settings_obj, *, compact=False, show_secrets, **pydantic_export_options +): typer.echo( - settings_obj.json(indent=None if compact else 2, **pydantic_export_options) + json.dumps( + model_dump_with_secrets( + settings_obj, show_secrets=show_secrets, **pydantic_export_options + ), + indent=None if compact else 2, + ) ) @@ -128,14 +139,14 @@ def settings( raise pydantic_export_options: dict[str, Any] = {"exclude_unset": exclude_unset} - if show_secrets: - # NOTE: this option is for json-only - pydantic_export_options["encoder"] = create_json_encoder_wo_secrets( - settings_cls - ) if as_json: - print_as_json(settings_obj, compact=compact, **pydantic_export_options) + print_as_json( + settings_obj, + compact=compact, + show_secrets=show_secrets, + **pydantic_export_options, + ) else: print_as_envfile( settings_obj, diff --git a/packages/settings-library/src/settings_library/utils_encoders.py b/packages/settings-library/src/settings_library/utils_encoders.py index 71ea960bf78..f38e156b6a5 100644 --- a/packages/settings-library/src/settings_library/utils_encoders.py +++ b/packages/settings-library/src/settings_library/utils_encoders.py @@ -12,7 +12,7 @@ def create_json_encoder_wo_secrets(model_cls: type[BaseModel]): show_secrets_encoder = create_json_encoder_wo_secrets(type(model)) model.dict(encoder=show_secrets_encoder)['my_secret'] == "secret" """ - current_encoders = getattr(model_cls.Config, "json_encoders", {}) + current_encoders = getattr(model_cls.model_config, "json_encoders", {}) return partial( custom_pydantic_encoder, { diff --git a/packages/settings-library/src/settings_library/utils_service.py b/packages/settings-library/src/settings_library/utils_service.py index e7bb66057c5..17746487a6f 100644 --- a/packages/settings-library/src/settings_library/utils_service.py +++ b/packages/settings-library/src/settings_library/utils_service.py @@ -4,14 +4,13 @@ """ from enum import Enum, auto -from pydantic import parse_obj_as from pydantic.networks import AnyUrl from pydantic.types import SecretStr from .basic_types import PortInt -DEFAULT_AIOHTTP_PORT: PortInt = parse_obj_as(PortInt, 8080) -DEFAULT_FASTAPI_PORT: PortInt = parse_obj_as(PortInt, 8000) +DEFAULT_AIOHTTP_PORT: PortInt = 8080 +DEFAULT_FASTAPI_PORT: PortInt = 8000 class URLPart(Enum): @@ -96,6 +95,8 @@ def _compose_url( assert prefix # nosec prefix = prefix.upper() + port_value = self._safe_getattr(f"{prefix}_PORT", port) + parts = { "scheme": ( "https" @@ -103,30 +104,32 @@ def _compose_url( else "http" ), "host": self._safe_getattr(f"{prefix}_HOST", URLPart.REQUIRED), - "user": self._safe_getattr(f"{prefix}_USER", user), + "port": int(port_value) if port_value is not None else None, + "username": self._safe_getattr(f"{prefix}_USER", user), "password": self._safe_getattr(f"{prefix}_PASSWORD", password), - "port": self._safe_getattr(f"{prefix}_PORT", port), } if vtag != URLPart.EXCLUDE: # noqa: SIM102 if v := self._safe_getattr(f"{prefix}_VTAG", vtag): - parts["path"] = f"/{v}" + parts["path"] = f"{v}" # post process parts dict kwargs = {} - for k, v in parts.items(): - value = v + for k, v in parts.items(): # type: ignore[assignment] if isinstance(v, SecretStr): value = v.get_secret_value() - elif v is not None: - value = f"{v}" + else: + value = v - kwargs[k] = value + if value is not None: + kwargs[k] = value - assert all(isinstance(v, str) or v is None for v in kwargs.values()) # nosec + assert all( + isinstance(v, (str, int)) or v is None for v in kwargs.values() + ) # nosec - composed_url: str = AnyUrl.build(**kwargs) - return composed_url + composed_url: str = str(AnyUrl.build(**kwargs)) # type: ignore[arg-type] + return composed_url.rstrip("/") def _build_api_base_url(self, *, prefix: str) -> str: return self._compose_url( diff --git a/packages/settings-library/src/settings_library/webserver.py b/packages/settings-library/src/settings_library/webserver.py index 4da2c41d699..c32bdbeb0c5 100644 --- a/packages/settings-library/src/settings_library/webserver.py +++ b/packages/settings-library/src/settings_library/webserver.py @@ -1,7 +1,5 @@ from functools import cached_property -from pydantic import parse_obj_as - from .base import BaseCustomSettings from .basic_types import PortInt, VersionTag from .utils_service import DEFAULT_AIOHTTP_PORT, MixinServiceSettings, URLPart @@ -10,7 +8,7 @@ class WebServerSettings(BaseCustomSettings, MixinServiceSettings): WEBSERVER_HOST: str = "webserver" WEBSERVER_PORT: PortInt = DEFAULT_AIOHTTP_PORT - WEBSERVER_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + WEBSERVER_VTAG: VersionTag = "v0" @cached_property def base_url(self) -> str: diff --git a/packages/settings-library/tests/conftest.py b/packages/settings-library/tests/conftest.py index 725f19c534a..0431a6c6748 100644 --- a/packages/settings-library/tests/conftest.py +++ b/packages/settings-library/tests/conftest.py @@ -96,9 +96,13 @@ class _ApplicationSettings(BaseCustomSettings): # NOTE: by convention, an addon is disabled when APP_ADDON=None, so we make this # entry nullable as well - APP_OPTIONAL_ADDON: _ModuleSettings | None = Field(auto_default_from_env=True) + APP_OPTIONAL_ADDON: _ModuleSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) # NOTE: example of a group that cannot be disabled (not nullable) - APP_REQUIRED_PLUGIN: PostgresSettings | None = Field(auto_default_from_env=True) + APP_REQUIRED_PLUGIN: PostgresSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) return _ApplicationSettings diff --git a/packages/settings-library/tests/test__models_examples.py b/packages/settings-library/tests/test__models_examples.py index c60a6c08261..96ffc7135b2 100644 --- a/packages/settings-library/tests/test__models_examples.py +++ b/packages/settings-library/tests/test__models_examples.py @@ -14,6 +14,6 @@ def test_all_settings_library_models_config_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): - assert model_cls.parse_obj( + assert model_cls.model_validate( example_data ), f"Failed {example_name} : {json.dumps(example_data)}" diff --git a/packages/settings-library/tests/test__pydantic_settings.py b/packages/settings-library/tests/test__pydantic_settings.py index 8cf3eadc30f..bdc536387fc 100644 --- a/packages/settings-library/tests/test__pydantic_settings.py +++ b/packages/settings-library/tests/test__pydantic_settings.py @@ -12,45 +12,50 @@ """ - -from pydantic import BaseSettings, validator -from pydantic.fields import ModelField, Undefined +from common_library.pydantic_fields_extension import is_nullable +from pydantic import ValidationInfo, field_validator +from pydantic.fields import PydanticUndefined +from pydantic_settings import BaseSettings def assert_field_specs( - model_cls, name, is_required, is_nullable, explicit_default, defaults + model_cls: type[BaseSettings], + name: str, + required: bool, + nullable: bool, + explicit_default, ): - field: ModelField = model_cls.__fields__[name] - print(field, field.field_info) + info = model_cls.model_fields[name] + print(info) - assert field.required == is_required - assert field.allow_none == is_nullable - assert field.field_info.default == explicit_default + assert info.is_required() == required + assert is_nullable(info) == nullable - assert field.default == defaults - if field.required: + if info.is_required(): # in this case, default is not really used - assert field.default is None + assert info.default is PydanticUndefined + else: + assert info.default == explicit_default class Settings(BaseSettings): VALUE: int VALUE_DEFAULT: int = 42 - VALUE_NULLABLE_REQUIRED: int | None = ... # type: ignore - VALUE_NULLABLE_OPTIONAL: int | None + VALUE_NULLABLE_REQUIRED: int | None = ... # type: ignore[assignment] + VALUE_NULLABLE_REQUIRED_AS_WELL: int | None VALUE_NULLABLE_DEFAULT_VALUE: int | None = 42 VALUE_NULLABLE_DEFAULT_NULL: int | None = None # Other ways to write down "required" is using ... - VALUE_ALSO_REQUIRED: int = ... # type: ignore + VALUE_REQUIRED_AS_WELL: int = ... # type: ignore[assignment] - @validator("*", pre=True) + @field_validator("*", mode="before") @classmethod - def parse_none(cls, v, values, field: ModelField): + def parse_none(cls, v, info: ValidationInfo): # WARNING: In nullable fields, envs equal to null or none are parsed as None !! - if field.allow_none: + if info.field_name and is_nullable(cls.model_fields[info.field_name]): if isinstance(v, str) and v.lower() in ("null", "none"): return None return v @@ -64,37 +69,33 @@ def test_fields_declarations(): assert_field_specs( Settings, "VALUE", - is_required=True, - is_nullable=False, - explicit_default=Undefined, - defaults=None, + required=True, + nullable=False, + explicit_default=PydanticUndefined, ) assert_field_specs( Settings, "VALUE_DEFAULT", - is_required=False, - is_nullable=False, + required=False, + nullable=False, explicit_default=42, - defaults=42, ) assert_field_specs( Settings, "VALUE_NULLABLE_REQUIRED", - is_required=True, - is_nullable=True, + required=True, + nullable=True, explicit_default=Ellipsis, - defaults=None, ) assert_field_specs( Settings, - "VALUE_NULLABLE_OPTIONAL", - is_required=False, - is_nullable=True, - explicit_default=Undefined, # <- difference wrt VALUE_NULLABLE_DEFAULT_NULL - defaults=None, + "VALUE_NULLABLE_REQUIRED_AS_WELL", + required=True, + nullable=True, + explicit_default=PydanticUndefined, # <- difference wrt VALUE_NULLABLE_DEFAULT_NULL ) # VALUE_NULLABLE_OPTIONAL interpretation has always been confusing @@ -104,54 +105,53 @@ def test_fields_declarations(): assert_field_specs( Settings, "VALUE_NULLABLE_DEFAULT_VALUE", - is_required=False, - is_nullable=True, + required=False, + nullable=True, explicit_default=42, - defaults=42, ) assert_field_specs( Settings, "VALUE_NULLABLE_DEFAULT_NULL", - is_required=False, - is_nullable=True, - explicit_default=None, # <- difference wrt VALUE_NULLABLE_OPTIONAL - defaults=None, + required=False, + nullable=True, + explicit_default=None, ) assert_field_specs( Settings, - "VALUE_ALSO_REQUIRED", - is_required=True, - is_nullable=False, + "VALUE_REQUIRED_AS_WELL", + required=True, + nullable=False, explicit_default=Ellipsis, - defaults=None, ) def test_construct(monkeypatch): # from __init__ settings_from_init = Settings( - VALUE=1, VALUE_ALSO_REQUIRED=10, VALUE_NULLABLE_REQUIRED=None + VALUE=1, + VALUE_NULLABLE_REQUIRED=None, + VALUE_NULLABLE_REQUIRED_AS_WELL=None, + VALUE_REQUIRED_AS_WELL=10, ) - print(settings_from_init.json(exclude_unset=True, indent=1)) + print(settings_from_init.model_dump_json(exclude_unset=True, indent=1)) # from env vars monkeypatch.setenv("VALUE", "1") - monkeypatch.setenv("VALUE_ALSO_REQUIRED", "10") - monkeypatch.setenv( - "VALUE_NULLABLE_REQUIRED", "null" - ) # WARNING: set this env to None would not work w/o ``parse_none`` validator! bug??? + monkeypatch.setenv("VALUE_REQUIRED_AS_WELL", "10") + monkeypatch.setenv("VALUE_NULLABLE_REQUIRED", "null") + monkeypatch.setenv("VALUE_NULLABLE_REQUIRED_AS_WELL", None) - settings_from_env = Settings() - print(settings_from_env.json(exclude_unset=True, indent=1)) + settings_from_env = Settings() # type: ignore[call-arg] + print(settings_from_env.model_dump_json(exclude_unset=True, indent=1)) assert settings_from_init == settings_from_env # mixed - settings_from_both = Settings(VALUE_NULLABLE_REQUIRED=3) - print(settings_from_both.json(exclude_unset=True, indent=1)) + settings_from_both = Settings(VALUE_NULLABLE_REQUIRED=3) # type: ignore[call-arg] + print(settings_from_both.model_dump_json(exclude_unset=True, indent=1)) - assert settings_from_both == settings_from_init.copy( + assert settings_from_both == settings_from_init.model_copy( update={"VALUE_NULLABLE_REQUIRED": 3} ) diff --git a/packages/settings-library/tests/test_base.py b/packages/settings-library/tests/test_base.py index 7cbd9fa8773..3344aa6b35a 100644 --- a/packages/settings-library/tests/test_base.py +++ b/packages/settings-library/tests/test_base.py @@ -10,8 +10,9 @@ import pytest import settings_library.base -from pydantic import BaseModel, BaseSettings, ValidationError +from pydantic import BaseModel, ValidationError from pydantic.fields import Field +from pydantic_settings import BaseSettings from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_envfile from settings_library.base import ( @@ -38,17 +39,17 @@ def _get_attrs_tree(obj: Any) -> dict[str, Any]: def _print_defaults(model_cls: type[BaseModel]): - for field in model_cls.__fields__.values(): - print(field.name, ":", end="") + for name, field in model_cls.model_fields.items(): + print(name, ":", end="") try: - default = field.get_default() + default = field.get_default(call_default_factory=True) # new in Pydatic v2 print(default, type(default)) except ValidationError as err: print(err) def _dumps_model_class(model_cls: type[BaseModel]): - d = {field.name: _get_attrs_tree(field) for field in model_cls.__fields__.values()} + d = {name: _get_attrs_tree(field) for name, field in model_cls.model_fields.items()} return json.dumps(d, indent=1) @@ -61,16 +62,19 @@ class S(BaseCustomSettings): class M1(BaseCustomSettings): VALUE: S VALUE_DEFAULT: S = S(S_VALUE=42) - VALUE_CONFUSING: S = None # type: ignore + # VALUE_CONFUSING: S = None # type: ignore VALUE_NULLABLE_REQUIRED: S | None = ... # type: ignore - VALUE_NULLABLE_OPTIONAL: S | None VALUE_NULLABLE_DEFAULT_VALUE: S | None = S(S_VALUE=42) VALUE_NULLABLE_DEFAULT_NULL: S | None = None - VALUE_NULLABLE_DEFAULT_ENV: S | None = Field(auto_default_from_env=True) - VALUE_DEFAULT_ENV: S = Field(auto_default_from_env=True) + VALUE_NULLABLE_DEFAULT_ENV: S | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) + VALUE_DEFAULT_ENV: S = Field( + json_schema_extra={"auto_default_from_env": True} + ) class M2(BaseCustomSettings): # @@ -82,10 +86,14 @@ class M2(BaseCustomSettings): VALUE_NULLABLE_DEFAULT_NULL: S | None = None # defaults enabled but if not exists, it disables - VALUE_NULLABLE_DEFAULT_ENV: S | None = Field(auto_default_from_env=True) + VALUE_NULLABLE_DEFAULT_ENV: S | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) # cannot be disabled - VALUE_DEFAULT_ENV: S = Field(auto_default_from_env=True) + VALUE_DEFAULT_ENV: S = Field( + json_schema_extra={"auto_default_from_env": True} + ) # Changed in version 3.7: Dictionary order is guaranteed to be insertion order _classes = {"M1": M1, "M2": M2, "S": S} @@ -101,14 +109,14 @@ def test_create_settings_class( # DEV: Path("M1.ignore.json").write_text(dumps_model_class(M)) - assert M.__fields__["VALUE_NULLABLE_DEFAULT_ENV"].default_factory + assert M.model_fields["VALUE_NULLABLE_DEFAULT_ENV"].default_factory - assert M.__fields__["VALUE_NULLABLE_DEFAULT_ENV"].get_default() is None + assert M.model_fields["VALUE_NULLABLE_DEFAULT_ENV"].get_default() is None - assert M.__fields__["VALUE_DEFAULT_ENV"].default_factory + assert M.model_fields["VALUE_DEFAULT_ENV"].default_factory with pytest.raises(DefaultFromEnvFactoryError): - M.__fields__["VALUE_DEFAULT_ENV"].get_default() + M.model_fields["VALUE_DEFAULT_ENV"].get_default(call_default_factory=True) def test_create_settings_class_with_environment( @@ -136,20 +144,19 @@ def test_create_settings_class_with_environment( instance = SettingsClass() - print(instance.json(indent=2)) + print(instance.model_dump_json(indent=2)) # checks - assert instance.dict(exclude_unset=True) == { + assert instance.model_dump(exclude_unset=True) == { "VALUE": {"S_VALUE": 2}, "VALUE_NULLABLE_REQUIRED": {"S_VALUE": 3}, } - assert instance.dict() == { + assert instance.model_dump() == { "VALUE": {"S_VALUE": 2}, "VALUE_DEFAULT": {"S_VALUE": 42}, - "VALUE_CONFUSING": None, + # "VALUE_CONFUSING": None, "VALUE_NULLABLE_REQUIRED": {"S_VALUE": 3}, - "VALUE_NULLABLE_OPTIONAL": None, "VALUE_NULLABLE_DEFAULT_VALUE": {"S_VALUE": 42}, "VALUE_NULLABLE_DEFAULT_NULL": None, "VALUE_NULLABLE_DEFAULT_ENV": {"S_VALUE": 1}, @@ -163,13 +170,15 @@ def test_create_settings_class_without_environ_fails( # now defining S_VALUE M2_outside_context = create_settings_class("M2") - with pytest.raises(ValidationError) as err_info: + with pytest.raises(DefaultFromEnvFactoryError) as err_info: M2_outside_context.create_from_envs() - assert err_info.value.errors()[0] == { - "loc": ("VALUE_DEFAULT_ENV", "S_VALUE"), - "msg": "field required", - "type": "value_error.missing", + assert err_info.value.errors[0] == { + "input": {}, + "loc": ("S_VALUE",), + "msg": "Field required", + "type": "missing", + "url": "https://errors.pydantic.dev/2.9/v/missing", } @@ -202,7 +211,9 @@ def test_auto_default_to_none_logs_a_warning( class SettingsClass(BaseCustomSettings): VALUE_NULLABLE_DEFAULT_NULL: S | None = None - VALUE_NULLABLE_DEFAULT_ENV: S | None = Field(auto_default_from_env=True) + VALUE_NULLABLE_DEFAULT_ENV: S | None = Field( + json_schema_extra={"auto_default_from_env": True}, + ) instance = SettingsClass.create_from_envs() assert instance.VALUE_NULLABLE_DEFAULT_NULL is None @@ -224,7 +235,9 @@ def test_auto_default_to_not_none( class SettingsClass(BaseCustomSettings): VALUE_NULLABLE_DEFAULT_NULL: S | None = None - VALUE_NULLABLE_DEFAULT_ENV: S | None = Field(auto_default_from_env=True) + VALUE_NULLABLE_DEFAULT_ENV: S | None = Field( + json_schema_extra={"auto_default_from_env": True}, + ) instance = SettingsClass.create_from_envs() assert instance.VALUE_NULLABLE_DEFAULT_NULL is None @@ -286,9 +299,11 @@ class SettingsClassExt(SettingsClass): error = err_info.value.errors()[0] assert error == { + "input": "", "loc": ("INT_VALUE_TO_NOTHING",), - "msg": "value is not a valid integer", - "type": "type_error.integer", + "msg": "Input should be a valid integer, unable to parse string as an integer", + "type": "int_parsing", + "url": "https://errors.pydantic.dev/2.9/v/int_parsing", } diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index d54d40bf925..85fd98c7522 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -3,10 +3,11 @@ # pylint: disable=unused-variable +import os from collections.abc import Callable import pytest -from pydantic import Field, ValidationError +from pydantic import AliasChoices, Field, ValidationError from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_envfile from settings_library.base import BaseCustomSettings, DefaultFromEnvFactoryError from settings_library.basic_types import PortInt @@ -22,6 +23,13 @@ # +@pytest.fixture +def postgres_envvars_unset(monkeypatch: pytest.MonkeyPatch) -> None: + for name in os.environ: + if name.startswith("POSTGRES_"): + monkeypatch.delenv(name) + + @pytest.fixture def model_classes_factory() -> Callable: # @@ -49,7 +57,9 @@ class _FakePostgresSettings(BaseCustomSettings): POSTGRES_CLIENT_NAME: str | None = Field( None, - env=["HOST", "HOSTNAME", "POSTGRES_CLIENT_NAME"], + validation_alias=AliasChoices( + "HOST", "HOSTNAME", "POSTGRES_CLIENT_NAME" + ), ) # @@ -60,18 +70,18 @@ class S1(BaseCustomSettings): WEBSERVER_POSTGRES: _FakePostgresSettings class S2(BaseCustomSettings): - WEBSERVER_POSTGRES_NULLABLE_OPTIONAL: _FakePostgresSettings | None + WEBSERVER_POSTGRES_NULLABLE_OPTIONAL: _FakePostgresSettings | None = None class S3(BaseCustomSettings): # cannot be disabled!! WEBSERVER_POSTGRES_DEFAULT_ENV: _FakePostgresSettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) class S4(BaseCustomSettings): # defaults enabled but if cannot be resolved, it disables WEBSERVER_POSTGRES_NULLABLE_DEFAULT_ENV: _FakePostgresSettings | None = ( - Field(auto_default_from_env=True) + Field(json_schema_extra={"auto_default_from_env": True}) ) class S5(BaseCustomSettings): @@ -104,7 +114,9 @@ class S5(BaseCustomSettings): # -def test_parse_from_empty_envs(model_classes_factory: Callable): +def test_parse_from_empty_envs( + postgres_envvars_unset: None, model_classes_factory: Callable +): S1, S2, S3, S4, S5 = model_classes_factory() @@ -115,7 +127,7 @@ def test_parse_from_empty_envs(model_classes_factory: Callable): assert s2.WEBSERVER_POSTGRES_NULLABLE_OPTIONAL is None with pytest.raises(DefaultFromEnvFactoryError): - # NOTE: cannot hae a default or assignment + # NOTE: cannot have a default or assignment S3() # auto default factory resolves to None (because is nullable) @@ -126,7 +138,11 @@ def test_parse_from_empty_envs(model_classes_factory: Callable): assert s5.WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL is None -def test_parse_from_individual_envs(monkeypatch, model_classes_factory): +def test_parse_from_individual_envs( + postgres_envvars_unset: None, + monkeypatch: pytest.MonkeyPatch, + model_classes_factory: Callable, +): S1, S2, S3, S4, S5 = model_classes_factory() @@ -146,18 +162,20 @@ def test_parse_from_individual_envs(monkeypatch, model_classes_factory): S1() assert exc_info.value.errors()[0] == { + "input": {}, "loc": ("WEBSERVER_POSTGRES",), - "msg": "field required", - "type": "value_error.missing", + "msg": "Field required", + "type": "missing", + "url": "https://errors.pydantic.dev/2.9/v/missing", } s2 = S2() - assert s2.dict(exclude_unset=True) == {} - assert s2.dict() == {"WEBSERVER_POSTGRES_NULLABLE_OPTIONAL": None} + assert s2.model_dump(exclude_unset=True) == {} + assert s2.model_dump() == {"WEBSERVER_POSTGRES_NULLABLE_OPTIONAL": None} s3 = S3() - assert s3.dict(exclude_unset=True) == {} - assert s3.dict() == { + assert s3.model_dump(exclude_unset=True) == {} + assert s3.model_dump() == { "WEBSERVER_POSTGRES_DEFAULT_ENV": { "POSTGRES_HOST": "pg", "POSTGRES_USER": "test", @@ -171,8 +189,8 @@ def test_parse_from_individual_envs(monkeypatch, model_classes_factory): } s4 = S4() - assert s4.dict(exclude_unset=True) == {} - assert s4.dict() == { + assert s4.model_dump(exclude_unset=True) == {} + assert s4.model_dump() == { "WEBSERVER_POSTGRES_NULLABLE_DEFAULT_ENV": { "POSTGRES_HOST": "pg", "POSTGRES_USER": "test", @@ -186,11 +204,13 @@ def test_parse_from_individual_envs(monkeypatch, model_classes_factory): } s5 = S5() - assert s5.dict(exclude_unset=True) == {} - assert s5.dict() == {"WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL": None} + assert s5.model_dump(exclude_unset=True) == {} + assert s5.model_dump() == {"WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL": None} -def test_parse_compact_env(monkeypatch, model_classes_factory): +def test_parse_compact_env( + postgres_envvars_unset: None, monkeypatch, model_classes_factory +): S1, S2, S3, S4, S5 = model_classes_factory() @@ -209,7 +229,7 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): # test s1 = S1() - assert s1.dict(exclude_unset=True) == { + assert s1.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -217,7 +237,7 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): "POSTGRES_DB": "db2", } } - assert s1.dict() == { + assert s1.model_dump() == { "WEBSERVER_POSTGRES": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -238,7 +258,7 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): """, ) s2 = S2() - assert s2.dict(exclude_unset=True) == { + assert s2.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_NULLABLE_OPTIONAL": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -258,7 +278,7 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): # default until it is really needed. Here before it would # fail because default cannot be computed even if the final value can! s3 = S3() - assert s3.dict(exclude_unset=True) == { + assert s3.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_DEFAULT_ENV": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -275,7 +295,7 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): """, ) s4 = S4() - assert s4.dict(exclude_unset=True) == { + assert s4.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_NULLABLE_DEFAULT_ENV": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -292,7 +312,7 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): """, ) s5 = S5() - assert s5.dict(exclude_unset=True) == { + assert s5.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -302,7 +322,9 @@ def test_parse_compact_env(monkeypatch, model_classes_factory): } -def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): +def test_parse_from_mixed_envs( + postgres_envvars_unset: None, monkeypatch, model_classes_factory +): S1, S2, S3, S4, S5 = model_classes_factory() @@ -326,7 +348,7 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): s1 = S1() - assert s1.dict() == { + assert s1.model_dump() == { "WEBSERVER_POSTGRES": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -341,7 +363,7 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): # NOTE how unset marks also applies to embedded fields # NOTE: (1) priority of json-compact over granulated # NOTE: (2) json-compact did not define this but granulated did - assert s1.dict(exclude_unset=True) == { + assert s1.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES": { "POSTGRES_HOST": "pg2", # <- (1) "POSTGRES_USER": "test2", # <- (1) @@ -358,7 +380,7 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): ) s2 = S2() - assert s2.dict(exclude_unset=True) == { + assert s2.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_NULLABLE_OPTIONAL": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -375,7 +397,7 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): ) s3 = S3() - assert s3.dict(exclude_unset=True) == { + assert s3.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_DEFAULT_ENV": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -392,7 +414,7 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): ) s4 = S4() - assert s4.dict(exclude_unset=True) == { + assert s4.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_NULLABLE_DEFAULT_ENV": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -409,7 +431,7 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): ) s5 = S5() - assert s5.dict(exclude_unset=True) == { + assert s5.model_dump(exclude_unset=True) == { "WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL": { "POSTGRES_HOST": "pg2", "POSTGRES_USER": "test2", @@ -436,7 +458,9 @@ def test_parse_from_mixed_envs(monkeypatch, model_classes_factory): # -def test_toggle_plugin_1(monkeypatch, model_classes_factory): +def test_toggle_plugin_1( + postgres_envvars_unset: None, monkeypatch, model_classes_factory +): *_, S4, S5 = model_classes_factory() @@ -449,7 +473,9 @@ def test_toggle_plugin_1(monkeypatch, model_classes_factory): assert s5.WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL is None -def test_toggle_plugin_2(monkeypatch, model_classes_factory): +def test_toggle_plugin_2( + postgres_envvars_unset: None, monkeypatch, model_classes_factory +): *_, S4, S5 = model_classes_factory() # minimal @@ -470,7 +496,9 @@ def test_toggle_plugin_2(monkeypatch, model_classes_factory): assert s5.WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL is None -def test_toggle_plugin_3(monkeypatch, model_classes_factory): +def test_toggle_plugin_3( + postgres_envvars_unset: None, monkeypatch, model_classes_factory +): *_, S4, S5 = model_classes_factory() # explicitly disables @@ -493,7 +521,9 @@ def test_toggle_plugin_3(monkeypatch, model_classes_factory): assert s5.WEBSERVER_POSTGRES_NULLABLE_DEFAULT_NULL is None -def test_toggle_plugin_4(monkeypatch, model_classes_factory): +def test_toggle_plugin_4( + postgres_envvars_unset: None, monkeypatch, model_classes_factory +): *_, S4, S5 = model_classes_factory() JSON_VALUE = '{"POSTGRES_HOST":"pg2", "POSTGRES_USER":"test2", "POSTGRES_PASSWORD":"shh2", "POSTGRES_DB":"db2"}' diff --git a/packages/settings-library/tests/test_email.py b/packages/settings-library/tests/test_email.py index 1cd3978503e..acb9d607c89 100644 --- a/packages/settings-library/tests/test_email.py +++ b/packages/settings-library/tests/test_email.py @@ -67,7 +67,7 @@ def all_env_devel_undefined( ], ) def test_smtp_configuration_ok(cfg: dict[str, Any], all_env_devel_undefined: None): - assert SMTPSettings.parse_obj(cfg) + assert SMTPSettings.model_validate(cfg) @pytest.mark.parametrize( diff --git a/packages/settings-library/tests/test_postgres.py b/packages/settings-library/tests/test_postgres.py index 1708acc7808..c191f0ea37e 100644 --- a/packages/settings-library/tests/test_postgres.py +++ b/packages/settings-library/tests/test_postgres.py @@ -14,23 +14,20 @@ def env_file(): def test_cached_property_dsn(mock_environment: dict): - settings = PostgresSettings() + settings = PostgresSettings() # type: ignore[call-arg] # all are upper-case - assert all(key == key.upper() for key in settings.dict()) - - # dsn is computed from the other fields - assert "dsn" not in settings.dict() - - # causes cached property to be computed and stored on the instance + assert all(key == key.upper() for key in settings.model_dump()) + assert settings.dsn - assert "dsn" in settings.dict() + # dsn is computed from the other fields + assert "dsn" not in settings.model_dump() def test_dsn_with_query(mock_environment: dict, monkeypatch): - settings = PostgresSettings() + settings = PostgresSettings() # type: ignore[call-arg] assert not settings.POSTGRES_CLIENT_NAME assert settings.dsn == "postgresql://foo:secret@localhost:5432/foodb" @@ -38,7 +35,7 @@ def test_dsn_with_query(mock_environment: dict, monkeypatch): # now with app monkeypatch.setenv("POSTGRES_CLIENT_NAME", "Some &43 funky name") - settings_with_app = PostgresSettings() + settings_with_app = PostgresSettings() # type: ignore[call-arg] assert settings_with_app.POSTGRES_CLIENT_NAME assert ( diff --git a/packages/settings-library/tests/test_twilio.py b/packages/settings-library/tests/test_twilio.py index 6f2830ea4aa..1989fbe6a9f 100644 --- a/packages/settings-library/tests/test_twilio.py +++ b/packages/settings-library/tests/test_twilio.py @@ -20,7 +20,7 @@ def test_twilio_settings_within_envdevel( }, ) settings = TwilioSettings.create_from_envs() - print(settings.json(indent=2)) + print(settings.model_dump_json(indent=2)) assert settings diff --git a/packages/settings-library/tests/test_utils_cli.py b/packages/settings-library/tests/test_utils_cli.py index 611ccf2509f..b4e50902124 100644 --- a/packages/settings-library/tests/test_utils_cli.py +++ b/packages/settings-library/tests/test_utils_cli.py @@ -18,10 +18,10 @@ from settings_library.utils_cli import ( create_settings_command, create_version_callback, + model_dump_with_secrets, print_as_envfile, print_as_json, ) -from settings_library.utils_encoders import create_json_encoder_wo_secrets from typer.testing import CliRunner log = logging.getLogger(__name__) @@ -84,12 +84,7 @@ def fake_granular_env_file_content() -> str: @pytest.fixture def export_as_dict() -> Callable: def _export(model_obj, **export_options): - return json.loads( - model_obj.json( - encoder=create_json_encoder_wo_secrets(model_obj.__class__), - **export_options, - ) - ) + return model_dump_with_secrets(model_obj, show_secrets=True, **export_options) return _export @@ -136,7 +131,7 @@ def test_settings_as_json( # reuse resulting json to build settings settings: dict = json.loads(result.stdout) - assert fake_settings_class.parse_obj(settings) + assert fake_settings_class.model_validate(settings) def test_settings_as_json_schema( @@ -439,7 +434,7 @@ class FakeSettings(BaseCustomSettings): assert "secret" not in captured.out assert "Some info" not in captured.out - print_as_json(settings_obj, compact=True) + print_as_json(settings_obj, compact=True, show_secrets=False) captured = capsys.readouterr() assert "secret" not in captured.out assert "**" in captured.out diff --git a/packages/settings-library/tests/test_utils_logging.py b/packages/settings-library/tests/test_utils_logging.py index 9054b391333..d7f4cf31569 100644 --- a/packages/settings-library/tests/test_utils_logging.py +++ b/packages/settings-library/tests/test_utils_logging.py @@ -1,6 +1,6 @@ import logging -from pydantic import Field, validator +from pydantic import AliasChoices, Field, field_validator from settings_library.base import BaseCustomSettings from settings_library.basic_types import BootMode from settings_library.utils_logging import MixinLoggingSettings @@ -14,20 +14,20 @@ def test_mixin_logging(monkeypatch): class Settings(BaseCustomSettings, MixinLoggingSettings): # DOCKER - SC_BOOT_MODE: BootMode | None + SC_BOOT_MODE: BootMode | None = None # LOGGING LOG_LEVEL: str = Field( "WARNING", - env=[ + validation_alias=AliasChoices( "APPNAME_LOG_LEVEL", "LOG_LEVEL", - ], + ), ) APPNAME_DEBUG: bool = Field(False, description="Starts app in debug mode") - @validator("LOG_LEVEL") + @field_validator("LOG_LEVEL") @classmethod def _v(cls, value) -> str: return cls.validate_log_level(value) @@ -40,14 +40,10 @@ def _v(cls, value) -> str: assert settings.LOG_LEVEL == "DEBUG" assert ( - settings.json() - == '{"SC_BOOT_MODE": null, "LOG_LEVEL": "DEBUG", "APPNAME_DEBUG": false}' + settings.model_dump_json() + == '{"SC_BOOT_MODE":null,"LOG_LEVEL":"DEBUG","APPNAME_DEBUG":false}' ) # test cached-property assert settings.log_level == logging.DEBUG - # log_level is cached-property (notice that is lower-case!), and gets added after first use - assert ( - settings.json() - == '{"SC_BOOT_MODE": null, "LOG_LEVEL": "DEBUG", "APPNAME_DEBUG": false, "log_level": 10}' - ) + diff --git a/packages/settings-library/tests/test_utils_service.py b/packages/settings-library/tests/test_utils_service.py index a3638f9b31e..8ecd9835893 100644 --- a/packages/settings-library/tests/test_utils_service.py +++ b/packages/settings-library/tests/test_utils_service.py @@ -5,7 +5,7 @@ from functools import cached_property import pytest -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pydantic.types import SecretStr from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag @@ -24,9 +24,9 @@ class MySettings(BaseCustomSettings, MixinServiceSettings): MY_VTAG: VersionTag | None = None MY_SECURE: bool = False - # optional - MY_USER: str | None - MY_PASSWORD: SecretStr | None + # optional (in Pydantic v2 requires a default) + MY_USER: str | None = None + MY_PASSWORD: SecretStr | None = None @cached_property def api_base_url(self) -> str: @@ -88,8 +88,8 @@ def test_service_settings_base_urls(service_settings_cls: type): settings_with_defaults = service_settings_cls() - base_url = parse_obj_as(AnyHttpUrl, settings_with_defaults.base_url) - api_base_url = parse_obj_as(AnyHttpUrl, settings_with_defaults.api_base_url) + base_url = TypeAdapter(AnyHttpUrl).validate_python(settings_with_defaults.base_url) + api_base_url = TypeAdapter(AnyHttpUrl).validate_python(settings_with_defaults.api_base_url) assert base_url.path != api_base_url.path assert (base_url.scheme, base_url.host, base_url.port) == ( diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 14712a97fc2..28d31ff4fb2 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -31,6 +31,8 @@ aiosignal==1.3.1 # via aiohttp alembic==1.13.2 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # fast-depends @@ -44,10 +46,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in async-timeout==4.0.3 # via - # aiohttp # aiopg # asyncpg - # redis asyncpg==0.29.0 # via sqlalchemy attrs==24.2.0 @@ -61,8 +61,6 @@ dnspython==2.6.1 # via email-validator email-validator==2.2.0 # via pydantic -exceptiongroup==1.2.2 - # via anyio fast-depends==2.4.8 # via faststream faststream==0.5.18 @@ -129,7 +127,7 @@ psycopg2-binary==2.9.9 # via # aiopg # sqlalchemy -pydantic==1.10.17 +pydantic==2.9.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -146,12 +144,28 @@ pydantic==1.10.17 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.7.2 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -225,12 +239,12 @@ typing-extensions==4.12.2 # via # aiodebug # alembic - # anyio # faststream # flexcache # flexparser # pint # pydantic + # pydantic-core # typer yarl==1.9.4 # via diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index db22f856cee..fa838e38766 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -28,12 +28,12 @@ alembic==1.13.2 # via # -c requirements/_base.txt # -r requirements/_test.in -antlr4-python3-runtime==4.13.2 - # via moto -async-timeout==4.0.3 +annotated-types==0.7.0 # via # -c requirements/_base.txt - # aiohttp + # pydantic +antlr4-python3-runtime==4.13.2 + # via moto attrs==24.2.0 # via # -c requirements/_base.txt @@ -88,10 +88,6 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -exceptiongroup==1.2.2 - # via - # -c requirements/_base.txt - # pytest execnet==2.1.1 # via pytest-xdist faker==27.0.0 @@ -207,11 +203,15 @@ py-partiql-parser==0.5.5 # via moto pycparser==2.22 # via cffi -pydantic==1.10.17 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via moto pytest==8.3.2 @@ -249,7 +249,9 @@ python-dateutil==2.9.0.post0 # faker # moto python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt @@ -304,11 +306,6 @@ sympy==1.13.2 # via cfn-lint termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # mypy - # pytest types-aiobotocore==2.13.2 # via -r requirements/_test.in types-aiobotocore-s3==2.13.2 @@ -327,6 +324,7 @@ typing-extensions==4.12.2 # cfn-lint # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # types-aiobotocore # types-aiobotocore-s3 diff --git a/packages/simcore-sdk/requirements/_tools.txt b/packages/simcore-sdk/requirements/_tools.txt index 8ca413e037f..5a573bd4848 100644 --- a/packages/simcore-sdk/requirements/_tools.txt +++ b/packages/simcore-sdk/requirements/_tools.txt @@ -76,22 +76,12 @@ setuptools==73.0.1 # via # -c requirements/_test.txt # pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py index 7b8b810ba38..7579c3eeb0c 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, StorageFileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.archiving_utils import unarchive_dir from servicelib.logging_utils import log_context from servicelib.progress_bar import ProgressBarData @@ -25,7 +25,9 @@ def __create_s3_object_key( project_id: ProjectID, node_uuid: NodeID, file_path: Path | str ) -> StorageFileID: file_name = file_path.name if isinstance(file_path, Path) else file_path - return parse_obj_as(StorageFileID, f"{project_id}/{node_uuid}/{file_name}") # type: ignore[arg-type] + return TypeAdapter(StorageFileID).validate_python( + f"{project_id}/{node_uuid}/{file_name}" + ) def __get_s3_name(path: Path, *, is_archive: bool) -> str: diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py index 7b5467c2851..a5305dd5b93 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py @@ -14,7 +14,7 @@ from models_library.projects_nodes_io import LocationID, LocationName from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from settings_library.node_ports import NodePortsSettings from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -42,7 +42,7 @@ async def _get_location_id_from_location_name( raise exceptions.S3InvalidStore(store) -def _get_https_link_if_storage_secure(url: AnyUrl) -> str: +def _get_https_link_if_storage_secure(url: str) -> str: # NOTE: links generated by storage are http only. # WEBSERVER -> STORAGE (http requests) # DY-SIDECAR (simcore-sdk) -> STORAGE (httpS requests) @@ -69,18 +69,18 @@ async def _complete_upload( :rtype: ETag """ async with session.post( - _get_https_link_if_storage_secure(upload_completion_link), + _get_https_link_if_storage_secure(str(upload_completion_link)), json=jsonable_encoder(FileUploadCompletionBody(parts=parts)), auth=get_basic_auth(), ) as resp: resp.raise_for_status() # now poll for state - file_upload_complete_response = parse_obj_as( - Envelope[FileUploadCompleteResponse], await resp.json() + file_upload_complete_response = TypeAdapter(Envelope[FileUploadCompleteResponse]).validate_python( + await resp.json() ) assert file_upload_complete_response.data # nosec state_url = _get_https_link_if_storage_secure( - file_upload_complete_response.data.links.state + str(file_upload_complete_response.data.links.state) ) _logger.info("completed upload of %s", f"{len(parts)} parts, received {state_url}") @@ -96,8 +96,8 @@ async def _complete_upload( with attempt: async with session.post(state_url, auth=get_basic_auth()) as resp: resp.raise_for_status() - future_enveloped = parse_obj_as( - Envelope[FileUploadCompleteFutureResponse], await resp.json() + future_enveloped = TypeAdapter(Envelope[FileUploadCompleteFutureResponse]).validate_python( + await resp.json() ) assert future_enveloped.data # nosec if future_enveloped.data.state == FileUploadCompleteState.NOK: @@ -142,7 +142,7 @@ async def _abort_upload( # abort the upload correctly, so it can revert back to last version try: async with session.post( - _get_https_link_if_storage_secure(abort_upload_link), auth=get_basic_auth() + _get_https_link_if_storage_secure(str(abort_upload_link)), auth=get_basic_auth() ) as resp: resp.raise_for_status() except ClientError: diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli.py index 35d1d7c71f8..320cfd7e25f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli.py @@ -6,10 +6,11 @@ from asyncio.streams import StreamReader from pathlib import Path +from common_library.errors_classes import OsparcErrorMixin + from aiocache import cached # type: ignore[import-untyped] from models_library.basic_types import IDStr from pydantic import AnyUrl, ByteSize -from pydantic.errors import PydanticErrorMixin from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather from settings_library.aws_s3_cli import AwsS3CliSettings @@ -24,7 +25,7 @@ _OSPARC_SYMLINK_EXTENSION = ".rclonelink" # named `rclonelink` to maintain backwards -class BaseAwsS3CliError(PydanticErrorMixin, RuntimeError): +class BaseAwsS3CliError(OsparcErrorMixin, RuntimeError): ... diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py index 695b710c8f8..5feefab82f8 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py @@ -253,7 +253,7 @@ def _check_for_aws_http_errors(exc: BaseException) -> bool: async def _session_put( session: ClientSession, file_part_size: int, - upload_url: AnyUrl, + upload_url: str, pbar: tqdm, io_log_redirect_cb: LogRedirectCB | None, progress_bar: ProgressBarData, @@ -314,7 +314,7 @@ async def _upload_file_part( received_e_tag = await _session_put( session=session, file_part_size=file_part_size, - upload_url=upload_url, + upload_url=str(upload_url), pbar=pbar, io_log_redirect_cb=io_log_redirect_cb, progress_bar=progress_bar, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py index 6a5609c7eb5..f3e2587fab7 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py @@ -15,7 +15,7 @@ from models_library.basic_types import IDStr, SHA256Str from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter, parse_obj_as from servicelib.file_utils import create_sha256_checksum from servicelib.progress_bar import ProgressBarData from settings_library.aws_s3_cli import AwsS3CliSettings @@ -189,14 +189,16 @@ async def download_path_from_s3( aws_s3_cli_settings, progress_bar, local_directory_path=local_path, - download_s3_link=parse_obj_as(AnyUrl, f"{download_link}"), + download_s3_link=TypeAdapter(AnyUrl).validate_python(f"{download_link}"), ) elif r_clone_settings: await r_clone.sync_s3_to_local( r_clone_settings, progress_bar, local_directory_path=local_path, - download_s3_link=parse_obj_as(AnyUrl, f"{download_link}"), + download_s3_link=str( + TypeAdapter(AnyUrl).validate_python(f"{download_link}") + ), ) else: msg = "Unexpected configuration" diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py index 18e15139493..bbfe14e7f39 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py @@ -8,11 +8,12 @@ from pathlib import Path from typing import Final +from common_library.errors_classes import OsparcErrorMixin + from aiocache import cached # type: ignore[import-untyped] from aiofiles import tempfile from models_library.basic_types import IDStr from pydantic import AnyUrl, BaseModel, ByteSize -from pydantic.errors import PydanticErrorMixin from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather from settings_library.r_clone import RCloneSettings @@ -31,7 +32,7 @@ _logger = logging.getLogger(__name__) -class BaseRCloneError(PydanticErrorMixin, RuntimeError): +class BaseRCloneError(OsparcErrorMixin, RuntimeError): ... @@ -166,7 +167,7 @@ async def _get_folder_size( cwd=f"{local_dir.resolve()}", ) - rclone_folder_size_result = _RCloneSize.parse_raw(result) + rclone_folder_size_result = _RCloneSize.model_validate_json(result) _logger.debug( "RClone size call for %s: %s", f"{folder}", f"{rclone_folder_size_result}" ) @@ -259,7 +260,7 @@ async def sync_local_to_s3( """ _raise_if_directory_is_file(local_directory_path) - upload_s3_path = re.sub(r"^s3://", "", upload_s3_link) + upload_s3_path = re.sub(r"^s3://", "", str(upload_s3_link)) _logger.debug(" %s; %s", f"{upload_s3_link=}", f"{upload_s3_path=}") await _sync_sources( @@ -279,7 +280,7 @@ async def sync_s3_to_local( progress_bar: ProgressBarData, *, local_directory_path: Path, - download_s3_link: AnyUrl, + download_s3_link: str, exclude_patterns: set[str] | None = None, debug_logs: bool = False, ) -> None: diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py index f539e451026..75ed54ec686 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py @@ -3,7 +3,7 @@ from typing import Union from models_library.utils.change_case import snake_to_camel -from pydantic import BaseModel, ByteSize, Field, parse_raw_as +from pydantic import BaseModel, ByteSize, ConfigDict, Field, TypeAdapter from servicelib.logging_utils import log_catch from servicelib.progress_bar import ProgressBarData @@ -12,6 +12,8 @@ _logger = logging.getLogger(__name__) + + class _RCloneSyncMessageBase(BaseModel): level: str = Field(..., description="log level") msg: str @@ -31,9 +33,7 @@ class _RCloneSyncTransferCompletedMessage(_RCloneSyncMessageBase): class _RCloneSyncTransferringStats(BaseModel): bytes: ByteSize total_bytes: ByteSize - - class Config: - alias_generator = snake_to_camel + model_config = ConfigDict(alias_generator=snake_to_camel) class _RCloneSyncTransferringMessage(_RCloneSyncMessageBase): @@ -77,8 +77,7 @@ def __init__(self, progress_bar: ProgressBarData) -> None: async def __call__(self, logs: str) -> None: _logger.debug("received logs: %s", logs) with log_catch(_logger, reraise=False): - rclone_message: _RCloneSyncMessages = parse_raw_as( - _RCloneSyncMessages, # type: ignore[arg-type] + rclone_message: _RCloneSyncMessages = TypeAdapter(_RCloneSyncMessages).validate_strings( logs, ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py index c249cbcf830..b7a394a6dbd 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py @@ -144,7 +144,7 @@ async def get_storage_locations( expected_status=status.HTTP_200_OK, params={"user_id": f"{user_id}"}, ) as response: - locations_enveloped = Envelope[FileLocationArray].parse_obj( + locations_enveloped = Envelope[FileLocationArray].model_validate( await response.json() ) if locations_enveloped.data is None: @@ -173,7 +173,7 @@ async def get_download_file_link( expected_status=status.HTTP_200_OK, params={"user_id": f"{user_id}", "link_type": link_type.value}, ) as response: - presigned_link_enveloped = Envelope[PresignedLink].parse_obj( + presigned_link_enveloped = Envelope[PresignedLink].model_validate( await response.json() ) if not presigned_link_enveloped.data or not presigned_link_enveloped.data.link: @@ -215,7 +215,7 @@ async def get_upload_file_links( expected_status=status.HTTP_200_OK, params=query_params, ) as response: - file_upload_links_enveloped = Envelope[FileUploadSchema].parse_obj( + file_upload_links_enveloped = Envelope[FileUploadSchema].model_validate( await response.json() ) if file_upload_links_enveloped.data is None: @@ -245,7 +245,7 @@ async def get_file_metadata( # NOTE: keeps backwards compatibility raise exceptions.S3InvalidPathError(file_id) - file_metadata_enveloped = Envelope[FileMetaDataGet].parse_obj(payload) + file_metadata_enveloped = Envelope[FileMetaDataGet].model_validate(payload) assert file_metadata_enveloped.data # nosec return file_metadata_enveloped.data @@ -265,7 +265,7 @@ async def list_file_metadata( expected_status=status.HTTP_200_OK, params={"user_id": f"{user_id}", "uuid_filter": uuid_filter}, ) as resp: - envelope = Envelope[list[FileMetaDataGet]].parse_obj(await resp.json()) + envelope = Envelope[list[FileMetaDataGet]].model_validate(await resp.json()) assert envelope.data is not None # nosec file_meta_data: list[FileMetaDataGet] = envelope.data return file_meta_data diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py index d8eb1d99349..ad94884c3b0 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -4,18 +4,27 @@ from models_library.basic_regex import UUID_RE from models_library.projects_nodes_io import BaseFileLink, DownloadLink from models_library.projects_nodes_io import PortLink as BasePortLink -from pydantic import AnyUrl, Extra, Field, StrictBool, StrictFloat, StrictInt, StrictStr +from pydantic import ( + AnyUrl, + ConfigDict, + Field, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, +) class PortLink(BasePortLink): - node_uuid: str = Field(..., regex=UUID_RE, alias="nodeUuid") # type: ignore[assignment] # This overrides the base class it is ugly but needs its own PR to fix it + node_uuid: str = Field(..., pattern=UUID_RE, alias="nodeUuid") # type: ignore[assignment] # This overrides the base class it is ugly but needs its own PR to fix it class FileLink(BaseFileLink): """allow all kind of file links""" - class Config: - extra = Extra.allow + model_config = ConfigDict( + extra="allow", + ) # TODO: needs to be in sync with project_nodes.InputTypes and project_nodes.OutputTypes diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 8c78e28a066..d0ef9eb14bf 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -1,16 +1,19 @@ +from asyncio import Task +import traceback import logging from collections.abc import Callable, Coroutine from pathlib import Path from typing import Any +from pydantic_core import InitErrorDetails + from models_library.api_schemas_storage import LinkType from models_library.basic_types import IDStr from models_library.projects import ProjectIDStr from models_library.projects_nodes_io import NodeIDStr from models_library.services_types import ServicePortKey from models_library.users import UserID -from pydantic import BaseModel, Field, ValidationError -from pydantic.error_wrappers import flatten_errors +from pydantic import BaseModel, ConfigDict, Field, ValidationError from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather from settings_library.aws_s3_cli import AwsS3CliSettings @@ -27,6 +30,16 @@ log = logging.getLogger(__name__) +def _format_error(task:Task)-> str: + # pylint:disable=protected-access + assert task._exception #nosec + error_list= traceback.format_exception(type(task._exception), task._exception, task._exception.__traceback__) + return "\n".join(error_list) + +def _get_error_details(task:Task, port_key:str)->InitErrorDetails: + # pylint:disable=protected-access + return InitErrorDetails(type="value_error", loc=(f"{port_key}",), input=_format_error(task), ctx={"error":task._exception}) + class Nodeports(BaseModel): """ Represents a node in a project and all its input/output ports @@ -47,9 +60,9 @@ class Nodeports(BaseModel): r_clone_settings: RCloneSettings | None = None io_log_redirect_cb: LogRedirectCB | None aws_s3_cli_settings: AwsS3CliSettings | None = None - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict( + arbitrary_types_allowed=True, + ) def __init__(self, **data: Any): super().__init__(**data) @@ -181,9 +194,9 @@ async def set_multiple( await self.save_to_db_cb(self) # groups all ValidationErrors pre-pending 'port_key' to loc and raises ValidationError - if errors := [ - list(flatten_errors([r], self.__config__, loc=(f"{port_key}",))) + if error_details:= [ + _get_error_details(r, port_key) for port_key, r in zip(port_values.keys(), results) - if isinstance(r, ValidationError) + if r is not None ]: - raise ValidationError(errors, model=type(self)) + raise ValidationError.from_exception_data(title="Multiple port_key errors",line_errors=error_details) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 2338563dcdb..3ddab6a29d3 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -10,8 +10,16 @@ from models_library.basic_types import IDStr from models_library.services_io import BaseServiceIOModel from models_library.services_types import ServicePortKey -from pydantic import AnyUrl, Field, PrivateAttr, ValidationError, validator -from pydantic.tools import parse_obj_as +from pydantic import ( + AnyUrl, + ConfigDict, + Field, + PrivateAttr, + TypeAdapter, + ValidationError, + ValidationInfo, + field_validator, +) from servicelib.progress_bar import ProgressBarData from ..node_ports_common.exceptions import ( @@ -56,7 +64,7 @@ def _check_if_symlink_is_valid(symlink: Path) -> None: def can_parse_as(v, *types) -> bool: try: for type_ in types: - parse_obj_as(type_, v) + TypeAdapter(type_).validate_python(v) return True except ValidationError: return False @@ -70,17 +78,23 @@ class SetKWargs: class Port(BaseServiceIOModel): key: ServicePortKey widget: dict[str, Any] | None = None - default_value: DataItemValue | None = Field(None, alias="defaultValue") + default_value: DataItemValue | None = Field( + None, alias="defaultValue", union_mode="left_to_right" + ) - value: DataItemValue | None = None + value: DataItemValue | None = Field( + None, validate_default=True, union_mode="left_to_right" + ) # Different states of "value" # - e.g. typically after resolving a port's link, a download link, ... # - lazy evaluation using get_* members # - used to run validation & conversion of resolved PortContentTypes values # - excluded from all model export - value_item: ItemValue | None = Field(None, exclude=True) - value_concrete: ItemConcreteValue | None = Field(None, exclude=True) + value_item: ItemValue | None = Field(None, exclude=True, union_mode="left_to_right") + value_concrete: ItemConcreteValue | None = Field( + None, exclude=True, union_mode="left_to_right" + ) # Function to convert from ItemValue -> ItemConcreteValue _py_value_converter: Callable[[Any], ItemConcreteValue] = PrivateAttr() @@ -90,15 +104,14 @@ class Port(BaseServiceIOModel): # flags _used_default_value: bool = PrivateAttr(False) - class Config(BaseServiceIOModel.Config): - validate_assignment = True + model_config = ConfigDict(validate_assignment=True) - @validator("value", always=True) + @field_validator("value") @classmethod - def check_value(cls, v: DataItemValue, values: dict[str, Any]) -> DataItemValue: + def check_value(cls, v: DataItemValue, info: ValidationInfo) -> DataItemValue: if ( v is not None - and (property_type := values.get("property_type")) + and (property_type := info.data.get("property_type")) and not isinstance(v, PortLink) ): if port_utils.is_file_type(property_type): @@ -108,10 +121,10 @@ def check_value(cls, v: DataItemValue, values: dict[str, Any]) -> DataItemValue: ) elif property_type == "ref_contentSchema": v, _ = validate_port_content( - port_key=values.get("key"), + port_key=info.data.get("key"), value=v, unit=None, - content_schema=values.get("content_schema", {}), + content_schema=info.data.get("content_schema", {}), ) elif isinstance(v, (list, dict)): raise TypeError( @@ -119,21 +132,21 @@ def check_value(cls, v: DataItemValue, values: dict[str, Any]) -> DataItemValue: ) return v - @validator("value_item", "value_concrete", pre=True) + @field_validator("value_item", "value_concrete", mode="before") @classmethod - def check_item_or_concrete_value(cls, v, values): + def check_item_or_concrete_value(cls, v, info: ValidationInfo): if ( v - and v != values["value"] - and (property_type := values.get("property_type")) + and v != info.data["value"] + and (property_type := info.data.get("property_type")) and property_type == "ref_contentSchema" and not can_parse_as(v, Path, AnyUrl) ): v, _ = validate_port_content( - port_key=values.get("key"), + port_key=info.data.get("key"), value=v, unit=None, - content_schema=values.get("content_schema", {}), + content_schema=info.data.get("content_schema", {}), ) return v @@ -209,7 +222,9 @@ async def _evaluate() -> ItemValue | None: if isinstance(self.value, DownloadLink): # generic download link for a file - url: AnyUrl = self.value.download_link + url: AnyUrl = TypeAdapter(AnyUrl).validate_python( + self.value.download_link + ) return url # otherwise, this is a BasicValueTypes diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py index c2ebb56986d..b33e677c0bf 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py @@ -1,7 +1,8 @@ import logging import re -from typing import Any, Dict, Optional, Tuple +from typing import Any +from common_library.errors_classes import OsparcErrorMixin from models_library.projects_nodes import UnitStr from models_library.utils.json_schema import ( JsonSchemaValidationError, @@ -9,9 +10,8 @@ jsonschema_validate_schema, ) from pint import PintError, UnitRegistry -from pydantic.errors import PydanticValueError -JsonSchemaDict = Dict[str, Any] +JsonSchemaDict = dict[str, Any] log = logging.getLogger(__name__) @@ -22,8 +22,8 @@ # - Use 'code' to discriminate port_validation errors -class PortValueError(PydanticValueError): - code = "port_validation.schema_error" +class PortValueError(OsparcErrorMixin, ValueError): + code = "port_validation.schema_error" # type: ignore msg_template = "Invalid value in port {port_key!r}: {schema_error_message}" # pylint: disable=useless-super-delegation @@ -37,8 +37,8 @@ def __init__(self, *, port_key: str, schema_error: JsonSchemaValidationError): ) -class PortUnitError(PydanticValueError): - code = "port_validation.unit_error" +class PortUnitError(OsparcErrorMixin, ValueError): + code = "port_validation.unit_error" # type: ignore msg_template = "Invalid unit in port {port_key!r}: {pint_error_msg}" # pylint: disable=useless-super-delegation @@ -72,7 +72,7 @@ def _validate_port_value(value, content_schema: JsonSchemaDict): def _validate_port_unit( value, unit, content_schema: JsonSchemaDict, *, ureg: UnitRegistry -) -> Tuple[Any, Optional[UnitStr]]: +) -> tuple[Any, UnitStr | None]: """ - Checks valid 'value' against content_schema - Converts 'value' with 'unit' to unit expected in content_schema @@ -101,7 +101,7 @@ def _validate_port_unit( def validate_port_content( port_key, value: Any, - unit: Optional[UnitStr], + unit: UnitStr | None, content_schema: JsonSchemaDict, ): """A port content is all datasets injected to a given port. Currently only diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py index 2855e8a253e..9fb13510afb 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py @@ -1,38 +1,35 @@ from collections.abc import ItemsView, Iterator, KeysView, ValuesView from models_library.services_types import ServicePortKey -from pydantic import BaseModel +from pydantic import RootModel from ..node_ports_common.exceptions import UnboundPortError from .port import Port -class BasePortsMapping(BaseModel): - __root__: dict[ServicePortKey, Port] - +class BasePortsMapping(RootModel[dict[ServicePortKey, Port]]): def __getitem__(self, key: int | ServicePortKey) -> Port: - if isinstance(key, int): - if key < len(self.__root__): - key = list(self.__root__.keys())[key] - if key not in self.__root__: + if isinstance(key, int) and key < len(self.root): + key = list(self.root.keys())[key] + if key not in self.root: raise UnboundPortError(key) assert isinstance(key, str) # nosec - return self.__root__[key] + return self.root[key] def __iter__(self) -> Iterator[ServicePortKey]: # type: ignore - return iter(self.__root__) + return iter(self.root) def keys(self) -> KeysView[ServicePortKey]: - return self.__root__.keys() + return self.root.keys() def items(self) -> ItemsView[ServicePortKey, Port]: - return self.__root__.items() + return self.root.items() def values(self) -> ValuesView[Port]: - return self.__root__.values() + return self.root.values() def __len__(self) -> int: - return self.__root__.__len__() + return self.root.__len__() class InputsList(BasePortsMapping): diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index daa4c9aaa3e..f4d74711e18 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -115,7 +115,7 @@ async def dump(nodeports: Nodeports) -> None: "dumping node_ports_v2 object %s", pformat(nodeports, indent=2), ) - _nodeports_cfg = nodeports.dict( + _nodeports_cfg = nodeports.model_dump( include={"internal_inputs", "internal_outputs"}, by_alias=True, exclude_unset=True, diff --git a/packages/simcore-sdk/tests/helpers/utils_port_v2.py b/packages/simcore-sdk/tests/helpers/utils_port_v2.py index 556e0eb4ced..23298f6b175 100644 --- a/packages/simcore-sdk/tests/helpers/utils_port_v2.py +++ b/packages/simcore-sdk/tests/helpers/utils_port_v2.py @@ -45,5 +45,5 @@ def create_valid_port_mapping( key=key_for_file_port, fileToKeyMap={file_to_key: key_for_file_port} if file_to_key else None, ) - port_mapping = mapping_class(**{"__root__": port_cfgs}) + port_mapping = mapping_class(**{"root": port_cfgs}) return port_mapping diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index d5f6cd7227a..92b6afaa81b 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -142,7 +142,7 @@ async def _create(file_path: Path) -> dict[str, Any]: async with ClientSession() as session: async with session.put(url) as resp: resp.raise_for_status() - presigned_links_enveloped = Envelope[FileUploadSchema].parse_obj( + presigned_links_enveloped = Envelope[FileUploadSchema].model_validate( await resp.json() ) assert presigned_links_enveloped.data diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py index 9cd1ce32de4..56f696bb46d 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py @@ -1,699 +1,699 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name -# pylint:disable=too-many-arguments -# pylint:disable=protected-access - -import filecmp -from collections.abc import Awaitable, Callable -from pathlib import Path -from typing import Any -from uuid import uuid4 - -import pytest -from aiohttp import ClientError -from faker import Faker -from models_library.projects_nodes_io import ( - LocationID, - SimcoreS3DirectoryID, - SimcoreS3FileID, -) -from models_library.users import UserID -from pydantic import BaseModel, ByteSize, parse_obj_as -from pytest_mock import MockerFixture -from pytest_simcore.helpers.parametrizations import byte_size_ids -from servicelib.progress_bar import ProgressBarData -from settings_library.aws_s3_cli import AwsS3CliSettings -from settings_library.r_clone import RCloneSettings -from simcore_sdk.node_ports_common import exceptions, filemanager -from simcore_sdk.node_ports_common.aws_s3_cli import AwsS3CliFailedError -from simcore_sdk.node_ports_common.filemanager import UploadedFile, UploadedFolder -from simcore_sdk.node_ports_common.r_clone import RCloneFailedError -from yarl import URL - -pytest_simcore_core_services_selection = [ - "migration", - "postgres", - "storage", - "redis", -] - -pytest_simcore_ops_services_selection = ["minio", "adminer"] - - -class _SyncSettings(BaseModel): - r_clone_settings: RCloneSettings | None - aws_s3_cli_settings: AwsS3CliSettings | None - - -@pytest.fixture( - params=[(True, False), (False, True), (False, False)], - ids=[ - "RClone enabled", - "AwsS3Cli enabled", - "Both RClone and AwsS3Cli disabled", - ], -) -def optional_sync_settings( - r_clone_settings: RCloneSettings, - aws_s3_cli_settings: AwsS3CliSettings, - request: pytest.FixtureRequest, -) -> _SyncSettings: - _rclone_enabled, _aws_s3_cli_enabled = request.param - - _r_clone_settings = r_clone_settings if _rclone_enabled else None - _aws_s3_cli_settings = aws_s3_cli_settings if _aws_s3_cli_enabled else None - - return _SyncSettings( - r_clone_settings=_r_clone_settings, aws_s3_cli_settings=_aws_s3_cli_settings - ) - - -def _file_size(size_str: str, **pytest_params): - return pytest.param(parse_obj_as(ByteSize, size_str), id=size_str, **pytest_params) - - -@pytest.mark.parametrize( - "file_size", - [ - _file_size("10Mib"), - _file_size("103Mib"), - _file_size("1003Mib", marks=pytest.mark.heavy_load), - _file_size("7Gib", marks=pytest.mark.heavy_load), - ], - ids=byte_size_ids, -) -async def test_valid_upload_download( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - file_size: ByteSize, - create_file_of_size: Callable[[ByteSize, str], Path], - optional_sync_settings: _SyncSettings, - simcore_services_ready: None, - storage_service: URL, - faker: Faker, -): - file_path = create_file_of_size(file_size, "test.test") - - file_id = create_valid_file_uuid("", file_path) - async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: - upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=file_path, - r_clone_settings=optional_sync_settings.r_clone_settings, - io_log_redirect_cb=None, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - assert isinstance(upload_result, UploadedFile) - store_id, e_tag = upload_result.store_id, upload_result.etag - # pylint: disable=protected-access - assert progress_bar._current_steps == pytest.approx(1) # noqa: SLF001 - assert store_id == s3_simcore_location - assert e_tag - file_metadata = await filemanager.get_file_metadata( - user_id=user_id, store_id=store_id, s3_object=file_id - ) - assert file_metadata.location == store_id - assert file_metadata.etag == e_tag - - download_folder = Path(tmpdir) / "downloads" - download_file_path = await filemanager.download_path_from_s3( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - local_path=download_folder, - io_log_redirect_cb=None, - r_clone_settings=optional_sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - assert progress_bar._current_steps == pytest.approx(2) # noqa: SLF001 - assert download_file_path.exists() - assert download_file_path.name == "test.test" - assert filecmp.cmp(download_file_path, file_path) - - -@pytest.mark.parametrize( - "file_size", - [ - _file_size("10Mib"), - _file_size("103Mib"), - ], - ids=byte_size_ids, -) -async def test_valid_upload_download_using_file_object( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - file_size: ByteSize, - create_file_of_size: Callable[[ByteSize, str], Path], - optional_sync_settings: _SyncSettings, - faker: Faker, -): - file_path = create_file_of_size(file_size, "test.test") - - file_id = create_valid_file_uuid("", file_path) - with file_path.open("rb") as file_object: - upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=filemanager.UploadableFileObject( - file_object, file_path.name, file_path.stat().st_size - ), - r_clone_settings=optional_sync_settings.r_clone_settings, - io_log_redirect_cb=None, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - assert isinstance(upload_result, UploadedFile) - store_id, e_tag = upload_result.store_id, upload_result.etag - assert store_id == s3_simcore_location - assert e_tag - file_metadata = await filemanager.get_file_metadata( - user_id=user_id, store_id=store_id, s3_object=file_id - ) - assert file_metadata.location == store_id - assert file_metadata.etag == e_tag - - download_folder = Path(tmpdir) / "downloads" - async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: - download_file_path = await filemanager.download_path_from_s3( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - local_path=download_folder, - io_log_redirect_cb=None, - r_clone_settings=optional_sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - assert progress_bar._current_steps == pytest.approx(1) # noqa: SLF001 - assert download_file_path.exists() - assert download_file_path.name == "test.test" - assert filecmp.cmp(download_file_path, file_path) - - -@pytest.fixture -def mocked_upload_file_raising_exceptions(mocker: MockerFixture) -> None: - mocker.patch( - "simcore_sdk.node_ports_common.filemanager.r_clone.sync_local_to_s3", - autospec=True, - side_effect=RCloneFailedError, - ) - mocker.patch( - "simcore_sdk.node_ports_common.file_io_utils._upload_file_part", - autospec=True, - side_effect=ClientError, - ) - mocker.patch( - "simcore_sdk.node_ports_common.filemanager.aws_s3_cli.sync_local_to_s3", - autospec=True, - side_effect=AwsS3CliFailedError, - ) - - -@pytest.mark.parametrize( - "file_size", - [ - _file_size("10Mib"), - ], - ids=byte_size_ids, -) -async def test_failed_upload_is_properly_removed_from_storage( - node_ports_config: None, - create_file_of_size: Callable[[ByteSize], Path], - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - optional_sync_settings: _SyncSettings, - file_size: ByteSize, - user_id: UserID, - mocked_upload_file_raising_exceptions: None, -): - file_path = create_file_of_size(file_size) - file_id = create_valid_file_uuid("", file_path) - with pytest.raises(exceptions.S3TransferError): - await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=file_path, - r_clone_settings=optional_sync_settings.r_clone_settings, - io_log_redirect_cb=None, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - with pytest.raises(exceptions.S3InvalidPathError): - await filemanager.get_file_metadata( - user_id=user_id, store_id=s3_simcore_location, s3_object=file_id - ) - - -@pytest.mark.parametrize( - "file_size", - [ - _file_size("10Mib"), - ], - ids=byte_size_ids, -) -async def test_failed_upload_after_valid_upload_keeps_last_valid_state( - node_ports_config: None, - create_file_of_size: Callable[[ByteSize], Path], - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - optional_sync_settings: _SyncSettings, - file_size: ByteSize, - user_id: UserID, - mocker: MockerFixture, -): - # upload a valid file - file_path = create_file_of_size(file_size) - file_id = create_valid_file_uuid("", file_path) - upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=file_path, - r_clone_settings=optional_sync_settings.r_clone_settings, - io_log_redirect_cb=None, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - assert isinstance(upload_result, UploadedFile) - store_id, e_tag = upload_result.store_id, upload_result.etag - assert store_id == s3_simcore_location - assert e_tag - # check the file is correctly uploaded - file_metadata = await filemanager.get_file_metadata( - user_id=user_id, store_id=store_id, s3_object=file_id - ) - assert file_metadata.location == store_id - assert file_metadata.etag == e_tag - # now start an invalid update by generating an exception while uploading the same file - mocker.patch( - "simcore_sdk.node_ports_common.filemanager.r_clone.sync_local_to_s3", - autospec=True, - side_effect=RCloneFailedError, - ) - mocker.patch( - "simcore_sdk.node_ports_common.file_io_utils._upload_file_part", - autospec=True, - side_effect=ClientError, - ) - with pytest.raises(exceptions.S3TransferError): - await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=file_path, - r_clone_settings=optional_sync_settings.r_clone_settings, - io_log_redirect_cb=None, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - # the file shall be back to its original state - file_metadata = await filemanager.get_file_metadata( - user_id=user_id, store_id=s3_simcore_location, s3_object=file_id - ) - assert file_metadata.location == store_id - assert file_metadata.etag == e_tag - - -async def test_invalid_file_path( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - optional_sync_settings: _SyncSettings, - faker: Faker, -): - file_path = Path(tmpdir) / "test.test" - file_path.write_text("I am a test file") - assert file_path.exists() - - file_id = create_valid_file_uuid("", file_path) - store = s3_simcore_location - with pytest.raises(FileNotFoundError): - await filemanager.upload_path( - user_id=user_id, - store_id=store, - store_name=None, - s3_object=file_id, - path_to_upload=Path(tmpdir) / "some other file.txt", - io_log_redirect_cb=None, - ) - - download_folder = Path(tmpdir) / "downloads" - with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 - async with ProgressBarData( - num_steps=1, description=faker.pystr() - ) as progress_bar: - await filemanager.download_path_from_s3( - user_id=user_id, - store_id=store, - store_name=None, - s3_object=file_id, - local_path=download_folder, - io_log_redirect_cb=None, - r_clone_settings=optional_sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - - -async def test_errors_upon_invalid_file_identifiers( - node_ports_config: None, - tmpdir: Path, - user_id: UserID, - project_id: str, - s3_simcore_location: LocationID, - optional_sync_settings: _SyncSettings, - faker: Faker, -): - file_path = Path(tmpdir) / "test.test" - file_path.write_text("I am a test file") - assert file_path.exists() - - store = s3_simcore_location - with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 - invalid_s3_path = SimcoreS3FileID("") - await filemanager.upload_path( - user_id=user_id, - store_id=store, - store_name=None, - s3_object=invalid_s3_path, - path_to_upload=file_path, - io_log_redirect_cb=None, - ) - - with pytest.raises(exceptions.StorageInvalidCall): # noqa: PT012 - invalid_file_id = SimcoreS3FileID("file_id") - await filemanager.upload_path( - user_id=user_id, - store_id=store, - store_name=None, - s3_object=invalid_file_id, - path_to_upload=file_path, - io_log_redirect_cb=None, - ) - - download_folder = Path(tmpdir) / "downloads" - with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 - async with ProgressBarData( - num_steps=1, description=faker.pystr() - ) as progress_bar: - invalid_s3_path = SimcoreS3FileID("") - await filemanager.download_path_from_s3( - user_id=user_id, - store_id=store, - store_name=None, - s3_object=invalid_s3_path, - local_path=download_folder, - io_log_redirect_cb=None, - r_clone_settings=optional_sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - - with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 - async with ProgressBarData( - num_steps=1, description=faker.pystr() - ) as progress_bar: - await filemanager.download_path_from_s3( - user_id=user_id, - store_id=store, - store_name=None, - s3_object=SimcoreS3FileID(f"{project_id}/{uuid4()}/invisible.txt"), - local_path=download_folder, - io_log_redirect_cb=None, - r_clone_settings=optional_sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - - -async def test_invalid_store( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - optional_sync_settings: _SyncSettings, - faker: Faker, -): - file_path = Path(tmpdir) / "test.test" - file_path.write_text("I am a test file") - assert file_path.exists() - - file_id = create_valid_file_uuid("", file_path) - store = "somefunkystore" - with pytest.raises(exceptions.S3InvalidStore): - await filemanager.upload_path( - user_id=user_id, - store_id=None, - store_name=store, # type: ignore - s3_object=file_id, - path_to_upload=file_path, - io_log_redirect_cb=None, - ) - - download_folder = Path(tmpdir) / "downloads" - with pytest.raises(exceptions.S3InvalidStore): # noqa: PT012 - async with ProgressBarData( - num_steps=1, description=faker.pystr() - ) as progress_bar: - await filemanager.download_path_from_s3( - user_id=user_id, - store_id=None, - store_name=store, # type: ignore - s3_object=file_id, - local_path=download_folder, - io_log_redirect_cb=None, - r_clone_settings=optional_sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, - ) - - -@pytest.fixture( - params=[True, False], - ids=["with RClone", "with AwsS3Cli"], -) -def sync_settings( - r_clone_settings: RCloneSettings, - aws_s3_cli_settings: AwsS3CliSettings, - request: pytest.FixtureRequest, -) -> _SyncSettings: - is_rclone_enabled = request.param - - return _SyncSettings( - r_clone_settings=r_clone_settings if is_rclone_enabled else None, - aws_s3_cli_settings=aws_s3_cli_settings if not is_rclone_enabled else None, - ) - - -@pytest.mark.parametrize("is_directory", [False, True]) -async def test_valid_metadata( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - sync_settings: _SyncSettings, - is_directory: bool, -): - # first we go with a non-existing file - file_path = Path(tmpdir) / "a-subdir" / "test.test" - file_path.parent.mkdir(parents=True, exist_ok=True) - - path_to_upload = file_path.parent if is_directory else file_path - - file_id = create_valid_file_uuid("", path_to_upload) - assert file_path.exists() is False - - is_metadata_present = await filemanager.entry_exists( - user_id=user_id, - store_id=s3_simcore_location, - s3_object=file_id, - is_directory=is_directory, - ) - assert is_metadata_present is False - - # now really create the file and upload it - file_path.write_text("I am a test file") - assert file_path.exists() - - file_id = create_valid_file_uuid("", path_to_upload) - upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=path_to_upload, - io_log_redirect_cb=None, - r_clone_settings=sync_settings.r_clone_settings, - aws_s3_cli_settings=sync_settings.aws_s3_cli_settings, - ) - if is_directory: - assert isinstance(upload_result, UploadedFolder) - else: - assert isinstance(upload_result, UploadedFile) - assert upload_result.store_id == s3_simcore_location - assert upload_result.etag - - is_metadata_present = await filemanager.entry_exists( - user_id=user_id, - store_id=s3_simcore_location, - s3_object=file_id, - is_directory=is_directory, - ) - - assert is_metadata_present is True - - -@pytest.mark.parametrize( - "fct, extra_kwargs", - [ - (filemanager.entry_exists, {"is_directory": False}), - (filemanager.delete_file, {}), - (filemanager.get_file_metadata, {}), - ], -) -async def test_invalid_call_raises_exception( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - fct: Callable[[int, str, str, Any | None], Awaitable], - extra_kwargs: dict[str, Any], -): - file_path = Path(tmpdir) / "test.test" - file_id = create_valid_file_uuid("", file_path) - assert file_path.exists() is False - - with pytest.raises(exceptions.StorageInvalidCall): - await fct( - user_id=None, store_id=s3_simcore_location, s3_object=file_id, **extra_kwargs # type: ignore - ) - with pytest.raises(exceptions.StorageInvalidCall): - await fct(user_id=user_id, store_id=None, s3_object=file_id, **extra_kwargs) # type: ignore - with pytest.raises(exceptions.StorageInvalidCall): - await fct( - user_id=user_id, store_id=s3_simcore_location, s3_object="bing", **extra_kwargs # type: ignore - ) - - -async def test_delete_file( - node_ports_config: None, - tmpdir: Path, - user_id: int, - create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], - s3_simcore_location: LocationID, - storage_service: URL, -): - file_path = Path(tmpdir) / "test.test" - file_path.write_text("I am a test file") - assert file_path.exists() - - file_id = create_valid_file_uuid("", file_path) - upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=file_id, - path_to_upload=file_path, - io_log_redirect_cb=None, - ) - assert isinstance(upload_result, UploadedFile) - store_id, e_tag = upload_result.store_id, upload_result.etag - assert store_id == s3_simcore_location - assert e_tag - - is_metadata_present = await filemanager.entry_exists( - user_id=user_id, store_id=store_id, s3_object=file_id, is_directory=False - ) - assert is_metadata_present is True - - await filemanager.delete_file( - user_id=user_id, store_id=s3_simcore_location, s3_object=file_id - ) - - # check that it disappeared - assert ( - await filemanager.entry_exists( - user_id=user_id, store_id=store_id, s3_object=file_id, is_directory=False - ) - is False - ) - - -@pytest.mark.parametrize("files_in_folder", [1, 10]) -async def test_upload_path_source_is_a_folder( - node_ports_config: None, - project_id: str, - tmp_path: Path, - faker: Faker, - user_id: int, - s3_simcore_location: LocationID, - files_in_folder: int, - sync_settings: _SyncSettings, -): - source_dir = tmp_path / f"source-{faker.uuid4()}" - source_dir.mkdir(parents=True, exist_ok=True) - - download_dir = tmp_path / f"download-{faker.uuid4()}" - download_dir.mkdir(parents=True, exist_ok=True) - - for i in range(files_in_folder): - (source_dir / f"file-{i}.txt").write_text("1") - - directory_id = SimcoreS3DirectoryID.from_simcore_s3_object( - f"{project_id}/{faker.uuid4()}/some-dir-in-node-root/" - ) - s3_object = SimcoreS3FileID(directory_id) - - upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( - user_id=user_id, - store_id=s3_simcore_location, - store_name=None, - s3_object=s3_object, - path_to_upload=source_dir, - io_log_redirect_cb=None, - r_clone_settings=sync_settings.r_clone_settings, - aws_s3_cli_settings=sync_settings.aws_s3_cli_settings, - ) - assert isinstance(upload_result, UploadedFolder) - assert source_dir.exists() - - async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: - await filemanager.download_path_from_s3( - user_id=user_id, - store_name=None, - store_id=s3_simcore_location, - s3_object=s3_object, - local_path=download_dir, - io_log_redirect_cb=None, - r_clone_settings=sync_settings.r_clone_settings, - progress_bar=progress_bar, - aws_s3_cli_settings=sync_settings.aws_s3_cli_settings, - ) - assert download_dir.exists() - - # ensure all files in download and source directory are the same - file_names: set = {f.name for f in source_dir.glob("*")} & { - f.name for f in download_dir.glob("*") - } - for file_name in file_names: - filecmp.cmp(source_dir / file_name, download_dir / file_name, shallow=False) +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=protected-access + +import filecmp +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest +from aiohttp import ClientError +from faker import Faker +from models_library.projects_nodes_io import ( + LocationID, + SimcoreS3DirectoryID, + SimcoreS3FileID, +) +from models_library.users import UserID +from pydantic import BaseModel, ByteSize, parse_obj_as +from pytest_mock import MockerFixture +from pytest_simcore.helpers.parametrizations import byte_size_ids +from servicelib.progress_bar import ProgressBarData +from settings_library.aws_s3_cli import AwsS3CliSettings +from settings_library.r_clone import RCloneSettings +from simcore_sdk.node_ports_common import exceptions, filemanager +from simcore_sdk.node_ports_common.aws_s3_cli import AwsS3CliFailedError +from simcore_sdk.node_ports_common.filemanager import UploadedFile, UploadedFolder +from simcore_sdk.node_ports_common.r_clone import RCloneFailedError +from yarl import URL + +pytest_simcore_core_services_selection = [ + "migration", + "postgres", + "storage", + "redis", +] + +pytest_simcore_ops_services_selection = ["minio", "adminer"] + + +class _SyncSettings(BaseModel): + r_clone_settings: RCloneSettings | None + aws_s3_cli_settings: AwsS3CliSettings | None + + +@pytest.fixture( + params=[(True, False), (False, True), (False, False)], + ids=[ + "RClone enabled", + "AwsS3Cli enabled", + "Both RClone and AwsS3Cli disabled", + ], +) +def optional_sync_settings( + r_clone_settings: RCloneSettings, + aws_s3_cli_settings: AwsS3CliSettings, + request: pytest.FixtureRequest, +) -> _SyncSettings: + _rclone_enabled, _aws_s3_cli_enabled = request.param + + _r_clone_settings = r_clone_settings if _rclone_enabled else None + _aws_s3_cli_settings = aws_s3_cli_settings if _aws_s3_cli_enabled else None + + return _SyncSettings( + r_clone_settings=_r_clone_settings, aws_s3_cli_settings=_aws_s3_cli_settings + ) + + +def _file_size(size_str: str, **pytest_params): + return pytest.param(parse_obj_as(ByteSize, size_str), id=size_str, **pytest_params) + + +@pytest.mark.parametrize( + "file_size", + [ + _file_size("10Mib"), + _file_size("103Mib"), + _file_size("1003Mib", marks=pytest.mark.heavy_load), + _file_size("7Gib", marks=pytest.mark.heavy_load), + ], + ids=byte_size_ids, +) +async def test_valid_upload_download( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + file_size: ByteSize, + create_file_of_size: Callable[[ByteSize, str], Path], + optional_sync_settings: _SyncSettings, + simcore_services_ready: None, + storage_service: URL, + faker: Faker, +): + file_path = create_file_of_size(file_size, "test.test") + + file_id = create_valid_file_uuid("", file_path) + async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: + upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=file_path, + r_clone_settings=optional_sync_settings.r_clone_settings, + io_log_redirect_cb=None, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + assert isinstance(upload_result, UploadedFile) + store_id, e_tag = upload_result.store_id, upload_result.etag + # pylint: disable=protected-access + assert progress_bar._current_steps == pytest.approx(1) # noqa: SLF001 + assert store_id == s3_simcore_location + assert e_tag + file_metadata = await filemanager.get_file_metadata( + user_id=user_id, store_id=store_id, s3_object=file_id + ) + assert file_metadata.location == store_id + assert file_metadata.etag == e_tag + + download_folder = Path(tmpdir) / "downloads" + download_file_path = await filemanager.download_path_from_s3( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + local_path=download_folder, + io_log_redirect_cb=None, + r_clone_settings=optional_sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + assert progress_bar._current_steps == pytest.approx(2) # noqa: SLF001 + assert download_file_path.exists() + assert download_file_path.name == "test.test" + assert filecmp.cmp(download_file_path, file_path) + + +@pytest.mark.parametrize( + "file_size", + [ + _file_size("10Mib"), + _file_size("103Mib"), + ], + ids=byte_size_ids, +) +async def test_valid_upload_download_using_file_object( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + file_size: ByteSize, + create_file_of_size: Callable[[ByteSize, str], Path], + optional_sync_settings: _SyncSettings, + faker: Faker, +): + file_path = create_file_of_size(file_size, "test.test") + + file_id = create_valid_file_uuid("", file_path) + with file_path.open("rb") as file_object: + upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=filemanager.UploadableFileObject( + file_object, file_path.name, file_path.stat().st_size + ), + r_clone_settings=optional_sync_settings.r_clone_settings, + io_log_redirect_cb=None, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + assert isinstance(upload_result, UploadedFile) + store_id, e_tag = upload_result.store_id, upload_result.etag + assert store_id == s3_simcore_location + assert e_tag + file_metadata = await filemanager.get_file_metadata( + user_id=user_id, store_id=store_id, s3_object=file_id + ) + assert file_metadata.location == store_id + assert file_metadata.etag == e_tag + + download_folder = Path(tmpdir) / "downloads" + async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: + download_file_path = await filemanager.download_path_from_s3( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + local_path=download_folder, + io_log_redirect_cb=None, + r_clone_settings=optional_sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + assert progress_bar._current_steps == pytest.approx(1) # noqa: SLF001 + assert download_file_path.exists() + assert download_file_path.name == "test.test" + assert filecmp.cmp(download_file_path, file_path) + + +@pytest.fixture +def mocked_upload_file_raising_exceptions(mocker: MockerFixture) -> None: + mocker.patch( + "simcore_sdk.node_ports_common.filemanager.r_clone.sync_local_to_s3", + autospec=True, + side_effect=RCloneFailedError, + ) + mocker.patch( + "simcore_sdk.node_ports_common.file_io_utils._upload_file_part", + autospec=True, + side_effect=ClientError, + ) + mocker.patch( + "simcore_sdk.node_ports_common.filemanager.aws_s3_cli.sync_local_to_s3", + autospec=True, + side_effect=AwsS3CliFailedError, + ) + + +@pytest.mark.parametrize( + "file_size", + [ + _file_size("10Mib"), + ], + ids=byte_size_ids, +) +async def test_failed_upload_is_properly_removed_from_storage( + node_ports_config: None, + create_file_of_size: Callable[[ByteSize], Path], + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + optional_sync_settings: _SyncSettings, + file_size: ByteSize, + user_id: UserID, + mocked_upload_file_raising_exceptions: None, +): + file_path = create_file_of_size(file_size) + file_id = create_valid_file_uuid("", file_path) + with pytest.raises(exceptions.S3TransferError): + await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=file_path, + r_clone_settings=optional_sync_settings.r_clone_settings, + io_log_redirect_cb=None, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + with pytest.raises(exceptions.S3InvalidPathError): + await filemanager.get_file_metadata( + user_id=user_id, store_id=s3_simcore_location, s3_object=file_id + ) + + +@pytest.mark.parametrize( + "file_size", + [ + _file_size("10Mib"), + ], + ids=byte_size_ids, +) +async def test_failed_upload_after_valid_upload_keeps_last_valid_state( + node_ports_config: None, + create_file_of_size: Callable[[ByteSize], Path], + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + optional_sync_settings: _SyncSettings, + file_size: ByteSize, + user_id: UserID, + mocker: MockerFixture, +): + # upload a valid file + file_path = create_file_of_size(file_size) + file_id = create_valid_file_uuid("", file_path) + upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=file_path, + r_clone_settings=optional_sync_settings.r_clone_settings, + io_log_redirect_cb=None, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + assert isinstance(upload_result, UploadedFile) + store_id, e_tag = upload_result.store_id, upload_result.etag + assert store_id == s3_simcore_location + assert e_tag + # check the file is correctly uploaded + file_metadata = await filemanager.get_file_metadata( + user_id=user_id, store_id=store_id, s3_object=file_id + ) + assert file_metadata.location == store_id + assert file_metadata.etag == e_tag + # now start an invalid update by generating an exception while uploading the same file + mocker.patch( + "simcore_sdk.node_ports_common.filemanager.r_clone.sync_local_to_s3", + autospec=True, + side_effect=RCloneFailedError, + ) + mocker.patch( + "simcore_sdk.node_ports_common.file_io_utils._upload_file_part", + autospec=True, + side_effect=ClientError, + ) + with pytest.raises(exceptions.S3TransferError): + await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=file_path, + r_clone_settings=optional_sync_settings.r_clone_settings, + io_log_redirect_cb=None, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + # the file shall be back to its original state + file_metadata = await filemanager.get_file_metadata( + user_id=user_id, store_id=s3_simcore_location, s3_object=file_id + ) + assert file_metadata.location == store_id + assert file_metadata.etag == e_tag + + +async def test_invalid_file_path( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + optional_sync_settings: _SyncSettings, + faker: Faker, +): + file_path = Path(tmpdir) / "test.test" + file_path.write_text("I am a test file") + assert file_path.exists() + + file_id = create_valid_file_uuid("", file_path) + store = s3_simcore_location + with pytest.raises(FileNotFoundError): + await filemanager.upload_path( + user_id=user_id, + store_id=store, + store_name=None, + s3_object=file_id, + path_to_upload=Path(tmpdir) / "some other file.txt", + io_log_redirect_cb=None, + ) + + download_folder = Path(tmpdir) / "downloads" + with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 + async with ProgressBarData( + num_steps=1, description=faker.pystr() + ) as progress_bar: + await filemanager.download_path_from_s3( + user_id=user_id, + store_id=store, + store_name=None, + s3_object=file_id, + local_path=download_folder, + io_log_redirect_cb=None, + r_clone_settings=optional_sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + + +async def test_errors_upon_invalid_file_identifiers( + node_ports_config: None, + tmpdir: Path, + user_id: UserID, + project_id: str, + s3_simcore_location: LocationID, + optional_sync_settings: _SyncSettings, + faker: Faker, +): + file_path = Path(tmpdir) / "test.test" + file_path.write_text("I am a test file") + assert file_path.exists() + + store = s3_simcore_location + with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 + invalid_s3_path = SimcoreS3FileID("") + await filemanager.upload_path( + user_id=user_id, + store_id=store, + store_name=None, + s3_object=invalid_s3_path, + path_to_upload=file_path, + io_log_redirect_cb=None, + ) + + with pytest.raises(exceptions.StorageInvalidCall): # noqa: PT012 + invalid_file_id = SimcoreS3FileID("file_id") + await filemanager.upload_path( + user_id=user_id, + store_id=store, + store_name=None, + s3_object=invalid_file_id, + path_to_upload=file_path, + io_log_redirect_cb=None, + ) + + download_folder = Path(tmpdir) / "downloads" + with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 + async with ProgressBarData( + num_steps=1, description=faker.pystr() + ) as progress_bar: + invalid_s3_path = SimcoreS3FileID("") + await filemanager.download_path_from_s3( + user_id=user_id, + store_id=store, + store_name=None, + s3_object=invalid_s3_path, + local_path=download_folder, + io_log_redirect_cb=None, + r_clone_settings=optional_sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + + with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 + async with ProgressBarData( + num_steps=1, description=faker.pystr() + ) as progress_bar: + await filemanager.download_path_from_s3( + user_id=user_id, + store_id=store, + store_name=None, + s3_object=SimcoreS3FileID(f"{project_id}/{uuid4()}/invisible.txt"), + local_path=download_folder, + io_log_redirect_cb=None, + r_clone_settings=optional_sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + + +async def test_invalid_store( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + optional_sync_settings: _SyncSettings, + faker: Faker, +): + file_path = Path(tmpdir) / "test.test" + file_path.write_text("I am a test file") + assert file_path.exists() + + file_id = create_valid_file_uuid("", file_path) + store = "somefunkystore" + with pytest.raises(exceptions.S3InvalidStore): + await filemanager.upload_path( + user_id=user_id, + store_id=None, + store_name=store, # type: ignore + s3_object=file_id, + path_to_upload=file_path, + io_log_redirect_cb=None, + ) + + download_folder = Path(tmpdir) / "downloads" + with pytest.raises(exceptions.S3InvalidStore): # noqa: PT012 + async with ProgressBarData( + num_steps=1, description=faker.pystr() + ) as progress_bar: + await filemanager.download_path_from_s3( + user_id=user_id, + store_id=None, + store_name=store, # type: ignore + s3_object=file_id, + local_path=download_folder, + io_log_redirect_cb=None, + r_clone_settings=optional_sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, + ) + + +@pytest.fixture( + params=[True, False], + ids=["with RClone", "with AwsS3Cli"], +) +def sync_settings( + r_clone_settings: RCloneSettings, + aws_s3_cli_settings: AwsS3CliSettings, + request: pytest.FixtureRequest, +) -> _SyncSettings: + is_rclone_enabled = request.param + + return _SyncSettings( + r_clone_settings=r_clone_settings if is_rclone_enabled else None, + aws_s3_cli_settings=aws_s3_cli_settings if not is_rclone_enabled else None, + ) + + +@pytest.mark.parametrize("is_directory", [False, True]) +async def test_valid_metadata( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + sync_settings: _SyncSettings, + is_directory: bool, +): + # first we go with a non-existing file + file_path = Path(tmpdir) / "a-subdir" / "test.test" + file_path.parent.mkdir(parents=True, exist_ok=True) + + path_to_upload = file_path.parent if is_directory else file_path + + file_id = create_valid_file_uuid("", path_to_upload) + assert file_path.exists() is False + + is_metadata_present = await filemanager.entry_exists( + user_id=user_id, + store_id=s3_simcore_location, + s3_object=file_id, + is_directory=is_directory, + ) + assert is_metadata_present is False + + # now really create the file and upload it + file_path.write_text("I am a test file") + assert file_path.exists() + + file_id = create_valid_file_uuid("", path_to_upload) + upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=path_to_upload, + io_log_redirect_cb=None, + r_clone_settings=sync_settings.r_clone_settings, + aws_s3_cli_settings=sync_settings.aws_s3_cli_settings, + ) + if is_directory: + assert isinstance(upload_result, UploadedFolder) + else: + assert isinstance(upload_result, UploadedFile) + assert upload_result.store_id == s3_simcore_location + assert upload_result.etag + + is_metadata_present = await filemanager.entry_exists( + user_id=user_id, + store_id=s3_simcore_location, + s3_object=file_id, + is_directory=is_directory, + ) + + assert is_metadata_present is True + + +@pytest.mark.parametrize( + "fct, extra_kwargs", + [ + (filemanager.entry_exists, {"is_directory": False}), + (filemanager.delete_file, {}), + (filemanager.get_file_metadata, {}), + ], +) +async def test_invalid_call_raises_exception( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + fct: Callable[[int, str, str, Any | None], Awaitable], + extra_kwargs: dict[str, Any], +): + file_path = Path(tmpdir) / "test.test" + file_id = create_valid_file_uuid("", file_path) + assert file_path.exists() is False + + with pytest.raises(exceptions.StorageInvalidCall): + await fct( + user_id=None, store_id=s3_simcore_location, s3_object=file_id, **extra_kwargs # type: ignore + ) + with pytest.raises(exceptions.StorageInvalidCall): + await fct(user_id=user_id, store_id=None, s3_object=file_id, **extra_kwargs) # type: ignore + with pytest.raises(exceptions.StorageInvalidCall): + await fct( + user_id=user_id, store_id=s3_simcore_location, s3_object="bing", **extra_kwargs # type: ignore + ) + + +async def test_delete_file( + node_ports_config: None, + tmpdir: Path, + user_id: int, + create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], + s3_simcore_location: LocationID, + storage_service: URL, +): + file_path = Path(tmpdir) / "test.test" + file_path.write_text("I am a test file") + assert file_path.exists() + + file_id = create_valid_file_uuid("", file_path) + upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=file_id, + path_to_upload=file_path, + io_log_redirect_cb=None, + ) + assert isinstance(upload_result, UploadedFile) + store_id, e_tag = upload_result.store_id, upload_result.etag + assert store_id == s3_simcore_location + assert e_tag + + is_metadata_present = await filemanager.entry_exists( + user_id=user_id, store_id=store_id, s3_object=file_id, is_directory=False + ) + assert is_metadata_present is True + + await filemanager.delete_file( + user_id=user_id, store_id=s3_simcore_location, s3_object=file_id + ) + + # check that it disappeared + assert ( + await filemanager.entry_exists( + user_id=user_id, store_id=store_id, s3_object=file_id, is_directory=False + ) + is False + ) + + +@pytest.mark.parametrize("files_in_folder", [1, 10]) +async def test_upload_path_source_is_a_folder( + node_ports_config: None, + project_id: str, + tmp_path: Path, + faker: Faker, + user_id: int, + s3_simcore_location: LocationID, + files_in_folder: int, + sync_settings: _SyncSettings, +): + source_dir = tmp_path / f"source-{faker.uuid4()}" + source_dir.mkdir(parents=True, exist_ok=True) + + download_dir = tmp_path / f"download-{faker.uuid4()}" + download_dir.mkdir(parents=True, exist_ok=True) + + for i in range(files_in_folder): + (source_dir / f"file-{i}.txt").write_text("1") + + directory_id = SimcoreS3DirectoryID.from_simcore_s3_object( + f"{project_id}/{faker.uuid4()}/some-dir-in-node-root/" + ) + s3_object = SimcoreS3FileID(directory_id) + + upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( + user_id=user_id, + store_id=s3_simcore_location, + store_name=None, + s3_object=s3_object, + path_to_upload=source_dir, + io_log_redirect_cb=None, + r_clone_settings=sync_settings.r_clone_settings, + aws_s3_cli_settings=sync_settings.aws_s3_cli_settings, + ) + assert isinstance(upload_result, UploadedFolder) + assert source_dir.exists() + + async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: + await filemanager.download_path_from_s3( + user_id=user_id, + store_name=None, + store_id=s3_simcore_location, + s3_object=s3_object, + local_path=download_dir, + io_log_redirect_cb=None, + r_clone_settings=sync_settings.r_clone_settings, + progress_bar=progress_bar, + aws_s3_cli_settings=sync_settings.aws_s3_cli_settings, + ) + assert download_dir.exists() + + # ensure all files in download and source directory are the same + file_names: set = {f.name for f in source_dir.glob("*")} & { + f.name for f in download_dir.glob("*") + } + for file_name in file_names: + filecmp.cmp(source_dir / file_name, download_dir / file_name, shallow=False) diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py index a9016609d13..ec24f271394 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py @@ -28,6 +28,7 @@ SimcoreS3FileID, ) from models_library.services_types import ServicePortKey +from pydantic import TypeAdapter from servicelib.progress_bar import ProgressBarData from settings_library.r_clone import RCloneSettings from simcore_sdk import node_ports_v2 @@ -90,7 +91,7 @@ async def _check_port_valid( assert port.value assert isinstance(port.value, DownloadLink | PortLink | BaseFileLink) assert ( - port.value.dict(by_alias=True, exclude_unset=True) + port.value.model_dump(by_alias=True, exclude_unset=True) == port_values[key_name] ) else: @@ -224,7 +225,7 @@ async def test_port_value_accessors( item_pytype: type, option_r_clone_settings: RCloneSettings | None, ): # pylint: disable=W0613, W0621 - item_key = ServicePortKey("some_key") + item_key = TypeAdapter(ServicePortKey).validate_python("some_key") config_dict, _, _ = create_special_configuration( inputs=[(item_key, item_type, item_value)], outputs=[(item_key, item_type, None)], @@ -299,17 +300,26 @@ async def test_port_file_accessors( ) await check_config_valid(PORTS, config_dict) assert ( - await (await PORTS.outputs)[ServicePortKey("out_34")].get() is None + await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_34") + ].get() + is None ) # check emptyness with pytest.raises(exceptions.S3InvalidPathError): - await (await PORTS.inputs)[ServicePortKey("in_1")].get() + await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_1") + ].get() # this triggers an upload to S3 + configuration change - await (await PORTS.outputs)[ServicePortKey("out_34")].set(item_value) + await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_34") + ].set(item_value) # this is the link to S3 storage - value = (await PORTS.outputs)[ServicePortKey("out_34")].value + value = (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_34") + ].value assert isinstance(value, DownloadLink | PortLink | BaseFileLink) - received_file_link = value.dict(by_alias=True, exclude_unset=True) + received_file_link = value.model_dump(by_alias=True, exclude_unset=True) assert received_file_link["store"] == s3_simcore_location assert ( received_file_link["path"] @@ -322,12 +332,21 @@ async def test_port_file_accessors( # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key assert isinstance( - await (await PORTS.outputs)[ServicePortKey("out_34")].get(), item_pytype + await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_34") + ].get(), + item_pytype, ) - downloaded_file = await (await PORTS.outputs)[ServicePortKey("out_34")].get() + downloaded_file = await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_34") + ].get() assert isinstance(downloaded_file, Path) assert downloaded_file.exists() - assert str(await (await PORTS.outputs)[ServicePortKey("out_34")].get()).startswith( + assert str( + await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_34") + ].get() + ).startswith( str( Path( tempfile.gettempdir(), @@ -472,9 +491,16 @@ async def test_get_value_from_previous_node( ) await check_config_valid(PORTS, config_dict) - input_value = await (await PORTS.inputs)[ServicePortKey("in_15")].get() + input_value = await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_15") + ].get() assert isinstance(input_value, item_pytype) - assert await (await PORTS.inputs)[ServicePortKey("in_15")].get() == item_value + assert ( + await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_15") + ].get() + == item_value + ) @pytest.mark.parametrize( @@ -516,7 +542,9 @@ async def test_get_file_from_previous_node( r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) - file_path = await (await PORTS.inputs)[ServicePortKey("in_15")].get() + file_path = await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_15") + ].get() assert isinstance(file_path, item_pytype) assert file_path == Path( tempfile.gettempdir(), @@ -577,7 +605,9 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( postgres_db, project_id, this_node_uuid, config_dict ) # pylint: disable=E1101 await check_config_valid(PORTS, config_dict) - file_path = await (await PORTS.inputs)[ServicePortKey("in_15")].get() + file_path = await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_15") + ].get() assert isinstance(file_path, item_pytype) assert file_path == Path( tempfile.gettempdir(), @@ -637,7 +667,9 @@ async def test_file_mapping( postgres_db, project_id, node_uuid, config_dict ) # pylint: disable=E1101 await check_config_valid(PORTS, config_dict) - file_path = await (await PORTS.inputs)[ServicePortKey("in_1")].get() + file_path = await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_1") + ].get() assert isinstance(file_path, item_pytype) assert file_path == Path( tempfile.gettempdir(), @@ -648,7 +680,9 @@ async def test_file_mapping( ) # let's get it a second time to see if replacing works - file_path = await (await PORTS.inputs)[ServicePortKey("in_1")].get() + file_path = await (await PORTS.inputs)[ + TypeAdapter(ServicePortKey).validate_python("in_1") + ].get() assert isinstance(file_path, item_pytype) assert file_path == Path( tempfile.gettempdir(), @@ -665,9 +699,11 @@ async def test_file_mapping( assert isinstance(file_path, Path) await PORTS.set_file_by_keymap(file_path) file_id = create_valid_file_uuid("out_1", file_path) - value = (await PORTS.outputs)[ServicePortKey("out_1")].value + value = (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python("out_1") + ].value assert isinstance(value, DownloadLink | PortLink | BaseFileLink) - received_file_link = value.dict(by_alias=True, exclude_unset=True) + received_file_link = value.model_dump(by_alias=True, exclude_unset=True) assert received_file_link["store"] == s3_simcore_location assert received_file_link["path"] == file_id # received a new eTag @@ -720,15 +756,19 @@ async def test_regression_concurrent_port_update_fails( # when writing in serial these are expected to work for item_key, _, _ in outputs: - await (await PORTS.outputs)[ServicePortKey(item_key)].set(int_item_value) - assert (await PORTS.outputs)[ServicePortKey(item_key)].value == int_item_value + await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python(item_key) + ].set(int_item_value) + assert (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python(item_key) + ].value == int_item_value # when writing in parallel and reading back, # they fail, with enough concurrency async def _upload_create_task(item_key: str) -> None: - await (await PORTS.outputs)[ServicePortKey(item_key)].set( - parallel_int_item_value - ) + await (await PORTS.outputs)[ + TypeAdapter(ServicePortKey).validate_python(item_key) + ].set(parallel_int_item_value) # updating in parallel creates a race condition results = await gather( @@ -741,7 +781,7 @@ async def _upload_create_task(item_key: str) -> None: with pytest.raises(AssertionError) as exc_info: # noqa: PT012 for item_key, _, _ in outputs: assert (await PORTS.outputs)[ - ServicePortKey(item_key) + TypeAdapter(ServicePortKey).validate_python(item_key) ].value == parallel_int_item_value assert exc_info.value.args[0].startswith( @@ -773,7 +813,7 @@ async def test_batch_update_inputs_outputs( async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: await PORTS.set_multiple( { - ServicePortKey(port.key): (k, None) + TypeAdapter(ServicePortKey).validate_python(port.key): (k, None) for k, port in enumerate((await PORTS.outputs).values()) }, progress_bar=progress_bar, @@ -782,7 +822,7 @@ async def test_batch_update_inputs_outputs( assert progress_bar._current_steps == pytest.approx(1) # noqa: SLF001 await PORTS.set_multiple( { - ServicePortKey(port.key): (k, None) + TypeAdapter(ServicePortKey).validate_python(port.key): (k, None) for k, port in enumerate((await PORTS.inputs).values(), start=1000) }, progress_bar=progress_bar, @@ -793,18 +833,38 @@ async def test_batch_update_inputs_outputs( ports_inputs = await PORTS.inputs for k, asd in enumerate(outputs): item_key, _, _ = asd - assert ports_outputs[ServicePortKey(item_key)].value == k - assert await ports_outputs[ServicePortKey(item_key)].get() == k + assert ( + ports_outputs[TypeAdapter(ServicePortKey).validate_python(item_key)].value + == k + ) + assert ( + await ports_outputs[ + TypeAdapter(ServicePortKey).validate_python(item_key) + ].get() + == k + ) for k, asd in enumerate(inputs, start=1000): item_key, _, _ = asd - assert ports_inputs[ServicePortKey(item_key)].value == k - assert await ports_inputs[ServicePortKey(item_key)].get() == k + assert ( + ports_inputs[TypeAdapter(ServicePortKey).validate_python(item_key)].value + == k + ) + assert ( + await ports_inputs[ + TypeAdapter(ServicePortKey).validate_python(item_key) + ].get() + == k + ) # test missing key raises error async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: with pytest.raises(UnboundPortError): await PORTS.set_multiple( - {ServicePortKey("missing_key_in_both"): (123132, None)}, + { + TypeAdapter(ServicePortKey).validate_python( + "missing_key_in_both" + ): (123132, None) + }, progress_bar=progress_bar, ) diff --git a/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py b/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py index c1edb4f183c..a578d410605 100644 --- a/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py +++ b/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py @@ -40,7 +40,7 @@ def _create_files(number: int, folder: Path) -> list[Path]: @pytest.fixture def r_clone_settings(faker: Faker) -> RCloneSettings: - return RCloneSettings.parse_obj( + return RCloneSettings.model_validate( { "R_CLONE_S3": { "S3_ENDPOINT": faker.url(), diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py index 2d32d345ffa..0540daa58d1 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py @@ -19,7 +19,7 @@ UploadedPart, ) from moto.server import ThreadedMotoServer -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_mock import MockerFixture from servicelib.aiohttp import status from servicelib.progress_bar import ProgressBarData @@ -212,8 +212,7 @@ async def _creator(num_upload_links: int, chunk_size: ByteSize) -> FileUploadSch assert "UploadId" in response upload_id = response["UploadId"] - upload_links = parse_obj_as( - list[AnyUrl], + upload_links = TypeAdapter(list[AnyUrl]).validate_python( await asyncio.gather( *[ aiobotocore_s3_client.generate_presigned_url( @@ -234,8 +233,8 @@ async def _creator(num_upload_links: int, chunk_size: ByteSize) -> FileUploadSch chunk_size=chunk_size, urls=upload_links, links=FileUploadLinks( - abort_upload=parse_obj_as(AnyUrl, faker.uri()), - complete_upload=parse_obj_as(AnyUrl, faker.uri()), + abort_upload=TypeAdapter(AnyUrl).validate_python(faker.uri()), + complete_upload=TypeAdapter(AnyUrl).validate_python(faker.uri()), ), ) @@ -245,7 +244,12 @@ async def _creator(num_upload_links: int, chunk_size: ByteSize) -> FileUploadSch @pytest.mark.skip(reason="this will allow to reproduce an issue") @pytest.mark.parametrize( "file_size,used_chunk_size", - [(parse_obj_as(ByteSize, 21800510238), parse_obj_as(ByteSize, 10485760))], + [ + ( + TypeAdapter(ByteSize).validate_python(21800510238), + TypeAdapter(ByteSize).validate_python(10485760), + ) + ], ) async def test_upload_file_to_presigned_links( client_session: ClientSession, @@ -253,6 +257,7 @@ async def test_upload_file_to_presigned_links( create_file_of_size: Callable[[ByteSize], Path], file_size: ByteSize, used_chunk_size: ByteSize, + faker: Faker, ): """This test is here to reproduce the issue https://github.com/ITISFoundation/osparc-simcore/issues/3531 One theory is that something might be wrong in how the chunking is done and that AWS times out @@ -267,11 +272,11 @@ async def test_upload_file_to_presigned_links( """ local_file = create_file_of_size(file_size) num_links = 2080 - effective_chunk_size = parse_obj_as(ByteSize, local_file.stat().st_size / num_links) + effective_chunk_size = TypeAdapter(ByteSize).validate_python(local_file.stat().st_size / num_links) assert effective_chunk_size <= used_chunk_size upload_links = await create_upload_links(num_links, used_chunk_size) assert len(upload_links.urls) == num_links - async with ProgressBarData(num_steps=1) as progress_bar: + async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: uploaded_parts: list[UploadedPart] = await upload_file_to_presigned_links( session=client_session, file_upload_links=upload_links, diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py index 5116311ae01..95b114ae563 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py @@ -2,7 +2,7 @@ from uuid import uuid4 import pytest -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py index 91609476b9c..856b4b268b1 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py @@ -3,9 +3,11 @@ # pylint:disable=redefined-outer-name # pylint:disable=protected-access +import asyncio from pathlib import Path from typing import Any, Callable +from pydantic import ValidationError import pytest from faker import Faker from pytest_mock import MockFixture @@ -50,6 +52,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): user_id=user_id, project_id=project_id, node_uuid=node_uuid, + io_log_redirect_cb=None, save_to_db_cb=mock_save_db_cb, node_port_creator_cb=mock_node_port_creator_cb, auto_update=False, @@ -62,6 +65,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): user_id=user_id, project_id=project_id, node_uuid=node_uuid, + io_log_redirect_cb=None, save_to_db_cb=mock_save_db_cb, node_port_creator_cb=mock_node_port_creator_cb, auto_update=auto_update, @@ -101,6 +105,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): user_id=user_id, project_id=project_id, node_uuid=node_uuid, + io_log_redirect_cb=None, save_to_db_cb=mock_save_db_cb, node_port_creator_cb=mock_node_port_creator_cb, auto_update=False, @@ -113,6 +118,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): user_id=user_id, project_id=project_id, node_uuid=node_uuid, + io_log_redirect_cb=None, save_to_db_cb=mock_save_db_cb, node_port_creator_cb=mock_node_port_creator_cb, auto_update=False, @@ -182,6 +188,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): user_id=user_id, project_id=project_id, node_uuid=node_uuid, + io_log_redirect_cb=None, save_to_db_cb=mock_save_db_cb, node_port_creator_cb=mock_node_port_creator_cb, auto_update=False, @@ -194,6 +201,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): user_id=user_id, project_id=project_id, node_uuid=node_uuid, + io_log_redirect_cb=None, save_to_db_cb=mock_save_db_cb, node_port_creator_cb=mock_node_port_creator_cb, auto_update=False, @@ -215,3 +223,54 @@ async def test_node_ports_v2_packages( db_manager = mock_db_manager(default_configuration) node_ports = await ports(user_id, project_id, node_uuid) node_ports = await ports(user_id, project_id, node_uuid, db_manager=db_manager) + + +@pytest.fixture +def mock_port_set(mocker: MockFixture)->None: + async def _always_raise_error(*args, **kwargs): + async def _i_raise_errors(): + raise ValidationError("invalid") + return asyncio.create_task(_i_raise_errors()) + + mocker.patch("simcore_sdk.node_ports_v2.port.Port._set", side_effect=_always_raise_error) + +async def test_node_ports_v2_set_multiple_catch_multiple_failing_set_ports( + mock_port_set:None, + mock_db_manager: Callable, + default_configuration: dict[str, Any], + user_id: int, + project_id: str, + node_uuid: str, + faker: Faker, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping(OutputsList, suffix="original") + + + async def _mock_callback(*args,**kwargs): + pass + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + io_log_redirect_cb=None, + save_to_db_cb=_mock_callback, + node_port_creator_cb=_mock_callback, + auto_update=False, + ) + async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: + with pytest.raises(ValidationError): + await node_ports.set_multiple( + { + port.key: (port.value, None) + for port in list(original_inputs.values()) + + list(original_outputs.values()) + }, + progress_bar=progress_bar, + ) diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py index 8485e19b74b..f8dbe5d0e8d 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py @@ -22,8 +22,7 @@ from faker import Faker from models_library.api_schemas_storage import FileMetaDataGet from models_library.projects_nodes_io import LocationID -from pydantic import parse_obj_as -from pydantic.error_wrappers import ValidationError +from pydantic import TypeAdapter, ValidationError from pytest_mock.plugin import MockerFixture from servicelib.progress_bar import ProgressBarData from simcore_sdk.node_ports_common.file_io_utils import LogRedirectCB @@ -218,8 +217,8 @@ def e_tag_fixture() -> str: async def mock_filemanager(mocker: MockerFixture, e_tag: str, faker: Faker) -> None: mocker.patch( "simcore_sdk.node_ports_common.filemanager._get_file_meta_data", - return_value=parse_obj_as( - FileMetaDataGet, FileMetaDataGet.Config.schema_extra["examples"][0] + return_value=TypeAdapter(FileMetaDataGet).validate_python( + FileMetaDataGet.model_config["json_schema_extra"]["examples"][0], ), ) mocker.patch( diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_mapping.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_mapping.py index 10c074591fc..3746520f42c 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_mapping.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_mapping.py @@ -4,20 +4,21 @@ from collections import deque from pprint import pprint -from typing import Any, Dict, List, Type, Union +from typing import Any import pytest from models_library.services import ServiceInput -from pydantic import ValidationError, confloat, schema_of +from pydantic import Field, ValidationError, schema_of from simcore_sdk.node_ports_v2 import exceptions from simcore_sdk.node_ports_v2.port import Port from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList +from typing_extensions import Annotated from utils_port_v2 import create_valid_port_config @pytest.mark.parametrize("port_class", [InputsList, OutputsList]) -def test_empty_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): - port_mapping = port_class(__root__={}) +def test_empty_ports_mapping(port_class: type[InputsList | OutputsList]): + port_mapping = port_class(root={}) assert not port_mapping.items() assert not port_mapping.values() assert not port_mapping.keys() @@ -28,17 +29,17 @@ def test_empty_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): @pytest.mark.parametrize("port_class", [InputsList, OutputsList]) -def test_filled_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): - port_cfgs: Dict[str, Any] = {} +def test_filled_ports_mapping(port_class: type[InputsList | OutputsList]): + port_cfgs: dict[str, Any] = {} for t in ["integer", "number", "boolean", "string"]: port = create_valid_port_config(t) port_cfgs[port["key"]] = port port_cfgs["some_file"] = create_valid_port_config("data:*/*", key="some_file") - port_mapping = port_class(__root__=port_cfgs) + port_mapping = port_class(root=port_cfgs) # two ways to construct instances of __root__ - assert port_class.parse_obj(port_cfgs) == port_mapping + assert port_class.model_validate(port_cfgs) == port_mapping assert len(port_mapping) == len(port_cfgs) for port_key, port_value in port_mapping.items(): @@ -60,8 +61,8 @@ def test_filled_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): def test_io_ports_are_not_aliases(): # prevents creating alises as InputsList = PortsMappings - inputs = InputsList(__root__={}) - outputs = OutputsList(__root__={}) + inputs = InputsList(root={}) + outputs = OutputsList(root={}) assert isinstance(inputs, InputsList) assert not isinstance(inputs, OutputsList) @@ -71,10 +72,10 @@ def test_io_ports_are_not_aliases(): @pytest.fixture -def fake_port_meta() -> Dict[str, Any]: +def fake_port_meta() -> dict[str, Any]: """Service port metadata: defines a list of non-negative numbers""" schema = schema_of( - List[confloat(ge=0)], + list[Annotated[float, Field(ge=0)]], title="list[non-negative number]", ) schema.update( @@ -83,10 +84,10 @@ def fake_port_meta() -> Dict[str, Any]: ) port_model = ServiceInput.from_json_schema(port_schema=schema) - return port_model.dict(exclude_unset=True, by_alias=True) + return port_model.model_dump(exclude_unset=True, by_alias=True) -def test_validate_port_value_against_schema(fake_port_meta: Dict[str, Any]): +def test_validate_port_value_against_schema(fake_port_meta: dict[str, Any]): # A simcore-sdk Port instance is a combination of both # - the port's metadata # - the port's value @@ -109,19 +110,19 @@ def test_validate_port_value_against_schema(fake_port_meta: Dict[str, Any]): assert error["loc"] == ("value",) assert "-2 is less than the minimum of 0" in error["msg"] - assert error["type"] == "value_error.port_validation.schema_error" + assert error["type"] == "value_error" assert "ctx" in error - assert error["ctx"]["port_key"] == "port_1" + assert error["ctx"]["error"].port_key == "port_1" - schema_error_message = error["ctx"]["schema_error_message"] - schema_error_path = error["ctx"]["schema_error_path"] + schema_error_message = error["ctx"]["error"].schema_error_message + schema_error_path = error["ctx"]["error"].schema_error_path assert schema_error_message in error["msg"] assert schema_error_path == deque([1]) -def test_validate_iolist_against_schema(fake_port_meta: Dict[str, Any]): +def test_validate_iolist_against_schema(fake_port_meta: dict[str, Any]): # Check how errors propagate from a single Port to InputsList # reference port @@ -151,7 +152,7 @@ def test_validate_iolist_against_schema(fake_port_meta: Dict[str, Any]): # ---- with pytest.raises(ValidationError) as err_info: - InputsList.parse_obj({p["key"]: p for p in ports}) + InputsList.model_validate({p["key"]: p for p in ports}) # --- assert isinstance(err_info.value, ValidationError) @@ -161,14 +162,13 @@ def test_validate_iolist_against_schema(fake_port_meta: Dict[str, Any]): for error in err_info.value.errors(): error_loc = error["loc"] assert "ctx" in error - port_key = error["ctx"].get("port_key") + port_key = error["ctx"]["error"].port_key # path hierachy - assert error_loc[0] == "__root__", f"{error_loc=}" - assert error_loc[1] == port_key, f"{error_loc=}" - assert error_loc[-1] == "value", f"{error_loc=}" + assert error_loc[0] == port_key, f"{error_loc=}" + assert error_loc[1] == "value", f"{error_loc=}" - assert error["type"] == "value_error.port_validation.schema_error" + assert error["type"] == "value_error" port_with_errors.append(port_key) pprint(error) diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_validation.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_validation.py index 41e61669fe5..ee0d19cec90 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_validation.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port_validation.py @@ -13,13 +13,14 @@ from unittest.mock import AsyncMock import pytest -from pydantic import BaseModel, conint, schema_of -from pydantic.error_wrappers import ValidationError +from pydantic import BaseModel, Field, schema_of +from pydantic import ValidationError from simcore_sdk.node_ports_v2.port import Port from simcore_sdk.node_ports_v2.port_validation import ( PortUnitError, validate_port_content, ) +from typing_extensions import Annotated def _replace_value_in_dict(item: Any, original_schema: dict[str, Any]): @@ -128,7 +129,7 @@ async def test_port_with_array_of_object(mocker): mocker.patch.object(Port, "_node_ports", new=AsyncMock()) class A(BaseModel): - i: conint(gt=3) + i: Annotated[int, Field(gt=3)] b: bool = False s: str l: list[int] @@ -142,7 +143,7 @@ class A(BaseModel): "contentSchema": content_schema, } sample = [{"i": 5, "s": "x", "l": [1, 2]}, {"i": 6, "s": "y", "l": [2]}] - expected_value = [A(**i).dict() for i in sample] + expected_value = [A(**i).model_dump() for i in sample] print(json.dumps(port_meta, indent=1)) print(json.dumps(expected_value, indent=1)) @@ -244,7 +245,7 @@ async def test_port_with_units_and_constraints(mocker): print(validation_error) assert validation_error["loc"] == ("value",) # starts with value,! - assert validation_error["type"] == "value_error.port_validation.schema_error" + assert validation_error["type"] == "value_error" assert "-3.14 is less than the minimum of 0" in validation_error["msg"] # inits with None + set_value @@ -256,8 +257,6 @@ async def test_port_with_units_and_constraints(mocker): with pytest.raises(ValidationError) as exc_info: await port.set_value(-3.14) - assert exc_info.value.errors()[0] == validation_error - def test_incident__port_validator_check_value(): # SEE incident https://git.speag.com/oSparc/e2e-testing/-/issues/1) diff --git a/packages/simcore-sdk/tests/unit/test_storage_client.py b/packages/simcore-sdk/tests/unit/test_storage_client.py index 91e46c5bd61..7786aafe494 100644 --- a/packages/simcore-sdk/tests/unit/test_storage_client.py +++ b/packages/simcore-sdk/tests/unit/test_storage_client.py @@ -20,7 +20,7 @@ ) from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.aiohttp import status from simcore_sdk.node_ports_common import exceptions @@ -176,8 +176,8 @@ async def test_get_file_metada( session=session, file_id=file_id, location_id=location_id, user_id=user_id ) assert file_metadata - assert file_metadata == FileMetaDataGet.parse_obj( - FileMetaDataGet.Config.schema_extra["examples"][0] + assert file_metadata == FileMetaDataGet.model_validate( + FileMetaDataGet.model_config["json_schema_extra"]["examples"][0] ) @@ -362,12 +362,28 @@ def test_mode_ports_storage_without_auth( [ (True, _HTTP_URL, _HTTPS_URL), (False, _HTTP_URL, _HTTP_URL), - (True, parse_obj_as(AnyUrl, _HTTP_URL), _HTTPS_URL), - (False, parse_obj_as(AnyUrl, _HTTP_URL), _HTTP_URL), + ( + True, + str(TypeAdapter(AnyUrl).validate_python(_HTTP_URL)).rstrip("/"), + _HTTPS_URL, + ), + ( + False, + str(TypeAdapter(AnyUrl).validate_python(_HTTP_URL)).rstrip("/"), + _HTTP_URL, + ), (True, _HTTPS_URL, _HTTPS_URL), (False, _HTTPS_URL, _HTTPS_URL), - (True, parse_obj_as(AnyUrl, _HTTPS_URL), _HTTPS_URL), - (False, parse_obj_as(AnyUrl, _HTTPS_URL), _HTTPS_URL), + ( + True, + str(TypeAdapter(AnyUrl).validate_python(_HTTPS_URL)).rstrip("/"), + _HTTPS_URL, + ), + ( + False, + str(TypeAdapter(AnyUrl).validate_python(_HTTPS_URL)).rstrip("/"), + _HTTPS_URL, + ), (True, "http://http", "https://http"), (True, "https://http", "https://http"), ], @@ -382,4 +398,4 @@ def test__get_secure_link( is_storage_secure.cache_clear() setenvs_from_dict(monkeypatch, {"STORAGE_SECURE": "1" if storage_secure else "0"}) - assert _get_https_link_if_storage_secure(provided) == expected + assert _get_https_link_if_storage_secure(str(provided)) == expected diff --git a/tests/environment-setup/requirements/requirements.txt b/tests/environment-setup/requirements/requirements.txt index d1641743383..1ae402aba4e 100644 --- a/tests/environment-setup/requirements/requirements.txt +++ b/tests/environment-setup/requirements/requirements.txt @@ -1,5 +1,5 @@ -exceptiongroup==1.2.2 - # via pytest +annotated-types==0.7.0 + # via pydantic iniconfig==2.0.0 # via pytest packaging==24.1 @@ -8,16 +8,20 @@ packaging==24.1 # pytest-sugar pluggy==1.5.0 # via pytest -pydantic==1.10.17 +pydantic==2.9.2 # via + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../requirements/constraints.txt # -r requirements/requirements.in +pydantic-core==2.23.4 + # via pydantic pytest==8.3.2 # via # -r requirements/requirements.in @@ -26,9 +30,13 @@ pytest==8.3.2 # pytest-sugar pytest-asyncio==0.23.8 # via + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/requirements.in @@ -40,15 +48,19 @@ pytest-sugar==1.0.0 # via -r requirements/requirements.in pyyaml==6.0.2 # via + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/requirements.in termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via pytest typing-extensions==4.12.2 - # via pydantic + # via + # pydantic + # pydantic-core diff --git a/tests/public-api/requirements/_base.txt b/tests/public-api/requirements/_base.txt index fe4fab279b4..4735ddc8cac 100644 --- a/tests/public-api/requirements/_base.txt +++ b/tests/public-api/requirements/_base.txt @@ -1,21 +1,23 @@ +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via httpx certifi==2024.7.4 # via + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # httpcore # httpx # osparc-client click==8.1.7 # via typer -exceptiongroup==1.2.2 - # via anyio h11==0.14.0 # via httpcore httpcore==1.0.5 # via httpx httpx==0.27.0 # via + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # osparc idna==3.7 @@ -34,14 +36,23 @@ osparc-client==0.6.6 # via osparc packaging==24.1 # via osparc -pydantic==1.10.17 +pydantic==2.9.2 # via + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-settings==2.5.2 + # via -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich python-dateutil==2.9.0.post0 # via osparc-client +python-dotenv==1.0.1 + # via pydantic-settings rich==13.7.1 # via # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -64,8 +75,8 @@ typer==0.12.4 # via -r requirements/../../../packages/settings-library/requirements/_base.in typing-extensions==4.12.2 # via - # anyio # pydantic + # pydantic-core # typer urllib3==2.2.2 # via diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 68e7f7e9a96..a85e38aa920 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -23,16 +23,28 @@ aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -49,6 +61,8 @@ alembic==1.13.2 # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/_test.in +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # fast-depends @@ -65,10 +79,8 @@ arrow==1.3.0 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in async-timeout==4.0.3 # via - # aiohttp # aiopg # asyncpg - # redis asyncpg==0.29.0 # via sqlalchemy attrs==24.2.0 @@ -78,16 +90,28 @@ attrs==24.2.0 # referencing certifi==2024.7.4 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -110,10 +134,6 @@ docker==7.1.0 # -r requirements/_test.in email-validator==2.2.0 # via pydantic -exceptiongroup==1.2.2 - # via - # anyio - # pytest fast-depends==2.4.8 # via faststream faststream==0.5.18 @@ -152,16 +172,28 @@ jsonschema-specifications==2023.7.1 # via jsonschema mako==1.3.5 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -181,16 +213,28 @@ multidict==6.0.5 # yarl orjson==3.10.7 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -213,34 +257,78 @@ psycopg2-binary==2.9.9 # via # aiopg # sqlalchemy -pydantic==1.10.17 +pydantic==2.9.2 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.7.2 @@ -256,16 +344,28 @@ pytest==8.3.2 # pytest-sugar pytest-asyncio==0.23.8 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -281,19 +381,33 @@ pytest-sugar==1.0.0 python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -r requirements/_test.in + # pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -302,16 +416,28 @@ pyyaml==6.0.2 # -r requirements/_test.in redis==5.0.8 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -346,16 +472,28 @@ sniffio==1.3.1 # via anyio sqlalchemy==1.4.53 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -372,8 +510,6 @@ tenacity==9.0.0 # -r requirements/_test.in termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via pytest toolz==0.12.1 # via # -r requirements/../../../packages/service-library/requirements/_base.in @@ -397,25 +533,37 @@ typing-extensions==4.12.2 # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # aiodebug # alembic - # anyio # faststream # flexcache # flexparser # pint # pydantic + # pydantic-core # typer urllib3==2.2.2 # via + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt diff --git a/tests/swarm-deploy/requirements/_tools.txt b/tests/swarm-deploy/requirements/_tools.txt index 24faa87728a..14b66aa5089 100644 --- a/tests/swarm-deploy/requirements/_tools.txt +++ b/tests/swarm-deploy/requirements/_tools.txt @@ -69,21 +69,11 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 # via # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit From 329b00da0b1f5a1042054029be59a066747bce31 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 10 Oct 2024 10:25:05 +0200 Subject: [PATCH 04/53] Upgrade agent service (Pydantic v2) (#6508) --- services/agent/requirements/_base.txt | 35 ++++++++++++------- services/agent/requirements/_test.txt | 19 +++++----- services/agent/requirements/_tools.txt | 12 +------ .../simcore_service_agent/core/application.py | 2 +- .../simcore_service_agent/core/settings.py | 15 +++++--- services/agent/tests/conftest.py | 8 ++--- .../unit/test_modules_volumes_cleanup_s3.py | 2 +- 7 files changed, 50 insertions(+), 43 deletions(-) diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 394611702aa..04f6020c874 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -11,6 +11,8 @@ aiohttp==3.8.5 # aiodocker aiosignal==1.2.0 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==3.6.2 # via # httpx @@ -41,16 +43,10 @@ click==8.1.3 # uvicorn dnspython==2.2.1 # via email-validator -email-validator==1.3.0 +email-validator==2.2.0 # via pydantic -fastapi==0.96.0 +fastapi==0.115.0 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -106,12 +102,11 @@ prometheus-client==0.19.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -pydantic==1.10.2 +pydantic==2.9.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -119,6 +114,16 @@ pydantic==1.10.2 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.15.1 # via rich pyrsistent==0.19.2 @@ -126,7 +131,9 @@ pyrsistent==0.19.2 python-dateutil==2.8.2 # via arrow python-dotenv==1.0.0 - # via -r requirements/_base.in + # via + # -r requirements/_base.in + # pydantic-settings rich==13.4.2 # via # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -143,7 +150,7 @@ sniffio==1.3.0 # via # anyio # httpx -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -154,10 +161,12 @@ starlette==0.27.0 # fastapi typer==0.12.3 # via -r requirements/../../../packages/settings-library/requirements/_base.in -typing-extensions==4.4.0 +typing-extensions==4.12.2 # via # aiodocker + # fastapi # pydantic + # pydantic-core # typer uvicorn==0.19.0 # via diff --git a/services/agent/requirements/_test.txt b/services/agent/requirements/_test.txt index fd7c0889fce..c3e2c6ee2fc 100644 --- a/services/agent/requirements/_test.txt +++ b/services/agent/requirements/_test.txt @@ -13,6 +13,10 @@ aiosignal==1.2.0 # via # -c requirements/_base.txt # aiohttp +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic anyio==3.6.2 # via # -c requirements/_base.txt @@ -81,8 +85,6 @@ ecdsa==0.19.0 # moto # python-jose # sshpubkeys -exceptiongroup==1.2.2 - # via pytest faker==27.0.0 # via -r requirements/_test.in flask==3.0.3 @@ -178,11 +180,15 @@ pyasn1==0.6.0 # rsa pycparser==2.22 # via cffi -pydantic==1.10.2 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via moto pyrsistent==0.19.2 @@ -257,16 +263,13 @@ sshpubkeys==3.3.1 # via moto sympy==1.13.2 # via cfn-lint -tomli==2.0.1 - # via - # coverage - # pytest -typing-extensions==4.4.0 +typing-extensions==4.12.2 # via # -c requirements/_base.txt # aws-sam-translator # cfn-lint # pydantic + # pydantic-core urllib3==2.2.2 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/agent/requirements/_tools.txt b/services/agent/requirements/_tools.txt index 69a9671437c..8c9416c12a5 100644 --- a/services/agent/requirements/_tools.txt +++ b/services/agent/requirements/_tools.txt @@ -77,22 +77,12 @@ setuptools==69.2.0 # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint -typing-extensions==4.4.0 +typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/services/agent/src/simcore_service_agent/core/application.py b/services/agent/src/simcore_service_agent/core/application.py index 1c2211b16f3..3ac90922dc1 100644 --- a/services/agent/src/simcore_service_agent/core/application.py +++ b/services/agent/src/simcore_service_agent/core/application.py @@ -38,7 +38,7 @@ def create_app() -> FastAPI: # SETTINGS settings = ApplicationSettings.create_from_envs() _setup_logger(settings) - logger.debug(settings.json(indent=2)) + logger.debug(settings.model_dump_json(indent=2)) assert settings.SC_BOOT_MODE # nosec app = FastAPI( diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index 882217f9a5f..d57d8d0af5b 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -1,7 +1,7 @@ from typing import Final from models_library.basic_types import BootModeEnum, LogLevel -from pydantic import AnyHttpUrl, Field, NonNegativeInt, validator +from pydantic import AliasChoices, AnyHttpUrl, Field, NonNegativeInt, field_validator from settings_library.base import BaseCustomSettings from settings_library.r_clone import S3Provider from settings_library.utils_logging import MixinLoggingSettings @@ -11,16 +11,21 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): LOGLEVEL: LogLevel = Field( - LogLevel.WARNING.value, env=["AGENT_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + LogLevel.WARNING.value, + validation_alias=AliasChoices( + "AGENT_LOGLEVEL", + "LOG_LEVEL", + "LOGLEVEL", + ), ) SC_BOOT_MODE: BootModeEnum | None AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field( @@ -47,7 +52,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True - @validator("LOGLEVEL") + @field_validator("LOGLEVEL") @classmethod def valid_log_level(cls, value) -> LogLevel: return LogLevel(cls.validate_log_level(value)) diff --git a/services/agent/tests/conftest.py b/services/agent/tests/conftest.py index bd0d1a8964f..5e85be5a035 100644 --- a/services/agent/tests/conftest.py +++ b/services/agent/tests/conftest.py @@ -11,10 +11,11 @@ import pytest import simcore_service_agent from aiodocker.volumes import DockerVolume +from common_library.pydantic_networks_extension import HttpUrlLegacy from models_library.basic_types import BootModeEnum from models_library.services import RunID from moto.server import ThreadedMotoServer -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from settings_library.r_clone import S3Provider from simcore_service_agent.core.settings import ApplicationSettings @@ -187,9 +188,8 @@ def caplog_info_debug( @pytest.fixture(scope="module") -def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrl: +def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrlLegacy: # pylint: disable=protected-access - return parse_obj_as( - HttpUrl, + return TypeAdapter(HttpUrlLegacy).validate_python( f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # noqa: SLF001 ) diff --git a/services/agent/tests/unit/test_modules_volumes_cleanup_s3.py b/services/agent/tests/unit/test_modules_volumes_cleanup_s3.py index 1728d0bb029..862f447fc1a 100644 --- a/services/agent/tests/unit/test_modules_volumes_cleanup_s3.py +++ b/services/agent/tests/unit/test_modules_volumes_cleanup_s3.py @@ -149,7 +149,7 @@ async def test_store_to_s3( ) await _download_files_from_bucket( - endpoint=mocked_s3_server_url, + endpoint=f"{mocked_s3_server_url}", access_key="xxx", secret_key="xxx", bucket_name=bucket, From 6786b92e9b5829fbfdb786d0cc5b512cc6bf6fb0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 11 Oct 2024 10:28:58 +0200 Subject: [PATCH 05/53] Upgrade EFS guardian service (Pydantic v2) (#6516) --- services/efs-guardian/requirements/_base.txt | 53 ++++++++++--------- services/efs-guardian/requirements/_test.txt | 31 +++++------ services/efs-guardian/requirements/_tools.txt | 10 ---- .../core/application.py | 2 +- .../core/settings.py | 23 +++++--- .../exceptions/_base.py | 7 +-- 6 files changed, 60 insertions(+), 66 deletions(-) diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 0559ab670cf..551a43f5c07 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -43,6 +43,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # fast-depends @@ -54,13 +56,10 @@ arrow==1.3.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==23.2.0 # via # aiohttp @@ -99,24 +98,10 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via pydantic -exceptiongroup==1.2.1 - # via anyio fast-depends==2.4.3 # via faststream -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -203,7 +188,7 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -214,7 +199,6 @@ pydantic==1.10.15 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -231,6 +215,26 @@ pydantic==1.10.15 # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.6.2 @@ -241,6 +245,8 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -302,7 +308,7 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -352,16 +358,15 @@ typing-extensions==4.11.0 # via # aiodebug # aiodocker - # anyio # fastapi # faststream # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm - # uvicorn urllib3==2.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/efs-guardian/requirements/_test.txt b/services/efs-guardian/requirements/_test.txt index 847b0b88781..e2086e769ae 100644 --- a/services/efs-guardian/requirements/_test.txt +++ b/services/efs-guardian/requirements/_test.txt @@ -11,6 +11,10 @@ aiosignal==1.3.1 # via # -c requirements/_base.txt # aiohttp +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto anyio==4.4.0 @@ -19,11 +23,6 @@ anyio==4.4.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -async-timeout==4.0.3 - # via - # -c requirements/_base.txt - # aiohttp - # redis attrs==23.2.0 # via # -c requirements/_base.txt @@ -82,11 +81,6 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -exceptiongroup==1.2.1 - # via - # -c requirements/_base.txt - # anyio - # pytest faker==27.0.0 # via -r requirements/_test.in fakeredis==2.23.5 @@ -205,11 +199,15 @@ py-partiql-parser==0.5.5 # via moto pycparser==2.22 # via cffi -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via moto pytest==8.3.2 @@ -235,7 +233,9 @@ python-dateutil==2.9.0.post0 # faker # moto python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt @@ -296,19 +296,14 @@ sortedcontainers==2.4.0 # via fakeredis sympy==1.13.2 # via cfn-lint -tomli==2.0.1 - # via - # coverage - # pytest typing-extensions==4.11.0 # via # -c requirements/_base.txt # aiodocker - # anyio # aws-sam-translator # cfn-lint - # fakeredis # pydantic + # pydantic-core urllib3==2.2.1 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/efs-guardian/requirements/_tools.txt b/services/efs-guardian/requirements/_tools.txt index db86636a373..7a2c3f9d91b 100644 --- a/services/efs-guardian/requirements/_tools.txt +++ b/services/efs-guardian/requirements/_tools.txt @@ -74,22 +74,12 @@ setuptools==73.0.1 # via # -c requirements/_test.txt # pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.11.0 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py index 88c20f25ea3..e93b071761c 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -20,7 +20,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: - logger.info("app settings: %s", settings.json(indent=1)) + logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.EFS_GUARDIAN_DEBUG, diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index 57528d7a13d..2b62ca16931 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -8,7 +8,7 @@ LogLevel, VersionTag, ) -from pydantic import Field, PositiveInt, validator +from pydantic import AliasChoices, Field, PositiveInt, field_validator from settings_library.base import BaseCustomSettings from settings_library.efs import AwsEfsSettings from settings_library.rabbit import RabbitSettings @@ -59,28 +59,35 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # RUNTIME ----------------------------------------------------------- EFS_GUARDIAN_DEBUG: bool = Field( - default=False, description="Debug mode", env=["EFS_GUARDIAN_DEBUG", "DEBUG"] + default=False, + description="Debug mode", + validation_alias=AliasChoices("EFS_GUARDIAN_DEBUG", "DEBUG"), ) EFS_GUARDIAN_LOGLEVEL: LogLevel = Field( - LogLevel.INFO, env=["EFS_GUARDIAN_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + LogLevel.INFO, + validation_alias=AliasChoices("EFS_GUARDIAN_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings = Field(auto_default_from_env=True) - EFS_GUARDIAN_RABBITMQ: RabbitSettings = Field(auto_default_from_env=True) + EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) + EFS_GUARDIAN_RABBITMQ: RabbitSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) @cached_property def LOG_LEVEL(self) -> LogLevel: # noqa: N802 return self.EFS_GUARDIAN_LOGLEVEL - @validator("EFS_GUARDIAN_LOGLEVEL") + @field_validator("EFS_GUARDIAN_LOGLEVEL") @classmethod def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py index 61a92118c92..9c8c45d0933 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/exceptions/_base.py @@ -1,8 +1,5 @@ -from typing import Any - -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class EfsGuardianBaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + """EFS guardian base error class.""" From bbe92a2fe60411418e2a966bc637acafee7f3d56 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 11 Oct 2024 11:43:56 +0200 Subject: [PATCH 06/53] Upgrade invitations service (Pydantic v2) (#6513) --- .../src/common_library/serialization.py | 7 +- .../src/models_library/invitations.py | 2 +- services/invitations/openapi.json | 118 ++++++++++++------ services/invitations/requirements/_base.txt | 42 ++++--- services/invitations/requirements/_test.txt | 14 --- services/invitations/requirements/_tools.txt | 11 -- .../api/_invitations.py | 4 +- .../simcore_service_invitations/api/_meta.py | 5 +- .../src/simcore_service_invitations/cli.py | 16 +-- .../core/settings.py | 18 ++- .../services/invitations.py | 56 ++++----- .../invitations/tests/unit/api/conftest.py | 2 +- .../tests/unit/api/test_api_invitations.py | 12 +- .../tests/unit/api/test_api_meta.py | 4 +- services/invitations/tests/unit/conftest.py | 2 +- .../tests/unit/test__model_examples.py | 2 +- services/invitations/tests/unit/test_cli.py | 10 +- .../tests/unit/test_invitations.py | 27 ++-- 18 files changed, 193 insertions(+), 159 deletions(-) diff --git a/packages/common-library/src/common_library/serialization.py b/packages/common-library/src/common_library/serialization.py index a2178cc3ae6..787e88123eb 100644 --- a/packages/common-library/src/common_library/serialization.py +++ b/packages/common-library/src/common_library/serialization.py @@ -3,6 +3,7 @@ from common_library.pydantic_fields_extension import get_type from pydantic import BaseModel, SecretStr +from pydantic_core import Url def model_dump_with_secrets( @@ -19,11 +20,15 @@ def model_dump_with_secrets( if isinstance(field_data, timedelta): data[field_name] = field_data.total_seconds() - if isinstance(field_data, SecretStr): + elif isinstance(field_data, SecretStr): if show_secrets: data[field_name] = field_data.get_secret_value() else: data[field_name] = str(field_data) + + elif isinstance(field_data, Url): + data[field_name] = str(field_data) + elif isinstance(field_data, dict): field_type = get_type(settings_obj.model_fields[field_name]) if issubclass(field_type, BaseModel): diff --git a/packages/models-library/src/models_library/invitations.py b/packages/models-library/src/models_library/invitations.py index b1a8e4154e9..595c09b6012 100644 --- a/packages/models-library/src/models_library/invitations.py +++ b/packages/models-library/src/models_library/invitations.py @@ -50,7 +50,7 @@ class InvitationContent(InvitationInputs): created: datetime = Field(..., description="Timestamp for creation") def as_invitation_inputs(self) -> InvitationInputs: - return self.copy(exclude={"created"}) + return self.model_validate(self.model_dump(exclude={"created"})) # copy excluding "created" @classmethod def create_from_inputs( diff --git a/services/invitations/openapi.json b/services/invitations/openapi.json index 96630799e46..508191b0419 100644 --- a/services/invitations/openapi.json +++ b/services/invitations/openapi.json @@ -175,18 +175,32 @@ "description": "Invitee's email. Note that the registration can ONLY be used with this email" }, "trial_account_days": { - "type": "integer", - "exclusiveMinimum": true, + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Trial Account Days", - "description": "If set, this invitation will activate a trial account.Sets the number of days from creation until the account expires", - "minimum": 0 + "description": "If set, this invitation will activate a trial account.Sets the number of days from creation until the account expires" }, "extra_credits_in_usd": { - "type": "integer", - "exclusiveMinimum": true, + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Extra Credits In Usd", - "description": "If set, the account's primary wallet will add extra credits corresponding to this ammount in USD", - "minimum": 0 + "description": "If set, the account's primary wallet will add extra credits corresponding to this ammount in USD" }, "product": { "type": "string", @@ -208,13 +222,12 @@ "created" ], "title": "ApiInvitationContent", - "description": "Data in an invitation", "example": { - "issuer": "issuerid", + "created": "2023-01-11 13:11:47.293595", "guest": "invitedguest@company.com", - "trial_account_days": 2, + "issuer": "issuerid", "product": "osparc", - "created": "2023-01-11 13:11:47.293595" + "trial_account_days": 2 } }, "ApiInvitationContentAndLink": { @@ -233,18 +246,32 @@ "description": "Invitee's email. Note that the registration can ONLY be used with this email" }, "trial_account_days": { - "type": "integer", - "exclusiveMinimum": true, + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Trial Account Days", - "description": "If set, this invitation will activate a trial account.Sets the number of days from creation until the account expires", - "minimum": 0 + "description": "If set, this invitation will activate a trial account.Sets the number of days from creation until the account expires" }, "extra_credits_in_usd": { - "type": "integer", - "exclusiveMinimum": true, + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Extra Credits In Usd", - "description": "If set, the account's primary wallet will add extra credits corresponding to this ammount in USD", - "minimum": 0 + "description": "If set, the account's primary wallet will add extra credits corresponding to this ammount in USD" }, "product": { "type": "string", @@ -275,14 +302,13 @@ "invitation_url" ], "title": "ApiInvitationContentAndLink", - "description": "Data in an invitation", "example": { - "issuer": "issuerid", + "created": "2023-01-11 13:11:47.293595", "guest": "invitedguest@company.com", - "trial_account_days": 2, + "invitation_url": "https://foo.com/#/registration?invitation=1234", + "issuer": "issuerid", "product": "osparc", - "created": "2023-01-11 13:11:47.293595", - "invitation_url": "https://foo.com/#/registration?invitation=1234" + "trial_account_days": 2 } }, "ApiInvitationInputs": { @@ -301,21 +327,42 @@ "description": "Invitee's email. Note that the registration can ONLY be used with this email" }, "trial_account_days": { - "type": "integer", - "exclusiveMinimum": true, + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Trial Account Days", - "description": "If set, this invitation will activate a trial account.Sets the number of days from creation until the account expires", - "minimum": 0 + "description": "If set, this invitation will activate a trial account.Sets the number of days from creation until the account expires" }, "extra_credits_in_usd": { - "type": "integer", - "exclusiveMinimum": true, + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Extra Credits In Usd", - "description": "If set, the account's primary wallet will add extra credits corresponding to this ammount in USD", - "minimum": 0 + "description": "If set, the account's primary wallet will add extra credits corresponding to this ammount in USD" }, "product": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Product", "description": "If None, it will use INVITATIONS_DEFAULT_PRODUCT" } @@ -326,10 +373,9 @@ "guest" ], "title": "ApiInvitationInputs", - "description": "Input data necessary to create an invitation", "example": { - "issuer": "issuerid", "guest": "invitedguest@company.com", + "issuer": "issuerid", "trial_account_days": 2 } }, diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 68a381d3f12..413ac0b5d35 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -19,6 +19,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -31,10 +33,6 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==23.2.0 # via # aiohttp @@ -69,18 +67,10 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via pydantic -exceptiongroup==1.2.0 - # via anyio fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -149,12 +139,11 @@ prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in pycparser==2.21 # via cffi -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -165,6 +154,20 @@ pydantic==1.10.14 # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -172,7 +175,9 @@ pyinstrument==4.6.2 python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 - # via uvicorn + # via + # pydantic-settings + # uvicorn pyyaml==6.0.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -214,7 +219,7 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -241,12 +246,11 @@ typing-extensions==4.10.0 # via # aiodebug # aiodocker - # anyio # fastapi # faststream # pydantic + # pydantic-core # typer - # uvicorn uvicorn==0.29.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in diff --git a/services/invitations/requirements/_test.txt b/services/invitations/requirements/_test.txt index 4571dba8ffa..7b029ccb0de 100644 --- a/services/invitations/requirements/_test.txt +++ b/services/invitations/requirements/_test.txt @@ -16,12 +16,6 @@ coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov -exceptiongroup==1.2.0 - # via - # -c requirements/_base.txt - # anyio - # hypothesis - # pytest faker==27.0.0 # via -r requirements/_test.in h11==0.14.0 @@ -90,11 +84,3 @@ sortedcontainers==2.4.0 # via hypothesis termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # pytest -typing-extensions==4.10.0 - # via - # -c requirements/_base.txt - # anyio diff --git a/services/invitations/requirements/_tools.txt b/services/invitations/requirements/_tools.txt index b17c8a2c2d8..f3932792cef 100644 --- a/services/invitations/requirements/_tools.txt +++ b/services/invitations/requirements/_tools.txt @@ -70,22 +70,11 @@ ruff==0.6.1 # via -r requirements/../../../requirements/devenv.txt setuptools==73.0.1 # via pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.10.0 # via # -c requirements/_base.txt - # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/services/invitations/src/simcore_service_invitations/api/_invitations.py b/services/invitations/src/simcore_service_invitations/api/_invitations.py index 4422bda3135..0a680189c61 100644 --- a/services/invitations/src/simcore_service_invitations/api/_invitations.py +++ b/services/invitations/src/simcore_service_invitations/api/_invitations.py @@ -48,11 +48,11 @@ async def create_invitation( default_product=settings.INVITATIONS_DEFAULT_PRODUCT, ) invitation = ApiInvitationContentAndLink( - **invitation_content.dict(), + **invitation_content.model_dump(), invitation_url=invitation_link, ) - _logger.info("New invitation: %s", f"{invitation.json(indent=1)}") + _logger.info("New invitation: %s", f"{invitation.model_dump_json(indent=1)}") return invitation diff --git a/services/invitations/src/simcore_service_invitations/api/_meta.py b/services/invitations/src/simcore_service_invitations/api/_meta.py index 621f3b4445e..49d3f4b48cf 100644 --- a/services/invitations/src/simcore_service_invitations/api/_meta.py +++ b/services/invitations/src/simcore_service_invitations/api/_meta.py @@ -2,7 +2,8 @@ from collections.abc import Callable from fastapi import APIRouter, Depends -from pydantic import BaseModel, HttpUrl +from pydantic import BaseModel +from common_library.pydantic_networks_extension import HttpUrlLegacy from .._meta import API_VERSION, PROJECT_NAME from ._dependencies import get_reverse_url_mapper @@ -20,7 +21,7 @@ class _Meta(BaseModel): name: str version: str - docs_url: HttpUrl + docs_url: HttpUrlLegacy # diff --git a/services/invitations/src/simcore_service_invitations/cli.py b/services/invitations/src/simcore_service_invitations/cli.py index dffb1dca32f..cfb7cf2717a 100644 --- a/services/invitations/src/simcore_service_invitations/cli.py +++ b/services/invitations/src/simcore_service_invitations/cli.py @@ -5,7 +5,7 @@ from cryptography.fernet import Fernet from models_library.emails import LowerCaseEmailStr from models_library.invitations import InvitationContent, InvitationInputs -from pydantic import EmailStr, HttpUrl, ValidationError, parse_obj_as +from pydantic import EmailStr, HttpUrl, TypeAdapter, ValidationError from rich.console import Console from servicelib.utils_secrets import generate_password from settings_library.utils_cli import ( @@ -96,19 +96,19 @@ def invite( ctx: typer.Context, email: str = typer.Argument( ..., - callback=lambda v: parse_obj_as(LowerCaseEmailStr, v), + callback=lambda v: TypeAdapter(LowerCaseEmailStr).validate_python(v), help="Custom invitation for a given guest", ), issuer: str = typer.Option( - ..., help=InvitationInputs.__fields__["issuer"].field_info.description + ..., help=InvitationInputs.model_fields["issuer"].description ), trial_account_days: int = typer.Option( None, - help=InvitationInputs.__fields__["trial_account_days"].field_info.description, + help=InvitationInputs.model_fields["trial_account_days"].description, ), product: str = typer.Option( None, - help=InvitationInputs.__fields__["product"].field_info.description, + help=InvitationInputs.model_fields["product"].description, ), ): """Creates an invitation link for user with 'email' and issued by 'issuer'""" @@ -117,7 +117,7 @@ def invite( invitation_data = InvitationInputs( issuer=issuer, - guest=parse_obj_as(EmailStr, email), + guest=TypeAdapter(EmailStr).validate_python(email), trial_account_days=trial_account_days, extra_credits_in_usd=None, product=product, @@ -142,14 +142,14 @@ def extract(ctx: typer.Context, invitation_url: str): try: invitation: InvitationContent = extract_invitation_content( invitation_code=extract_invitation_code_from_query( - parse_obj_as(HttpUrl, invitation_url) + TypeAdapter(HttpUrl).validate_python(invitation_url) ), secret_key=settings.INVITATIONS_SECRET_KEY.get_secret_value().encode(), default_product=settings.INVITATIONS_DEFAULT_PRODUCT, ) assert invitation.product is not None # nosec - print(invitation.json(indent=1)) # noqa: T201 + print(invitation.model_dump_json(indent=1)) # noqa: T201 except (InvalidInvitationCodeError, ValidationError): _err_console.print("[bold red]Invalid code[/bold red]") diff --git a/services/invitations/src/simcore_service_invitations/core/settings.py b/services/invitations/src/simcore_service_invitations/core/settings.py index 6af076f4229..29d286f1020 100644 --- a/services/invitations/src/simcore_service_invitations/core/settings.py +++ b/services/invitations/src/simcore_service_invitations/core/settings.py @@ -1,7 +1,14 @@ from functools import cached_property from models_library.products import ProductName -from pydantic import Field, HttpUrl, PositiveInt, SecretStr, validator +from pydantic import ( + AliasChoices, + Field, + HttpUrl, + PositiveInt, + SecretStr, + field_validator, +) from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.utils_logging import MixinLoggingSettings @@ -38,14 +45,15 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # RUNTIME ----------------------------------------------------------- INVITATIONS_LOGLEVEL: LogLevel = Field( - default=LogLevel.INFO, env=["INVITATIONS_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + default=LogLevel.INFO, + validation_alias=AliasChoices("INVITATIONS_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -53,7 +61,7 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def LOG_LEVEL(self): return self.INVITATIONS_LOGLEVEL - @validator("INVITATIONS_LOGLEVEL") + @field_validator("INVITATIONS_LOGLEVEL") @classmethod def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) diff --git a/services/invitations/src/simcore_service_invitations/services/invitations.py b/services/invitations/src/simcore_service_invitations/services/invitations.py index d5bed8662cf..1cdc33f4085 100644 --- a/services/invitations/src/simcore_service_invitations/services/invitations.py +++ b/services/invitations/src/simcore_service_invitations/services/invitations.py @@ -1,18 +1,22 @@ import base64 import binascii import logging -from typing import Any, ClassVar, cast from urllib import parse +from common_library.pydantic_networks_extension import HttpUrlLegacy from cryptography.fernet import Fernet, InvalidToken from models_library.invitations import InvitationContent, InvitationInputs from models_library.products import ProductName -from pydantic import HttpUrl, ValidationError, parse_obj_as +from pydantic import ConfigDict, HttpUrl, TypeAdapter, ValidationError from starlette.datastructures import URL _logger = logging.getLogger(__name__) +def _to_initial(v: str): + return v[0] + + class InvalidInvitationCodeError(Exception): ... @@ -23,9 +27,9 @@ class _ContentWithShortNames(InvitationContent): @classmethod def serialize(cls, model_obj: InvitationContent) -> str: """Exports to json using *short* aliases and values in order to produce shorter codes""" - model_w_short_aliases_json: str = cls.construct( - **model_obj.dict(exclude_unset=True) - ).json(exclude_unset=True, by_alias=True) + model_w_short_aliases_json: str = cls.model_construct( + **model_obj.model_dump(exclude_unset=True) + ).model_dump_json(exclude_unset=True, by_alias=True) # NOTE: json arguments try to minimize the amount of data # serialized. The CONS is that it relies on models in the code # that might change over time. This might lead to some datasets in codes @@ -35,36 +39,18 @@ def serialize(cls, model_obj: InvitationContent) -> str: @classmethod def deserialize(cls, raw_json: str) -> InvitationContent: """Parses a json string and returns InvitationContent model""" - model_w_short_aliases = cls.parse_raw(raw_json) - return InvitationContent.construct( - **model_w_short_aliases.dict(exclude_unset=True) + model_w_short_aliases = cls.model_validate_json(raw_json) + return InvitationContent.model_construct( + **model_w_short_aliases.model_dump(exclude_unset=True) ) - class Config: - allow_population_by_field_name = True # NOTE: can parse using field names - allow_mutation = False - anystr_strip_whitespace = True + model_config = ConfigDict( # NOTE: Can export with alias: short aliases to minimize the size of serialization artifact - fields: ClassVar[dict[str, Any]] = { - "issuer": { - "alias": "i", - }, - "guest": { - "alias": "g", - }, - "trial_account_days": { - "alias": "t", - }, - "extra_credits_in_usd": { - "alias": "e", - }, - "product": { - "alias": "p", - }, - "created": { - "alias": "c", - }, - } + alias_generator=_to_initial, + populate_by_name=True, # NOTE: can parse using field names + frozen=True, + str_strip_whitespace=True, + ) # @@ -79,9 +65,9 @@ def _build_link( r = URL("/registration").include_query_params(invitation=code_url_safe) # Adds query to fragment - base_url = f"{base_url.rstrip('/')}/" + base_url = f"{base_url}/" url = URL(base_url).replace(fragment=f"{r}") - return cast(HttpUrl, parse_obj_as(HttpUrl, f"{url}")) + return TypeAdapter(HttpUrlLegacy).validate_python(f"{url}") def _fernet_encrypt_as_urlsafe_code( @@ -124,7 +110,7 @@ def create_invitation_link_and_content( code = _create_invitation_code(content, secret_key) # Adds message as the invitation in query link = _build_link( - base_url=base_url, + base_url=f"{base_url}", code_url_safe=code.decode(), ) return link, content diff --git a/services/invitations/tests/unit/api/conftest.py b/services/invitations/tests/unit/api/conftest.py index f4151fcc519..c558ac496ad 100644 --- a/services/invitations/tests/unit/api/conftest.py +++ b/services/invitations/tests/unit/api/conftest.py @@ -18,7 +18,7 @@ def client(app_environment: EnvVarsDict) -> Iterator[TestClient]: print(f"app_environment={json.dumps(app_environment)}") app = create_app() - print("settings:\n", app.state.settings.json(indent=1)) + print("settings:\n", app.state.settings.model_dump_json(indent=1)) with TestClient(app, base_url="http://testserver.test") as client: yield client diff --git a/services/invitations/tests/unit/api/test_api_invitations.py b/services/invitations/tests/unit/api/test_api_invitations.py index 572f30f8173..95d5eeeece0 100644 --- a/services/invitations/tests/unit/api/test_api_invitations.py +++ b/services/invitations/tests/unit/api/test_api_invitations.py @@ -23,7 +23,7 @@ @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) -@given(invitation_input=st.builds(InvitationInputs)) +@given(invitation_input=st.builds(InvitationInputs, guest=st.emails())) def test_create_invitation( invitation_input: InvitationInputs, client: TestClient, @@ -31,7 +31,7 @@ def test_create_invitation( ): response = client.post( f"/{API_VTAG}/invitations", - json=invitation_input.dict(exclude_none=True), + json=invitation_input.model_dump(exclude_none=True), auth=basic_auth, ) assert response.status_code == status.HTTP_200_OK, f"{response.json()=}" @@ -64,20 +64,20 @@ def test_check_invitation( # up ot here, identifcal to above. # Let's use invitation link - invitation_url = ApiInvitationContentAndLink.parse_obj( + invitation_url = ApiInvitationContentAndLink.model_validate( response.json() ).invitation_url # check invitation_url response = client.post( f"/{API_VTAG}/invitations:extract", - json={"invitation_url": invitation_url}, + json={"invitation_url": f"{invitation_url}"}, auth=basic_auth, ) assert response.status_code == 200, f"{response.json()=}" # decrypted invitation should be identical to request above - invitation = InvitationContent.parse_obj(response.json()) + invitation = InvitationContent.model_validate(response.json()) assert invitation.issuer == invitation_data.issuer assert invitation.guest == invitation_data.guest assert invitation.trial_account_days == invitation_data.trial_account_days @@ -106,7 +106,7 @@ def test_check_valid_invitation( assert response.status_code == 200, f"{response.json()=}" # decrypted invitation should be identical to request above - invitation = InvitationContent.parse_obj(response.json()) + invitation = InvitationContent.model_validate(response.json()) assert invitation.issuer == invitation_data.issuer assert invitation.guest == invitation_data.guest diff --git a/services/invitations/tests/unit/api/test_api_meta.py b/services/invitations/tests/unit/api/test_api_meta.py index cee4afd13c9..4fe4f39b22c 100644 --- a/services/invitations/tests/unit/api/test_api_meta.py +++ b/services/invitations/tests/unit/api/test_api_meta.py @@ -19,7 +19,7 @@ def test_healthcheck(client: TestClient): def test_meta(client: TestClient): response = client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK - meta = _Meta.parse_obj(response.json()) + meta = _Meta.model_validate(response.json()) - response = client.get(meta.docs_url) + response = client.get(f"{meta.docs_url}") assert response.status_code == status.HTTP_200_OK diff --git a/services/invitations/tests/unit/conftest.py b/services/invitations/tests/unit/conftest.py index 1b6ea4ee6e9..fde239d5332 100644 --- a/services/invitations/tests/unit/conftest.py +++ b/services/invitations/tests/unit/conftest.py @@ -109,4 +109,4 @@ def invitation_data( if product: kwargs["product"] = product - return InvitationInputs.parse_obj(kwargs) + return InvitationInputs.model_validate(kwargs) diff --git a/services/invitations/tests/unit/test__model_examples.py b/services/invitations/tests/unit/test__model_examples.py index 31ed0dfc603..78dfdd96669 100644 --- a/services/invitations/tests/unit/test__model_examples.py +++ b/services/invitations/tests/unit/test__model_examples.py @@ -26,4 +26,4 @@ def test_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): print(example_name, ":", json.dumps(example_data)) - assert model_cls.parse_obj(example_data) + assert model_cls.model_validate(example_data) diff --git a/services/invitations/tests/unit/test_cli.py b/services/invitations/tests/unit/test_cli.py index e77247115cf..0ca095f1915 100644 --- a/services/invitations/tests/unit/test_cli.py +++ b/services/invitations/tests/unit/test_cli.py @@ -8,6 +8,7 @@ import pytest from faker import Faker from models_library.products import ProductName +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import load_dotenv, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_invitations._meta import API_VERSION @@ -71,7 +72,10 @@ def test_invite_user_and_check_invitation( env=environs, ) assert result.exit_code == os.EX_OK, result.output - assert expected == InvitationInputs.parse_raw(result.stdout).dict() + assert ( + expected + == TypeAdapter(InvitationInputs).validate_json(result.stdout).model_dump() + ) def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): @@ -82,7 +86,7 @@ def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): environs = load_dotenv(result.stdout) envs = setenvs_from_dict(monkeypatch, environs) - settings_from_obj = ApplicationSettings.parse_obj(envs) + settings_from_obj = ApplicationSettings.model_validate(envs) settings_from_envs = ApplicationSettings.create_from_envs() assert settings_from_envs == settings_from_obj @@ -93,5 +97,5 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, result.output print(result.output) - settings = ApplicationSettings.parse_raw(result.output) + settings = ApplicationSettings.model_validate_json(result.output) assert settings == ApplicationSettings.create_from_envs() diff --git a/services/invitations/tests/unit/test_invitations.py b/services/invitations/tests/unit/test_invitations.py index edd9c01556b..0e6ea34b18c 100644 --- a/services/invitations/tests/unit/test_invitations.py +++ b/services/invitations/tests/unit/test_invitations.py @@ -5,6 +5,7 @@ import binascii from datetime import datetime, timezone +from typing import Counter from urllib import parse import cryptography.fernet @@ -28,7 +29,7 @@ def test_all_invitation_fields_have_short_and_unique_aliases(): # all have short alias all_alias = [] - for field in _ContentWithShortNames.__fields__.values(): + for field in _ContentWithShortNames.model_fields.values(): assert field.alias assert field.alias not in all_alias all_alias.append(field.alias) @@ -38,7 +39,7 @@ def test_import_and_export_invitation_alias_by_alias( invitation_data: InvitationInputs, ): expected_content = InvitationContent( - **invitation_data.dict(), + **invitation_data.model_dump(), created=datetime.now(tz=timezone.utc), ) raw_data = _ContentWithShortNames.serialize(expected_content) @@ -51,13 +52,13 @@ def test_export_by_alias_produces_smaller_strings( invitation_data: InvitationInputs, ): content = InvitationContent( - **invitation_data.dict(), + **invitation_data.model_dump(), created=datetime.now(tz=timezone.utc), ) raw_data = _ContentWithShortNames.serialize(content) # export by alias produces smaller strings - assert len(raw_data) < len(content.json()) + assert len(raw_data) < len(content.model_dump_json()) def test_create_and_decrypt_invitation( @@ -72,8 +73,8 @@ def test_create_and_decrypt_invitation( base_url=faker.url(), default_product=default_product, ) - assert invitation_link.fragment - query_params = dict(parse.parse_qsl(URL(invitation_link.fragment).query)) + assert URL(invitation_link).fragment + query_params = dict(parse.parse_qsl((URL(URL(invitation_link).fragment).query))) # will raise TokenError or ValidationError invitation = decrypt_invitation( @@ -85,9 +86,9 @@ def test_create_and_decrypt_invitation( assert isinstance(invitation, InvitationContent) assert invitation.product is not None - expected = invitation_data.dict(exclude_none=True) + expected = invitation_data.model_dump(exclude_none=True) expected.setdefault("product", default_product) - assert invitation.dict(exclude={"created"}, exclude_none=True) == expected + assert invitation.model_dump(exclude={"created"}, exclude_none=True) == expected # @@ -116,9 +117,9 @@ def test_valid_invitation_code( default_product=default_product, ) - expected = invitation_data.dict(exclude_none=True) + expected = invitation_data.model_dump(exclude_none=True) expected.setdefault("product", default_product) - assert invitation.dict(exclude={"created"}, exclude_none=True) == expected + assert invitation.model_dump(exclude={"created"}, exclude_none=True) == expected def test_invalid_invitation_encoding( @@ -176,7 +177,7 @@ class OtherModel(BaseModel): secret = secret_key.encode() other_code = _fernet_encrypt_as_urlsafe_code( - data=OtherModel().json().encode(), secret_key=secret + data=OtherModel().model_dump_json().encode(), secret_key=secret ) with pytest.raises(ValidationError): @@ -192,3 +193,7 @@ class OtherModel(BaseModel): secret_key=secret, default_product=default_product, ) + + +def test_aliases_uniqueness(): + assert not [item for item, count in Counter([field.alias for field in _ContentWithShortNames.model_fields.values()]).items() if count > 1] #nosec From 42c0a68a9872d961f0be08d10d1a04b66279b7c2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Oct 2024 11:16:42 +0200 Subject: [PATCH 07/53] Upgrade clusters keeper service (Pydantic v2) (#6519) --- .../clusters-keeper/requirements/_base.txt | 53 +++++++------ .../clusters-keeper/requirements/_test.txt | 31 ++++---- .../clusters-keeper/requirements/_tools.txt | 10 --- .../simcore_service_clusters_keeper/_meta.py | 12 ++- .../core/application.py | 2 +- .../core/errors.py | 4 +- .../core/settings.py | 75 ++++++++++++------- .../modules/clusters_management_core.py | 8 +- .../modules/dask.py | 4 +- .../utils/clusters.py | 2 +- .../utils/dask.py | 6 +- .../utils/ec2.py | 10 ++- .../clusters-keeper/tests/unit/conftest.py | 10 ++- .../tests/unit/test_api_health.py | 6 +- .../tests/unit/test_core_settings.py | 4 +- .../tests/unit/test_modules_dask.py | 14 ++-- .../tests/unit/test_modules_rabbitmq.py | 2 +- .../tests/unit/test_utils_clusters.py | 10 ++- 18 files changed, 147 insertions(+), 116 deletions(-) diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 5c2cff64acf..2b7f7c6b234 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -43,6 +43,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -54,13 +56,10 @@ arrow==1.3.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==23.2.0 # via # aiohttp @@ -116,24 +115,10 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via pydantic -exceptiongroup==1.2.1 - # via anyio fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -269,7 +254,7 @@ psutil==5.9.8 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -280,7 +265,6 @@ pydantic==1.10.15 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -297,6 +281,26 @@ pydantic==1.10.15 # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.6.2 @@ -307,6 +311,8 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -375,7 +381,7 @@ sortedcontainers==2.4.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -437,16 +443,15 @@ typing-extensions==4.11.0 # via # aiodebug # aiodocker - # anyio # fastapi # faststream # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm - # uvicorn urllib3==2.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 3539d24ebad..491a8027fd1 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -11,6 +11,10 @@ aiosignal==1.3.1 # via # -c requirements/_base.txt # aiohttp +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto anyio==4.3.0 @@ -19,11 +23,6 @@ anyio==4.3.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -async-timeout==4.0.3 - # via - # -c requirements/_base.txt - # aiohttp - # redis attrs==23.2.0 # via # -c requirements/_base.txt @@ -82,11 +81,6 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -exceptiongroup==1.2.1 - # via - # -c requirements/_base.txt - # anyio - # pytest faker==27.0.0 # via -r requirements/_test.in fakeredis==2.23.5 @@ -209,11 +203,15 @@ py-partiql-parser==0.5.5 # via moto pycparser==2.22 # via cffi -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via moto pytest==8.3.2 @@ -239,7 +237,9 @@ python-dateutil==2.9.0.post0 # faker # moto python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt @@ -302,21 +302,16 @@ sortedcontainers==2.4.0 # fakeredis sympy==1.13.2 # via cfn-lint -tomli==2.0.1 - # via - # coverage - # pytest types-pyyaml==6.0.12.20240808 # via -r requirements/_test.in typing-extensions==4.11.0 # via # -c requirements/_base.txt # aiodocker - # anyio # aws-sam-translator # cfn-lint - # fakeredis # pydantic + # pydantic-core urllib3==2.2.1 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_tools.txt b/services/clusters-keeper/requirements/_tools.txt index db86636a373..7a2c3f9d91b 100644 --- a/services/clusters-keeper/requirements/_tools.txt +++ b/services/clusters-keeper/requirements/_tools.txt @@ -74,22 +74,12 @@ setuptools==73.0.1 # via # -c requirements/_test.txt # pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.11.0 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py index 828216222aa..58d79f3b9ba 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py @@ -9,17 +9,21 @@ from models_library.basic_types import VersionStr, VersionTag from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter _current_distribution = distribution("simcore-service-clusters-keeper") __version__: str = version("simcore-service-clusters-keeper") APP_NAME: Final[str] = _current_distribution.metadata["Name"] -API_VERSION: Final[VersionStr] = parse_obj_as(VersionStr, __version__) +API_VERSION: Final[VersionStr] = TypeAdapter(VersionStr).validate_python(__version__) VERSION: Final[Version] = Version(__version__) -API_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") -RPC_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") +API_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + f"v{VERSION.major}" +) +RPC_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + f"v{VERSION.major}" +) def get_summary() -> str: diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py index 4683725e52f..2e528e1b30b 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py @@ -25,7 +25,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: - logger.info("app settings: %s", settings.json(indent=1)) + logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.CLUSTERS_KEEPER_DEBUG, diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py index 068a13f702e..02824102d43 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class ClustersKeeperRuntimeError(PydanticErrorMixin, RuntimeError): +class ClustersKeeperRuntimeError(OsparcErrorMixin, RuntimeError): msg_template: str = "clusters-keeper unexpected error" diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 3c1e69d85a2..106ca40b5a7 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -1,6 +1,6 @@ import datetime from functools import cached_property -from typing import Any, ClassVar, Final, Literal, cast +from typing import Annotated, Final, Literal, cast from aws_library.ec2 import EC2InstanceBootSpecific, EC2Tags from fastapi import FastAPI @@ -12,14 +12,18 @@ ) from models_library.clusters import InternalClusterAuthentication from pydantic import ( + AliasChoices, + BeforeValidator, Field, NonNegativeFloat, NonNegativeInt, PositiveInt, SecretStr, - parse_obj_as, - validator, + TypeAdapter, + WrapValidator, + field_validator, ) +from pydantic_settings import SettingsConfigDict from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings @@ -34,10 +38,9 @@ class ClustersKeeperEC2Settings(EC2Settings): - class Config(EC2Settings.Config): - env_prefix = CLUSTERS_KEEPER_ENV_PREFIX - - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + env_prefix=CLUSTERS_KEEPER_ENV_PREFIX, + json_schema_extra={ "examples": [ { f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", @@ -46,7 +49,8 @@ class Config(EC2Settings.Config): f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", } ], - } + }, + ) class WorkersEC2InstancesSettings(BaseCustomSettings): @@ -77,7 +81,7 @@ class WorkersEC2InstancesSettings(BaseCustomSettings): # NAME PREFIX is not exposed since we override it anyway WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., - min_items=1, + min_length=1, description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " " this is required to start a new EC2 instance", @@ -108,14 +112,14 @@ class WorkersEC2InstancesSettings(BaseCustomSettings): "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]", ) - @validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def check_valid_instance_names( cls, value: dict[str, EC2InstanceBootSpecific] ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(list[InstanceTypeType], list(value)) + TypeAdapter(list[InstanceTypeType]).validate_python(list(value)) return value @@ -130,7 +134,7 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings): ) PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., - min_items=1, + min_length=1, description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " " this is required to start a new EC2 instance", @@ -182,17 +186,17 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings): "that take longer than this time will be terminated as sometimes it happens that EC2 machine fail on start.", ) - @validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def check_valid_instance_names( cls, value: dict[str, EC2InstanceBootSpecific] ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(list[InstanceTypeType], list(value)) + TypeAdapter(list[InstanceTypeType]).validate_python(list(value)) return value - @validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def check_only_one_value( cls, value: dict[str, EC2InstanceBootSpecific] @@ -231,30 +235,35 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # RUNTIME ----------------------------------------------------------- CLUSTERS_KEEPER_DEBUG: bool = Field( - default=False, description="Debug mode", env=["CLUSTERS_KEEPER_DEBUG", "DEBUG"] + default=False, + description="Debug mode", + validation_alias=AliasChoices("CLUSTERS_KEEPER_DEBUG", "DEBUG"), ) CLUSTERS_KEEPER_LOGLEVEL: LogLevel = Field( - LogLevel.INFO, env=["CLUSTERS_KEEPER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + LogLevel.INFO, + validation_alias=AliasChoices( + "CLUSTERS_KEEPER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) CLUSTERS_KEEPER_EC2_ACCESS: ClustersKeeperEC2Settings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: PrimaryEC2InstancesSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES: WorkersEC2InstancesSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: str = Field( @@ -262,14 +271,18 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="set a prefix to all machines created (useful for testing)", ) - CLUSTERS_KEEPER_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True) + CLUSTERS_KEEPER_RABBITMQ: RabbitSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) CLUSTERS_KEEPER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True - CLUSTERS_KEEPER_REDIS: RedisSettings = Field(auto_default_from_env=True) + CLUSTERS_KEEPER_REDIS: RedisSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) CLUSTERS_KEEPER_REGISTRY: RegistrySettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_TASK_INTERVAL: datetime.timedelta = Field( @@ -320,10 +333,18 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def LOG_LEVEL(self) -> LogLevel: # noqa: N802 return self.CLUSTERS_KEEPER_LOGLEVEL - @validator("CLUSTERS_KEEPER_LOGLEVEL") + @field_validator("CLUSTERS_KEEPER_LOGLEVEL") @classmethod - def valid_log_level(cls, value: str) -> str: + def _valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + + + @field_validator("CLUSTERS_KEEPER_TASK_INTERVAL", "SERVICE_TRACKING_HEARTBEAT", mode="before") + @classmethod + def _validate_interval(cls, value: str | datetime.timedelta) -> int | datetime.timedelta: + if isinstance(value, str): + return int(value) + return value def get_application_settings(app: FastAPI) -> ApplicationSettings: diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py index a7c23143a0b..799b724cdcf 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py @@ -8,7 +8,7 @@ from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.logging_utils import log_catch from ..core.settings import get_application_settings @@ -35,8 +35,10 @@ def _get_instance_last_heartbeat(instance: EC2InstanceData) -> datetime.datetime return None -_USER_ID_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "user_id") -_WALLET_ID_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "wallet_id") +_USER_ID_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python("user_id") +_WALLET_ID_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python( + "wallet_id" +) async def _get_all_associated_worker_instances( diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py index de585dc654f..af1d0df0e66 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py @@ -34,7 +34,7 @@ async def ping_scheduler( require_encryption=True, ) async with distributed.Client( - url, asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security + f"{url}", asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security ): ... return True @@ -59,7 +59,7 @@ async def is_scheduler_busy( require_encryption=True, ) async with distributed.Client( - url, asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security + f"{url}", asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security ) as client: datasets_on_scheduler = await _wrap_client_async_routine(client.list_datasets()) _logger.info("cluster currently has %s datasets", len(datasets_on_scheduler)) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index 48eb4dee380..a2c037075cb 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -94,7 +94,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str: f"EC2_INSTANCES_NAME_PREFIX={cluster_machines_name_prefix}", f"LOG_LEVEL={app_settings.LOG_LEVEL}", f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", - f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}", # type: ignore[arg-type] + f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}", f"WORKERS_EC2_INSTANCES_KEY_NAME={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_KEY_NAME}", f"WORKERS_EC2_INSTANCES_MAX_INSTANCES={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_MAX_INSTANCES}", f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}", diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py index 957644f6346..266557358b7 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py @@ -1,13 +1,15 @@ from aws_library.ec2 import EC2InstanceData from fastapi import FastAPI from models_library.clusters import InternalClusterAuthentication -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from ..core.settings import get_application_settings def get_scheduler_url(ec2_instance: EC2InstanceData) -> AnyUrl: - url: AnyUrl = parse_obj_as(AnyUrl, f"tls://{ec2_instance.aws_private_dns}:8786") + url: AnyUrl = TypeAdapter(AnyUrl).validate_python( + f"tls://{ec2_instance.aws_private_dns}:8786" + ) return url diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py index c74bbc554d9..a19dac17d32 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py @@ -4,17 +4,19 @@ from aws_library.ec2 import AWSTagKey, AWSTagValue, EC2Tags from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from .._meta import VERSION from ..core.settings import ApplicationSettings _APPLICATION_TAG_KEY: Final[str] = "io.simcore.clusters-keeper" -_APPLICATION_VERSION_TAG: Final[EC2Tags] = parse_obj_as( - EC2Tags, {f"{_APPLICATION_TAG_KEY}.version": f"{VERSION}"} +_APPLICATION_VERSION_TAG: Final[EC2Tags] = TypeAdapter(EC2Tags).validate_python( + {f"{_APPLICATION_TAG_KEY}.version": f"{VERSION}"} ) -HEARTBEAT_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "last_heartbeat") +HEARTBEAT_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python( + "last_heartbeat" +) CLUSTER_NAME_PREFIX: Final[str] = "osparc-computational-cluster-" diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index a8f4913d4bb..3f5a72cca06 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -81,7 +81,7 @@ def mocked_ec2_server_envs( # NOTE: overrides the EC2Settings with what clusters-keeper expects changed_envs: EnvVarsDict = { f"{CLUSTERS_KEEPER_ENV_PREFIX}{k}": v - for k, v in mocked_ec2_server_settings.dict().items() + for k, v in mocked_ec2_server_settings.model_dump().items() } return setenvs_from_dict(monkeypatch, changed_envs) @@ -119,7 +119,9 @@ def app_environment( { random.choice( # noqa: S311 ec2_instances - ): EC2InstanceBootSpecific.Config.schema_extra["examples"][ + ): EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ][ 1 ] # NOTE: we use example with custom script } @@ -137,7 +139,9 @@ def app_environment( "WORKERS_EC2_INSTANCES_ALLOWED_TYPES": json.dumps( { ec2_type_name: random.choice( # noqa: S311 - EC2InstanceBootSpecific.Config.schema_extra["examples"] + EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ] ) for ec2_type_name in ec2_instances } diff --git a/services/clusters-keeper/tests/unit/test_api_health.py b/services/clusters-keeper/tests/unit/test_api_health.py index 734620afa1b..a2a14bcf72c 100644 --- a/services/clusters-keeper/tests/unit/test_api_health.py +++ b/services/clusters-keeper/tests/unit/test_api_health.py @@ -40,7 +40,7 @@ async def test_status_no_rabbit( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is False @@ -60,7 +60,7 @@ async def test_status( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is True @@ -75,7 +75,7 @@ async def test_status( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is True diff --git a/services/clusters-keeper/tests/unit/test_core_settings.py b/services/clusters-keeper/tests/unit/test_core_settings.py index 0e467dc1e67..d734bf32cff 100644 --- a/services/clusters-keeper/tests/unit/test_core_settings.py +++ b/services/clusters-keeper/tests/unit/test_core_settings.py @@ -45,7 +45,9 @@ def test_multiple_primary_ec2_instances_raises( "PRIMARY_EC2_INSTANCES_ALLOWED_TYPES": json.dumps( { ec2_type_name: random.choice( # noqa: S311 - EC2InstanceBootSpecific.Config.schema_extra["examples"] + EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ] ) for ec2_type_name in ec2_instances } diff --git a/services/clusters-keeper/tests/unit/test_modules_dask.py b/services/clusters-keeper/tests/unit/test_modules_dask.py index db1833ffd91..7f0408d7057 100644 --- a/services/clusters-keeper/tests/unit/test_modules_dask.py +++ b/services/clusters-keeper/tests/unit/test_modules_dask.py @@ -12,7 +12,7 @@ NoAuthentication, TLSAuthentication, ) -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from simcore_service_clusters_keeper.modules.dask import ( is_scheduler_busy, ping_scheduler, @@ -24,7 +24,9 @@ _authentication_types = [ NoAuthentication(), - TLSAuthentication.construct(**TLSAuthentication.Config.schema_extra["examples"][0]), + TLSAuthentication.model_construct( + **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + ), ] @@ -36,7 +38,7 @@ async def test_ping_scheduler_non_existing_scheduler( ): assert ( await ping_scheduler( - parse_obj_as(AnyUrl, f"tcp://{faker.ipv4()}:{faker.port_number()}"), + TypeAdapter(AnyUrl).validate_python(f"tcp://{faker.ipv4()}:{faker.port_number()}"), authentication, ) is False @@ -46,7 +48,7 @@ async def test_ping_scheduler_non_existing_scheduler( async def test_ping_scheduler(dask_spec_local_cluster: SpecCluster): assert ( await ping_scheduler( - parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address), + TypeAdapter(AnyUrl).validate_python(dask_spec_local_cluster.scheduler_address), NoAuthentication(), ) is True @@ -69,7 +71,7 @@ async def test_is_scheduler_busy( dask_spec_cluster_client: distributed.Client, ): # nothing runs right now - scheduler_address = parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address) + scheduler_address = TypeAdapter(AnyUrl).validate_python(dask_spec_local_cluster.scheduler_address) assert await is_scheduler_busy(scheduler_address, NoAuthentication()) is False _SLEEP_TIME = 5 @@ -84,5 +86,5 @@ def _some_long_running_fct(sleep_time: int) -> str: busy=True, ) - result = await future.result(timeout=2 * _SLEEP_TIME) # type: ignore + result = await future.result(timeout=2 * _SLEEP_TIME) assert "seconds" in result diff --git a/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py b/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py index a2c23ac0602..110d4fe4827 100644 --- a/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py +++ b/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py @@ -116,7 +116,7 @@ async def test_post_message( f"--> checking for message in rabbit exchange {rabbit_message.channel_name}, {attempt.retry_state.retry_object.statistics}" ) mocked_message_handler.assert_called_once_with( - rabbit_message.json().encode() + rabbit_message.model_dump_json().encode() ) print("... message received") diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index a6592ed1fa4..7191b1a23f9 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -23,7 +23,7 @@ TLSAuthentication, ) from models_library.utils.json_serialization import json_dumps -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from simcore_service_clusters_keeper.core.settings import ApplicationSettings from simcore_service_clusters_keeper.utils.clusters import ( @@ -60,7 +60,7 @@ def app_environment( monkeypatch, { "CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": json_dumps( - TLSAuthentication.Config.schema_extra["examples"][0] + TLSAuthentication.model_config["json_schema_extra"]["examples"][0] ) }, ) @@ -178,7 +178,7 @@ def test_create_startup_script_script_size_below_16kb( script_size_in_bytes = len(startup_script.encode("utf-8")) print( - f"current script size is {parse_obj_as(ByteSize, script_size_in_bytes).human_readable()}" + f"current script size is {TypeAdapter(ByteSize).validate_python(script_size_in_bytes).human_readable()}" ) # NOTE: EC2 user data cannot be above 16KB, we keep some margin here assert script_size_in_bytes < 15 * 1024 @@ -240,7 +240,9 @@ def test_startup_script_defines_all_envs_for_docker_compose( "authentication", [ NoAuthentication(), - TLSAuthentication(**TLSAuthentication.Config.schema_extra["examples"][0]), + TLSAuthentication( + **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + ), ], ) def test_create_cluster_from_ec2_instance( From cac813475b12df509e740351a63a5ab1fc9fc462 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:23:47 +0200 Subject: [PATCH 08/53] =?UTF-8?q?=E2=99=BB=EF=B8=8FPydantic=20V2=20upgrade?= =?UTF-8?q?:=20autoscaling=20(#6535)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env-devel | 2 +- .../src/aws_library/ec2/_models.py | 4 +- packages/models-library/Makefile | 2 +- .../src/models_library/docker.py | 6 +- .../generated_models/docker_rest_api.py | 3948 +++++++++++------ .../openapi-pydantic-models-generator.bash | 17 +- services/autoscaling/requirements/_base.txt | 44 +- services/autoscaling/requirements/_test.txt | 15 +- .../src/simcore_service_autoscaling/_meta.py | 5 +- .../simcore_service_autoscaling/constants.py | 42 +- .../core/application.py | 2 +- .../core/errors.py | 7 +- .../core/settings.py | 83 +- .../modules/auto_scaling_core.py | 22 +- .../auto_scaling_mode_computational.py | 8 +- .../modules/dask.py | 34 +- .../utils/auto_scaling_core.py | 4 +- .../utils/buffer_machines_pool_core.py | 33 +- .../utils/rabbitmq.py | 10 +- .../utils/redis.py | 4 +- .../utils/utils_docker.py | 296 +- services/autoscaling/tests/manual/.env-devel | 2 +- services/autoscaling/tests/unit/conftest.py | 141 +- .../autoscaling/tests/unit/test_api_health.py | 8 +- .../autoscaling/tests/unit/test_models.py | 20 +- ...test_modules_auto_scaling_computational.py | 84 +- .../unit/test_modules_auto_scaling_dynamic.py | 176 +- .../unit/test_modules_auto_scaling_task.py | 10 +- .../unit/test_modules_buffer_machine_core.py | 21 +- .../tests/unit/test_modules_dask.py | 26 +- .../tests/unit/test_modules_rabbitmq.py | 2 +- .../unit/test_utils_auto_scaling_core.py | 25 +- .../test_utils_buffer_machines_pool_core.py | 18 +- .../unit/test_utils_computational_scaling.py | 15 +- .../tests/unit/test_utils_docker.py | 259 +- .../tests/unit/test_utils_rabbitmq.py | 42 +- .../data/docker-compose.yml | 2 +- 37 files changed, 3302 insertions(+), 2137 deletions(-) diff --git a/.env-devel b/.env-devel index 8ae1e5856d5..dc71eb30f76 100644 --- a/.env-devel +++ b/.env-devel @@ -31,7 +31,7 @@ AUTOSCALING_EC2_ACCESS=null AUTOSCALING_EC2_INSTANCES=null AUTOSCALING_LOGLEVEL=WARNING AUTOSCALING_NODES_MONITORING=null -AUTOSCALING_POLL_INTERVAL=10 +AUTOSCALING_POLL_INTERVAL="00:00:10" AUTOSCALING_SSM_ACCESS=null AWS_S3_CLI_S3=null diff --git a/packages/aws-library/src/aws_library/ec2/_models.py b/packages/aws-library/src/aws_library/ec2/_models.py index ad15a74f0eb..621adc0f4ee 100644 --- a/packages/aws-library/src/aws_library/ec2/_models.py +++ b/packages/aws-library/src/aws_library/ec2/_models.py @@ -88,8 +88,8 @@ class EC2InstanceType: # quotes []{} were added as it allows to json encode. it seems to be accepted as a value str, StringConstraints( - min_length=0, - max_length=256, + min_length=AWS_TAG_VALUE_MIN_LENGTH, + max_length=AWS_TAG_VALUE_MAX_LENGTH, pattern=r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$", ), ] diff --git a/packages/models-library/Makefile b/packages/models-library/Makefile index b41bafd2f2c..8a18102f51a 100644 --- a/packages/models-library/Makefile +++ b/packages/models-library/Makefile @@ -61,7 +61,7 @@ DOCKER_API_VERSION ?= 1.41 docker_rest_api.py: ## auto-generates pydantic models for Docker REST API models # auto-generates $@ from $< @$(SCRIPTS_DIR)/openapi-pydantic-models-generator.bash \ - --url https://docs.docker.com/engine/api/v$(DOCKER_API_VERSION).yaml \ + --url https://docs.docker.com/reference/api/engine/version/v$(DOCKER_API_VERSION).yaml \ --output $@ # formats diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index a75f2110546..b16c4ae13cc 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -146,9 +146,9 @@ def to_simcore_runtime_docker_labels(self) -> dict[DockerLabelKey, str]: @classmethod def from_docker_task(cls, docker_task: Task) -> "StandardSimcoreDockerLabels": - assert docker_task.Spec # nosec - assert docker_task.Spec.ContainerSpec_ # nosec - task_labels = docker_task.Spec.ContainerSpec_.Labels or {} + assert docker_task.spec # nosec + assert docker_task.spec.container_spec # nosec + task_labels = docker_task.spec.container_spec.labels or {} return cls.model_validate(task_labels) model_config = ConfigDict( diff --git a/packages/models-library/src/models_library/generated_models/docker_rest_api.py b/packages/models-library/src/models_library/generated_models/docker_rest_api.py index 0170672cb26..9083641e020 100644 --- a/packages/models-library/src/models_library/generated_models/docker_rest_api.py +++ b/packages/models-library/src/models_library/generated_models/docker_rest_api.py @@ -1,18 +1,17 @@ # generated by datamodel-codegen: -# filename: https://docs.docker.com/engine/api/v1.41.yaml -# timestamp: 2022-11-28T14:56:37+00:00 +# filename: https://docs.docker.com/reference/api/engine/version/v1.41.yaml +# timestamp: 2024-10-15T11:03:37+00:00 from __future__ import annotations -from datetime import datetime from enum import Enum from typing import Any -from pydantic import BaseModel, ConfigDict, Field, RootModel +from pydantic import AwareDatetime, BaseModel, Field, RootModel class Model(RootModel[Any]): - ... + root: Any class Type(str, Enum): @@ -26,17 +25,23 @@ class Port(BaseModel): An open port on a container """ - IP: str | None = Field( - None, description="Host IP address that the container's port is mapped to" + ip: str | None = Field( + default=None, + alias="IP", + description="Host IP address that the container's port is mapped to", ) - PrivatePort: int = Field(..., description="Port on the container") - PublicPort: int | None = Field(None, description="Port exposed on the host") - Type: Type + private_port: int = Field( + ..., alias="PrivatePort", description="Port on the container" + ) + public_port: int | None = Field( + default=None, alias="PublicPort", description="Port exposed on the host" + ) + type: Type = Field(..., alias="Type") class Type1(str, Enum): """ - The mount type: + The mount type: - `bind` a mount of a file or directory from the host into the container. - `volume` a docker volume with the given `Name`. @@ -53,48 +58,56 @@ class Type1(str, Enum): class MountPoint(BaseModel): """ - MountPoint represents a mount point configuration inside the container. + MountPoint represents a mount point configuration inside the container. This is used for reporting the mountpoints in use by a container. """ - Type: Type1 | None = Field( - None, + type: Type1 | None = Field( + default=None, + alias="Type", description="The mount type:\n\n- `bind` a mount of a file or directory from the host into the container.\n- `volume` a docker volume with the given `Name`.\n- `tmpfs` a `tmpfs`.\n- `npipe` a named pipe from the host into the container.\n", examples=["volume"], ) - Name: str | None = Field( - None, + name: str | None = Field( + default=None, + alias="Name", description="Name is the name reference to the underlying data defined by `Source`\ne.g., the volume name.\n", examples=["myvolume"], ) - Source: str | None = Field( - None, + source: str | None = Field( + default=None, + alias="Source", description="Source location of the mount.\n\nFor volumes, this contains the storage location of the volume (within\n`/var/lib/docker/volumes/`). For bind-mounts, and `npipe`, this contains\nthe source (host) part of the bind-mount. For `tmpfs` mount points, this\nfield is empty.\n", examples=["/var/lib/docker/volumes/myvolume/_data"], ) - Destination: str | None = Field( - None, + destination: str | None = Field( + default=None, + alias="Destination", description="Destination is the path relative to the container root (`/`) where\nthe `Source` is mounted inside the container.\n", examples=["/usr/share/nginx/html/"], ) - Driver: str | None = Field( - None, + driver: str | None = Field( + default=None, + alias="Driver", description="Driver is the volume driver used to create the volume (if it is a volume).\n", examples=["local"], ) - Mode: str | None = Field( - None, + mode: str | None = Field( + default=None, + alias="Mode", description='Mode is a comma separated list of options supplied by the user when\ncreating the bind/volume mount.\n\nThe default is platform-specific (`"z"` on Linux, empty on Windows).\n', examples=["z"], ) - RW: bool | None = Field( - None, + rw: bool | None = Field( + default=None, + alias="RW", description="Whether the mount is mounted writable (read-write).\n", examples=[True], ) - Propagation: str | None = Field( - None, + propagation: str | None = Field( + default=None, + alias="Propagation", description="Propagation describes how mounts are propagated from the host into the\nmount point, and vice-versa. Refer to the [Linux kernel documentation](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt)\nfor details. This field is not used on Windows.\n", examples=[""], ) @@ -105,9 +118,9 @@ class DeviceMapping(BaseModel): A device mapping between the host and container """ - PathOnHost: str | None = None - PathInContainer: str | None = None - CgroupPermissions: str | None = None + path_on_host: str | None = Field(default=None, alias="PathOnHost") + path_in_container: str | None = Field(default=None, alias="PathInContainer") + cgroup_permissions: str | None = Field(default=None, alias="CgroupPermissions") class DeviceRequest(BaseModel): @@ -115,25 +128,46 @@ class DeviceRequest(BaseModel): A request for devices to be sent to device drivers """ - Driver: str | None = Field(None, examples=["nvidia"]) - Count: int | None = Field(None, examples=[-1]) - DeviceIDs: list[str] | None = Field( - None, examples=[["0", "1", "GPU-fef8089b-4820-abfc-e83e-94318197576e"]] + driver: str | None = Field(default=None, alias="Driver", examples=["nvidia"]) + count: int | None = Field(default=None, alias="Count", examples=[-1]) + device_i_ds: list[str] | None = Field( + default=None, + alias="DeviceIDs", + examples=[["0", "1", "GPU-fef8089b-4820-abfc-e83e-94318197576e"]], ) - Capabilities: list[list[str]] | None = Field( - None, + capabilities: list[list[str]] | None = Field( + default=None, + alias="Capabilities", description="A list of capabilities; an OR list of AND lists of capabilities.\n", examples=[[["gpu", "nvidia", "compute"]]], ) - Options: dict[str, str] | None = Field( - None, + options: dict[str, str] | None = Field( + default=None, + alias="Options", description="Driver-specific options, specified as a key/value pairs. These options\nare passed directly to the driver.\n", ) class ThrottleDevice(BaseModel): - Path: str | None = Field(None, description="Device path") - Rate: int | None = Field(None, description="Rate", ge=0) + path: str | None = Field(default=None, alias="Path", description="Device path") + rate: int | None = Field(default=None, alias="Rate", description="Rate", ge=0) + + +class Type2(str, Enum): + """ + The mount type. Available types: + + - `bind` Mounts a file or directory from the host into the container. Must exist prior to creating the container. + - `volume` Creates a volume with the given name and options (or uses a pre-existing volume with the same name and options). These are **not** removed when the container is removed. + - `tmpfs` Create a tmpfs with the given options. The mount source cannot be specified for tmpfs. + - `npipe` Mounts a named pipe from the host into the container. Must exist prior to creating the container. + + """ + + bind = "bind" + volume = "volume" + tmpfs = "tmpfs" + npipe = "npipe" class Propagation(str, Enum): @@ -154,13 +188,13 @@ class BindOptions(BaseModel): Optional configuration for the `bind` type. """ - Propagation_: Propagation | None = Field( - None, + propagation: Propagation | None = Field( + default=None, alias="Propagation", description="A propagation mode with the value `[r]private`, `[r]shared`, or `[r]slave`.", ) - NonRecursive: bool | None = Field( - False, description="Disable recursive bind mount." + non_recursive: bool | None = Field( + default=False, alias="NonRecursive", description="Disable recursive bind mount." ) @@ -169,11 +203,15 @@ class DriverConfig(BaseModel): Map of driver specific options """ - Name: str | None = Field( - None, description="Name of the driver to use to create the volume." + name: str | None = Field( + default=None, + alias="Name", + description="Name of the driver to use to create the volume.", ) - Options: dict[str, str] | None = Field( - None, description="key/value map of driver specific options." + options: dict[str, str] | None = Field( + default=None, + alias="Options", + description="key/value map of driver specific options.", ) @@ -182,14 +220,16 @@ class VolumeOptions(BaseModel): Optional configuration for the `volume` type. """ - NoCopy: bool | None = Field( - False, description="Populate volume with data from the target." + no_copy: bool | None = Field( + default=False, + alias="NoCopy", + description="Populate volume with data from the target.", ) - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value metadata." + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value metadata." ) - DriverConfig_: DriverConfig | None = Field( - None, alias="DriverConfig", description="Map of driver specific options" + driver_config: DriverConfig | None = Field( + default=None, alias="DriverConfig", description="Map of driver specific options" ) @@ -198,42 +238,54 @@ class TmpfsOptions(BaseModel): Optional configuration for the `tmpfs` type. """ - SizeBytes: int | None = Field( - None, description="The size for the tmpfs mount in bytes." + size_bytes: int | None = Field( + default=None, + alias="SizeBytes", + description="The size for the tmpfs mount in bytes.", ) - Mode: int | None = Field( - None, description="The permission mode for the tmpfs mount in an integer." + mode: int | None = Field( + default=None, + alias="Mode", + description="The permission mode for the tmpfs mount in an integer.", ) class Mount(BaseModel): - Target: str | None = Field(None, description="Container path.") - Source: str | None = Field( - None, description="Mount source (e.g. a volume name, a host path)." + target: str | None = Field( + default=None, alias="Target", description="Container path." ) - Type: Type1 | None = Field( - None, + source: str | None = Field( + default=None, + alias="Source", + description="Mount source (e.g. a volume name, a host path).", + ) + type: Type2 | None = Field( + default=None, + alias="Type", description="The mount type. Available types:\n\n- `bind` Mounts a file or directory from the host into the container. Must exist prior to creating the container.\n- `volume` Creates a volume with the given name and options (or uses a pre-existing volume with the same name and options). These are **not** removed when the container is removed.\n- `tmpfs` Create a tmpfs with the given options. The mount source cannot be specified for tmpfs.\n- `npipe` Mounts a named pipe from the host into the container. Must exist prior to creating the container.\n", ) - ReadOnly: bool | None = Field( - None, description="Whether the mount should be read-only." + read_only: bool | None = Field( + default=None, + alias="ReadOnly", + description="Whether the mount should be read-only.", ) - Consistency: str | None = Field( - None, + consistency: str | None = Field( + default=None, + alias="Consistency", description="The consistency requirement for the mount: `default`, `consistent`, `cached`, or `delegated`.", ) - BindOptions_: BindOptions | None = Field( - None, + bind_options: BindOptions | None = Field( + default=None, alias="BindOptions", description="Optional configuration for the `bind` type.", ) - VolumeOptions_: VolumeOptions | None = Field( - None, + volume_options: VolumeOptions | None = Field( + default=None, alias="VolumeOptions", description="Optional configuration for the `volume` type.", ) - TmpfsOptions_: TmpfsOptions | None = Field( - None, + tmpfs_options: TmpfsOptions | None = Field( + default=None, alias="TmpfsOptions", description="Optional configuration for the `tmpfs` type.", ) @@ -241,7 +293,7 @@ class Mount(BaseModel): class Name(str, Enum): """ - - Empty string means not to restart + - Empty string means not to restart - `no` Do not automatically restart - `always` Always restart - `unless-stopped` Restart always except when the user has manually stopped the container @@ -249,7 +301,7 @@ class Name(str, Enum): """ - _ = "" + field_ = "" no = "no" always = "always" unless_stopped = "unless-stopped" @@ -258,7 +310,7 @@ class Name(str, Enum): class RestartPolicy(BaseModel): """ - The behavior to apply when the container exits. The default is not to + The behavior to apply when the container exits. The default is not to restart. An ever increasing delay (double the previous delay, starting at 100ms) is @@ -266,26 +318,27 @@ class RestartPolicy(BaseModel): """ - Name_: Name | None = Field( - None, + name: Name | None = Field( + default=None, alias="Name", description="- Empty string means not to restart\n- `no` Do not automatically restart\n- `always` Always restart\n- `unless-stopped` Restart always except when the user has manually stopped the container\n- `on-failure` Restart only when the container exit code is non-zero\n", ) - MaximumRetryCount: int | None = Field( - None, + maximum_retry_count: int | None = Field( + default=None, + alias="MaximumRetryCount", description="If `on-failure` is used, the number of times to retry before giving up.\n", ) class BlkioWeightDeviceItem(BaseModel): - Path: str | None = None - Weight: int | None = Field(None, ge=0) + path: str | None = Field(default=None, alias="Path") + weight: int | None = Field(default=None, alias="Weight", ge=0) class Ulimit(BaseModel): - Name: str | None = Field(None, description="Name of ulimit") - Soft: int | None = Field(None, description="Soft limit") - Hard: int | None = Field(None, description="Hard limit") + name: str | None = Field(default=None, alias="Name", description="Name of ulimit") + soft: int | None = Field(default=None, alias="Soft", description="Soft limit") + hard: int | None = Field(default=None, alias="Hard", description="Hard limit") class Resources(BaseModel): @@ -293,124 +346,168 @@ class Resources(BaseModel): A container's resources (cgroups config, ulimits, etc) """ - CpuShares: int | None = Field( - None, + cpu_shares: int | None = Field( + default=None, + alias="CpuShares", description="An integer value representing this container's relative CPU weight\nversus other containers.\n", ) - Memory: int | None = Field(0, description="Memory limit in bytes.") - CgroupParent: str | None = Field( - None, + memory: int | None = Field( + default=0, alias="Memory", description="Memory limit in bytes." + ) + cgroup_parent: str | None = Field( + default=None, + alias="CgroupParent", description="Path to `cgroups` under which the container's `cgroup` is created. If\nthe path is not absolute, the path is considered to be relative to the\n`cgroups` path of the init process. Cgroups are created if they do not\nalready exist.\n", ) - BlkioWeight: int | None = Field( - None, description="Block IO weight (relative weight).", ge=0, le=1000 + blkio_weight: int | None = Field( + default=None, + alias="BlkioWeight", + description="Block IO weight (relative weight).", + ge=0, + le=1000, ) - BlkioWeightDevice: list[BlkioWeightDeviceItem] | None = Field( - None, + blkio_weight_device: list[BlkioWeightDeviceItem] | None = Field( + default=None, + alias="BlkioWeightDevice", description='Block IO weight (relative device weight) in the form:\n\n```\n[{"Path": "device_path", "Weight": weight}]\n```\n', ) - BlkioDeviceReadBps: list[ThrottleDevice] | None = Field( - None, + blkio_device_read_bps: list[ThrottleDevice] | None = Field( + default=None, + alias="BlkioDeviceReadBps", description='Limit read rate (bytes per second) from a device, in the form:\n\n```\n[{"Path": "device_path", "Rate": rate}]\n```\n', ) - BlkioDeviceWriteBps: list[ThrottleDevice] | None = Field( - None, + blkio_device_write_bps: list[ThrottleDevice] | None = Field( + default=None, + alias="BlkioDeviceWriteBps", description='Limit write rate (bytes per second) to a device, in the form:\n\n```\n[{"Path": "device_path", "Rate": rate}]\n```\n', ) - BlkioDeviceReadIOps: list[ThrottleDevice] | None = Field( - None, + blkio_device_read_i_ops: list[ThrottleDevice] | None = Field( + default=None, + alias="BlkioDeviceReadIOps", description='Limit read rate (IO per second) from a device, in the form:\n\n```\n[{"Path": "device_path", "Rate": rate}]\n```\n', ) - BlkioDeviceWriteIOps: list[ThrottleDevice] | None = Field( - None, + blkio_device_write_i_ops: list[ThrottleDevice] | None = Field( + default=None, + alias="BlkioDeviceWriteIOps", description='Limit write rate (IO per second) to a device, in the form:\n\n```\n[{"Path": "device_path", "Rate": rate}]\n```\n', ) - CpuPeriod: int | None = Field( - None, description="The length of a CPU period in microseconds." + cpu_period: int | None = Field( + default=None, + alias="CpuPeriod", + description="The length of a CPU period in microseconds.", ) - CpuQuota: int | None = Field( - None, + cpu_quota: int | None = Field( + default=None, + alias="CpuQuota", description="Microseconds of CPU time that the container can get in a CPU period.\n", ) - CpuRealtimePeriod: int | None = Field( - None, + cpu_realtime_period: int | None = Field( + default=None, + alias="CpuRealtimePeriod", description="The length of a CPU real-time period in microseconds. Set to 0 to\nallocate no time allocated to real-time tasks.\n", ) - CpuRealtimeRuntime: int | None = Field( - None, + cpu_realtime_runtime: int | None = Field( + default=None, + alias="CpuRealtimeRuntime", description="The length of a CPU real-time runtime in microseconds. Set to 0 to\nallocate no time allocated to real-time tasks.\n", ) - CpusetCpus: str | None = Field( - None, + cpuset_cpus: str | None = Field( + default=None, + alias="CpusetCpus", description="CPUs in which to allow execution (e.g., `0-3`, `0,1`).\n", examples=["0-3"], ) - CpusetMems: str | None = Field( - None, + cpuset_mems: str | None = Field( + default=None, + alias="CpusetMems", description="Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only\neffective on NUMA systems.\n", ) - Devices: list[DeviceMapping] | None = Field( - None, description="A list of devices to add to the container." + devices: list[DeviceMapping] | None = Field( + default=None, + alias="Devices", + description="A list of devices to add to the container.", ) - DeviceCgroupRules: list[str] | None = Field( - None, description="a list of cgroup rules to apply to the container" + device_cgroup_rules: list[str] | None = Field( + default=None, + alias="DeviceCgroupRules", + description="a list of cgroup rules to apply to the container", ) - DeviceRequests: list[DeviceRequest] | None = Field( - None, + device_requests: list[DeviceRequest] | None = Field( + default=None, + alias="DeviceRequests", description="A list of requests for devices to be sent to device drivers.\n", ) - KernelMemory: int | None = Field( - None, + kernel_memory: int | None = Field( + default=None, + alias="KernelMemory", description="Kernel memory limit in bytes.\n\n


\n\n> **Deprecated**: This field is deprecated as the kernel 5.4 deprecated\n> `kmem.limit_in_bytes`.\n", examples=[209715200], ) - KernelMemoryTCP: int | None = Field( - None, description="Hard limit for kernel TCP buffer memory (in bytes)." + kernel_memory_tcp: int | None = Field( + default=None, + alias="KernelMemoryTCP", + description="Hard limit for kernel TCP buffer memory (in bytes).", ) - MemoryReservation: int | None = Field( - None, description="Memory soft limit in bytes." + memory_reservation: int | None = Field( + default=None, + alias="MemoryReservation", + description="Memory soft limit in bytes.", ) - MemorySwap: int | None = Field( - None, + memory_swap: int | None = Field( + default=None, + alias="MemorySwap", description="Total memory limit (memory + swap). Set as `-1` to enable unlimited\nswap.\n", ) - MemorySwappiness: int | None = Field( - None, + memory_swappiness: int | None = Field( + default=None, + alias="MemorySwappiness", description="Tune a container's memory swappiness behavior. Accepts an integer\nbetween 0 and 100.\n", ge=0, le=100, ) - NanoCpus: int | None = Field( - None, description="CPU quota in units of 10-9 CPUs." + nano_cpus: int | None = Field( + default=None, + alias="NanoCpus", + description="CPU quota in units of 10-9 CPUs.", ) - OomKillDisable: bool | None = Field( - None, description="Disable OOM Killer for the container." + oom_kill_disable: bool | None = Field( + default=None, + alias="OomKillDisable", + description="Disable OOM Killer for the container.", ) - Init: bool | None = Field( - None, + init: bool | None = Field( + default=None, + alias="Init", description="Run an init inside the container that forwards signals and reaps\nprocesses. This field is omitted if empty, and the default (as\nconfigured on the daemon) is used.\n", ) - PidsLimit: int | None = Field( - None, + pids_limit: int | None = Field( + default=None, + alias="PidsLimit", description="Tune a container's PIDs limit. Set `0` or `-1` for unlimited, or `null`\nto not change.\n", ) - Ulimits: list[Ulimit] | None = Field( - None, + ulimits: list[Ulimit] | None = Field( + default=None, + alias="Ulimits", description='A list of resource limits to set in the container. For example:\n\n```\n{"Name": "nofile", "Soft": 1024, "Hard": 2048}\n```\n', ) - CpuCount: int | None = Field( - None, + cpu_count: int | None = Field( + default=None, + alias="CpuCount", description="The number of usable CPUs (Windows only).\n\nOn Windows Server containers, the processor resource controls are\nmutually exclusive. The order of precedence is `CPUCount` first, then\n`CPUShares`, and `CPUPercent` last.\n", ) - CpuPercent: int | None = Field( - None, + cpu_percent: int | None = Field( + default=None, + alias="CpuPercent", description="The usable percentage of the available CPUs (Windows only).\n\nOn Windows Server containers, the processor resource controls are\nmutually exclusive. The order of precedence is `CPUCount` first, then\n`CPUShares`, and `CPUPercent` last.\n", ) - IOMaximumIOps: int | None = Field( - None, description="Maximum IOps for the container system drive (Windows only)" + io_maximum_i_ops: int | None = Field( + default=None, + alias="IOMaximumIOps", + description="Maximum IOps for the container system drive (Windows only)", ) - IOMaximumBandwidth: int | None = Field( - None, + io_maximum_bandwidth: int | None = Field( + default=None, + alias="IOMaximumBandwidth", description="Maximum IO in bytes per second for the container system drive\n(Windows only).\n", ) @@ -421,37 +518,42 @@ class Limit(BaseModel): """ - NanoCPUs: int | None = Field(None, examples=[4000000000]) - MemoryBytes: int | None = Field(None, examples=[8272408576]) - Pids: int | None = Field( - 0, + nano_cp_us: int | None = Field( + default=None, alias="NanoCPUs", examples=[4000000000] + ) + memory_bytes: int | None = Field( + default=None, alias="MemoryBytes", examples=[8272408576] + ) + pids: int | None = Field( + default=0, + alias="Pids", description="Limits the maximum number of PIDs in the container. Set `0` for unlimited.\n", examples=[100], ) class NamedResourceSpec(BaseModel): - Kind: str | None = None - Value: str | None = None + kind: str | None = Field(default=None, alias="Kind") + value: str | None = Field(default=None, alias="Value") class DiscreteResourceSpec(BaseModel): - Kind: str | None = None - Value: int | None = None + kind: str | None = Field(default=None, alias="Kind") + value: int | None = Field(default=None, alias="Value") class GenericResource(BaseModel): - NamedResourceSpec_: NamedResourceSpec | None = Field( - None, alias="NamedResourceSpec" + named_resource_spec: NamedResourceSpec | None = Field( + default=None, alias="NamedResourceSpec" ) - DiscreteResourceSpec_: DiscreteResourceSpec | None = Field( - None, alias="DiscreteResourceSpec" + discrete_resource_spec: DiscreteResourceSpec | None = Field( + default=None, alias="DiscreteResourceSpec" ) -class GenericResources(RootModel): +class GenericResources(RootModel[list[GenericResource]]): """ - User-defined resources can be either Integer resources (e.g, `SSD=3`) or + User-defined resources can be either Integer resources (e.g, `SSD=3`) or String resources (e.g, `GPU=UUID1`). """ @@ -474,31 +576,36 @@ class HealthConfig(BaseModel): A test to perform to check that the container is healthy. """ - Test: list[str] | None = Field( - None, + test: list[str] | None = Field( + default=None, + alias="Test", description='The test to perform. Possible values are:\n\n- `[]` inherit healthcheck from image or parent image\n- `["NONE"]` disable healthcheck\n- `["CMD", args...]` exec arguments directly\n- `["CMD-SHELL", command]` run command with system\'s default shell\n', ) - Interval: int | None = Field( - None, + interval: int | None = Field( + default=None, + alias="Interval", description="The time to wait between checks in nanoseconds. It should be 0 or at\nleast 1000000 (1 ms). 0 means inherit.\n", ) - Timeout: int | None = Field( - None, + timeout: int | None = Field( + default=None, + alias="Timeout", description="The time to wait before considering the check to have hung. It should\nbe 0 or at least 1000000 (1 ms). 0 means inherit.\n", ) - Retries: int | None = Field( - None, + retries: int | None = Field( + default=None, + alias="Retries", description="The number of consecutive failures needed to consider a container as\nunhealthy. 0 means inherit.\n", ) - StartPeriod: int | None = Field( - None, + start_period: int | None = Field( + default=None, + alias="StartPeriod", description="Start period for the container to initialize before starting\nhealth-retries countdown in nanoseconds. It should be 0 or at least\n1000000 (1 ms). 0 means inherit.\n", ) class Status(str, Enum): """ - Status is one of `none`, `starting`, `healthy` or `unhealthy` + Status is one of `none`, `starting`, `healthy` or `unhealthy` - "none" Indicates there is no healthcheck - "starting" Starting indicates that the container is not yet ready @@ -519,22 +626,27 @@ class HealthcheckResult(BaseModel): """ - Start: datetime | None = Field( - None, + start: AwareDatetime | None = Field( + default=None, + alias="Start", description="Date and time at which this check started in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2020-01-04T10:44:24.496525531Z"], ) - End: str | None = Field( - None, + end: str | None = Field( + default=None, + alias="End", description="Date and time at which this check ended in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2020-01-04T10:45:21.364524523Z"], ) - ExitCode: int | None = Field( - None, + exit_code: int | None = Field( + default=None, + alias="ExitCode", description="ExitCode meanings:\n\n- `0` healthy\n- `1` unhealthy\n- `2` reserved (considered unhealthy)\n- other values: error running probe\n", examples=[0], ) - Output: str | None = Field(None, description="Output from last check") + output: str | None = Field( + default=None, alias="Output", description="Output from last check" + ) class Type3(str, Enum): @@ -554,13 +666,13 @@ class LogConfig(BaseModel): The logging configuration for this container """ - Type: Type3 | None = None - Config_: dict[str, str] | None = Field(None, alias="Config") + type: Type3 | None = Field(default=None, alias="Type") + config: dict[str, str] | None = Field(default=None, alias="Config") class CgroupnsMode(str, Enum): """ - cgroup namespace mode for the container. Possible values are: + cgroup namespace mode for the container. Possible values are: - `"private"`: the container runs in its own private cgroup namespace - `"host"`: use the host system's cgroup namespace @@ -574,7 +686,7 @@ class CgroupnsMode(str, Enum): host = "host" -class ConsoleSizeItem(RootModel): +class ConsoleSizeItem(RootModel[int]): root: int = Field(..., ge=0) @@ -591,7 +703,7 @@ class Isolation(str, Enum): class ContainerConfig(BaseModel): """ - Configuration for a container that is portable between hosts. + Configuration for a container that is portable between hosts. When used as `ContainerConfig` field in an image, `ContainerConfig` is an optional field containing the configuration of the container that was last @@ -602,83 +714,111 @@ class ContainerConfig(BaseModel): """ - Hostname: str | None = Field( - None, + hostname: str | None = Field( + default=None, + alias="Hostname", description="The hostname to use for the container, as a valid RFC 1123 hostname.\n", examples=["439f4e91bd1d"], ) - Domainname: str | None = Field( - None, description="The domain name to use for the container.\n" + domainname: str | None = Field( + default=None, + alias="Domainname", + description="The domain name to use for the container.\n", ) - User: str | None = Field( - None, description="The user that commands are run as inside the container." + user: str | None = Field( + default=None, + alias="User", + description="The user that commands are run as inside the container.", ) - AttachStdin: bool | None = Field(False, description="Whether to attach to `stdin`.") - AttachStdout: bool | None = Field( - True, description="Whether to attach to `stdout`." + attach_stdin: bool | None = Field( + default=False, alias="AttachStdin", description="Whether to attach to `stdin`." ) - AttachStderr: bool | None = Field( - True, description="Whether to attach to `stderr`." + attach_stdout: bool | None = Field( + default=True, alias="AttachStdout", description="Whether to attach to `stdout`." ) - ExposedPorts: dict[str, dict[str, Any]] | None = Field( - None, + attach_stderr: bool | None = Field( + default=True, alias="AttachStderr", description="Whether to attach to `stderr`." + ) + exposed_ports: dict[str, dict[str, Any]] | None = Field( + default=None, + alias="ExposedPorts", description='An object mapping ports to an empty object in the form:\n\n`{"/": {}}`\n', examples=[{"80/tcp": {}, "443/tcp": {}}], ) - Tty: bool | None = Field( - False, + tty: bool | None = Field( + default=False, + alias="Tty", description="Attach standard streams to a TTY, including `stdin` if it is not closed.\n", ) - OpenStdin: bool | None = Field(False, description="Open `stdin`") - StdinOnce: bool | None = Field( - False, description="Close `stdin` after one attached client disconnects" + open_stdin: bool | None = Field( + default=False, alias="OpenStdin", description="Open `stdin`" + ) + stdin_once: bool | None = Field( + default=False, + alias="StdinOnce", + description="Close `stdin` after one attached client disconnects", ) - Env: list[str] | None = Field( - None, + env: list[str] | None = Field( + default=None, + alias="Env", description='A list of environment variables to set inside the container in the\nform `["VAR=value", ...]`. A variable without `=` is removed from the\nenvironment, rather than to have an empty value.\n', examples=[ ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] ], ) - Cmd: list[str] | None = Field( - None, + cmd: list[str] | None = Field( + default=None, + alias="Cmd", description="Command to run specified as a string or an array of strings.\n", examples=[["/bin/sh"]], ) - Healthcheck: HealthConfig | None = None - ArgsEscaped: bool | None = Field( - False, description="Command is already escaped (Windows only)", examples=[False] + healthcheck: HealthConfig | None = Field(default=None, alias="Healthcheck") + args_escaped: bool | None = Field( + default=False, + alias="ArgsEscaped", + description="Command is already escaped (Windows only)", + examples=[False], ) - Image: str | None = Field( - None, + image: str | None = Field( + default=None, + alias="Image", description="The name (or reference) of the image to use when creating the container,\nor which was used when the container was created.\n", examples=["example-image:1.0"], ) - Volumes: dict[str, dict[str, Any]] | None = Field( - None, + volumes: dict[str, dict[str, Any]] | None = Field( + default=None, + alias="Volumes", description="An object mapping mount point paths inside the container to empty\nobjects.\n", ) - WorkingDir: str | None = Field( - None, + working_dir: str | None = Field( + default=None, + alias="WorkingDir", description="The working directory for commands to run in.", examples=["/public/"], ) - Entrypoint: list[str] | None = Field( - None, + entrypoint: list[str] | None = Field( + default=None, + alias="Entrypoint", description='The entry point for the container as a string or an array of strings.\n\nIf the array consists of exactly one empty string (`[""]`) then the\nentry point is reset to system default (i.e., the entry point used by\ndocker when there is no `ENTRYPOINT` instruction in the `Dockerfile`).\n', examples=[[]], ) - NetworkDisabled: bool | None = Field( - None, description="Disable networking for the container." + network_disabled: bool | None = Field( + default=None, + alias="NetworkDisabled", + description="Disable networking for the container.", + ) + mac_address: str | None = Field( + default=None, alias="MacAddress", description="MAC address of the container." ) - MacAddress: str | None = Field(None, description="MAC address of the container.") - OnBuild: list[str] | None = Field( - None, + on_build: list[str] | None = Field( + default=None, + alias="OnBuild", description="`ONBUILD` metadata that were defined in the image's `Dockerfile`.\n", examples=[[]], ) - Labels: dict[str, str] | None = Field( - None, + labels: dict[str, str] | None = Field( + default=None, + alias="Labels", description="User-defined key/value metadata.", examples=[ { @@ -687,58 +827,212 @@ class ContainerConfig(BaseModel): } ], ) - StopSignal: str | None = Field( - None, + stop_signal: str | None = Field( + default=None, + alias="StopSignal", description="Signal to stop a container as a string or unsigned integer.\n", examples=["SIGTERM"], ) - StopTimeout: int | None = Field( - 10, description="Timeout to stop a container in seconds." + stop_timeout: int | None = Field( + default=10, + alias="StopTimeout", + description="Timeout to stop a container in seconds.", ) - Shell: list[str] | None = Field( - None, + shell: list[str] | None = Field( + default=None, + alias="Shell", description="Shell for when `RUN`, `CMD`, and `ENTRYPOINT` uses a shell.\n", examples=[["/bin/sh", "-c"]], ) -class Address(BaseModel): +class ImageConfig(BaseModel): """ - Address represents an IPv4 or IPv6 IP address. + Configuration of the image. These fields are used as defaults + when starting a container from the image. + """ - Addr: str | None = Field(None, description="IP address.") - PrefixLen: int | None = Field(None, description="Mask length of the IP address.") + hostname: str | None = Field( + default=None, + alias="Hostname", + description="The hostname to use for the container, as a valid RFC 1123 hostname.\n\n


\n\n> **Note**: this field is always empty and must not be used.\n", + examples=[""], + ) + domainname: str | None = Field( + default=None, + alias="Domainname", + description="The domain name to use for the container.\n\n


\n\n> **Note**: this field is always empty and must not be used.\n", + examples=[""], + ) + user: str | None = Field( + default=None, + alias="User", + description="The user that commands are run as inside the container.", + examples=["web:web"], + ) + attach_stdin: bool | None = Field( + default=False, + alias="AttachStdin", + description="Whether to attach to `stdin`.\n\n


\n\n> **Note**: this field is always false and must not be used.\n", + examples=[False], + ) + attach_stdout: bool | None = Field( + default=False, + alias="AttachStdout", + description="Whether to attach to `stdout`.\n\n


\n\n> **Note**: this field is always false and must not be used.\n", + examples=[False], + ) + attach_stderr: bool | None = Field( + default=False, + alias="AttachStderr", + description="Whether to attach to `stderr`.\n\n


\n\n> **Note**: this field is always false and must not be used.\n", + examples=[False], + ) + exposed_ports: dict[str, dict[str, Any]] | None = Field( + default=None, + alias="ExposedPorts", + description='An object mapping ports to an empty object in the form:\n\n`{"/": {}}`\n', + examples=[{"80/tcp": {}, "443/tcp": {}}], + ) + tty: bool | None = Field( + default=False, + alias="Tty", + description="Attach standard streams to a TTY, including `stdin` if it is not closed.\n\n


\n\n> **Note**: this field is always false and must not be used.\n", + examples=[False], + ) + open_stdin: bool | None = Field( + default=False, + alias="OpenStdin", + description="Open `stdin`\n\n


\n\n> **Note**: this field is always false and must not be used.\n", + examples=[False], + ) + stdin_once: bool | None = Field( + default=False, + alias="StdinOnce", + description="Close `stdin` after one attached client disconnects.\n\n


\n\n> **Note**: this field is always false and must not be used.\n", + examples=[False], + ) + env: list[str] | None = Field( + default=None, + alias="Env", + description='A list of environment variables to set inside the container in the\nform `["VAR=value", ...]`. A variable without `=` is removed from the\nenvironment, rather than to have an empty value.\n', + examples=[ + ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] + ], + ) + cmd: list[str] | None = Field( + default=None, + alias="Cmd", + description="Command to run specified as a string or an array of strings.\n", + examples=[["/bin/sh"]], + ) + healthcheck: HealthConfig | None = Field(default=None, alias="Healthcheck") + args_escaped: bool | None = Field( + default=False, + alias="ArgsEscaped", + description="Command is already escaped (Windows only)", + examples=[False], + ) + image: str | None = Field( + default="", + alias="Image", + description="The name (or reference) of the image to use when creating the container,\nor which was used when the container was created.\n\n


\n\n> **Note**: this field is always empty and must not be used.\n", + examples=[""], + ) + volumes: dict[str, dict[str, Any]] | None = Field( + default=None, + alias="Volumes", + description="An object mapping mount point paths inside the container to empty\nobjects.\n", + examples=[{"/app/data": {}, "/app/config": {}}], + ) + working_dir: str | None = Field( + default=None, + alias="WorkingDir", + description="The working directory for commands to run in.", + examples=["/public/"], + ) + entrypoint: list[str] | None = Field( + default=None, + alias="Entrypoint", + description='The entry point for the container as a string or an array of strings.\n\nIf the array consists of exactly one empty string (`[""]`) then the\nentry point is reset to system default (i.e., the entry point used by\ndocker when there is no `ENTRYPOINT` instruction in the `Dockerfile`).\n', + examples=[[]], + ) + network_disabled: bool | None = Field( + default=False, + alias="NetworkDisabled", + description="Disable networking for the container.\n\n


\n\n> **Note**: this field is always omitted and must not be used.\n", + examples=[False], + ) + mac_address: str | None = Field( + default="", + alias="MacAddress", + description="MAC address of the container.\n\n


\n\n> **Note**: this field is always omitted and must not be used.\n", + examples=[""], + ) + on_build: list[str] | None = Field( + default=None, + alias="OnBuild", + description="`ONBUILD` metadata that were defined in the image's `Dockerfile`.\n", + examples=[[]], + ) + labels: dict[str, str] | None = Field( + default=None, + alias="Labels", + description="User-defined key/value metadata.", + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], + ) + stop_signal: str | None = Field( + default=None, + alias="StopSignal", + description="Signal to stop a container as a string or unsigned integer.\n", + examples=["SIGTERM"], + ) + stop_timeout: int | None = Field( + default=10, + alias="StopTimeout", + description="Timeout to stop a container in seconds.\n\n


\n\n> **Note**: this field is always omitted and must not be used.\n", + ) + shell: list[str] | None = Field( + default=None, + alias="Shell", + description="Shell for when `RUN`, `CMD`, and `ENTRYPOINT` uses a shell.\n", + examples=[["/bin/sh", "-c"]], + ) -class PortMap(BaseModel): +class Address(BaseModel): """ - PortMap describes the mapping of container ports to host ports, using the - container's port-number and protocol as key in the format `/`, - for example, `80/udp`. - - If a container's port is mapped for multiple protocols, separate entries - are added to the mapping table. - + Address represents an IPv4 or IPv6 IP address. """ - model_config = ConfigDict(extra="allow") + addr: str | None = Field(default=None, alias="Addr", description="IP address.") + prefix_len: int | None = Field( + default=None, alias="PrefixLen", description="Mask length of the IP address." + ) class PortBinding(BaseModel): """ - PortBinding represents a binding between a host IP address and a host + PortBinding represents a binding between a host IP address and a host port. """ - HostIp: str | None = Field( - None, + host_ip: str | None = Field( + default=None, + alias="HostIp", description="Host IP address that the container's port is mapped to.", examples=["127.0.0.1"], ) - HostPort: str | None = Field( - None, + host_port: str | None = Field( + default=None, + alias="HostPort", description="Host port number that the container's port is mapped to.", examples=["4443"], ) @@ -746,16 +1040,20 @@ class PortBinding(BaseModel): class GraphDriverData(BaseModel): """ - Information about the storage driver used to store the container's and + Information about the storage driver used to store the container's and image's filesystem. """ - Name: str = Field( - ..., description="Name of the storage driver.", examples=["overlay2"] + name: str = Field( + ..., + alias="Name", + description="Name of the storage driver.", + examples=["overlay2"], ) - Data: dict[str, str] = Field( + data: dict[str, str] = Field( ..., + alias="Data", description="Low-level storage metadata, provided as key/value pairs.\n\nThis information is driver-specific, and depends on the storage-driver\nin use, and should be used for informational purposes only.\n", examples=[ { @@ -767,15 +1065,16 @@ class GraphDriverData(BaseModel): ) -class RootFS(BaseModel): +class RootFs(BaseModel): """ Information about the image's RootFS, including the layer IDs. """ - Type: str = Field(..., examples=["layers"]) - Layers: list[str] | None = Field( - None, + type: str = Field(..., alias="Type", examples=["layers"]) + layers: list[str] | None = Field( + default=None, + alias="Layers", examples=[ [ "sha256:1834950e52ce4d5a88a1bbd131c537f4d0e56d10ff0dd69e66be3b7dfa9df7e6", @@ -787,13 +1086,14 @@ class RootFS(BaseModel): class Metadata(BaseModel): """ - Additional metadata of the image in the local cache. This information + Additional metadata of the image in the local cache. This information is local to the daemon, and not part of the image itself. """ - LastTagTime: str | None = Field( - None, + last_tag_time: str | None = Field( + default=None, + alias="LastTagTime", description="Date and time at which the image was last tagged in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n\nThis information is only available if the image was tagged locally,\nand omitted otherwise.\n", examples=["2022-02-28T14:40:02.623929178Z"], ) @@ -805,16 +1105,18 @@ class ImageInspect(BaseModel): """ - Id: str | None = Field( - None, + id: str | None = Field( + default=None, + alias="Id", description="ID is the content-addressable ID of an image.\n\nThis identifier is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n", examples=[ "sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710" ], ) - RepoTags: list[str] | None = Field( - None, - description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same imagem and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n', + repo_tags: list[str] | None = Field( + default=None, + alias="RepoTags", + description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same image, and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n', examples=[ [ "example:1.0", @@ -824,8 +1126,9 @@ class ImageInspect(BaseModel): ] ], ) - RepoDigests: list[str] | None = Field( - None, + repo_digests: list[str] | None = Field( + default=None, + alias="RepoDigests", description="List of content-addressable digests of locally available image manifests\nthat the image is referenced from. Multiple manifests can refer to the\nsame image.\n\nThese digests are usually only available if the image was either pulled\nfrom a registry, or if the image was pushed to a registry, which is when\nthe manifest is generated and its digest calculated.\n", examples=[ [ @@ -834,97 +1137,114 @@ class ImageInspect(BaseModel): ] ], ) - Parent: str | None = Field( - None, + parent: str | None = Field( + default=None, + alias="Parent", description="ID of the parent image.\n\nDepending on how the image was created, this field may be empty and\nis only set for images that were built/created locally. This field\nis empty if the image was pulled from an image registry.\n", examples=[""], ) - Comment: str | None = Field( - None, + comment: str | None = Field( + default=None, + alias="Comment", description="Optional message that was set when committing or importing the image.\n", examples=[""], ) - Created: str | None = Field( - None, + created: str | None = Field( + default=None, + alias="Created", description="Date and time at which the image was created, formatted in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2022-02-04T21:20:12.497794809Z"], ) - Container: str | None = Field( - None, + container: str | None = Field( + default=None, + alias="Container", description="The ID of the container that was used to create the image.\n\nDepending on how the image was created, this field may be empty.\n", examples=["65974bc86f1770ae4bff79f651ebdbce166ae9aada632ee3fa9af3a264911735"], ) - ContainerConfig_: ContainerConfig | None = Field(None, alias="ContainerConfig") - DockerVersion: str | None = Field( - None, + container_config: ContainerConfig | None = Field( + default=None, alias="ContainerConfig" + ) + docker_version: str | None = Field( + default=None, + alias="DockerVersion", description="The version of Docker that was used to build the image.\n\nDepending on how the image was created, this field may be empty.\n", examples=["20.10.7"], ) - Author: str | None = Field( - None, + author: str | None = Field( + default=None, + alias="Author", description="Name of the author that was specified when committing the image, or as\nspecified through MAINTAINER (deprecated) in the Dockerfile.\n", examples=[""], ) - Config_: ContainerConfig | None = Field(None, alias="Config") - Architecture: str | None = Field( - None, + config: ImageConfig | None = Field(default=None, alias="Config") + architecture: str | None = Field( + default=None, + alias="Architecture", description="Hardware CPU architecture that the image runs on.\n", examples=["arm"], ) - Variant: str | None = Field( - None, + variant: str | None = Field( + default=None, + alias="Variant", description="CPU architecture variant (presently ARM-only).\n", examples=["v7"], ) - Os: str | None = Field( - None, + os: str | None = Field( + default=None, + alias="Os", description="Operating System the image is built to run on.\n", examples=["linux"], ) - OsVersion: str | None = Field( - None, + os_version: str | None = Field( + default=None, + alias="OsVersion", description="Operating System version the image is built to run on (especially\nfor Windows).\n", examples=[""], ) - Size: int | None = Field( - None, + size: int | None = Field( + default=None, + alias="Size", description="Total size of the image including all layers it is composed of.\n", examples=[1239828], ) - VirtualSize: int | None = Field( - None, + virtual_size: int | None = Field( + default=None, + alias="VirtualSize", description="Total size of the image including all layers it is composed of.\n\nIn versions of Docker before v1.10, this field was calculated from\nthe image itself and all of its parent images. Docker v1.10 and up\nstore images self-contained, and no longer use a parent-chain, making\nthis field an equivalent of the Size field.\n\nThis field is kept for backward compatibility, but may be removed in\na future version of the API.\n", examples=[1239828], ) - GraphDriver: GraphDriverData | None = None - RootFS_: RootFS | None = Field( - None, + graph_driver: GraphDriverData | None = Field(default=None, alias="GraphDriver") + root_fs: RootFs | None = Field( + default=None, alias="RootFS", description="Information about the image's RootFS, including the layer IDs.\n", ) - Metadata_: Metadata | None = Field( - None, + metadata: Metadata | None = Field( + default=None, alias="Metadata", description="Additional metadata of the image in the local cache. This information\nis local to the daemon, and not part of the image itself.\n", ) class ImageSummary(BaseModel): - Id: str = Field( + id: str = Field( ..., + alias="Id", description="ID is the content-addressable ID of an image.\n\nThis identifier is a content-addressable digest calculated from the\nimage's configuration (which includes the digests of layers used by\nthe image).\n\nNote that this digest differs from the `RepoDigests` below, which\nholds digests of image manifests that reference the image.\n", examples=[ "sha256:ec3f0931a6e6b6855d76b2d7b0be30e81860baccd891b2e243280bf1cd8ad710" ], ) - ParentId: str = Field( + parent_id: str = Field( ..., + alias="ParentId", description="ID of the parent image.\n\nDepending on how the image was created, this field may be empty and\nis only set for images that were built/created locally. This field\nis empty if the image was pulled from an image registry.\n", examples=[""], ) - RepoTags: list[str] = Field( + repo_tags: list[str] = Field( ..., - description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same imagem and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n', + alias="RepoTags", + description='List of image names/tags in the local image cache that reference this\nimage.\n\nMultiple image tags can refer to the same image, and this list may be\nempty if no tags reference the image, in which case the image is\n"untagged", in which case it can still be referenced by its ID.\n', examples=[ [ "example:1.0", @@ -934,8 +1254,9 @@ class ImageSummary(BaseModel): ] ], ) - RepoDigests: list[str] = Field( + repo_digests: list[str] = Field( ..., + alias="RepoDigests", description="List of content-addressable digests of locally available image manifests\nthat the image is referenced from. Multiple manifests can refer to the\nsame image.\n\nThese digests are usually only available if the image was either pulled\nfrom a registry, or if the image was pushed to a registry, which is when\nthe manifest is generated and its digest calculated.\n", examples=[ [ @@ -944,28 +1265,33 @@ class ImageSummary(BaseModel): ] ], ) - Created: int = Field( + created: int = Field( ..., - description="Date and time at which the image was created as a Unix timestamp\n(number of seconds sinds EPOCH).\n", + alias="Created", + description="Date and time at which the image was created as a Unix timestamp\n(number of seconds since EPOCH).\n", examples=["1644009612"], ) - Size: int = Field( + size: int = Field( ..., + alias="Size", description="Total size of the image including all layers it is composed of.\n", examples=[172064416], ) - SharedSize: int = Field( + shared_size: int = Field( ..., + alias="SharedSize", description="Total size of image layers that are shared between this image and other\nimages.\n\nThis size is not calculated by default. `-1` indicates that the value\nhas not been set / calculated.\n", examples=[1239828], ) - VirtualSize: int = Field( + virtual_size: int = Field( ..., + alias="VirtualSize", description="Total size of the image including all layers it is composed of.\n\nIn versions of Docker before v1.10, this field was calculated from\nthe image itself and all of its parent images. Docker v1.10 and up\nstore images self-contained, and no longer use a parent-chain, making\nthis field an equivalent of the Size field.\n\nThis field is kept for backward compatibility, but may be removed in\na future version of the API.\n", examples=[172064416], ) - Labels: dict[str, str] = Field( + labels: dict[str, str] = Field( ..., + alias="Labels", description="User-defined key/value metadata.", examples=[ { @@ -974,8 +1300,9 @@ class ImageSummary(BaseModel): } ], ) - Containers: int = Field( + containers: int = Field( ..., + alias="Containers", description="Number of containers using this image. Includes both stopped and running\ncontainers.\n\nThis size is not calculated by default, and depends on which API endpoint\nis used. `-1` indicates that the value has not been set / calculated.\n", examples=[2], ) @@ -998,7 +1325,7 @@ class ProcessConfig(BaseModel): class Scope(str, Enum): """ - The level at which the volume exists. Either `global` for cluster-wide, + The level at which the volume exists. Either `global` for cluster-wide, or `local` for machine level. """ @@ -1009,45 +1336,54 @@ class Scope(str, Enum): class UsageData(BaseModel): """ - Usage details about the volume. This information is used by the + Usage details about the volume. This information is used by the `GET /system/df` endpoint, and omitted in other endpoints. """ - Size: int = Field( + size: int = Field( ..., + alias="Size", description='Amount of disk space used by the volume (in bytes). This information\nis only available for volumes created with the `"local"` volume\ndriver. For volumes created with other volume drivers, this field\nis set to `-1` ("not available")\n', ) - RefCount: int = Field( + ref_count: int = Field( ..., + alias="RefCount", description="The number of containers referencing this volume. This field\nis set to `-1` if the reference-count is not available.\n", ) class Volume(BaseModel): - Name: str = Field(..., description="Name of the volume.", examples=["tardis"]) - Driver: str = Field( + name: str = Field( + ..., alias="Name", description="Name of the volume.", examples=["tardis"] + ) + driver: str = Field( ..., + alias="Driver", description="Name of the volume driver used by the volume.", examples=["custom"], ) - Mountpoint: str = Field( + mountpoint: str = Field( ..., + alias="Mountpoint", description="Mount path of the volume on the host.", examples=["/var/lib/docker/volumes/tardis"], ) - CreatedAt: str | None = Field( - None, + created_at: str | None = Field( + default=None, + alias="CreatedAt", description="Date/Time the volume was created.", examples=["2016-06-07T20:31:11.853781916Z"], ) - Status: dict[str, dict[str, Any]] | None = Field( - None, + status: dict[str, dict[str, Any]] | None = Field( + default=None, + alias="Status", description='Low-level details about the volume, provided by the volume driver.\nDetails are returned as a map with key/value pairs:\n`{"key":"value","key2":"value2"}`.\n\nThe `Status` field is optional, and is omitted if the volume driver\ndoes not support this feature.\n', examples=[{"hello": "world"}], ) - Labels: dict[str, str] = Field( + labels: dict[str, str] = Field( ..., + alias="Labels", description="User-defined key/value metadata.", examples=[ { @@ -1056,19 +1392,20 @@ class Volume(BaseModel): } ], ) - Scope_: Scope = Field( + scope: Scope = Field( ..., alias="Scope", description="The level at which the volume exists. Either `global` for cluster-wide,\nor `local` for machine level.\n", examples=["local"], ) - Options: dict[str, str] = Field( + options: dict[str, str] = Field( ..., + alias="Options", description="The driver specific options used when creating the volume.\n", examples=[{"device": "tmpfs", "o": "size=100m,uid=1000", "type": "tmpfs"}], ) - UsageData_: UsageData | None = Field( - None, + usage_data: UsageData | None = Field( + default=None, alias="UsageData", description="Usage details about the volume. This information is used by the\n`GET /system/df` endpoint, and omitted in other endpoints.\n", ) @@ -1079,21 +1416,27 @@ class VolumeConfig(BaseModel): Volume configuration """ - Name: str | None = Field( - None, + name: str | None = Field( + default=None, + alias="Name", description="The new volume's name. If not specified, Docker generates a name.\n", examples=["tardis"], ) - Driver: str | None = Field( - "local", description="Name of the volume driver to use.", examples=["custom"] + driver: str | None = Field( + default="local", + alias="Driver", + description="Name of the volume driver to use.", + examples=["custom"], ) - DriverOpts: dict[str, str] | None = Field( - None, + driver_opts: dict[str, str] | None = Field( + default=None, + alias="DriverOpts", description="A mapping of driver options and values. These options are\npassed directly to the driver and are driver specific.\n", examples=[{"device": "tmpfs", "o": "size=100m,uid=1000", "type": "tmpfs"}], ) - Labels: dict[str, str] | None = Field( - None, + labels: dict[str, str] | None = Field( + default=None, + alias="Labels", description="User-defined key/value metadata.", examples=[ { @@ -1104,19 +1447,84 @@ class VolumeConfig(BaseModel): ) +class VolumeListResponse(BaseModel): + """ + Volume list response + """ + + volumes: list[Volume] | None = Field( + default=None, alias="Volumes", description="List of volumes" + ) + warnings: list[str] | None = Field( + default=None, + alias="Warnings", + description="Warnings that occurred when fetching the list of volumes.\n", + examples=[[]], + ) + + +class ConfigReference(BaseModel): + """ + The config-only network source to provide the configuration for + this network. + + """ + + network: str | None = Field( + default=None, + alias="Network", + description="The name of the config-only network that provides the network's\nconfiguration. The specified network must be an existing config-only\nnetwork. Only network names are allowed, not network IDs.\n", + examples=["config_only_network_01"], + ) + + class IPAMConfig(BaseModel): - Subnet: str | None = None - IPRange: str | None = None - Gateway: str | None = None - AuxiliaryAddresses: dict[str, str] | None = None + subnet: str | None = Field(default=None, alias="Subnet", examples=["172.20.0.0/16"]) + ip_range: str | None = Field( + default=None, alias="IPRange", examples=["172.20.10.0/24"] + ) + gateway: str | None = Field( + default=None, alias="Gateway", examples=["172.20.10.11"] + ) + auxiliary_addresses: dict[str, str] | None = Field( + default=None, alias="AuxiliaryAddresses" + ) class NetworkContainer(BaseModel): - Name: str | None = None - EndpointID: str | None = None - MacAddress: str | None = None - IPv4Address: str | None = None - IPv6Address: str | None = None + name: str | None = Field(default=None, alias="Name", examples=["container_1"]) + endpoint_id: str | None = Field( + default=None, + alias="EndpointID", + examples=["628cadb8bcb92de107b2a1e516cbffe463e321f548feb37697cce00ad694f21a"], + ) + mac_address: str | None = Field( + default=None, alias="MacAddress", examples=["02:42:ac:13:00:02"] + ) + i_pv4_address: str | None = Field( + default=None, alias="IPv4Address", examples=["172.19.0.2/16"] + ) + i_pv6_address: str | None = Field(default=None, alias="IPv6Address", examples=[""]) + + +class PeerInfo(BaseModel): + """ + PeerInfo represents one peer of an overlay network. + + """ + + name: str | None = Field( + default=None, + alias="Name", + description="ID of the peer-node in the Swarm cluster.", + examples=["6869d7c1732b"], + ) + ip: str | None = Field( + default=None, + alias="IP", + description="IP-address of the peer-node in the Swarm cluster.", + examples=["10.133.77.91"], + ) class Type4(str, Enum): @@ -1139,48 +1547,63 @@ class BuildCache(BaseModel): """ - ID: str | None = Field( - None, + id: str | None = Field( + default=None, + alias="ID", description="Unique ID of the build cache record.\n", examples=["ndlpt0hhvkqcdfkputsk4cq9c"], ) - Parent: str | None = Field( - None, + parent: str | None = Field( + default=None, + alias="Parent", description="ID of the parent build cache record.\n", examples=["hw53o5aio51xtltp5xjp8v7fx"], ) - Type: Type4 | None = Field( - None, description="Cache record type.\n", examples=["regular"] + type: Type4 | None = Field( + default=None, + alias="Type", + description="Cache record type.\n", + examples=["regular"], ) - Description: str | None = Field( - None, + description: str | None = Field( + default=None, + alias="Description", description="Description of the build-step that produced the build cache.\n", examples=[ "mount / from exec /bin/sh -c echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/keep-cache" ], ) - InUse: bool | None = Field( - None, description="Indicates if the build cache is in use.\n", examples=[False] + in_use: bool | None = Field( + default=None, + alias="InUse", + description="Indicates if the build cache is in use.\n", + examples=[False], ) - Shared: bool | None = Field( - None, description="Indicates if the build cache is shared.\n", examples=[True] + shared: bool | None = Field( + default=None, + alias="Shared", + description="Indicates if the build cache is shared.\n", + examples=[True], ) - Size: int | None = Field( - None, + size: int | None = Field( + default=None, + alias="Size", description="Amount of disk space used by the build cache (in bytes).\n", examples=[51], ) - CreatedAt: str | None = Field( - None, + created_at: str | None = Field( + default=None, + alias="CreatedAt", description="Date and time at which the build cache was created in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2016-08-18T10:44:24.496525531Z"], ) - LastUsedAt: str | None = Field( - None, + last_used_at: str | None = Field( + default=None, + alias="LastUsedAt", description="Date and time at which the build cache was last used in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2017-08-09T07:09:37.632105588Z"], ) - UsageCount: int | None = Field(None, examples=[26]) + usage_count: int | None = Field(default=None, alias="UsageCount", examples=[26]) class ImageID(BaseModel): @@ -1188,7 +1611,7 @@ class ImageID(BaseModel): Image ID or Digest """ - ID: str | None = None + id: str | None = Field(default=None, alias="ID") class ErrorDetail(BaseModel): @@ -1214,7 +1637,7 @@ class IdResponse(BaseModel): Response to an API call that returns just an Id """ - Id: str = Field(..., description="The id of the newly created object.") + id: str = Field(..., alias="Id", description="The id of the newly created object.") class EndpointIPAMConfig(BaseModel): @@ -1223,55 +1646,61 @@ class EndpointIPAMConfig(BaseModel): """ - IPv4Address: str | None = Field(None, examples=["172.20.30.33"]) - IPv6Address: str | None = Field(None, examples=["2001:db8:abcd::3033"]) - LinkLocalIPs: list[str] | None = Field( - None, examples=[["169.254.34.68", "fe80::3468"]] + i_pv4_address: str | None = Field( + default=None, alias="IPv4Address", examples=["172.20.30.33"] + ) + i_pv6_address: str | None = Field( + default=None, alias="IPv6Address", examples=["2001:db8:abcd::3033"] + ) + link_local_i_ps: list[str] | None = Field( + default=None, alias="LinkLocalIPs", examples=[["169.254.34.68", "fe80::3468"]] ) class PluginMount(BaseModel): - Name: str = Field(..., examples=["some-mount"]) - Description: str = Field( - ..., examples=["This is a mount that's used by the plugin."] + name: str = Field(..., alias="Name", examples=["some-mount"]) + description: str = Field( + ..., + alias="Description", + examples=["This is a mount that's used by the plugin."], ) - Settable: list[str] - Source: str = Field(..., examples=["/var/lib/docker/plugins/"]) - Destination: str = Field(..., examples=["/mnt/state"]) - Type: str = Field(..., examples=["bind"]) - Options: list[str] = Field(..., examples=[["rbind", "rw"]]) + settable: list[str] = Field(..., alias="Settable") + source: str = Field(..., alias="Source", examples=["/var/lib/docker/plugins/"]) + destination: str = Field(..., alias="Destination", examples=["/mnt/state"]) + type: str = Field(..., alias="Type", examples=["bind"]) + options: list[str] = Field(..., alias="Options", examples=[["rbind", "rw"]]) class PluginDevice(BaseModel): - Name: str - Description: str - Settable: list[str] - Path: str = Field(..., examples=["/dev/fuse"]) + name: str = Field(..., alias="Name") + description: str = Field(..., alias="Description") + settable: list[str] = Field(..., alias="Settable") + path: str = Field(..., alias="Path", examples=["/dev/fuse"]) class PluginEnv(BaseModel): - Name: str - Description: str - Settable: list[str] - Value: str + name: str = Field(..., alias="Name") + description: str = Field(..., alias="Description") + settable: list[str] = Field(..., alias="Settable") + value: str = Field(..., alias="Value") class PluginInterfaceType(BaseModel): - Prefix: str - Capability: str - Version: str + prefix: str = Field(..., alias="Prefix") + capability: str = Field(..., alias="Capability") + version: str = Field(..., alias="Version") class PluginPrivilege(BaseModel): """ - Describes a permission the user has to accept upon installing + Describes a permission the user has to accept upon installing the plugin. """ - Name: str | None = Field(None, examples=["network"]) - Description: str | None = None - Value: list[str] | None = Field(None, examples=[["host"]]) + name: str | None = Field(default=None, alias="Name", examples=["network"]) + description: str | None = Field(default=None, alias="Description") + value: list[str] | None = Field(default=None, alias="Value", examples=[["host"]]) class Settings(BaseModel): @@ -1279,10 +1708,10 @@ class Settings(BaseModel): Settings that can be modified by users. """ - Mounts: list[PluginMount] - Env: list[str] = Field(..., examples=[["DEBUG=0"]]) - Args: list[str] - Devices: list[PluginDevice] + mounts: list[PluginMount] = Field(..., alias="Mounts") + env: list[str] = Field(..., alias="Env", examples=[["DEBUG=0"]]) + args: list[str] = Field(..., alias="Args") + devices: list[PluginDevice] = Field(..., alias="Devices") class ProtocolScheme(str, Enum): @@ -1290,7 +1719,7 @@ class ProtocolScheme(str, Enum): Protocol to use for clients connecting to the plugin. """ - _ = "" + field_ = "" moby_plugins_http_v1 = "moby.plugins.http/v1" @@ -1299,12 +1728,12 @@ class Interface(BaseModel): The interface between Docker and the plugin """ - Types: list[PluginInterfaceType] = Field( - ..., examples=[["docker.volumedriver/1.0"]] + types: list[PluginInterfaceType] = Field( + ..., alias="Types", examples=[["docker.volumedriver/1.0"]] ) - Socket: str = Field(..., examples=["plugins.sock"]) - ProtocolScheme_: ProtocolScheme | None = Field( - None, + socket: str = Field(..., alias="Socket", examples=["plugins.sock"]) + protocol_scheme: ProtocolScheme | None = Field( + default=None, alias="ProtocolScheme", description="Protocol to use for clients connecting to the plugin.", examples=["some.protocol/v1.0"], @@ -1312,31 +1741,35 @@ class Interface(BaseModel): class User(BaseModel): - UID: int | None = Field(None, examples=[1000]) - GID: int | None = Field(None, examples=[1000]) + uid: int | None = Field(default=None, alias="UID", examples=[1000]) + gid: int | None = Field(default=None, alias="GID", examples=[1000]) class Network1(BaseModel): - Type: str = Field(..., examples=["host"]) + type: str = Field(..., alias="Type", examples=["host"]) class Linux(BaseModel): - Capabilities: list[str] = Field(..., examples=[["CAP_SYS_ADMIN", "CAP_SYSLOG"]]) - AllowAllDevices: bool = Field(..., examples=[False]) - Devices: list[PluginDevice] + capabilities: list[str] = Field( + ..., alias="Capabilities", examples=[["CAP_SYS_ADMIN", "CAP_SYSLOG"]] + ) + allow_all_devices: bool = Field(..., alias="AllowAllDevices", examples=[False]) + devices: list[PluginDevice] = Field(..., alias="Devices") class Args(BaseModel): - Name: str = Field(..., examples=["args"]) - Description: str = Field(..., examples=["command line arguments"]) - Settable: list[str] - Value: list[str] + name: str = Field(..., alias="Name", examples=["args"]) + description: str = Field( + ..., alias="Description", examples=["command line arguments"] + ) + settable: list[str] = Field(..., alias="Settable") + value: list[str] = Field(..., alias="Value") class Rootfs(BaseModel): - type: str | None = Field(None, examples=["layers"]) + type: str | None = Field(default=None, examples=["layers"]) diff_ids: list[str] | None = Field( - None, + default=None, examples=[ [ "sha256:675532206fbf3030b8458f88d6e26d4eb1577688a25efec97154c94e8b6b4887", @@ -1351,31 +1784,41 @@ class Config(BaseModel): The config of a plugin. """ - DockerVersion: str | None = Field( - None, + docker_version: str | None = Field( + default=None, + alias="DockerVersion", description="Docker Version used to create the plugin", examples=["17.06.0-ce"], ) - Description: str = Field(..., examples=["A sample volume plugin for Docker"]) - Documentation: str = Field(..., examples=["/engine/extend/plugins/"]) - Interface_: Interface = Field( + description: str = Field( + ..., alias="Description", examples=["A sample volume plugin for Docker"] + ) + documentation: str = Field( + ..., + alias="Documentation", + examples=["https://docs.docker.com/engine/extend/plugins/"], + ) + interface: Interface = Field( ..., alias="Interface", description="The interface between Docker and the plugin", ) - Entrypoint: list[str] = Field( - ..., examples=[["/usr/bin/sample-volume-plugin", "/data"]] - ) - WorkDir: str = Field(..., examples=["/bin/"]) - User_: User | None = Field(None, alias="User") - Network: Network1 - Linux: Linux - PropagatedMount: str = Field(..., examples=["/mnt/volumes"]) - IpcHost: bool = Field(..., examples=[False]) - PidHost: bool = Field(..., examples=[False]) - Mounts: list[PluginMount] - Env: list[PluginEnv] = Field( + entrypoint: list[str] = Field( + ..., alias="Entrypoint", examples=[["/usr/bin/sample-volume-plugin", "/data"]] + ) + work_dir: str = Field(..., alias="WorkDir", examples=["/bin/"]) + user: User | None = Field(default=None, alias="User") + network: Network1 = Field(..., alias="Network") + linux: Linux = Field(..., alias="Linux") + propagated_mount: str = Field( + ..., alias="PropagatedMount", examples=["/mnt/volumes"] + ) + ipc_host: bool = Field(..., alias="IpcHost", examples=[False]) + pid_host: bool = Field(..., alias="PidHost", examples=[False]) + mounts: list[PluginMount] = Field(..., alias="Mounts") + env: list[PluginEnv] = Field( ..., + alias="Env", examples=[ [ { @@ -1387,7 +1830,7 @@ class Config(BaseModel): ] ], ) - Args: Args + args: Args = Field(..., alias="Args") rootfs: Rootfs | None = None @@ -1396,30 +1839,33 @@ class Plugin(BaseModel): A plugin for the Engine API """ - Id: str | None = Field( - None, + id: str | None = Field( + default=None, + alias="Id", examples=["5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078"], ) - Name: str = Field(..., examples=["tiborvass/sample-volume-plugin"]) - Enabled: bool = Field( + name: str = Field(..., alias="Name", examples=["tiborvass/sample-volume-plugin"]) + enabled: bool = Field( ..., + alias="Enabled", description="True if the plugin is running. False if the plugin is not running, only installed.", examples=[True], ) - Settings_: Settings = Field( + settings: Settings = Field( ..., alias="Settings", description="Settings that can be modified by users." ) - PluginReference: str | None = Field( - None, + plugin_reference: str | None = Field( + default=None, + alias="PluginReference", description="plugin remote reference used to push/pull the plugin", examples=["localhost:5000/tiborvass/sample-volume-plugin:latest"], ) - Config_: Config = Field(..., alias="Config", description="The config of a plugin.") + config: Config = Field(..., alias="Config", description="The config of a plugin.") class ObjectVersion(BaseModel): """ - The version number of the object such as node, service, etc. This is needed + The version number of the object such as node, service, etc. This is needed to avoid conflicting writes. The client must send the version number along with the modified specification when updating these objects. @@ -1432,7 +1878,7 @@ class ObjectVersion(BaseModel): """ - Index: int | None = Field(None, examples=[373531]) + index: int | None = Field(default=None, alias="Index", examples=[373531]) class Role(str, Enum): @@ -1455,17 +1901,23 @@ class Availability(str, Enum): class NodeSpec(BaseModel): - Name: str | None = Field( - None, description="Name for the node.", examples=["my-node"] + name: str | None = Field( + default=None, + alias="Name", + description="Name for the node.", + examples=["my-node"], ) - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value metadata." + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value metadata." ) - Role_: Role | None = Field( - None, alias="Role", description="Role of the node.", examples=["manager"] + role: Role | None = Field( + default=None, + alias="Role", + description="Role of the node.", + examples=["manager"], ) - Availability_: Availability | None = Field( - None, + availability: Availability | None = Field( + default=None, alias="Availability", description="Availability of the node.", examples=["active"], @@ -1478,21 +1930,23 @@ class Platform(BaseModel): """ - Architecture: str | None = Field( - None, + architecture: str | None = Field( + default=None, + alias="Architecture", description="Architecture represents the hardware architecture (for example,\n`x86_64`).\n", examples=["x86_64"], ) - OS: str | None = Field( - None, + os: str | None = Field( + default=None, + alias="OS", description="OS represents the Operating System (for example, `linux` or `windows`).\n", examples=["linux"], ) class Plugin1(BaseModel): - Type: str | None = None - Name: str | None = None + type: str | None = Field(default=None, alias="Type") + name: str | None = Field(default=None, alias="Name") class EngineDescription(BaseModel): @@ -1500,10 +1954,15 @@ class EngineDescription(BaseModel): EngineDescription provides information about an engine. """ - EngineVersion: str | None = Field(None, examples=["17.06.0"]) - Labels: dict[str, str] | None = Field(None, examples=[{"foo": "bar"}]) - Plugins: list[Plugin1] | None = Field( - None, + engine_version: str | None = Field( + default=None, alias="EngineVersion", examples=["17.06.0"] + ) + labels: dict[str, str] | None = Field( + default=None, alias="Labels", examples=[{"foo": "bar"}] + ) + plugins: list[Plugin1] | None = Field( + default=None, + alias="Plugins", examples=[ [ {"Type": "Log", "Name": "awslogs"}, @@ -1512,7 +1971,6 @@ class EngineDescription(BaseModel): {"Type": "Log", "Name": "gelf"}, {"Type": "Log", "Name": "journald"}, {"Type": "Log", "Name": "json-file"}, - {"Type": "Log", "Name": "logentries"}, {"Type": "Log", "Name": "splunk"}, {"Type": "Log", "Name": "syslog"}, {"Type": "Network", "Name": "bridge"}, @@ -1531,20 +1989,24 @@ class EngineDescription(BaseModel): class TLSInfo(BaseModel): """ - Information about the issuer of leaf TLS certificates and the trusted root + Information about the issuer of leaf TLS certificates and the trusted root CA certificate. """ - TrustRoot: str | None = Field( - None, + trust_root: str | None = Field( + default=None, + alias="TrustRoot", description="The root CA certificate(s) that are used to validate leaf TLS\ncertificates.\n", ) - CertIssuerSubject: str | None = Field( - None, description="The base64-url-safe-encoded raw subject bytes of the issuer." + cert_issuer_subject: str | None = Field( + default=None, + alias="CertIssuerSubject", + description="The base64-url-safe-encoded raw subject bytes of the issuer.", ) - CertIssuerPublicKey: str | None = Field( - None, + cert_issuer_public_key: str | None = Field( + default=None, + alias="CertIssuerPublicKey", description="The base64-url-safe-encoded raw public key bytes of the issuer.\n", ) @@ -1575,8 +2037,9 @@ class Orchestration(BaseModel): Orchestration configuration. """ - TaskHistoryRetentionLimit: int | None = Field( - None, + task_history_retention_limit: int | None = Field( + default=None, + alias="TaskHistoryRetentionLimit", description="The number of historic tasks to keep per instance or node. If\nnegative, never remove completed or failed tasks.\n", examples=[10], ) @@ -1587,27 +2050,32 @@ class Raft(BaseModel): Raft configuration. """ - SnapshotInterval: int | None = Field( - None, + snapshot_interval: int | None = Field( + default=None, + alias="SnapshotInterval", description="The number of log entries between snapshots.", examples=[10000], ) - KeepOldSnapshots: int | None = Field( - None, + keep_old_snapshots: int | None = Field( + default=None, + alias="KeepOldSnapshots", description="The number of snapshots to keep beyond the current snapshot.\n", ) - LogEntriesForSlowFollowers: int | None = Field( - None, + log_entries_for_slow_followers: int | None = Field( + default=None, + alias="LogEntriesForSlowFollowers", description="The number of log entries to keep around to sync up slow followers\nafter a snapshot is created.\n", examples=[500], ) - ElectionTick: int | None = Field( - None, + election_tick: int | None = Field( + default=None, + alias="ElectionTick", description="The number of ticks that a follower will wait for a message from\nthe leader before becoming a candidate and starting an election.\n`ElectionTick` must be greater than `HeartbeatTick`.\n\nA tick currently defaults to one second, so these translate\ndirectly to seconds currently, but this is NOT guaranteed.\n", examples=[3], ) - HeartbeatTick: int | None = Field( - None, + heartbeat_tick: int | None = Field( + default=None, + alias="HeartbeatTick", description="The number of ticks between heartbeats. Every HeartbeatTick ticks,\nthe leader will send a heartbeat to the followers.\n\nA tick currently defaults to one second, so these translate\ndirectly to seconds currently, but this is NOT guaranteed.\n", examples=[1], ) @@ -1618,8 +2086,9 @@ class Dispatcher(BaseModel): Dispatcher configuration. """ - HeartbeatPeriod: int | None = Field( - None, + heartbeat_period: int | None = Field( + default=None, + alias="HeartbeatPeriod", description="The delay for an agent to send a heartbeat to the dispatcher.\n", examples=[5000000000], ) @@ -1627,7 +2096,7 @@ class Dispatcher(BaseModel): class Protocol(str, Enum): """ - Protocol for communication with the external CA (currently + Protocol for communication with the external CA (currently only `cfssl` is supported). """ @@ -1636,48 +2105,57 @@ class Protocol(str, Enum): class ExternalCA(BaseModel): - Protocol_: Protocol | None = Field( - Protocol.cfssl, + protocol: Protocol | None = Field( + default=Protocol.cfssl, alias="Protocol", description="Protocol for communication with the external CA (currently\nonly `cfssl` is supported).\n", ) - URL: str | None = Field( - None, description="URL where certificate signing requests should be sent.\n" + url: str | None = Field( + default=None, + alias="URL", + description="URL where certificate signing requests should be sent.\n", ) - Options: dict[str, str] | None = Field( - None, + options: dict[str, str] | None = Field( + default=None, + alias="Options", description="An object with key/value pairs that are interpreted as\nprotocol-specific options for the external CA driver.\n", ) - CACert: str | None = Field( - None, + ca_cert: str | None = Field( + default=None, + alias="CACert", description="The root CA certificate (in PEM format) this external CA uses\nto issue TLS certificates (assumed to be to the current swarm\nroot CA certificate if not provided).\n", ) -class CAConfig(BaseModel): +class CaConfig(BaseModel): """ CA configuration. """ - NodeCertExpiry: int | None = Field( - None, + node_cert_expiry: int | None = Field( + default=None, + alias="NodeCertExpiry", description="The duration node certificates are issued for.", examples=[7776000000000000], ) - ExternalCAs: list[ExternalCA] | None = Field( - None, + external_c_as: list[ExternalCA] | None = Field( + default=None, + alias="ExternalCAs", description="Configuration for forwarding signing requests to an external\ncertificate authority.\n", ) - SigningCACert: str | None = Field( - None, + signing_ca_cert: str | None = Field( + default=None, + alias="SigningCACert", description="The desired signing CA certificate for all swarm node TLS leaf\ncertificates, in PEM format.\n", ) - SigningCAKey: str | None = Field( - None, + signing_ca_key: str | None = Field( + default=None, + alias="SigningCAKey", description="The desired signing CA key for all swarm node TLS leaf certificates,\nin PEM format.\n", ) - ForceRotate: int | None = Field( - None, + force_rotate: int | None = Field( + default=None, + alias="ForceRotate", description="An integer whose purpose is to force swarm to generate a new\nsigning CA certificate and key, if none have been specified in\n`SigningCACert` and `SigningCAKey`\n", ) @@ -1687,8 +2165,9 @@ class EncryptionConfig(BaseModel): Parameters related to encryption-at-rest. """ - AutoLockManagers: bool | None = Field( - None, + auto_lock_managers: bool | None = Field( + default=None, + alias="AutoLockManagers", description="If set, generate a key and use it to lock data stored on the\nmanagers.\n", examples=[False], ) @@ -1696,7 +2175,7 @@ class EncryptionConfig(BaseModel): class LogDriver(BaseModel): """ - The log driver to use for tasks created in the orchestrator if + The log driver to use for tasks created in the orchestrator if unspecified by a service. Updating this value only affects new tasks. Existing tasks continue @@ -1704,14 +2183,16 @@ class LogDriver(BaseModel): """ - Name: str | None = Field( - None, + name: str | None = Field( + default=None, + alias="Name", description="The log driver to use as a default for new tasks.\n", examples=["json-file"], ) - Options: dict[str, str] | None = Field( - None, - description="Driver-specific options for the selectd log driver, specified\nas key/value pairs.\n", + options: dict[str, str] | None = Field( + default=None, + alias="Options", + description="Driver-specific options for the selected log driver, specified\nas key/value pairs.\n", examples=[{"max-file": "10", "max-size": "100m"}], ) @@ -1721,8 +2202,8 @@ class TaskDefaults(BaseModel): Defaults for creating tasks in this cluster. """ - LogDriver_: LogDriver | None = Field( - None, + log_driver: LogDriver | None = Field( + default=None, alias="LogDriver", description="The log driver to use for tasks created in the orchestrator if\nunspecified by a service.\n\nUpdating this value only affects new tasks. Existing tasks continue\nto use their previously configured log driver until recreated.\n", ) @@ -1733,11 +2214,15 @@ class SwarmSpec(BaseModel): User modifiable swarm configuration. """ - Name: str | None = Field( - None, description="Name of the swarm.", examples=["default"] + name: str | None = Field( + default=None, + alias="Name", + description="Name of the swarm.", + examples=["default"], ) - Labels: dict[str, str] | None = Field( - None, + labels: dict[str, str] | None = Field( + default=None, + alias="Labels", description="User-defined key/value metadata.", examples=[ { @@ -1746,23 +2231,25 @@ class SwarmSpec(BaseModel): } ], ) - Orchestration_: Orchestration | None = Field( - None, alias="Orchestration", description="Orchestration configuration." + orchestration: Orchestration | None = Field( + default=None, alias="Orchestration", description="Orchestration configuration." + ) + raft: Raft | None = Field( + default=None, alias="Raft", description="Raft configuration." ) - Raft_: Raft | None = Field(None, alias="Raft", description="Raft configuration.") - Dispatcher_: Dispatcher | None = Field( - None, alias="Dispatcher", description="Dispatcher configuration." + dispatcher: Dispatcher | None = Field( + default=None, alias="Dispatcher", description="Dispatcher configuration." ) - CAConfig_: CAConfig | None = Field( - None, alias="CAConfig", description="CA configuration." + ca_config: CaConfig | None = Field( + default=None, alias="CAConfig", description="CA configuration." ) - EncryptionConfig_: EncryptionConfig | None = Field( - None, + encryption_config: EncryptionConfig | None = Field( + default=None, alias="EncryptionConfig", description="Parameters related to encryption-at-rest.", ) - TaskDefaults_: TaskDefaults | None = Field( - None, + task_defaults: TaskDefaults | None = Field( + default=None, alias="TaskDefaults", description="Defaults for creating tasks in this cluster.", ) @@ -1770,43 +2257,52 @@ class SwarmSpec(BaseModel): class ClusterInfo(BaseModel): """ - ClusterInfo represents information about the swarm as is returned by the + ClusterInfo represents information about the swarm as is returned by the "/info" endpoint. Join-tokens are not included. """ - ID: str | None = Field( - None, description="The ID of the swarm.", examples=["abajmipo7b4xz5ip2nrla6b11"] + id: str | None = Field( + default=None, + alias="ID", + description="The ID of the swarm.", + examples=["abajmipo7b4xz5ip2nrla6b11"], ) - Version: ObjectVersion | None = None - CreatedAt: str | None = Field( - None, + version: ObjectVersion | None = Field(default=None, alias="Version") + created_at: str | None = Field( + default=None, + alias="CreatedAt", description="Date and time at which the swarm was initialised in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2016-08-18T10:44:24.496525531Z"], ) - UpdatedAt: str | None = Field( - None, + updated_at: str | None = Field( + default=None, + alias="UpdatedAt", description="Date and time at which the swarm was last updated in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2017-08-09T07:09:37.632105588Z"], ) - Spec: SwarmSpec | None = None - TLSInfo_: TLSInfo | None = Field(None, alias="TLSInfo") - RootRotationInProgress: bool | None = Field( - None, + spec: SwarmSpec | None = Field(default=None, alias="Spec") + tls_info: TLSInfo | None = Field(default=None, alias="TLSInfo") + root_rotation_in_progress: bool | None = Field( + default=None, + alias="RootRotationInProgress", description="Whether there is currently a root CA rotation in progress for the swarm\n", examples=[False], ) - DataPathPort: int | None = Field( - 4789, + data_path_port: int | None = Field( + default=4789, + alias="DataPathPort", description="DataPathPort specifies the data path port number for data traffic.\nAcceptable port range is 1024 to 49151.\nIf no port is set or is set to 0, the default port (4789) is used.\n", examples=[4789], ) - DefaultAddrPool: list[str] | None = Field( - None, + default_addr_pool: list[str] | None = Field( + default=None, + alias="DefaultAddrPool", description="Default Address Pool specifies default subnet pools for global scope\nnetworks.\n", ) - SubnetSize: int | None = Field( - 24, + subnet_size: int | None = Field( + default=24, + alias="SubnetSize", description="SubnetSize specifies the subnet size of the networks created from the\ndefault subnet pool.\n", examples=[24], le=29, @@ -1819,15 +2315,17 @@ class JoinTokens(BaseModel): """ - Worker: str | None = Field( - None, + worker: str | None = Field( + default=None, + alias="Worker", description="The token workers can use to join the swarm.\n", examples=[ "SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx" ], ) - Manager: str | None = Field( - None, + manager: str | None = Field( + default=None, + alias="Manager", description="The token managers can use to join the swarm.\n", examples=[ "SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-7p73s1dx5in4tatdymyhg9hu2" @@ -1836,12 +2334,12 @@ class JoinTokens(BaseModel): class Swarm(ClusterInfo): - JoinTokens_: JoinTokens | None = Field(None, alias="JoinTokens") + join_tokens: JoinTokens | None = Field(default=None, alias="JoinTokens") class PluginSpec(BaseModel): """ - Plugin spec for the service. *(Experimental release only.)* + Plugin spec for the service. *(Experimental release only.)*


@@ -1852,14 +2350,20 @@ class PluginSpec(BaseModel): """ - Name: str | None = Field( - None, description="The name or 'alias' to use for the plugin." + name: str | None = Field( + default=None, + alias="Name", + description="The name or 'alias' to use for the plugin.", + ) + remote: str | None = Field( + default=None, alias="Remote", description="The plugin image reference to use." + ) + disabled: bool | None = Field( + default=None, alias="Disabled", description="Disable the plugin once scheduled." ) - Remote: str | None = Field(None, description="The plugin image reference to use.") - Disabled: bool | None = Field( - None, description="Disable the plugin once scheduled." + plugin_privilege: list[PluginPrivilege] | None = Field( + default=None, alias="PluginPrivilege" ) - PluginPrivilege: list[PluginPrivilege] | None = None class CredentialSpec(BaseModel): @@ -1867,33 +2371,45 @@ class CredentialSpec(BaseModel): CredentialSpec for managed service account (Windows only) """ - Config_: str | None = Field( - None, + config: str | None = Field( + default=None, alias="Config", description="Load credential spec from a Swarm Config with the given ID.\nThe specified config must also be present in the Configs\nfield with the Runtime property set.\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", examples=["0bt9dmxjvjiqermk6xrop3ekq"], ) - File: str | None = Field( - None, + file: str | None = Field( + default=None, + alias="File", description="Load credential spec from this file. The file is read by\nthe daemon, and must be present in the `CredentialSpecs`\nsubdirectory in the docker data directory, which defaults\nto `C:\\ProgramData\\Docker\\` on Windows.\n\nFor example, specifying `spec.json` loads\n`C:\\ProgramData\\Docker\\CredentialSpecs\\spec.json`.\n\n


\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", examples=["spec.json"], ) - Registry: str | None = Field( - None, + registry: str | None = Field( + default=None, + alias="Registry", description="Load credential spec from this value in the Windows\nregistry. The specified registry value must be located in:\n\n`HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Virtualization\\Containers\\CredentialSpecs`\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", ) -class SELinuxContext(BaseModel): +class SeLinuxContext(BaseModel): """ SELinux labels of the container """ - Disable: bool | None = Field(None, description="Disable SELinux") - User: str | None = Field(None, description="SELinux user label") - Role: str | None = Field(None, description="SELinux role label") - Type: str | None = Field(None, description="SELinux type label") - Level: str | None = Field(None, description="SELinux level label") + disable: bool | None = Field( + default=None, alias="Disable", description="Disable SELinux" + ) + user: str | None = Field( + default=None, alias="User", description="SELinux user label" + ) + role: str | None = Field( + default=None, alias="Role", description="SELinux role label" + ) + type: str | None = Field( + default=None, alias="Type", description="SELinux type label" + ) + level: str | None = Field( + default=None, alias="Level", description="SELinux level label" + ) class Privileges(BaseModel): @@ -1901,31 +2417,36 @@ class Privileges(BaseModel): Security options for the container """ - CredentialSpec_: CredentialSpec | None = Field( - None, + credential_spec: CredentialSpec | None = Field( + default=None, alias="CredentialSpec", description="CredentialSpec for managed service account (Windows only)", ) - SELinuxContext_: SELinuxContext | None = Field( - None, alias="SELinuxContext", description="SELinux labels of the container" + se_linux_context: SeLinuxContext | None = Field( + default=None, + alias="SELinuxContext", + description="SELinux labels of the container", ) -class DNSConfig(BaseModel): +class DnsConfig(BaseModel): """ - Specification for DNS related configurations in resolver configuration + Specification for DNS related configurations in resolver configuration file (`resolv.conf`). """ - Nameservers: list[str] | None = Field( - None, description="The IP addresses of the name servers." + nameservers: list[str] | None = Field( + default=None, + alias="Nameservers", + description="The IP addresses of the name servers.", ) - Search: list[str] | None = Field( - None, description="A search list for host-name lookup." + search: list[str] | None = Field( + default=None, alias="Search", description="A search list for host-name lookup." ) - Options: list[str] | None = Field( - None, + options: list[str] | None = Field( + default=None, + alias="Options", description="A list of internal resolver variables to be modified (e.g.,\n`debug`, `ndots:3`, etc.).\n", ) @@ -1936,35 +2457,45 @@ class File(BaseModel): """ - Name: str | None = Field( - None, description="Name represents the final filename in the filesystem.\n" + name: str | None = Field( + default=None, + alias="Name", + description="Name represents the final filename in the filesystem.\n", + ) + uid: str | None = Field( + default=None, alias="UID", description="UID represents the file UID." ) - UID: str | None = Field(None, description="UID represents the file UID.") - GID: str | None = Field(None, description="GID represents the file GID.") - Mode: int | None = Field( - None, description="Mode represents the FileMode of the file." + gid: str | None = Field( + default=None, alias="GID", description="GID represents the file GID." + ) + mode: int | None = Field( + default=None, + alias="Mode", + description="Mode represents the FileMode of the file.", ) class Secret(BaseModel): - File_: File | None = Field( - None, + file: File | None = Field( + default=None, alias="File", description="File represents a specific target that is backed by a file.\n", ) - SecretID: str | None = Field( - None, + secret_id: str | None = Field( + default=None, + alias="SecretID", description="SecretID represents the ID of the specific secret that we're\nreferencing.\n", ) - SecretName: str | None = Field( - None, + secret_name: str | None = Field( + default=None, + alias="SecretName", description="SecretName is the name of the secret that this references,\nbut this is just provided for lookup/display purposes. The\nsecret in the reference will be identified by its ID.\n", ) -class File1(File): +class File1(BaseModel): """ - File represents a specific target that is backed by a file. + File represents a specific target that is backed by a file.


@@ -1972,33 +2503,62 @@ class File1(File): """ + name: str | None = Field( + default=None, + alias="Name", + description="Name represents the final filename in the filesystem.\n", + ) + uid: str | None = Field( + default=None, alias="UID", description="UID represents the file UID." + ) + gid: str | None = Field( + default=None, alias="GID", description="GID represents the file GID." + ) + mode: int | None = Field( + default=None, + alias="Mode", + description="Mode represents the FileMode of the file.", + ) + class Config1(BaseModel): - File: File1 | None = Field( - None, + file: File1 | None = Field( + default=None, + alias="File", description="File represents a specific target that is backed by a file.\n\n


\n\n> **Note**: `Configs.File` and `Configs.Runtime` are mutually exclusive\n", ) - Runtime: dict[str, Any] | None = Field( - None, + runtime: dict[str, Any] | None = Field( + default=None, + alias="Runtime", description="Runtime represents a target that is not mounted into the\ncontainer but is used by the task\n\n


\n\n> **Note**: `Configs.File` and `Configs.Runtime` are mutually\n> exclusive\n", ) - ConfigID: str | None = Field( - None, + config_id: str | None = Field( + default=None, + alias="ConfigID", description="ConfigID represents the ID of the specific config that we're\nreferencing.\n", ) - ConfigName: str | None = Field( - None, + config_name: str | None = Field( + default=None, + alias="ConfigName", description="ConfigName is the name of the config that this references,\nbut this is just provided for lookup/display purposes. The\nconfig in the reference will be identified by its ID.\n", ) -class Ulimit1(Ulimit): - pass +class Isolation1(str, Enum): + """ + Isolation technology of the containers running the service. + (Windows only) + + """ + + default = "default" + process = "process" + hyperv = "hyperv" class ContainerSpec(BaseModel): """ - Container spec for the service. + Container spec for the service.


@@ -2009,100 +2569,132 @@ class ContainerSpec(BaseModel): """ - Image: str | None = Field( - None, description="The image name to use for the container" + image: str | None = Field( + default=None, + alias="Image", + description="The image name to use for the container", + ) + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value data." ) - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value data." + command: list[str] | None = Field( + default=None, alias="Command", description="The command to be run in the image." ) - Command: list[str] | None = Field( - None, description="The command to be run in the image." + args: list[str] | None = Field( + default=None, alias="Args", description="Arguments to the command." ) - Args: list[str] | None = Field(None, description="Arguments to the command.") - Hostname: str | None = Field( - None, + hostname: str | None = Field( + default=None, + alias="Hostname", description="The hostname to use for the container, as a valid\n[RFC 1123](https://tools.ietf.org/html/rfc1123) hostname.\n", ) - Env: list[str] | None = Field( - None, description="A list of environment variables in the form `VAR=value`.\n" + env: list[str] | None = Field( + default=None, + alias="Env", + description="A list of environment variables in the form `VAR=value`.\n", ) - Dir: str | None = Field( - None, description="The working directory for commands to run in." + dir: str | None = Field( + default=None, + alias="Dir", + description="The working directory for commands to run in.", ) - User: str | None = Field(None, description="The user inside the container.") - Groups: list[str] | None = Field( - None, + user: str | None = Field( + default=None, alias="User", description="The user inside the container." + ) + groups: list[str] | None = Field( + default=None, + alias="Groups", description="A list of additional groups that the container process will run as.\n", ) - Privileges_: Privileges | None = Field( - None, alias="Privileges", description="Security options for the container" + privileges: Privileges | None = Field( + default=None, + alias="Privileges", + description="Security options for the container", + ) + tty: bool | None = Field( + default=None, + alias="TTY", + description="Whether a pseudo-TTY should be allocated.", ) - TTY: bool | None = Field( - None, description="Whether a pseudo-TTY should be allocated." + open_stdin: bool | None = Field( + default=None, alias="OpenStdin", description="Open `stdin`" ) - OpenStdin: bool | None = Field(None, description="Open `stdin`") - ReadOnly: bool | None = Field( - None, description="Mount the container's root filesystem as read only." + read_only: bool | None = Field( + default=None, + alias="ReadOnly", + description="Mount the container's root filesystem as read only.", ) - Mounts: list[Mount] | None = Field( - None, + mounts: list[Mount] | None = Field( + default=None, + alias="Mounts", description="Specification for mounts to be added to containers created as part\nof the service.\n", ) - StopSignal: str | None = Field(None, description="Signal to stop the container.") - StopGracePeriod: int | None = Field( - None, + stop_signal: str | None = Field( + default=None, alias="StopSignal", description="Signal to stop the container." + ) + stop_grace_period: int | None = Field( + default=None, + alias="StopGracePeriod", description="Amount of time to wait for the container to terminate before\nforcefully killing it.\n", ) - HealthCheck: HealthConfig | None = None - Hosts: list[str] | None = Field( - None, + health_check: HealthConfig | None = Field(default=None, alias="HealthCheck") + hosts: list[str] | None = Field( + default=None, + alias="Hosts", description="A list of hostname/IP mappings to add to the container's `hosts`\nfile. The format of extra hosts is specified in the\n[hosts(5)](http://man7.org/linux/man-pages/man5/hosts.5.html)\nman page:\n\n IP_address canonical_hostname [aliases...]\n", ) - DNSConfig_: DNSConfig | None = Field( - None, + dns_config: DnsConfig | None = Field( + default=None, alias="DNSConfig", description="Specification for DNS related configurations in resolver configuration\nfile (`resolv.conf`).\n", ) - Secrets: list[Secret] | None = Field( - None, + secrets: list[Secret] | None = Field( + default=None, + alias="Secrets", description="Secrets contains references to zero or more secrets that will be\nexposed to the service.\n", ) - Configs: list[Config1] | None = Field( - None, + configs: list[Config1] | None = Field( + default=None, + alias="Configs", description="Configs contains references to zero or more configs that will be\nexposed to the service.\n", ) - Isolation_: Isolation | None = Field( - None, + isolation: Isolation1 | None = Field( + default=None, alias="Isolation", description="Isolation technology of the containers running the service.\n(Windows only)\n", ) - Init: bool | None = Field( - None, + init: bool | None = Field( + default=None, + alias="Init", description="Run an init inside the container that forwards signals and reaps\nprocesses. This field is omitted if empty, and the default (as\nconfigured on the daemon) is used.\n", ) - Sysctls: dict[str, str] | None = Field( - None, + sysctls: dict[str, str] | None = Field( + default=None, + alias="Sysctls", description="Set kernel namedspaced parameters (sysctls) in the container.\nThe Sysctls option on services accepts the same sysctls as the\nare supported on containers. Note that while the same sysctls are\nsupported, no guarantees or checks are made about their\nsuitability for a clustered environment, and it's up to the user\nto determine whether a given sysctl will work properly in a\nService.\n", ) - CapabilityAdd: list[str] | None = Field( - None, + capability_add: list[str] | None = Field( + default=None, + alias="CapabilityAdd", description="A list of kernel capabilities to add to the default set\nfor the container.\n", examples=[["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_CHROOT", "CAP_SYSLOG"]], ) - CapabilityDrop: list[str] | None = Field( - None, + capability_drop: list[str] | None = Field( + default=None, + alias="CapabilityDrop", description="A list of kernel capabilities to drop from the default set\nfor the container.\n", examples=[["CAP_NET_RAW"]], ) - Ulimits: list[Ulimit1] | None = Field( - None, + ulimits: list[Ulimit] | None = Field( + default=None, + alias="Ulimits", description='A list of resource limits to set in the container. For example: `{"Name": "nofile", "Soft": 1024, "Hard": 2048}`"\n', ) class NetworkAttachmentSpec(BaseModel): """ - Read-only spec type for non-swarm containers attached to swarm overlay + Read-only spec type for non-swarm containers attached to swarm overlay networks.


@@ -2114,8 +2706,10 @@ class NetworkAttachmentSpec(BaseModel): """ - ContainerID: str | None = Field( - None, description="ID of the container represented by this task" + container_id: str | None = Field( + default=None, + alias="ContainerID", + description="ID of the container represented by this task", ) @@ -2131,38 +2725,45 @@ class Condition(str, Enum): class RestartPolicy1(BaseModel): """ - Specification for the restart policy which applies to containers + Specification for the restart policy which applies to containers created as part of this service. """ - Condition_: Condition | None = Field( - None, alias="Condition", description="Condition for restart." + condition: Condition | None = Field( + default=None, alias="Condition", description="Condition for restart." ) - Delay: int | None = Field(None, description="Delay between restart attempts.") - MaxAttempts: int | None = Field( - 0, + delay: int | None = Field( + default=None, alias="Delay", description="Delay between restart attempts." + ) + max_attempts: int | None = Field( + default=0, + alias="MaxAttempts", description="Maximum attempts to restart a given container before giving up\n(default value is 0, which is ignored).\n", ) - Window: int | None = Field( - 0, + window: int | None = Field( + default=0, + alias="Window", description="Windows is the time window used to evaluate the restart policy\n(default value is 0, which is unbounded).\n", ) class Spread(BaseModel): - SpreadDescriptor: str | None = Field( - None, description="label descriptor, such as `engine.labels.az`.\n" + spread_descriptor: str | None = Field( + default=None, + alias="SpreadDescriptor", + description="label descriptor, such as `engine.labels.az`.\n", ) class Preference(BaseModel): - Spread_: Spread | None = Field(None, alias="Spread") + spread: Spread | None = Field(default=None, alias="Spread") class Placement(BaseModel): - Constraints: list[str] | None = Field( - None, + constraints: list[str] | None = Field( + default=None, + alias="Constraints", description="An array of constraint expressions to limit the set of nodes where\na task can be scheduled. Constraint expressions can either use a\n_match_ (`==`) or _exclude_ (`!=`) rule. Multiple constraints find\nnodes that satisfy every expression (AND match). Constraints can\nmatch node or Docker Engine labels as follows:\n\nnode attribute | matches | example\n---------------------|--------------------------------|-----------------------------------------------\n`node.id` | Node ID | `node.id==2ivku8v2gvtg4`\n`node.hostname` | Node hostname | `node.hostname!=node-2`\n`node.role` | Node role (`manager`/`worker`) | `node.role==manager`\n`node.platform.os` | Node operating system | `node.platform.os==windows`\n`node.platform.arch` | Node architecture | `node.platform.arch==x86_64`\n`node.labels` | User-defined node labels | `node.labels.security==high`\n`engine.labels` | Docker Engine's labels | `engine.labels.operatingsystem==ubuntu-14.04`\n\n`engine.labels` apply to Docker Engine labels like operating system,\ndrivers, etc. Swarm administrators add `node.labels` for operational\npurposes by using the [`node update endpoint`](#operation/NodeUpdate).\n", examples=[ [ @@ -2174,8 +2775,9 @@ class Placement(BaseModel): ] ], ) - Preferences: list[Preference] | None = Field( - None, + preferences: list[Preference] | None = Field( + default=None, + alias="Preferences", description="Preferences provide a way to make the scheduler aware of factors\nsuch as topology. They are provided in order from highest to\nlowest precedence.\n", examples=[ [ @@ -2184,26 +2786,28 @@ class Placement(BaseModel): ] ], ) - MaxReplicas: int | None = Field( - 0, + max_replicas: int | None = Field( + default=0, + alias="MaxReplicas", description="Maximum number of replicas for per node (default value is 0, which\nis unlimited)\n", ) - Platforms: list[Platform] | None = Field( - None, + platforms: list[Platform] | None = Field( + default=None, + alias="Platforms", description="Platforms stores all the platforms that the service's image can\nrun on. This field is used in the platform filter for scheduling.\nIf empty, then the platform filter is off, meaning there are no\nscheduling restrictions.\n", ) class LogDriver1(BaseModel): """ - Specifies the log driver to use for tasks created from this spec. If + Specifies the log driver to use for tasks created from this spec. If not present, the default one for the swarm will be used, finally falling back to the engine default if not specified. """ - Name: str | None = None - Options: dict[str, str] | None = None + name: str | None = Field(default=None, alias="Name") + options: dict[str, str] | None = Field(default=None, alias="Options") class TaskState(str, Enum): @@ -2225,35 +2829,40 @@ class TaskState(str, Enum): class ContainerStatus(BaseModel): - ContainerID: str | None = None - PID: int | None = None - ExitCode: int | None = None + container_id: str | None = Field(default=None, alias="ContainerID") + pid: int | None = Field(default=None, alias="PID") + exit_code: int | None = Field(default=None, alias="ExitCode") class Status1(BaseModel): - Timestamp: str | None = None - State: TaskState | None = None - Message: str | None = None - Err: str | None = None - ContainerStatus_: ContainerStatus | None = Field(None, alias="ContainerStatus") + timestamp: str | None = Field(default=None, alias="Timestamp") + state: TaskState | None = Field(default=None, alias="State") + message: str | None = Field(default=None, alias="Message") + err: str | None = Field(default=None, alias="Err") + container_status: ContainerStatus | None = Field( + default=None, alias="ContainerStatus" + ) class Replicated(BaseModel): - Replicas: int | None = None + replicas: int | None = Field(default=None, alias="Replicas") class ReplicatedJob(BaseModel): """ - The mode used for services with a finite number of tasks that run + The mode used for services with a finite number of tasks that run to a completed state. """ - MaxConcurrent: int | None = Field( - 1, description="The maximum number of replicas to run simultaneously.\n" + max_concurrent: int | None = Field( + default=1, + alias="MaxConcurrent", + description="The maximum number of replicas to run simultaneously.\n", ) - TotalCompletions: int | None = Field( - None, + total_completions: int | None = Field( + default=None, + alias="TotalCompletions", description="The total number of replicas desired to reach the Completed\nstate. If unset, will default to the value of `MaxConcurrent`\n", ) @@ -2263,22 +2872,23 @@ class Mode(BaseModel): Scheduling mode for the service. """ - Replicated_: Replicated | None = Field(None, alias="Replicated") - Global: dict[str, Any] | None = None - ReplicatedJob_: ReplicatedJob | None = Field( - None, + replicated: Replicated | None = Field(default=None, alias="Replicated") + global_: dict[str, Any] | None = Field(default=None, alias="Global") + replicated_job: ReplicatedJob | None = Field( + default=None, alias="ReplicatedJob", description="The mode used for services with a finite number of tasks that run\nto a completed state.\n", ) - GlobalJob: dict[str, Any] | None = Field( - None, + global_job: dict[str, Any] | None = Field( + default=None, + alias="GlobalJob", description="The mode used for services which run a task to the completed state\non each valid node.\n", ) class FailureAction(str, Enum): """ - Action to take if an updated task fails to run, or stops running + Action to take if an updated task fails to run, or stops running during the update. """ @@ -2290,7 +2900,7 @@ class FailureAction(str, Enum): class Order(str, Enum): """ - The order of operations when rolling out an updated task. Either + The order of operations when rolling out an updated task. Either the old task is shut down before the new task is started, or the new task is started before the old task is shut down. @@ -2305,28 +2915,33 @@ class UpdateConfig(BaseModel): Specification for the update strategy of the service. """ - Parallelism: int | None = Field( - None, + parallelism: int | None = Field( + default=None, + alias="Parallelism", description="Maximum number of tasks to be updated in one iteration (0 means\nunlimited parallelism).\n", ) - Delay: int | None = Field( - None, description="Amount of time between updates, in nanoseconds." + delay: int | None = Field( + default=None, + alias="Delay", + description="Amount of time between updates, in nanoseconds.", ) - FailureAction_: FailureAction | None = Field( - None, + failure_action: FailureAction | None = Field( + default=None, alias="FailureAction", description="Action to take if an updated task fails to run, or stops running\nduring the update.\n", ) - Monitor: int | None = Field( - None, + monitor: int | None = Field( + default=None, + alias="Monitor", description="Amount of time to monitor each updated task for failures, in\nnanoseconds.\n", ) - MaxFailureRatio: float | None = Field( - 0, + max_failure_ratio: float | None = Field( + default=0, + alias="MaxFailureRatio", description="The fraction of tasks that may fail during an update before the\nfailure action is invoked, specified as a floating point number\nbetween 0 and 1.\n", ) - Order_: Order | None = Field( - None, + order: Order | None = Field( + default=None, alias="Order", description="The order of operations when rolling out an updated task. Either\nthe old task is shut down before the new task is started, or the\nnew task is started before the old task is shut down.\n", ) @@ -2334,7 +2949,7 @@ class UpdateConfig(BaseModel): class FailureAction1(str, Enum): """ - Action to take if an rolled back task fails to run, or stops + Action to take if an rolled back task fails to run, or stops running during the rollback. """ @@ -2343,33 +2958,50 @@ class FailureAction1(str, Enum): pause = "pause" +class Order1(str, Enum): + """ + The order of operations when rolling back a task. Either the old + task is shut down before the new task is started, or the new task + is started before the old task is shut down. + + """ + + stop_first = "stop-first" + start_first = "start-first" + + class RollbackConfig(BaseModel): """ Specification for the rollback strategy of the service. """ - Parallelism: int | None = Field( - None, + parallelism: int | None = Field( + default=None, + alias="Parallelism", description="Maximum number of tasks to be rolled back in one iteration (0 means\nunlimited parallelism).\n", ) - Delay: int | None = Field( - None, + delay: int | None = Field( + default=None, + alias="Delay", description="Amount of time between rollback iterations, in nanoseconds.\n", ) - FailureAction: FailureAction1 | None = Field( - None, + failure_action: FailureAction1 | None = Field( + default=None, + alias="FailureAction", description="Action to take if an rolled back task fails to run, or stops\nrunning during the rollback.\n", ) - Monitor: int | None = Field( - None, + monitor: int | None = Field( + default=None, + alias="Monitor", description="Amount of time to monitor each rolled back task for failures, in\nnanoseconds.\n", ) - MaxFailureRatio: float | None = Field( - 0, + max_failure_ratio: float | None = Field( + default=0, + alias="MaxFailureRatio", description="The fraction of tasks that may fail during a rollback before the\nfailure action is invoked, specified as a floating point number\nbetween 0 and 1.\n", ) - Order_: Order | None = Field( - None, + order: Order1 | None = Field( + default=None, alias="Order", description="The order of operations when rolling back a task. Either the old\ntask is shut down before the new task is started, or the new task\nis started before the old task is shut down.\n", ) @@ -2377,7 +3009,7 @@ class RollbackConfig(BaseModel): class PublishMode(str, Enum): """ - The mode in which port is published. + The mode in which port is published.


@@ -2394,12 +3026,16 @@ class PublishMode(str, Enum): class EndpointPortConfig(BaseModel): - Name: str | None = None - Protocol: Type | None = None - TargetPort: int | None = Field(None, description="The port inside the container.") - PublishedPort: int | None = Field(None, description="The port on the swarm hosts.") - PublishMode_: PublishMode | None = Field( - PublishMode.ingress, + name: str | None = Field(default=None, alias="Name") + protocol: Type | None = Field(default=None, alias="Protocol") + target_port: int | None = Field( + default=None, alias="TargetPort", description="The port inside the container." + ) + published_port: int | None = Field( + default=None, alias="PublishedPort", description="The port on the swarm hosts." + ) + publish_mode: PublishMode | None = Field( + default=PublishMode.ingress, alias="PublishMode", description='The mode in which port is published.\n\n


\n\n- "ingress" makes the target port accessible on every node,\n regardless of whether there is a task for the service running on\n that node or not.\n- "host" bypasses the routing mesh and publish the port directly on\n the swarm node where that service is running.\n', examples=["ingress"], @@ -2421,25 +3057,27 @@ class EndpointSpec(BaseModel): Properties that can be configured to access and load balance a service. """ - Mode: Mode1 | None = Field( - Mode1.vip, + mode: Mode1 | None = Field( + default=Mode1.vip, + alias="Mode", description="The mode of resolution to use for internal load balancing between tasks.\n", ) - Ports: list[EndpointPortConfig] | None = Field( - None, + ports: list[EndpointPortConfig] | None = Field( + default=None, + alias="Ports", description="List of exposed ports that this service is accessible on from the\noutside. Ports can only be provided if `vip` resolution mode is used.\n", ) class VirtualIP(BaseModel): - NetworkID: str | None = None - Addr: str | None = None + network_id: str | None = Field(default=None, alias="NetworkID") + addr: str | None = Field(default=None, alias="Addr") class Endpoint(BaseModel): - Spec: EndpointSpec | None = None - Ports: list[EndpointPortConfig] | None = None - VirtualIPs: list[VirtualIP] | None = None + spec: EndpointSpec | None = Field(default=None, alias="Spec") + ports: list[EndpointPortConfig] | None = Field(default=None, alias="Ports") + virtual_i_ps: list[VirtualIP] | None = Field(default=None, alias="VirtualIPs") class State(str, Enum): @@ -2453,69 +3091,80 @@ class UpdateStatus(BaseModel): The status of a service update. """ - State_: State | None = Field(None, alias="State") - StartedAt: str | None = None - CompletedAt: str | None = None - Message: str | None = None + state: State | None = Field(default=None, alias="State") + started_at: str | None = Field(default=None, alias="StartedAt") + completed_at: str | None = Field(default=None, alias="CompletedAt") + message: str | None = Field(default=None, alias="Message") class ServiceStatus(BaseModel): """ - The status of the service's tasks. Provided only when requested as + The status of the service's tasks. Provided only when requested as part of a ServiceList operation. """ - RunningTasks: int | None = Field( - None, + running_tasks: int | None = Field( + default=None, + alias="RunningTasks", description="The number of tasks for the service currently in the Running state.\n", examples=[7], ) - DesiredTasks: int | None = Field( - None, + desired_tasks: int | None = Field( + default=None, + alias="DesiredTasks", description="The number of tasks for the service desired to be running.\nFor replicated services, this is the replica count from the\nservice spec. For global services, this is computed by taking\ncount of all tasks for the service with a Desired State other\nthan Shutdown.\n", examples=[10], ) - CompletedTasks: int | None = Field( - None, + completed_tasks: int | None = Field( + default=None, + alias="CompletedTasks", description="The number of tasks for a job that are in the Completed state.\nThis field must be cross-referenced with the service type, as the\nvalue of 0 may mean the service is not in a job mode, or it may\nmean the job-mode service has no tasks yet Completed.\n", ) class JobStatus(BaseModel): """ - The status of the service when it is in one of ReplicatedJob or + The status of the service when it is in one of ReplicatedJob or GlobalJob modes. Absent on Replicated and Global mode services. The JobIteration is an ObjectVersion, but unlike the Service's version, does not need to be sent with an update request. """ - JobIteration: ObjectVersion | None = Field( - None, + job_iteration: ObjectVersion | None = Field( + default=None, + alias="JobIteration", description='JobIteration is a value increased each time a Job is executed,\nsuccessfully or otherwise. "Executed", in this case, means the\njob as a whole has been started, not that an individual Task has\nbeen launched. A job is "Executed" when its ServiceSpec is\nupdated. JobIteration can be used to disambiguate Tasks belonging\nto different executions of a job. Though JobIteration will\nincrease with each subsequent execution, it may not necessarily\nincrease by 1, and so JobIteration should not be used to\n', ) - LastExecution: str | None = Field( - None, + last_execution: str | None = Field( + default=None, + alias="LastExecution", description="The last time, as observed by the server, that this job was\nstarted.\n", ) class ImageDeleteResponseItem(BaseModel): - Untagged: str | None = Field( - None, description="The image ID of an image that was untagged" + untagged: str | None = Field( + default=None, + alias="Untagged", + description="The image ID of an image that was untagged", ) - Deleted: str | None = Field( - None, description="The image ID of an image that was deleted" + deleted: str | None = Field( + default=None, + alias="Deleted", + description="The image ID of an image that was deleted", ) class ServiceUpdateResponse(BaseModel): - Warnings: list[str] | None = Field(None, description="Optional warning messages") + warnings: list[str] | None = Field( + default=None, alias="Warnings", description="Optional warning messages" + ) class HostConfig1(BaseModel): - NetworkMode: str | None = None + network_mode: str | None = Field(default=None, alias="NetworkMode") class Driver(BaseModel): @@ -2523,9 +3172,12 @@ class Driver(BaseModel): Driver represents a driver (network, logging, secrets). """ - name: str = Field(..., description="Name of the driver.") + name: str = Field( + ..., alias="Name", description="Name of the driver.", examples=["some-driver"] + ) options: dict[str, str] | None = Field( - None, + default=None, + alias="Options", description="Key/value map of driver-specific options.", examples=[ { @@ -2537,9 +3189,12 @@ class Driver(BaseModel): class SecretSpec(BaseModel): - name: str | None = Field(None, description="User-defined name of the secret.") + name: str | None = Field( + default=None, alias="Name", description="User-defined name of the secret." + ) labels: dict[str, str] | None = Field( - None, + default=None, + alias="Labels", description="User-defined key/value metadata.", examples=[ { @@ -2549,54 +3204,67 @@ class SecretSpec(BaseModel): ], ) data: str | None = Field( - None, + default=None, + alias="Data", description="Base64-url-safe-encoded ([RFC 4648](https://tools.ietf.org/html/rfc4648#section-5))\ndata to store as secret.\n\nThis field is only used to _create_ a secret, and is not returned by\nother endpoints.\n", examples=[""], ) driver: Driver | None = Field( - None, + default=None, + alias="Driver", description="Name of the secrets driver used to fetch the secret's value from an\nexternal secret store.\n", ) templating: Driver | None = Field( - None, + default=None, + alias="Templating", description="Templating driver, if applicable\n\nTemplating controls whether and how to evaluate the config payload as\na template. If no driver is set, no templating is used.\n", ) class Secret1(BaseModel): - ID: str | None = Field(None, examples=["blt1owaxmitz71s9v5zh81zun"]) - Version: ObjectVersion | None = None - CreatedAt: str | None = Field(None, examples=["2017-07-20T13:55:28.678958722Z"]) - UpdatedAt: str | None = Field(None, examples=["2017-07-20T13:55:28.678958722Z"]) - Spec: SecretSpec | None = None + id: str | None = Field( + default=None, alias="ID", examples=["blt1owaxmitz71s9v5zh81zun"] + ) + version: ObjectVersion | None = Field(default=None, alias="Version") + created_at: str | None = Field( + default=None, alias="CreatedAt", examples=["2017-07-20T13:55:28.678958722Z"] + ) + updated_at: str | None = Field( + default=None, alias="UpdatedAt", examples=["2017-07-20T13:55:28.678958722Z"] + ) + spec: SecretSpec | None = Field(default=None, alias="Spec") class ConfigSpec(BaseModel): - Name: str | None = Field(None, description="User-defined name of the config.") - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value metadata." + name: str | None = Field( + default=None, alias="Name", description="User-defined name of the config." ) - Data: str | None = Field( - None, + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value metadata." + ) + data: str | None = Field( + default=None, + alias="Data", description="Base64-url-safe-encoded ([RFC 4648](https://tools.ietf.org/html/rfc4648#section-5))\nconfig data.\n", ) - Templating: Driver | None = Field( - None, + templating: Driver | None = Field( + default=None, + alias="Templating", description="Templating driver, if applicable\n\nTemplating controls whether and how to evaluate the config payload as\na template. If no driver is set, no templating is used.\n", ) class Config2(BaseModel): - ID: str | None = None - Version: ObjectVersion | None = None - CreatedAt: str | None = None - UpdatedAt: str | None = None - Spec: ConfigSpec | None = None + id: str | None = Field(default=None, alias="ID") + version: ObjectVersion | None = Field(default=None, alias="Version") + created_at: str | None = Field(default=None, alias="CreatedAt") + updated_at: str | None = Field(default=None, alias="UpdatedAt") + spec: ConfigSpec | None = Field(default=None, alias="Spec") class Status2(str, Enum): """ - String representation of the container state. Can be one of "created", + String representation of the container state. Can be one of "created", "running", "paused", "restarting", "removing", "exited", or "dead". """ @@ -2615,20 +3283,28 @@ class ContainerWaitExitError(BaseModel): container waiting error, if any """ - Message: str | None = Field(None, description="Details of an error") + message: str | None = Field( + default=None, alias="Message", description="Details of an error" + ) class Platform1(BaseModel): - Name: str + name: str = Field(..., alias="Name") class Component(BaseModel): - Name: str = Field(..., description="Name of the component\n", examples=["Engine"]) - Version: str = Field( - ..., description="Version of the component\n", examples=["19.03.12"] + name: str = Field( + ..., alias="Name", description="Name of the component\n", examples=["Engine"] + ) + version: str = Field( + ..., + alias="Version", + description="Version of the component\n", + examples=["19.03.12"], ) - Details: dict[str, Any] | None = Field( - None, + details: dict[str, Any] | None = Field( + default=None, + alias="Details", description="Key/value pairs of strings with additional information about the\ncomponent. These values are intended for informational purposes\nonly, and their content is not defined, and not part of the API\nspecification.\n\nThese messages can be printed by the client as information to the user.\n", ) @@ -2639,55 +3315,69 @@ class SystemVersion(BaseModel): """ - Platform: Platform1 | None = None - Components: list[Component] | None = Field( - None, description="Information about system components\n" + platform: Platform1 | None = Field(default=None, alias="Platform") + components: list[Component] | None = Field( + default=None, + alias="Components", + description="Information about system components\n", ) - Version: str | None = Field( - None, description="The version of the daemon", examples=["19.03.12"] + version: str | None = Field( + default=None, + alias="Version", + description="The version of the daemon", + examples=["19.03.12"], ) - ApiVersion: str | None = Field( - None, + api_version: str | None = Field( + default=None, + alias="ApiVersion", description="The default (and highest) API version that is supported by the daemon\n", examples=["1.40"], ) - MinAPIVersion: str | None = Field( - None, + min_api_version: str | None = Field( + default=None, + alias="MinAPIVersion", description="The minimum API version that is supported by the daemon\n", examples=["1.12"], ) - GitCommit: str | None = Field( - None, + git_commit: str | None = Field( + default=None, + alias="GitCommit", description="The Git commit of the source code that was used to build the daemon\n", examples=["48a66213fe"], ) - GoVersion: str | None = Field( - None, + go_version: str | None = Field( + default=None, + alias="GoVersion", description="The version Go used to compile the daemon, and the version of the Go\nruntime in use.\n", examples=["go1.13.14"], ) - Os: str | None = Field( - None, + os: str | None = Field( + default=None, + alias="Os", description='The operating system that the daemon is running on ("linux" or "windows")\n', examples=["linux"], ) - Arch: str | None = Field( - None, + arch: str | None = Field( + default=None, + alias="Arch", description="The architecture that the daemon is running on\n", examples=["amd64"], ) - KernelVersion: str | None = Field( - None, + kernel_version: str | None = Field( + default=None, + alias="KernelVersion", description="The kernel version (`uname -r`) that the daemon is running on.\n\nThis field is omitted when empty.\n", examples=["4.19.76-linuxkit"], ) - Experimental: bool | None = Field( - None, + experimental: bool | None = Field( + default=None, + alias="Experimental", description="Indicates if the daemon is started with experimental features enabled.\n\nThis field is omitted when empty / false.\n", examples=[True], ) - BuildTime: str | None = Field( - None, + build_time: str | None = Field( + default=None, + alias="BuildTime", description="The date and time that the daemon was compiled.\n", examples=["2020-06-22T15:49:27.000000000+00:00"], ) @@ -2716,7 +3406,7 @@ class CgroupVersion(str, Enum): class Isolation2(str, Enum): """ - Represents the isolation technology to use as a default for containers. + Represents the isolation technology to use as a default for containers. The supported values are platform-specific. If no isolation value is specified on daemon start, on Windows client, @@ -2732,17 +3422,20 @@ class Isolation2(str, Enum): class DefaultAddressPool(BaseModel): - Base: str | None = Field( - None, + base: str | None = Field( + default=None, + alias="Base", description="The network address in CIDR format", examples=["10.10.0.0/16"], ) - Size: int | None = Field(None, description="The network pool size", examples=["24"]) + size: int | None = Field( + default=None, alias="Size", description="The network pool size", examples=["24"] + ) class PluginsInfo(BaseModel): """ - Available plugins per type. + Available plugins per type.


@@ -2752,23 +3445,27 @@ class PluginsInfo(BaseModel): """ - Volume: list[str] | None = Field( - None, + volume: list[str] | None = Field( + default=None, + alias="Volume", description="Names of available volume-drivers, and network-driver plugins.", examples=[["local"]], ) - Network: list[str] | None = Field( - None, + network: list[str] | None = Field( + default=None, + alias="Network", description="Names of available network-drivers, and network-driver plugins.", examples=[["bridge", "host", "ipvlan", "macvlan", "null", "overlay"]], ) - Authorization: list[str] | None = Field( - None, + authorization: list[str] | None = Field( + default=None, + alias="Authorization", description="Names of available authorization plugins.", examples=[["img-authz-plugin", "hbm"]], ) - Log: list[str] | None = Field( - None, + log: list[str] | None = Field( + default=None, + alias="Log", description="Names of available logging-drivers, and logging-driver plugins.", examples=[ [ @@ -2778,7 +3475,6 @@ class PluginsInfo(BaseModel): "gelf", "journald", "json-file", - "logentries", "splunk", "syslog", ] @@ -2791,13 +3487,15 @@ class IndexInfo(BaseModel): IndexInfo contains information about a registry. """ - Name: str | None = Field( - None, + name: str | None = Field( + default=None, + alias="Name", description='Name of the registry, such as "docker.io".\n', examples=["docker.io"], ) - Mirrors: list[str] | None = Field( - None, + mirrors: list[str] | None = Field( + default=None, + alias="Mirrors", description="List of mirrors, expressed as URIs.\n", examples=[ [ @@ -2807,13 +3505,15 @@ class IndexInfo(BaseModel): ] ], ) - Secure: bool | None = Field( - None, + secure: bool | None = Field( + default=None, + alias="Secure", description="Indicates if the registry is part of the list of insecure\nregistries.\n\nIf `false`, the registry is insecure. Insecure registries accept\nun-encrypted (HTTP) and/or untrusted (HTTPS with certificates from\nunknown CAs) communication.\n\n> **Warning**: Insecure registries can be useful when running a local\n> registry. However, because its use creates security vulnerabilities\n> it should ONLY be enabled for testing purposes. For increased\n> security, users should add their CA to their system's list of\n> trusted CAs instead of enabling this option.\n", examples=[True], ) - Official: bool | None = Field( - None, + official: bool | None = Field( + default=None, + alias="Official", description="Indicates whether this is an official registry (i.e., Docker Hub / docker.io)\n", examples=[True], ) @@ -2821,7 +3521,7 @@ class IndexInfo(BaseModel): class Runtime(BaseModel): """ - Runtime describes an [OCI compliant](https://github.com/opencontainers/runtime-spec) + Runtime describes an [OCI compliant](https://github.com/opencontainers/runtime-spec) runtime. The runtime is invoked by the daemon via the `containerd` daemon. OCI @@ -2831,12 +3531,13 @@ class Runtime(BaseModel): """ path: str | None = Field( - None, + default=None, description="Name and, optional, path, of the OCI executable binary.\n\nIf the path is omitted, the daemon searches the host's `$PATH` for the\nbinary and uses the first result.\n", examples=["/usr/local/bin/my-oci-runtime"], ) - runtimeArgs: list[str] | None = Field( - None, + runtime_args: list[str] | None = Field( + default=None, + alias="runtimeArgs", description="List of command-line arguments to pass to the runtime when invoked.\n", examples=[["--debug", "--systemd-cgroup=false"]], ) @@ -2844,19 +3545,21 @@ class Runtime(BaseModel): class Commit(BaseModel): """ - Commit holds the Git-commit (SHA1) that a binary was built from, as + Commit holds the Git-commit (SHA1) that a binary was built from, as reported in the version-string of external tools, such as `containerd`, or `runC`. """ - ID: str | None = Field( - None, + id: str | None = Field( + default=None, + alias="ID", description="Actual commit ID of external tool.", examples=["cfb82a876ecc11b5ca0977d1733adbe58599088a"], ) - Expected: str | None = Field( - None, + expected: str | None = Field( + default=None, + alias="Expected", description="Commit ID of external tool expected by dockerd as set at build time.\n", examples=["2d41c047c83e09a6d61d464906feb2a2f3c52aa4"], ) @@ -2867,7 +3570,7 @@ class LocalNodeState(str, Enum): Current local status of this node. """ - _ = "" + field_ = "" inactive = "inactive" pending = "pending" active = "active" @@ -2880,11 +3583,15 @@ class PeerNode(BaseModel): Represents a peer-node in the swarm """ - NodeID: str | None = Field( - None, description="Unique identifier of for this node in the swarm." + node_id: str | None = Field( + default=None, + alias="NodeID", + description="Unique identifier of for this node in the swarm.", ) - Addr: str | None = Field( - None, description="IP address and ports at which this node can be reached.\n" + addr: str | None = Field( + default=None, + alias="Addr", + description="IP address and ports at which this node can be reached.\n", ) @@ -2894,33 +3601,39 @@ class NetworkAttachmentConfig(BaseModel): """ - Target: str | None = Field( - None, + target: str | None = Field( + default=None, + alias="Target", description="The target network for attachment. Must be a network name or ID.\n", ) - Aliases: list[str] | None = Field( - None, + aliases: list[str] | None = Field( + default=None, + alias="Aliases", description="Discoverable alternate names for the service on this network.\n", ) - DriverOpts: dict[str, str] | None = Field( - None, description="Driver attachment options for the network target.\n" + driver_opts: dict[str, str] | None = Field( + default=None, + alias="DriverOpts", + description="Driver attachment options for the network target.\n", ) class EventActor(BaseModel): """ - Actor describes something that generates events, like a container, network, + Actor describes something that generates events, like a container, network, or a volume. """ - ID: str | None = Field( - None, + id: str | None = Field( + default=None, + alias="ID", description="The ID of the object emitting the event", examples=["ede54ee1afda366ab42f824e8a5ffd195155d853ceaec74a927f249ea270c743"], ) - Attributes: dict[str, str] | None = Field( - None, + attributes: dict[str, str] | None = Field( + default=None, + alias="Attributes", description="Various key/value attributes of the object, depending on its type.\n", examples=[ { @@ -2945,14 +3658,14 @@ class Type5(str, Enum): network = "network" node = "node" plugin = "plugin" - secret = "secret" # nosec + secret = "secret" service = "service" volume = "volume" class Scope1(str, Enum): """ - Scope of the event. Engine events are `local` scope. Cluster (Swarm) + Scope of the event. Engine events are `local` scope. Cluster (Swarm) events are `swarm` scope. """ @@ -2967,24 +3680,29 @@ class SystemEventsResponse(BaseModel): """ - Type: Type5 | None = Field( - None, + type: Type5 | None = Field( + default=None, + alias="Type", description="The type of object emitting the event", examples=["container"], ) - Action: str | None = Field( - None, description="The type of event", examples=["create"] + action: str | None = Field( + default=None, + alias="Action", + description="The type of event", + examples=["create"], ) - Actor: EventActor | None = None + actor: EventActor | None = Field(default=None, alias="Actor") scope: Scope1 | None = Field( - None, + default=None, description="Scope of the event. Engine events are `local` scope. Cluster (Swarm)\nevents are `swarm` scope.\n", ) time: int | None = Field( - None, description="Timestamp of event", examples=[1629574695] + default=None, description="Timestamp of event", examples=[1629574695] ) - timeNano: int | None = Field( - None, + time_nano: int | None = Field( + default=None, + alias="timeNano", description="Timestamp of event, with nanosecond accuracy", examples=[1629574695515050031], ) @@ -2992,59 +3710,60 @@ class SystemEventsResponse(BaseModel): class OCIDescriptor(BaseModel): """ - A descriptor struct containing digest, media type, and size, as defined in + A descriptor struct containing digest, media type, and size, as defined in the [OCI Content Descriptors Specification](https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md). """ - mediaType: str | None = Field( - None, + media_type: str | None = Field( + default=None, + alias="mediaType", description="The media type of the object this schema refers to.\n", examples=["application/vnd.docker.distribution.manifest.v2+json"], ) digest: str | None = Field( - None, + default=None, description="The digest of the targeted content.\n", examples=[ "sha256:c0537ff6a5218ef531ece93d4984efc99bbf3f7497c0a7726c88e2bb7584dc96" ], ) size: int | None = Field( - None, description="The size in bytes of the blob.\n", examples=[3987495] + default=None, description="The size in bytes of the blob.\n", examples=[3987495] ) class OCIPlatform(BaseModel): """ - Describes the platform which the image in the manifest runs on, as defined + Describes the platform which the image in the manifest runs on, as defined in the [OCI Image Index Specification](https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md). """ architecture: str | None = Field( - None, + default=None, description="The CPU architecture, for example `amd64` or `ppc64`.\n", examples=["arm"], ) os: str | None = Field( - None, + default=None, description="The operating system, for example `linux` or `windows`.\n", examples=["windows"], ) os_version: str | None = Field( - None, + default=None, alias="os.version", description="Optional field specifying the operating system version, for example on\nWindows `10.0.19041.1165`.\n", examples=["10.0.19041.1165"], ) os_features: list[str] | None = Field( - None, + default=None, alias="os.features", description="Optional field specifying an array of strings, each listing a required\nOS feature (for example on Windows `win32k`).\n", examples=[["win32k"]], ) variant: str | None = Field( - None, + default=None, description="Optional field specifying a variant of the CPU, for example `v7` to\nspecify ARMv7 when architecture is `arm`.\n", examples=["v7"], ) @@ -3052,27 +3771,35 @@ class OCIPlatform(BaseModel): class DistributionInspectResponse(BaseModel): """ - Describes the result obtained from contacting the registry to retrieve + Describes the result obtained from contacting the registry to retrieve image metadata. """ - Descriptor: OCIDescriptor - Platforms: list[OCIPlatform] = Field( - ..., description="An array containing all platforms supported by the image.\n" + descriptor: OCIDescriptor = Field(..., alias="Descriptor") + platforms: list[OCIPlatform] = Field( + ..., + alias="Platforms", + description="An array containing all platforms supported by the image.\n", ) class ResourceObject(BaseModel): """ - An object describing the resources which can be advertised by a node and + An object describing the resources which can be advertised by a node and requested by a task. """ - NanoCPUs: int | None = Field(None, examples=[4000000000]) - MemoryBytes: int | None = Field(None, examples=[8272408576]) - GenericResources_: GenericResources | None = Field(None, alias="GenericResources") + nano_cp_us: int | None = Field( + default=None, alias="NanoCPUs", examples=[4000000000] + ) + memory_bytes: int | None = Field( + default=None, alias="MemoryBytes", examples=[8272408576] + ) + generic_resources: GenericResources | None = Field( + default=None, alias="GenericResources" + ) class Health(BaseModel): @@ -3081,174 +3808,56 @@ class Health(BaseModel): """ - Status_: Status | None = Field( - None, + status: Status | None = Field( + default=None, alias="Status", description='Status is one of `none`, `starting`, `healthy` or `unhealthy`\n\n- "none" Indicates there is no healthcheck\n- "starting" Starting indicates that the container is not yet ready\n- "healthy" Healthy indicates that the container is running correctly\n- "unhealthy" Unhealthy indicates that the container has a problem\n', examples=["healthy"], ) - FailingStreak: int | None = Field( - None, + failing_streak: int | None = Field( + default=None, + alias="FailingStreak", description="FailingStreak is the number of consecutive failures", examples=[0], ) - Log: list[HealthcheckResult] | None = Field( - None, description="Log contains the last few results (oldest first)\n" + log: list[HealthcheckResult] | None = Field( + default=None, + alias="Log", + description="Log contains the last few results (oldest first)\n", ) -class HostConfig(Resources): +class PortMap(RootModel[dict[str, list[PortBinding]] | None]): """ - Container configuration that depends on the host we are running on + PortMap describes the mapping of container ports to host ports, using the + container's port-number and protocol as key in the format `/`, + for example, `80/udp`. + + If a container's port is mapped for multiple protocols, separate entries + are added to the mapping table. + """ - Binds: list[str] | None = Field( - None, - description="A list of volume bindings for this container. Each volume binding\nis a string in one of these forms:\n\n- `host-src:container-dest[:options]` to bind-mount a host path\n into the container. Both `host-src`, and `container-dest` must\n be an _absolute_ path.\n- `volume-name:container-dest[:options]` to bind-mount a volume\n managed by a volume driver into the container. `container-dest`\n must be an _absolute_ path.\n\n`options` is an optional, comma-delimited list of:\n\n- `nocopy` disables automatic copying of data from the container\n path to the volume. The `nocopy` flag only applies to named volumes.\n- `[ro|rw]` mounts a volume read-only or read-write, respectively.\n If omitted or set to `rw`, volumes are mounted read-write.\n- `[z|Z]` applies SELinux labels to allow or deny multiple containers\n to read and write to the same volume.\n - `z`: a _shared_ content label is applied to the content. This\n label indicates that multiple containers can share the volume\n content, for both reading and writing.\n - `Z`: a _private unshared_ label is applied to the content.\n This label indicates that only the current container can use\n a private volume. Labeling systems such as SELinux require\n proper labels to be placed on volume content that is mounted\n into a container. Without a label, the security system can\n prevent a container's processes from using the content. By\n default, the labels set by the host operating system are not\n modified.\n- `[[r]shared|[r]slave|[r]private]` specifies mount\n [propagation behavior](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt).\n This only applies to bind-mounted volumes, not internal volumes\n or named volumes. Mount propagation requires the source mount\n point (the location where the source directory is mounted in the\n host operating system) to have the correct propagation properties.\n For shared volumes, the source mount point must be set to `shared`.\n For slave volumes, the mount must be set to either `shared` or\n `slave`.\n", - ) - ContainerIDFile: str | None = Field( - None, description="Path to a file where the container ID is written" - ) - LogConfig_: LogConfig | None = Field( - None, - alias="LogConfig", - description="The logging configuration for this container", - ) - NetworkMode: str | None = Field( - None, - description="Network mode to use for this container. Supported standard values\nare: `bridge`, `host`, `none`, and `container:`. Any\nother value is taken as a custom network's name to which this\ncontainer should connect to.\n", - ) - PortBindings: PortMap | None = None - RestartPolicy_: RestartPolicy | None = Field(None, alias="RestartPolicy") - AutoRemove: bool | None = Field( - None, - description="Automatically remove the container when the container's process\nexits. This has no effect if `RestartPolicy` is set.\n", - ) - VolumeDriver: str | None = Field( - None, description="Driver that this container uses to mount volumes." - ) - VolumesFrom: list[str] | None = Field( - None, - description="A list of volumes to inherit from another container, specified in\nthe form `[:]`.\n", - ) - Mounts: list[Mount] | None = Field( - None, description="Specification for mounts to be added to the container.\n" - ) - CapAdd: list[str] | None = Field( - None, - description="A list of kernel capabilities to add to the container. Conflicts\nwith option 'Capabilities'.\n", - ) - CapDrop: list[str] | None = Field( - None, - description="A list of kernel capabilities to drop from the container. Conflicts\nwith option 'Capabilities'.\n", - ) - CgroupnsMode_: CgroupnsMode | None = Field( - None, - alias="CgroupnsMode", - description='cgroup namespace mode for the container. Possible values are:\n\n- `"private"`: the container runs in its own private cgroup namespace\n- `"host"`: use the host system\'s cgroup namespace\n\nIf not specified, the daemon default is used, which can either be `"private"`\nor `"host"`, depending on daemon version, kernel support and configuration.\n', - ) - Dns: list[str] | None = Field( - None, description="A list of DNS servers for the container to use." - ) - DnsOptions: list[str] | None = Field(None, description="A list of DNS options.") - DnsSearch: list[str] | None = Field( - None, description="A list of DNS search domains." - ) - ExtraHosts: list[str] | None = Field( - None, - description='A list of hostnames/IP mappings to add to the container\'s `/etc/hosts`\nfile. Specified in the form `["hostname:IP"]`.\n', - ) - GroupAdd: list[str] | None = Field( - None, - description="A list of additional groups that the container process will run as.\n", - ) - IpcMode: str | None = Field( - None, - description='IPC sharing mode for the container. Possible values are:\n\n- `"none"`: own private IPC namespace, with /dev/shm not mounted\n- `"private"`: own private IPC namespace\n- `"shareable"`: own private IPC namespace, with a possibility to share it with other containers\n- `"container:"`: join another (shareable) container\'s IPC namespace\n- `"host"`: use the host system\'s IPC namespace\n\nIf not specified, daemon default is used, which can either be `"private"`\nor `"shareable"`, depending on daemon version and configuration.\n', - ) - Cgroup: str | None = Field(None, description="Cgroup to use for the container.") - Links: list[str] | None = Field( - None, - description="A list of links for the container in the form `container_name:alias`.\n", - ) - OomScoreAdj: int | None = Field( - None, - description="An integer value containing the score given to the container in\norder to tune OOM killer preferences.\n", - examples=[500], - ) - PidMode: str | None = Field( - None, - description='Set the PID (Process) Namespace mode for the container. It can be\neither:\n\n- `"container:"`: joins another container\'s PID namespace\n- `"host"`: use the host\'s PID namespace inside the container\n', - ) - Privileged: bool | None = Field( - None, description="Gives the container full access to the host." - ) - PublishAllPorts: bool | None = Field( - None, - description="Allocates an ephemeral host port for all of a container's\nexposed ports.\n\nPorts are de-allocated when the container stops and allocated when\nthe container starts. The allocated port might be changed when\nrestarting the container.\n\nThe port is selected from the ephemeral port range that depends on\nthe kernel. For example, on Linux the range is defined by\n`/proc/sys/net/ipv4/ip_local_port_range`.\n", - ) - ReadonlyRootfs: bool | None = Field( - None, description="Mount the container's root filesystem as read only." - ) - SecurityOpt: list[str] | None = Field( - None, - description="A list of string values to customize labels for MLS systems, such\nas SELinux.\n", - ) - StorageOpt: dict[str, str] | None = Field( - None, - description='Storage driver options for this container, in the form `{"size": "120G"}`.\n', - ) - Tmpfs: dict[str, str] | None = Field( - None, - description='A map of container directories which should be replaced by tmpfs\nmounts, and their corresponding mount options. For example:\n\n```\n{ "/run": "rw,noexec,nosuid,size=65536k" }\n```\n', - ) - UTSMode: str | None = Field( - None, description="UTS namespace to use for the container." - ) - UsernsMode: str | None = Field( - None, - description="Sets the usernamespace mode for the container when usernamespace\nremapping option is enabled.\n", - ) - ShmSize: int | None = Field( - None, - description="Size of `/dev/shm` in bytes. If omitted, the system uses 64MB.\n", - ge=0, - ) - Sysctls: dict[str, str] | None = Field( - None, - description='A list of kernel parameters (sysctls) to set in the container.\nFor example:\n\n```\n{"net.ipv4.ip_forward": "1"}\n```\n', - ) - Runtime: str | None = Field(None, description="Runtime to use with this container.") - ConsoleSize: list[ConsoleSizeItem] | None = Field( - None, - description="Initial console size, as an `[height, width]` array. (Windows only)\n", - max_length=2, - min_length=2, - ) - Isolation_: Isolation | None = Field( - None, - alias="Isolation", - description="Isolation technology of the container. (Windows only)\n", - ) - MaskedPaths: list[str] | None = Field( - None, - description="The list of paths to be masked inside the container (this overrides\nthe default set of paths).\n", - ) - ReadonlyPaths: list[str] | None = Field( - None, - description="The list of paths to be set as read-only inside the container\n(this overrides the default set of paths).\n", - ) + root: dict[str, list[PortBinding]] | None = None class IPAM(BaseModel): - Driver: str | None = Field("default", description="Name of the IPAM driver to use.") - Config_: list[IPAMConfig] | None = Field( - None, + driver: str | None = Field( + default="default", + alias="Driver", + description="Name of the IPAM driver to use.", + examples=["default"], + ) + config: list[IPAMConfig] | None = Field( + default=None, alias="Config", description='List of IPAM configuration options, specified as a map:\n\n```\n{"Subnet": , "IPRange": , "Gateway": , "AuxAddress": }\n```\n', ) - Options: dict[str, str] | None = Field( - None, description="Driver-specific options, specified as a map." + options: dict[str, str] | None = Field( + default=None, + alias="Options", + description="Driver-specific options, specified as a map.", + examples=[{"foo": "bar"}], ) @@ -3256,26 +3865,27 @@ class BuildInfo(BaseModel): id: str | None = None stream: str | None = None error: str | None = None - errorDetail: ErrorDetail | None = None + error_detail: ErrorDetail | None = Field(default=None, alias="errorDetail") status: str | None = None progress: str | None = None - progressDetail: ProgressDetail | None = None + progress_detail: ProgressDetail | None = Field(default=None, alias="progressDetail") aux: ImageID | None = None class CreateImageInfo(BaseModel): id: str | None = None error: str | None = None + error_detail: ErrorDetail | None = Field(default=None, alias="errorDetail") status: str | None = None progress: str | None = None - progressDetail: ProgressDetail | None = None + progress_detail: ProgressDetail | None = Field(default=None, alias="progressDetail") class PushImageInfo(BaseModel): error: str | None = None status: str | None = None progress: str | None = None - progressDetail: ProgressDetail | None = None + progress_detail: ProgressDetail | None = Field(default=None, alias="progressDetail") class EndpointSettings(BaseModel): @@ -3283,44 +3893,70 @@ class EndpointSettings(BaseModel): Configuration for a network endpoint. """ - IPAMConfig: EndpointIPAMConfig | None = None - Links: list[str] | None = Field(None, examples=[["container_1", "container_2"]]) - Aliases: list[str] | None = Field(None, examples=[["server_x", "server_y"]]) - NetworkID: str | None = Field( - None, + ipam_config: EndpointIPAMConfig | None = Field(default=None, alias="IPAMConfig") + links: list[str] | None = Field( + default=None, alias="Links", examples=[["container_1", "container_2"]] + ) + aliases: list[str] | None = Field( + default=None, alias="Aliases", examples=[["server_x", "server_y"]] + ) + network_id: str | None = Field( + default=None, + alias="NetworkID", description="Unique ID of the network.\n", examples=["08754567f1f40222263eab4102e1c733ae697e8e354aa9cd6e18d7402835292a"], ) - EndpointID: str | None = Field( - None, + endpoint_id: str | None = Field( + default=None, + alias="EndpointID", description="Unique ID for the service endpoint in a Sandbox.\n", examples=["b88f5b905aabf2893f3cbc4ee42d1ea7980bbc0a92e2c8922b1e1795298afb0b"], ) - Gateway: str | None = Field( - None, description="Gateway address for this network.\n", examples=["172.17.0.1"] + gateway: str | None = Field( + default=None, + alias="Gateway", + description="Gateway address for this network.\n", + examples=["172.17.0.1"], ) - IPAddress: str | None = Field( - None, description="IPv4 address.\n", examples=["172.17.0.4"] + ip_address: str | None = Field( + default=None, + alias="IPAddress", + description="IPv4 address.\n", + examples=["172.17.0.4"], ) - IPPrefixLen: int | None = Field( - None, description="Mask length of the IPv4 address.\n", examples=[16] + ip_prefix_len: int | None = Field( + default=None, + alias="IPPrefixLen", + description="Mask length of the IPv4 address.\n", + examples=[16], ) - IPv6Gateway: str | None = Field( - None, description="IPv6 gateway address.\n", examples=["2001:db8:2::100"] + i_pv6_gateway: str | None = Field( + default=None, + alias="IPv6Gateway", + description="IPv6 gateway address.\n", + examples=["2001:db8:2::100"], ) - GlobalIPv6Address: str | None = Field( - None, description="Global IPv6 address.\n", examples=["2001:db8::5689"] + global_i_pv6_address: str | None = Field( + default=None, + alias="GlobalIPv6Address", + description="Global IPv6 address.\n", + examples=["2001:db8::5689"], ) - GlobalIPv6PrefixLen: int | None = Field( - None, description="Mask length of the global IPv6 address.\n", examples=[64] + global_i_pv6_prefix_len: int | None = Field( + default=None, + alias="GlobalIPv6PrefixLen", + description="Mask length of the global IPv6 address.\n", + examples=[64], ) - MacAddress: str | None = Field( - None, + mac_address: str | None = Field( + default=None, + alias="MacAddress", description="MAC address for the endpoint on this network.\n", examples=["02:42:ac:11:00:04"], ) - DriverOpts: dict[str, str] | None = Field( - None, + driver_opts: dict[str, str] | None = Field( + default=None, + alias="DriverOpts", description="DriverOpts is a mapping of driver options and values. These options\nare passed directly to the driver and are driver specific.\n", examples=[ { @@ -3333,46 +3969,52 @@ class EndpointSettings(BaseModel): class NodeDescription(BaseModel): """ - NodeDescription encapsulates the properties of the Node as reported by the + NodeDescription encapsulates the properties of the Node as reported by the agent. """ - Hostname: str | None = Field(None, examples=["bf3067039e47"]) - Platform_: Platform | None = Field(None, alias="Platform") - Resources: ResourceObject | None = None - Engine: EngineDescription | None = None - TLSInfo_: TLSInfo | None = Field(None, alias="TLSInfo") + hostname: str | None = Field( + default=None, alias="Hostname", examples=["bf3067039e47"] + ) + platform: Platform | None = Field(default=None, alias="Platform") + resources: ResourceObject | None = Field(default=None, alias="Resources") + engine: EngineDescription | None = Field(default=None, alias="Engine") + tls_info: TLSInfo | None = Field(default=None, alias="TLSInfo") class NodeStatus(BaseModel): """ - NodeStatus represents the status of a node. + NodeStatus represents the status of a node. It provides the current status of the node, as seen by the manager. """ - State: NodeState | None = None - Message: str | None = Field(None, examples=[""]) - Addr: str | None = Field( - None, description="IP address of the node.", examples=["172.17.0.2"] + state: NodeState | None = Field(default=None, alias="State") + message: str | None = Field(default=None, alias="Message", examples=[""]) + addr: str | None = Field( + default=None, + alias="Addr", + description="IP address of the node.", + examples=["172.17.0.2"], ) class ManagerStatus(BaseModel): """ - ManagerStatus represents the status of a manager. + ManagerStatus represents the status of a manager. It provides the current status of a node's manager component, if the node is a manager. """ - Leader: bool | None = Field(False, examples=[True]) - Reachability_: Reachability | None = Field(None, alias="Reachability") - Addr: str | None = Field( - None, + leader: bool | None = Field(default=False, alias="Leader", examples=[True]) + reachability: Reachability | None = Field(default=None, alias="Reachability") + addr: str | None = Field( + default=None, + alias="Addr", description="The IP address and port at which the manager is reachable.\n", examples=["10.0.0.46:2377"], ) @@ -3380,14 +4022,16 @@ class ManagerStatus(BaseModel): class Resources1(BaseModel): """ - Resource requirements which apply to each individual container created + Resource requirements which apply to each individual container created as part of the service. """ - Limits: Limit | None = Field(None, description="Define resources limits.") - Reservations: ResourceObject | None = Field( - None, description="Define resources reservation." + limits: Limit | None = Field( + default=None, alias="Limits", description="Define resources limits." + ) + reservations: ResourceObject | None = Field( + default=None, alias="Reservations", description="Define resources reservation." ) @@ -3396,69 +4040,85 @@ class TaskSpec(BaseModel): User modifiable task configuration. """ - PluginSpec_: PluginSpec | None = Field( - None, + plugin_spec: PluginSpec | None = Field( + default=None, alias="PluginSpec", description="Plugin spec for the service. *(Experimental release only.)*\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n", ) - ContainerSpec_: ContainerSpec | None = Field( - None, + container_spec: ContainerSpec | None = Field( + default=None, alias="ContainerSpec", description="Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n", ) - NetworkAttachmentSpec_: NetworkAttachmentSpec | None = Field( - None, + network_attachment_spec: NetworkAttachmentSpec | None = Field( + default=None, alias="NetworkAttachmentSpec", description="Read-only spec type for non-swarm containers attached to swarm overlay\nnetworks.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n", ) - Resources: Resources1 | None = Field( - None, + resources: Resources1 | None = Field( + default=None, + alias="Resources", description="Resource requirements which apply to each individual container created\nas part of the service.\n", ) - RestartPolicy: RestartPolicy1 | None = Field( - None, + restart_policy: RestartPolicy1 | None = Field( + default=None, + alias="RestartPolicy", description="Specification for the restart policy which applies to containers\ncreated as part of this service.\n", ) - Placement_: Placement | None = Field(None, alias="Placement") - ForceUpdate: int | None = Field( - None, + placement: Placement | None = Field(default=None, alias="Placement") + force_update: int | None = Field( + default=None, + alias="ForceUpdate", description="A counter that triggers an update even if no relevant parameters have\nbeen changed.\n", ) - Runtime: str | None = Field( - None, + runtime: str | None = Field( + default=None, + alias="Runtime", description="Runtime is the type of runtime specified for the task executor.\n", ) - Networks: list[NetworkAttachmentConfig] | None = Field( - None, description="Specifies which networks the service should attach to." + networks: list[NetworkAttachmentConfig] | None = Field( + default=None, + alias="Networks", + description="Specifies which networks the service should attach to.", ) - LogDriver: LogDriver1 | None = Field( - None, + log_driver: LogDriver1 | None = Field( + default=None, + alias="LogDriver", description="Specifies the log driver to use for tasks created from this spec. If\nnot present, the default one for the swarm will be used, finally\nfalling back to the engine default if not specified.\n", ) class Task(BaseModel): - ID: str | None = Field(None, description="The ID of the task.") - Version: ObjectVersion | None = None - CreatedAt: str | None = None - UpdatedAt: str | None = None - Name: str | None = Field(None, description="Name of the task.") - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value metadata." - ) - Spec: TaskSpec | None = None - ServiceID: str | None = Field( - None, description="The ID of the service this task is part of." - ) - Slot: int | None = None - NodeID: str | None = Field( - None, description="The ID of the node that this task is on." - ) - AssignedGenericResources: GenericResources | None = None - Status: Status1 | None = None - DesiredState: TaskState | None = None - JobIteration: ObjectVersion | None = Field( - None, + id: str | None = Field(default=None, alias="ID", description="The ID of the task.") + version: ObjectVersion | None = Field(default=None, alias="Version") + created_at: str | None = Field(default=None, alias="CreatedAt") + updated_at: str | None = Field(default=None, alias="UpdatedAt") + name: str | None = Field( + default=None, alias="Name", description="Name of the task." + ) + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value metadata." + ) + spec: TaskSpec | None = Field(default=None, alias="Spec") + service_id: str | None = Field( + default=None, + alias="ServiceID", + description="The ID of the service this task is part of.", + ) + slot: int | None = Field(default=None, alias="Slot") + node_id: str | None = Field( + default=None, + alias="NodeID", + description="The ID of the node that this task is on.", + ) + assigned_generic_resources: GenericResources | None = Field( + default=None, alias="AssignedGenericResources" + ) + status: Status1 | None = Field(default=None, alias="Status") + desired_state: TaskState | None = Field(default=None, alias="DesiredState") + job_iteration: ObjectVersion | None = Field( + default=None, + alias="JobIteration", description="If the Service this Task belongs to is a job-mode service, contains\nthe JobIteration of the Service this Task was created for. Absent if\nthe Task was created for a Replicated or Global Service.\n", ) @@ -3468,47 +4128,53 @@ class ServiceSpec(BaseModel): User modifiable configuration for a service. """ - Name: str | None = Field(None, description="Name of the service.") - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value metadata." + name: str | None = Field( + default=None, alias="Name", description="Name of the service." ) - TaskTemplate: TaskSpec | None = None - Mode_: Mode | None = Field( - None, alias="Mode", description="Scheduling mode for the service." + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value metadata." + ) + task_template: TaskSpec | None = Field(default=None, alias="TaskTemplate") + mode: Mode | None = Field( + default=None, alias="Mode", description="Scheduling mode for the service." ) - UpdateConfig_: UpdateConfig | None = Field( - None, + update_config: UpdateConfig | None = Field( + default=None, alias="UpdateConfig", description="Specification for the update strategy of the service.", ) - RollbackConfig_: RollbackConfig | None = Field( - None, + rollback_config: RollbackConfig | None = Field( + default=None, alias="RollbackConfig", description="Specification for the rollback strategy of the service.", ) - Networks: list[NetworkAttachmentConfig] | None = Field( - None, description="Specifies which networks the service should attach to." + networks: list[NetworkAttachmentConfig] | None = Field( + default=None, + alias="Networks", + description="Specifies which networks the service should attach to.", ) - EndpointSpec_: EndpointSpec | None = Field(None, alias="EndpointSpec") + endpoint_spec: EndpointSpec | None = Field(default=None, alias="EndpointSpec") class Service(BaseModel): - ID: str | None = None - Version: ObjectVersion | None = None - CreatedAt: str | None = None - UpdatedAt: str | None = None - Spec: ServiceSpec | None = None - Endpoint_: Endpoint | None = Field(None, alias="Endpoint") - UpdateStatus_: UpdateStatus | None = Field( - None, alias="UpdateStatus", description="The status of a service update." - ) - ServiceStatus_: ServiceStatus | None = Field( - None, + id: str | None = Field(default=None, alias="ID") + version: ObjectVersion | None = Field(default=None, alias="Version") + created_at: str | None = Field(default=None, alias="CreatedAt") + updated_at: str | None = Field(default=None, alias="UpdatedAt") + spec: ServiceSpec | None = Field(default=None, alias="Spec") + endpoint: Endpoint | None = Field(default=None, alias="Endpoint") + update_status: UpdateStatus | None = Field( + default=None, + alias="UpdateStatus", + description="The status of a service update.", + ) + service_status: ServiceStatus | None = Field( + default=None, alias="ServiceStatus", description="The status of the service's tasks. Provided only when requested as\npart of a ServiceList operation.\n", ) - JobStatus_: JobStatus | None = Field( - None, + job_status: JobStatus | None = Field( + default=None, alias="JobStatus", description="The status of the service when it is in one of ReplicatedJob or\nGlobalJob modes. Absent on Replicated and Global mode services. The\nJobIteration is an ObjectVersion, but unlike the Service's version,\ndoes not need to be sent with an update request.\n", ) @@ -3519,98 +4185,135 @@ class NetworkSettings1(BaseModel): A summary of the container's network settings """ - Networks: dict[str, EndpointSettings] | None = None + networks: dict[str, EndpointSettings] | None = Field(default=None, alias="Networks") class ContainerSummary(BaseModel): - Id: str | None = Field(None, description="The ID of this container") - Names: list[str] | None = Field( - None, description="The names that this container has been given" - ) - Image: str | None = Field( - None, description="The name of the image used when creating this container" - ) - ImageID: str | None = Field( - None, description="The ID of the image that this container was created from" - ) - Command: str | None = Field( - None, description="Command to run when starting the container" - ) - Created: int | None = Field(None, description="When the container was created") - Ports: list[Port] | None = Field( - None, description="The ports exposed by this container" - ) - SizeRw: int | None = Field( - None, + id: str | None = Field( + default=None, alias="Id", description="The ID of this container" + ) + names: list[str] | None = Field( + default=None, + alias="Names", + description="The names that this container has been given", + ) + image: str | None = Field( + default=None, + alias="Image", + description="The name of the image used when creating this container", + ) + image_id: str | None = Field( + default=None, + alias="ImageID", + description="The ID of the image that this container was created from", + ) + command: str | None = Field( + default=None, + alias="Command", + description="Command to run when starting the container", + ) + created: int | None = Field( + default=None, alias="Created", description="When the container was created" + ) + ports: list[Port] | None = Field( + default=None, alias="Ports", description="The ports exposed by this container" + ) + size_rw: int | None = Field( + default=None, + alias="SizeRw", description="The size of files that have been created or changed by this container", ) - SizeRootFs: int | None = Field( - None, description="The total size of all the files in this container" + size_root_fs: int | None = Field( + default=None, + alias="SizeRootFs", + description="The total size of all the files in this container", ) - Labels: dict[str, str] | None = Field( - None, description="User-defined key/value metadata." + labels: dict[str, str] | None = Field( + default=None, alias="Labels", description="User-defined key/value metadata." ) - State: str | None = Field( - None, description="The state of this container (e.g. `Exited`)" + state: str | None = Field( + default=None, + alias="State", + description="The state of this container (e.g. `Exited`)", ) - Status: str | None = Field( - None, + status: str | None = Field( + default=None, + alias="Status", description="Additional human-readable status of this container (e.g. `Exit 0`)", ) - HostConfig: HostConfig1 | None = None - NetworkSettings: NetworkSettings1 | None = Field( - None, description="A summary of the container's network settings" + host_config: HostConfig1 | None = Field(default=None, alias="HostConfig") + network_settings: NetworkSettings1 | None = Field( + default=None, + alias="NetworkSettings", + description="A summary of the container's network settings", ) - Mounts: list[MountPoint] | None = None + mounts: list[MountPoint] | None = Field(default=None, alias="Mounts") class ContainerState(BaseModel): """ - ContainerState stores container's running state. It's part of ContainerJSONBase + ContainerState stores container's running state. It's part of ContainerJSONBase and will be returned by the "inspect" command. """ - Status: Status2 | None = Field( - None, + status: Status2 | None = Field( + default=None, + alias="Status", description='String representation of the container state. Can be one of "created",\n"running", "paused", "restarting", "removing", "exited", or "dead".\n', examples=["running"], ) - Running: bool | None = Field( - None, + running: bool | None = Field( + default=None, + alias="Running", description='Whether this container is running.\n\nNote that a running container can be _paused_. The `Running` and `Paused`\nbooleans are not mutually exclusive:\n\nWhen pausing a container (on Linux), the freezer cgroup is used to suspend\nall processes in the container. Freezing the process requires the process to\nbe running. As a result, paused containers are both `Running` _and_ `Paused`.\n\nUse the `Status` field instead to determine if a container\'s state is "running".\n', examples=[True], ) - Paused: bool | None = Field( - None, description="Whether this container is paused.", examples=[False] + paused: bool | None = Field( + default=None, + alias="Paused", + description="Whether this container is paused.", + examples=[False], ) - Restarting: bool | None = Field( - None, description="Whether this container is restarting.", examples=[False] + restarting: bool | None = Field( + default=None, + alias="Restarting", + description="Whether this container is restarting.", + examples=[False], ) - OOMKilled: bool | None = Field( - None, + oom_killed: bool | None = Field( + default=None, + alias="OOMKilled", description="Whether this container has been killed because it ran out of memory.\n", examples=[False], ) - Dead: bool | None = Field(None, examples=[False]) - Pid: int | None = Field( - None, description="The process ID of this container", examples=[1234] - ) - ExitCode: int | None = Field( - None, description="The last exit code of this container", examples=[0] + dead: bool | None = Field(default=None, alias="Dead", examples=[False]) + pid: int | None = Field( + default=None, + alias="Pid", + description="The process ID of this container", + examples=[1234], + ) + exit_code: int | None = Field( + default=None, + alias="ExitCode", + description="The last exit code of this container", + examples=[0], ) - Error: str | None = None - StartedAt: str | None = Field( - None, + error: str | None = Field(default=None, alias="Error") + started_at: str | None = Field( + default=None, + alias="StartedAt", description="The time when this container was last started.", examples=["2020-01-06T09:06:59.461876391Z"], ) - FinishedAt: str | None = Field( - None, + finished_at: str | None = Field( + default=None, + alias="FinishedAt", description="The time when this container last exited.", examples=["2020-01-06T09:07:59.461876391Z"], ) - Health_: Health | None = Field(None, alias="Health") + health: Health | None = Field(default=None, alias="Health") class ContainerWaitResponse(BaseModel): @@ -3618,8 +4321,10 @@ class ContainerWaitResponse(BaseModel): OK response to ContainerWait operation """ - StatusCode: int = Field(..., description="Exit code of the container") - Error: ContainerWaitExitError | None = None + status_code: int = Field( + ..., alias="StatusCode", description="Exit code of the container" + ) + error: ContainerWaitExitError | None = Field(default=None, alias="Error") class RegistryServiceConfig(BaseModel): @@ -3628,28 +4333,29 @@ class RegistryServiceConfig(BaseModel): """ - AllowNondistributableArtifactsCIDRs: list[str] | None = Field( - None, + allow_nondistributable_artifacts_cid_rs: list[str] | None = Field( + default=None, + alias="AllowNondistributableArtifactsCIDRs", description="List of IP ranges to which nondistributable artifacts can be pushed,\nusing the CIDR syntax [RFC 4632](https://tools.ietf.org/html/4632).\n\nSome images (for example, Windows base images) contain artifacts\nwhose distribution is restricted by license. When these images are\npushed to a registry, restricted artifacts are not included.\n\nThis configuration override this behavior, and enables the daemon to\npush nondistributable artifacts to all registries whose resolved IP\naddress is within the subnet described by the CIDR syntax.\n\nThis option is useful when pushing images containing\nnondistributable artifacts to a registry on an air-gapped network so\nhosts on that network can pull the images without connecting to\nanother server.\n\n> **Warning**: Nondistributable artifacts typically have restrictions\n> on how and where they can be distributed and shared. Only use this\n> feature to push artifacts to private registries and ensure that you\n> are in compliance with any terms that cover redistributing\n> nondistributable artifacts.\n", examples=[["::1/128", "127.0.0.0/8"]], ) - AllowNondistributableArtifactsHostnames: list[str] | None = Field( - None, + allow_nondistributable_artifacts_hostnames: list[str] | None = Field( + default=None, + alias="AllowNondistributableArtifactsHostnames", description="List of registry hostnames to which nondistributable artifacts can be\npushed, using the format `[:]` or `[:]`.\n\nSome images (for example, Windows base images) contain artifacts\nwhose distribution is restricted by license. When these images are\npushed to a registry, restricted artifacts are not included.\n\nThis configuration override this behavior for the specified\nregistries.\n\nThis option is useful when pushing images containing\nnondistributable artifacts to a registry on an air-gapped network so\nhosts on that network can pull the images without connecting to\nanother server.\n\n> **Warning**: Nondistributable artifacts typically have restrictions\n> on how and where they can be distributed and shared. Only use this\n> feature to push artifacts to private registries and ensure that you\n> are in compliance with any terms that cover redistributing\n> nondistributable artifacts.\n", examples=[ - [ - "registry.internal.corp.example.com:3000", - "[2001:db8:a0b:12f0::1]:443", - ] + ["registry.internal.corp.example.com:3000", "[2001:db8:a0b:12f0::1]:443"] ], ) - InsecureRegistryCIDRs: list[str] | None = Field( - None, + insecure_registry_cid_rs: list[str] | None = Field( + default=None, + alias="InsecureRegistryCIDRs", description="List of IP ranges of insecure registries, using the CIDR syntax\n([RFC 4632](https://tools.ietf.org/html/4632)). Insecure registries\naccept un-encrypted (HTTP) and/or untrusted (HTTPS with certificates\nfrom unknown CAs) communication.\n\nBy default, local registries (`127.0.0.0/8`) are configured as\ninsecure. All other registries are secure. Communicating with an\ninsecure registry is not possible if the daemon assumes that registry\nis secure.\n\nThis configuration override this behavior, insecure communication with\nregistries whose resolved IP address is within the subnet described by\nthe CIDR syntax.\n\nRegistries can also be marked insecure by hostname. Those registries\nare listed under `IndexConfigs` and have their `Secure` field set to\n`false`.\n\n> **Warning**: Using this option can be useful when running a local\n> registry, but introduces security vulnerabilities. This option\n> should therefore ONLY be used for testing purposes. For increased\n> security, users should add their CA to their system's list of trusted\n> CAs instead of enabling this option.\n", examples=[["::1/128", "127.0.0.0/8"]], ) - IndexConfigs: dict[str, IndexInfo] | None = Field( - None, + index_configs: dict[str, IndexInfo] | None = Field( + default=None, + alias="IndexConfigs", examples=[ { "127.0.0.1:5000": { @@ -3679,8 +4385,9 @@ class RegistryServiceConfig(BaseModel): } ], ) - Mirrors: list[str] | None = Field( - None, + mirrors: list[str] | None = Field( + default=None, + alias="Mirrors", description="List of registry URLs that act as a mirror for the official\n(`docker.io`) registry.\n", examples=[ [ @@ -3697,21 +4404,26 @@ class SwarmInfo(BaseModel): """ - NodeID: str | None = Field( - "", + node_id: str | None = Field( + default="", + alias="NodeID", description="Unique identifier of for this node in the swarm.", examples=["k67qz4598weg5unwwffg6z1m1"], ) - NodeAddr: str | None = Field( - "", + node_addr: str | None = Field( + default="", + alias="NodeAddr", description="IP address at which this node can be reached by other nodes in the\nswarm.\n", examples=["10.0.0.46"], ) - LocalNodeState_: LocalNodeState | None = Field(None, alias="LocalNodeState") - ControlAvailable: bool | None = Field(False, examples=[True]) - Error: str | None = "" - RemoteManagers: list[PeerNode] | None = Field( - None, + local_node_state: LocalNodeState | None = Field(default="", alias="LocalNodeState") + control_available: bool | None = Field( + default=False, alias="ControlAvailable", examples=[True] + ) + error: str | None = Field(default="", alias="Error") + remote_managers: list[PeerNode] | None = Field( + default=None, + alias="RemoteManagers", description="List of ID's and addresses of other managers in the swarm.\n", examples=[ [ @@ -3721,26 +4433,218 @@ class SwarmInfo(BaseModel): ] ], ) - Nodes: int | None = Field( - None, description="Total number of nodes in the swarm.", examples=[4] + nodes: int | None = Field( + default=None, + alias="Nodes", + description="Total number of nodes in the swarm.", + examples=[4], ) - Managers: int | None = Field( - None, description="Total number of managers in the swarm.", examples=[3] + managers: int | None = Field( + default=None, + alias="Managers", + description="Total number of managers in the swarm.", + examples=[3], + ) + cluster: ClusterInfo | None = Field(default=None, alias="Cluster") + + +class HostConfig(Resources): + """ + Container configuration that depends on the host we are running on + """ + + binds: list[str] | None = Field( + default=None, + alias="Binds", + description="A list of volume bindings for this container. Each volume binding\nis a string in one of these forms:\n\n- `host-src:container-dest[:options]` to bind-mount a host path\n into the container. Both `host-src`, and `container-dest` must\n be an _absolute_ path.\n- `volume-name:container-dest[:options]` to bind-mount a volume\n managed by a volume driver into the container. `container-dest`\n must be an _absolute_ path.\n\n`options` is an optional, comma-delimited list of:\n\n- `nocopy` disables automatic copying of data from the container\n path to the volume. The `nocopy` flag only applies to named volumes.\n- `[ro|rw]` mounts a volume read-only or read-write, respectively.\n If omitted or set to `rw`, volumes are mounted read-write.\n- `[z|Z]` applies SELinux labels to allow or deny multiple containers\n to read and write to the same volume.\n - `z`: a _shared_ content label is applied to the content. This\n label indicates that multiple containers can share the volume\n content, for both reading and writing.\n - `Z`: a _private unshared_ label is applied to the content.\n This label indicates that only the current container can use\n a private volume. Labeling systems such as SELinux require\n proper labels to be placed on volume content that is mounted\n into a container. Without a label, the security system can\n prevent a container's processes from using the content. By\n default, the labels set by the host operating system are not\n modified.\n- `[[r]shared|[r]slave|[r]private]` specifies mount\n [propagation behavior](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt).\n This only applies to bind-mounted volumes, not internal volumes\n or named volumes. Mount propagation requires the source mount\n point (the location where the source directory is mounted in the\n host operating system) to have the correct propagation properties.\n For shared volumes, the source mount point must be set to `shared`.\n For slave volumes, the mount must be set to either `shared` or\n `slave`.\n", + ) + container_id_file: str | None = Field( + default=None, + alias="ContainerIDFile", + description="Path to a file where the container ID is written", + ) + log_config: LogConfig | None = Field( + default=None, + alias="LogConfig", + description="The logging configuration for this container", + ) + network_mode: str | None = Field( + default=None, + alias="NetworkMode", + description="Network mode to use for this container. Supported standard values\nare: `bridge`, `host`, `none`, and `container:`. Any\nother value is taken as a custom network's name to which this\ncontainer should connect to.\n", + ) + port_bindings: PortMap | None = Field(default=None, alias="PortBindings") + restart_policy: RestartPolicy | None = Field(default=None, alias="RestartPolicy") + auto_remove: bool | None = Field( + default=None, + alias="AutoRemove", + description="Automatically remove the container when the container's process\nexits. This has no effect if `RestartPolicy` is set.\n", + ) + volume_driver: str | None = Field( + default=None, + alias="VolumeDriver", + description="Driver that this container uses to mount volumes.", + ) + volumes_from: list[str] | None = Field( + default=None, + alias="VolumesFrom", + description="A list of volumes to inherit from another container, specified in\nthe form `[:]`.\n", + ) + mounts: list[Mount] | None = Field( + default=None, + alias="Mounts", + description="Specification for mounts to be added to the container.\n", + ) + cap_add: list[str] | None = Field( + default=None, + alias="CapAdd", + description="A list of kernel capabilities to add to the container. Conflicts\nwith option 'Capabilities'.\n", + ) + cap_drop: list[str] | None = Field( + default=None, + alias="CapDrop", + description="A list of kernel capabilities to drop from the container. Conflicts\nwith option 'Capabilities'.\n", + ) + cgroupns_mode: CgroupnsMode | None = Field( + default=None, + alias="CgroupnsMode", + description='cgroup namespace mode for the container. Possible values are:\n\n- `"private"`: the container runs in its own private cgroup namespace\n- `"host"`: use the host system\'s cgroup namespace\n\nIf not specified, the daemon default is used, which can either be `"private"`\nor `"host"`, depending on daemon version, kernel support and configuration.\n', + ) + dns: list[str] | None = Field( + default=None, + alias="Dns", + description="A list of DNS servers for the container to use.", + ) + dns_options: list[str] | None = Field( + default=None, alias="DnsOptions", description="A list of DNS options." + ) + dns_search: list[str] | None = Field( + default=None, alias="DnsSearch", description="A list of DNS search domains." + ) + extra_hosts: list[str] | None = Field( + default=None, + alias="ExtraHosts", + description='A list of hostnames/IP mappings to add to the container\'s `/etc/hosts`\nfile. Specified in the form `["hostname:IP"]`.\n', + ) + group_add: list[str] | None = Field( + default=None, + alias="GroupAdd", + description="A list of additional groups that the container process will run as.\n", + ) + ipc_mode: str | None = Field( + default=None, + alias="IpcMode", + description='IPC sharing mode for the container. Possible values are:\n\n- `"none"`: own private IPC namespace, with /dev/shm not mounted\n- `"private"`: own private IPC namespace\n- `"shareable"`: own private IPC namespace, with a possibility to share it with other containers\n- `"container:"`: join another (shareable) container\'s IPC namespace\n- `"host"`: use the host system\'s IPC namespace\n\nIf not specified, daemon default is used, which can either be `"private"`\nor `"shareable"`, depending on daemon version and configuration.\n', + ) + cgroup: str | None = Field( + default=None, alias="Cgroup", description="Cgroup to use for the container." + ) + links: list[str] | None = Field( + default=None, + alias="Links", + description="A list of links for the container in the form `container_name:alias`.\n", + ) + oom_score_adj: int | None = Field( + default=None, + alias="OomScoreAdj", + description="An integer value containing the score given to the container in\norder to tune OOM killer preferences.\n", + examples=[500], + ) + pid_mode: str | None = Field( + default=None, + alias="PidMode", + description='Set the PID (Process) Namespace mode for the container. It can be\neither:\n\n- `"container:"`: joins another container\'s PID namespace\n- `"host"`: use the host\'s PID namespace inside the container\n', + ) + privileged: bool | None = Field( + default=None, + alias="Privileged", + description="Gives the container full access to the host.", + ) + publish_all_ports: bool | None = Field( + default=None, + alias="PublishAllPorts", + description="Allocates an ephemeral host port for all of a container's\nexposed ports.\n\nPorts are de-allocated when the container stops and allocated when\nthe container starts. The allocated port might be changed when\nrestarting the container.\n\nThe port is selected from the ephemeral port range that depends on\nthe kernel. For example, on Linux the range is defined by\n`/proc/sys/net/ipv4/ip_local_port_range`.\n", + ) + readonly_rootfs: bool | None = Field( + default=None, + alias="ReadonlyRootfs", + description="Mount the container's root filesystem as read only.", + ) + security_opt: list[str] | None = Field( + default=None, + alias="SecurityOpt", + description="A list of string values to customize labels for MLS systems, such\nas SELinux.\n", + ) + storage_opt: dict[str, str] | None = Field( + default=None, + alias="StorageOpt", + description='Storage driver options for this container, in the form `{"size": "120G"}`.\n', + ) + tmpfs: dict[str, str] | None = Field( + default=None, + alias="Tmpfs", + description='A map of container directories which should be replaced by tmpfs\nmounts, and their corresponding mount options. For example:\n\n```\n{ "/run": "rw,noexec,nosuid,size=65536k" }\n```\n', + ) + uts_mode: str | None = Field( + default=None, + alias="UTSMode", + description="UTS namespace to use for the container.", + ) + userns_mode: str | None = Field( + default=None, + alias="UsernsMode", + description="Sets the usernamespace mode for the container when usernamespace\nremapping option is enabled.\n", + ) + shm_size: int | None = Field( + default=None, + alias="ShmSize", + description="Size of `/dev/shm` in bytes. If omitted, the system uses 64MB.\n", + ge=0, + ) + sysctls: dict[str, str] | None = Field( + default=None, + alias="Sysctls", + description='A list of kernel parameters (sysctls) to set in the container.\nFor example:\n\n```\n{"net.ipv4.ip_forward": "1"}\n```\n', + ) + runtime: str | None = Field( + default=None, alias="Runtime", description="Runtime to use with this container." + ) + console_size: list[ConsoleSizeItem] | None = Field( + default=None, + alias="ConsoleSize", + description="Initial console size, as an `[height, width]` array. (Windows only)\n", + max_length=2, + min_length=2, + ) + isolation: Isolation | None = Field( + default=None, + alias="Isolation", + description="Isolation technology of the container. (Windows only)\n", + ) + masked_paths: list[str] | None = Field( + default=None, + alias="MaskedPaths", + description="The list of paths to be masked inside the container (this overrides\nthe default set of paths).\n", + ) + readonly_paths: list[str] | None = Field( + default=None, + alias="ReadonlyPaths", + description="The list of paths to be set as read-only inside the container\n(this overrides the default set of paths).\n", ) - Cluster: ClusterInfo | None = None class NetworkingConfig(BaseModel): """ - NetworkingConfig represents the container's networking configuration for + NetworkingConfig represents the container's networking configuration for each of its interfaces. It is used for the networking configs specified in the `docker create` and `docker network connect` commands. """ - EndpointsConfig: dict[str, EndpointSettings] | None = Field( - None, + endpoints_config: dict[str, EndpointSettings] | None = Field( + default=None, + alias="EndpointsConfig", description="A mapping of network name to endpoint configuration for that network.\n", ) @@ -3750,150 +4654,282 @@ class NetworkSettings(BaseModel): NetworkSettings exposes the network settings in the API """ - Bridge: str | None = Field( - None, - description="Name of the network'a bridge (for example, `docker0`).", + bridge: str | None = Field( + default=None, + alias="Bridge", + description="Name of the network's bridge (for example, `docker0`).", examples=["docker0"], ) - SandboxID: str | None = Field( - None, + sandbox_id: str | None = Field( + default=None, + alias="SandboxID", description="SandboxID uniquely represents a container's network stack.", examples=["9d12daf2c33f5959c8bf90aa513e4f65b561738661003029ec84830cd503a0c3"], ) - HairpinMode: bool | None = Field( - None, + hairpin_mode: bool | None = Field( + default=None, + alias="HairpinMode", description="Indicates if hairpin NAT should be enabled on the virtual interface.\n", examples=[False], ) - LinkLocalIPv6Address: str | None = Field( - None, + link_local_i_pv6_address: str | None = Field( + default=None, + alias="LinkLocalIPv6Address", description="IPv6 unicast address using the link-local prefix.", examples=["fe80::42:acff:fe11:1"], ) - LinkLocalIPv6PrefixLen: int | None = Field( - None, description="Prefix length of the IPv6 unicast address.", examples=["64"] + link_local_i_pv6_prefix_len: int | None = Field( + default=None, + alias="LinkLocalIPv6PrefixLen", + description="Prefix length of the IPv6 unicast address.", + examples=["64"], ) - Ports: PortMap | None = None - SandboxKey: str | None = Field( - None, + ports: PortMap | None = Field(default=None, alias="Ports") + sandbox_key: str | None = Field( + default=None, + alias="SandboxKey", description="SandboxKey identifies the sandbox", examples=["/var/run/docker/netns/8ab54b426c38"], ) - SecondaryIPAddresses: list[Address] | None = Field(None, description="") - SecondaryIPv6Addresses: list[Address] | None = Field(None, description="") - EndpointID: str | None = Field( - None, + secondary_ip_addresses: list[Address] | None = Field( + default=None, alias="SecondaryIPAddresses", description="" + ) + secondary_i_pv6_addresses: list[Address] | None = Field( + default=None, alias="SecondaryIPv6Addresses", description="" + ) + endpoint_id: str | None = Field( + default=None, + alias="EndpointID", description='EndpointID uniquely represents a service endpoint in a Sandbox.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=["b88f5b905aabf2893f3cbc4ee42d1ea7980bbc0a92e2c8922b1e1795298afb0b"], ) - Gateway: str | None = Field( - None, + gateway: str | None = Field( + default=None, + alias="Gateway", description='Gateway address for the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=["172.17.0.1"], ) - GlobalIPv6Address: str | None = Field( - None, + global_i_pv6_address: str | None = Field( + default=None, + alias="GlobalIPv6Address", description='Global IPv6 address for the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=["2001:db8::5689"], ) - GlobalIPv6PrefixLen: int | None = Field( - None, + global_i_pv6_prefix_len: int | None = Field( + default=None, + alias="GlobalIPv6PrefixLen", description='Mask length of the global IPv6 address.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=[64], ) - IPAddress: str | None = Field( - None, + ip_address: str | None = Field( + default=None, + alias="IPAddress", description='IPv4 address for the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=["172.17.0.4"], ) - IPPrefixLen: int | None = Field( - None, + ip_prefix_len: int | None = Field( + default=None, + alias="IPPrefixLen", description='Mask length of the IPv4 address.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=[16], ) - IPv6Gateway: str | None = Field( - None, + i_pv6_gateway: str | None = Field( + default=None, + alias="IPv6Gateway", description='IPv6 gateway address for this network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=["2001:db8:2::100"], ) - MacAddress: str | None = Field( - None, + mac_address: str | None = Field( + default=None, + alias="MacAddress", description='MAC address for the container on the default "bridge" network.\n\n


\n\n> **Deprecated**: This field is only propagated when attached to the\n> default "bridge" network. Use the information from the "bridge"\n> network inside the `Networks` map instead, which contains the same\n> information. This field was deprecated in Docker 1.9 and is scheduled\n> to be removed in Docker 17.12.0\n', examples=["02:42:ac:11:00:04"], ) - Networks: dict[str, EndpointSettings] | None = Field( - None, + networks: dict[str, EndpointSettings] | None = Field( + default=None, + alias="Networks", description="Information about all networks that the container is connected to.\n", ) class Network(BaseModel): - Name: str | None = None - Id: str | None = None - Created: str | None = None - Scope: str | None = None - Driver: str | None = None - EnableIPv6: bool | None = None - IPAM_: IPAM | None = Field(None, alias="IPAM") - Internal: bool | None = None - Attachable: bool | None = None - Ingress: bool | None = None - Containers: dict[str, NetworkContainer] | None = None - Options: dict[str, str] | None = None - Labels: dict[str, str] | None = None + name: str | None = Field( + default=None, + alias="Name", + description="Name of the network.\n", + examples=["my_network"], + ) + id: str | None = Field( + default=None, + alias="Id", + description="ID that uniquely identifies a network on a single machine.\n", + examples=["7d86d31b1478e7cca9ebed7e73aa0fdeec46c5ca29497431d3007d2d9e15ed99"], + ) + created: str | None = Field( + default=None, + alias="Created", + description="Date and time at which the network was created in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", + examples=["2016-10-19T04:33:30.360899459Z"], + ) + scope: str | None = Field( + default=None, + alias="Scope", + description="The level at which the network exists (e.g. `swarm` for cluster-wide\nor `local` for machine level)\n", + examples=["local"], + ) + driver: str | None = Field( + default=None, + alias="Driver", + description="The name of the driver used to create the network (e.g. `bridge`,\n`overlay`).\n", + examples=["overlay"], + ) + enable_i_pv6: bool | None = Field( + default=None, + alias="EnableIPv6", + description="Whether the network was created with IPv6 enabled.\n", + examples=[False], + ) + ipam: IPAM | None = Field(default=None, alias="IPAM") + internal: bool | None = Field( + default=False, + alias="Internal", + description="Whether the network is created to only allow internal networking\nconnectivity.\n", + examples=[False], + ) + attachable: bool | None = Field( + default=False, + alias="Attachable", + description="Whether a global / swarm scope network is manually attachable by regular\ncontainers from workers in swarm mode.\n", + examples=[False], + ) + ingress: bool | None = Field( + default=False, + alias="Ingress", + description="Whether the network is providing the routing-mesh for the swarm cluster.\n", + examples=[False], + ) + config_from: ConfigReference | None = Field(default=None, alias="ConfigFrom") + config_only: bool | None = Field( + default=False, + alias="ConfigOnly", + description="Whether the network is a config-only network. Config-only networks are\nplaceholder networks for network configurations to be used by other\nnetworks. Config-only networks cannot be used directly to run containers\nor services.\n", + ) + containers: dict[str, NetworkContainer] | None = Field( + default=None, + alias="Containers", + description="Contains endpoints attached to the network.\n", + examples=[ + { + "19a4d5d687db25203351ed79d478946f861258f018fe384f229f2efa4b23513c": { + "Name": "test", + "EndpointID": "628cadb8bcb92de107b2a1e516cbffe463e321f548feb37697cce00ad694f21a", + "MacAddress": "02:42:ac:13:00:02", + "IPv4Address": "172.19.0.2/16", + "IPv6Address": "", + } + } + ], + ) + options: dict[str, str] | None = Field( + default=None, + alias="Options", + description="Network-specific options uses when creating the network.\n", + examples=[ + { + "com.docker.network.bridge.default_bridge": "true", + "com.docker.network.bridge.enable_icc": "true", + "com.docker.network.bridge.enable_ip_masquerade": "true", + "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", + "com.docker.network.bridge.name": "docker0", + "com.docker.network.driver.mtu": "1500", + } + ], + ) + labels: dict[str, str] | None = Field( + default=None, + alias="Labels", + description="User-defined key/value metadata.", + examples=[ + { + "com.example.some-label": "some-value", + "com.example.some-other-label": "some-other-value", + } + ], + ) + peers: list[PeerInfo] | None = Field( + default=None, + alias="Peers", + description="List of peer nodes for an overlay network. This field is only present\nfor overlay networks, and omitted for other network types.\n", + ) class Node(BaseModel): - ID: str | None = Field(None, examples=["24ifsmvkjbyhk"]) - Version: ObjectVersion | None = None - CreatedAt: str | None = Field( - None, + id: str | None = Field(default=None, alias="ID", examples=["24ifsmvkjbyhk"]) + version: ObjectVersion | None = Field(default=None, alias="Version") + created_at: str | None = Field( + default=None, + alias="CreatedAt", description="Date and time at which the node was added to the swarm in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2016-08-18T10:44:24.496525531Z"], ) - UpdatedAt: str | None = Field( - None, + updated_at: str | None = Field( + default=None, + alias="UpdatedAt", description="Date and time at which the node was last updated in\n[RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format with nano-seconds.\n", examples=["2017-08-09T07:09:37.632105588Z"], ) - Spec: NodeSpec | None = None - Description: NodeDescription | None = None - Status: NodeStatus | None = None - ManagerStatus_: ManagerStatus | None = Field(None, alias="ManagerStatus") + spec: NodeSpec | None = Field(default=None, alias="Spec") + description: NodeDescription | None = Field(default=None, alias="Description") + status: NodeStatus | None = Field(default=None, alias="Status") + manager_status: ManagerStatus | None = Field(default=None, alias="ManagerStatus") class SystemInfo(BaseModel): - ID: str | None = Field( - None, + id: str | None = Field( + default=None, + alias="ID", description="Unique identifier of the daemon.\n\n


\n\n> **Note**: The format of the ID itself is not part of the API, and\n> should not be considered stable.\n", examples=["7TRN:IPZB:QYBB:VPBQ:UMPP:KARE:6ZNR:XE6T:7EWV:PKF4:ZOJD:TPYS"], ) - Containers: int | None = Field( - None, description="Total number of containers on the host.", examples=[14] + containers: int | None = Field( + default=None, + alias="Containers", + description="Total number of containers on the host.", + examples=[14], ) - ContainersRunning: int | None = Field( - None, + containers_running: int | None = Field( + default=None, + alias="ContainersRunning", description='Number of containers with status `"running"`.\n', examples=[3], ) - ContainersPaused: int | None = Field( - None, description='Number of containers with status `"paused"`.\n', examples=[1] + containers_paused: int | None = Field( + default=None, + alias="ContainersPaused", + description='Number of containers with status `"paused"`.\n', + examples=[1], ) - ContainersStopped: int | None = Field( - None, + containers_stopped: int | None = Field( + default=None, + alias="ContainersStopped", description='Number of containers with status `"stopped"`.\n', examples=[10], ) - Images: int | None = Field( - None, + images: int | None = Field( + default=None, + alias="Images", description="Total number of images on the host.\n\nBoth _tagged_ and _untagged_ (dangling) images are counted.\n", examples=[508], ) - Driver: str | None = Field( - None, description="Name of the storage driver in use.", examples=["overlay2"] + driver: str | None = Field( + default=None, + alias="Driver", + description="Name of the storage driver in use.", + examples=["overlay2"], ) - DriverStatus: list[list[str]] | None = Field( - None, + driver_status: list[list[str]] | None = Field( + default=None, + alias="DriverStatus", description='Information specific to the storage driver, provided as\n"label" / "value" pairs.\n\nThis information is provided by the storage driver, and formatted\nin a way consistent with the output of `docker info` on the command\nline.\n\n


\n\n> **Note**: The information returned in this field, including the\n> formatting of values and labels, should not be considered stable,\n> and may change without notice.\n', examples=[ [ @@ -3903,198 +4939,248 @@ class SystemInfo(BaseModel): ] ], ) - DockerRootDir: str | None = Field( - None, + docker_root_dir: str | None = Field( + default=None, + alias="DockerRootDir", description="Root directory of persistent Docker state.\n\nDefaults to `/var/lib/docker` on Linux, and `C:\\ProgramData\\docker`\non Windows.\n", examples=["/var/lib/docker"], ) - Plugins: PluginsInfo | None = None - MemoryLimit: bool | None = Field( - None, + plugins: PluginsInfo | None = Field(default=None, alias="Plugins") + memory_limit: bool | None = Field( + default=None, + alias="MemoryLimit", description="Indicates if the host has memory limit support enabled.", examples=[True], ) - SwapLimit: bool | None = Field( - None, + swap_limit: bool | None = Field( + default=None, + alias="SwapLimit", description="Indicates if the host has memory swap limit support enabled.", examples=[True], ) - KernelMemory: bool | None = Field( - None, + kernel_memory: bool | None = Field( + default=None, + alias="KernelMemory", description="Indicates if the host has kernel memory limit support enabled.\n\n


\n\n> **Deprecated**: This field is deprecated as the kernel 5.4 deprecated\n> `kmem.limit_in_bytes`.\n", examples=[True], ) - KernelMemoryTCP: bool | None = Field( - None, + kernel_memory_tcp: bool | None = Field( + default=None, + alias="KernelMemoryTCP", description="Indicates if the host has kernel memory TCP limit support enabled.\n\nKernel memory TCP limits are not supported when using cgroups v2, which\ndoes not support the corresponding `memory.kmem.tcp.limit_in_bytes` cgroup.\n", examples=[True], ) - CpuCfsPeriod: bool | None = Field( - None, + cpu_cfs_period: bool | None = Field( + default=None, + alias="CpuCfsPeriod", description="Indicates if CPU CFS(Completely Fair Scheduler) period is supported by\nthe host.\n", examples=[True], ) - CpuCfsQuota: bool | None = Field( - None, + cpu_cfs_quota: bool | None = Field( + default=None, + alias="CpuCfsQuota", description="Indicates if CPU CFS(Completely Fair Scheduler) quota is supported by\nthe host.\n", examples=[True], ) - CPUShares: bool | None = Field( - None, + cpu_shares: bool | None = Field( + default=None, + alias="CPUShares", description="Indicates if CPU Shares limiting is supported by the host.\n", examples=[True], ) - CPUSet: bool | None = Field( - None, + cpu_set: bool | None = Field( + default=None, + alias="CPUSet", description="Indicates if CPUsets (cpuset.cpus, cpuset.mems) are supported by the host.\n\nSee [cpuset(7)](https://www.kernel.org/doc/Documentation/cgroup-v1/cpusets.txt)\n", examples=[True], ) - PidsLimit: bool | None = Field( - None, + pids_limit: bool | None = Field( + default=None, + alias="PidsLimit", description="Indicates if the host kernel has PID limit support enabled.", examples=[True], ) - OomKillDisable: bool | None = Field( - None, description="Indicates if OOM killer disable is supported on the host." + oom_kill_disable: bool | None = Field( + default=None, + alias="OomKillDisable", + description="Indicates if OOM killer disable is supported on the host.", ) - IPv4Forwarding: bool | None = Field( - None, description="Indicates IPv4 forwarding is enabled.", examples=[True] + i_pv4_forwarding: bool | None = Field( + default=None, + alias="IPv4Forwarding", + description="Indicates IPv4 forwarding is enabled.", + examples=[True], ) - BridgeNfIptables: bool | None = Field( - None, + bridge_nf_iptables: bool | None = Field( + default=None, + alias="BridgeNfIptables", description="Indicates if `bridge-nf-call-iptables` is available on the host.", examples=[True], ) - BridgeNfIp6tables: bool | None = Field( - None, + bridge_nf_ip6tables: bool | None = Field( + default=None, + alias="BridgeNfIp6tables", description="Indicates if `bridge-nf-call-ip6tables` is available on the host.", examples=[True], ) - Debug: bool | None = Field( - None, + debug: bool | None = Field( + default=None, + alias="Debug", description="Indicates if the daemon is running in debug-mode / with debug-level\nlogging enabled.\n", examples=[True], ) - NFd: int | None = Field( - None, + n_fd: int | None = Field( + default=None, + alias="NFd", description="The total number of file Descriptors in use by the daemon process.\n\nThis information is only returned if debug-mode is enabled.\n", examples=[64], ) - NGoroutines: int | None = Field( - None, + n_goroutines: int | None = Field( + default=None, + alias="NGoroutines", description="The number of goroutines that currently exist.\n\nThis information is only returned if debug-mode is enabled.\n", examples=[174], ) - SystemTime: str | None = Field( - None, + system_time: str | None = Field( + default=None, + alias="SystemTime", description="Current system-time in [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt)\nformat with nano-seconds.\n", examples=["2017-08-08T20:28:29.06202363Z"], ) - LoggingDriver: str | None = Field( - None, description="The logging driver to use as a default for new containers.\n" + logging_driver: str | None = Field( + default=None, + alias="LoggingDriver", + description="The logging driver to use as a default for new containers.\n", ) - CgroupDriver_: CgroupDriver | None = Field( - CgroupDriver.cgroupfs, + cgroup_driver: CgroupDriver | None = Field( + default=CgroupDriver.cgroupfs, alias="CgroupDriver", description="The driver to use for managing cgroups.\n", examples=["cgroupfs"], ) - CgroupVersion_: CgroupVersion | None = Field( - CgroupVersion.field_1, + cgroup_version: CgroupVersion | None = Field( + default=CgroupVersion.field_1, alias="CgroupVersion", description="The version of the cgroup.\n", examples=["1"], ) - NEventsListener: int | None = Field( - None, description="Number of event listeners subscribed.", examples=[30] + n_events_listener: int | None = Field( + default=None, + alias="NEventsListener", + description="Number of event listeners subscribed.", + examples=[30], ) - KernelVersion: str | None = Field( - None, + kernel_version: str | None = Field( + default=None, + alias="KernelVersion", description='Kernel version of the host.\n\nOn Linux, this information obtained from `uname`. On Windows this\ninformation is queried from the HKEY_LOCAL_MACHINE\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\\nregistry value, for example _"10.0 14393 (14393.1198.amd64fre.rs1_release_sec.170427-1353)"_.\n', examples=["4.9.38-moby"], ) - OperatingSystem: str | None = Field( - None, + operating_system: str | None = Field( + default=None, + alias="OperatingSystem", description='Name of the host\'s operating system, for example: "Ubuntu 16.04.2 LTS"\nor "Windows Server 2016 Datacenter"\n', examples=["Alpine Linux v3.5"], ) - OSVersion: str | None = Field( - None, + os_version: str | None = Field( + default=None, + alias="OSVersion", description="Version of the host's operating system\n\n


\n\n> **Note**: The information returned in this field, including its\n> very existence, and the formatting of values, should not be considered\n> stable, and may change without notice.\n", examples=["16.04"], ) - OSType: str | None = Field( - None, - description='Generic type of the operating system of the host, as returned by the\nGo runtime (`GOOS`).\n\nCurrently returned values are "linux" and "windows". A full list of\npossible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment).\n', + os_type: str | None = Field( + default=None, + alias="OSType", + description='Generic type of the operating system of the host, as returned by the\nGo runtime (`GOOS`).\n\nCurrently returned values are "linux" and "windows". A full list of\npossible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment).\n', examples=["linux"], ) - Architecture: str | None = Field( - None, - description="Hardware architecture of the host, as returned by the Go runtime\n(`GOARCH`).\n\nA full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment).\n", + architecture: str | None = Field( + default=None, + alias="Architecture", + description="Hardware architecture of the host, as returned by the Go runtime\n(`GOARCH`).\n\nA full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment).\n", examples=["x86_64"], ) - NCPU: int | None = Field( - None, + ncpu: int | None = Field( + default=None, + alias="NCPU", description="The number of logical CPUs usable by the daemon.\n\nThe number of available CPUs is checked by querying the operating\nsystem when the daemon starts. Changes to operating system CPU\nallocation after the daemon is started are not reflected.\n", examples=[4], ) - MemTotal: int | None = Field( - None, + mem_total: int | None = Field( + default=None, + alias="MemTotal", description="Total amount of physical memory available on the host, in bytes.\n", examples=[2095882240], ) - IndexServerAddress: str | None = Field( - "https://index.docker.io/v1/", + index_server_address: str | None = Field( + default="https://index.docker.io/v1/", + alias="IndexServerAddress", description="Address / URL of the index server that is used for image search,\nand as a default for user authentication for Docker Hub and Docker Cloud.\n", examples=["https://index.docker.io/v1/"], ) - RegistryConfig: RegistryServiceConfig | None = None - GenericResources_: GenericResources | None = Field(None, alias="GenericResources") - HttpProxy: str | None = Field( - None, + registry_config: RegistryServiceConfig | None = Field( + default=None, alias="RegistryConfig" + ) + generic_resources: GenericResources | None = Field( + default=None, alias="GenericResources" + ) + http_proxy: str | None = Field( + default=None, + alias="HttpProxy", description="HTTP-proxy configured for the daemon. This value is obtained from the\n[`HTTP_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html) environment variable.\nCredentials ([user info component](https://tools.ietf.org/html/rfc3986#section-3.2.1)) in the proxy URL\nare masked in the API response.\n\nContainers do not automatically inherit this configuration.\n", examples=["http://xxxxx:xxxxx@proxy.corp.example.com:8080"], ) - HttpsProxy: str | None = Field( - None, + https_proxy: str | None = Field( + default=None, + alias="HttpsProxy", description="HTTPS-proxy configured for the daemon. This value is obtained from the\n[`HTTPS_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html) environment variable.\nCredentials ([user info component](https://tools.ietf.org/html/rfc3986#section-3.2.1)) in the proxy URL\nare masked in the API response.\n\nContainers do not automatically inherit this configuration.\n", examples=["https://xxxxx:xxxxx@proxy.corp.example.com:4443"], ) - NoProxy: str | None = Field( - None, + no_proxy: str | None = Field( + default=None, + alias="NoProxy", description="Comma-separated list of domain extensions for which no proxy should be\nused. This value is obtained from the [`NO_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html)\nenvironment variable.\n\nContainers do not automatically inherit this configuration.\n", examples=["*.local, 169.254/16"], ) - Name: str | None = Field( - None, description="Hostname of the host.", examples=["node5.corp.example.com"] + name: str | None = Field( + default=None, + alias="Name", + description="Hostname of the host.", + examples=["node5.corp.example.com"], ) - Labels: list[str] | None = Field( - None, + labels: list[str] | None = Field( + default=None, + alias="Labels", description="User-defined labels (key/value metadata) as set on the daemon.\n\n


\n\n> **Note**: When part of a Swarm, nodes can both have _daemon_ labels,\n> set through the daemon configuration, and _node_ labels, set from a\n> manager node in the Swarm. Node labels are not included in this\n> field. Node labels can be retrieved using the `/nodes/(id)` endpoint\n> on a manager node in the Swarm.\n", examples=[["storage=ssd", "production"]], ) - ExperimentalBuild: bool | None = Field( - None, + experimental_build: bool | None = Field( + default=None, + alias="ExperimentalBuild", description="Indicates if experimental features are enabled on the daemon.\n", examples=[True], ) - ServerVersion: str | None = Field( - None, - description="Version string of the daemon.\n\n> **Note**: the [standalone Swarm API](/swarm/swarm-api/)\n> returns the Swarm version instead of the daemon version, for example\n> `swarm/1.2.8`.\n", - examples=["17.06.0-ce"], + server_version: str | None = Field( + default=None, + alias="ServerVersion", + description="Version string of the daemon.\n", + examples=["20.10.25"], ) - ClusterStore: str | None = Field( - None, + cluster_store: str | None = Field( + default=None, + alias="ClusterStore", description="URL of the distributed storage backend.\n\n\nThe storage backend is used for multihost networking (to store\nnetwork and endpoint information) and by the node discovery mechanism.\n\n


\n\n> **Deprecated**: This field is only propagated when using standalone Swarm\n> mode, and overlay networking using an external k/v store. Overlay\n> networks with Swarm mode enabled use the built-in raft store, and\n> this field will be empty.\n", examples=["consul://consul.corp.example.com:8600/some/path"], ) - ClusterAdvertise: str | None = Field( - None, + cluster_advertise: str | None = Field( + default=None, + alias="ClusterAdvertise", description="The network endpoint that the Engine advertises for the purpose of\nnode discovery. ClusterAdvertise is a `host:port` combination on which\nthe daemon is reachable by other hosts.\n\n


\n\n> **Deprecated**: This field is only propagated when using standalone Swarm\n> mode, and overlay networking using an external k/v store. Overlay\n> networks with Swarm mode enabled use the built-in raft store, and\n> this field will be empty.\n", examples=["node5.corp.example.com:8000"], ) - Runtimes: dict[str, Runtime] | None = Field( - {"runc": {"path": "runc"}}, + runtimes: dict[str, Runtime] | None = Field( + default_factory=lambda: Runtime.model_validate({"runc": {"path": "runc"}}), + alias="Runtimes", description='List of [OCI compliant](https://github.com/opencontainers/runtime-spec)\nruntimes configured on the daemon. Keys hold the "name" used to\nreference the runtime.\n\nThe Docker daemon relies on an OCI compliant runtime (invoked via the\n`containerd` daemon) as its interface to the Linux kernel namespaces,\ncgroups, and SELinux.\n\nThe default runtime is `runc`, and automatically configured. Additional\nruntimes can be configured by the user and will be listed here.\n', examples=[ { @@ -4107,31 +5193,36 @@ class SystemInfo(BaseModel): } ], ) - DefaultRuntime: str | None = Field( - "runc", + default_runtime: str | None = Field( + default="runc", + alias="DefaultRuntime", description="Name of the default OCI runtime that is used when starting containers.\n\nThe default can be overridden per-container at create time.\n", examples=["runc"], ) - Swarm: SwarmInfo | None = None - LiveRestoreEnabled: bool | None = Field( - False, + swarm: SwarmInfo | None = Field(default=None, alias="Swarm") + live_restore_enabled: bool | None = Field( + default=False, + alias="LiveRestoreEnabled", description="Indicates if live restore is enabled.\n\nIf enabled, containers are kept running when the daemon is shutdown\nor upon daemon start if running containers are detected.\n", examples=[False], ) - Isolation: Isolation2 | None = Field( - Isolation2.default, + isolation: Isolation2 | None = Field( + default=Isolation2.default, + alias="Isolation", description="Represents the isolation technology to use as a default for containers.\nThe supported values are platform-specific.\n\nIf no isolation value is specified on daemon start, on Windows client,\nthe default is `hyperv`, and on Windows server, the default is `process`.\n\nThis option is currently not used on other platforms.\n", ) - InitBinary: str | None = Field( - None, + init_binary: str | None = Field( + default=None, + alias="InitBinary", description="Name and, optional, path of the `docker-init` binary.\n\nIf the path is omitted, the daemon searches the host's `$PATH` for the\nbinary and uses the first result.\n", examples=["docker-init"], ) - ContainerdCommit: Commit | None = None - RuncCommit: Commit | None = None - InitCommit: Commit | None = None - SecurityOptions: list[str] | None = Field( - None, + containerd_commit: Commit | None = Field(default=None, alias="ContainerdCommit") + runc_commit: Commit | None = Field(default=None, alias="RuncCommit") + init_commit: Commit | None = Field(default=None, alias="InitCommit") + security_options: list[str] | None = Field( + default=None, + alias="SecurityOptions", description="List of security features that are enabled on the daemon, such as\napparmor, seccomp, SELinux, user-namespaces (userns), and rootless.\n\nAdditional configuration options for each security feature may\nbe present, and are included as a comma-separated list of key/value\npairs.\n", examples=[ [ @@ -4143,17 +5234,20 @@ class SystemInfo(BaseModel): ] ], ) - ProductLicense: str | None = Field( - None, + product_license: str | None = Field( + default=None, + alias="ProductLicense", description="Reports a summary of the product license on the daemon.\n\nIf a commercial license has been applied to the daemon, information\nsuch as number of nodes, and expiration are included.\n", examples=["Community Engine"], ) - DefaultAddressPools: list[DefaultAddressPool] | None = Field( - None, + default_address_pools: list[DefaultAddressPool] | None = Field( + default=None, + alias="DefaultAddressPools", description='List of custom default address pools for local networks, which can be\nspecified in the daemon.json file or dockerd option.\n\nExample: a Base "10.10.0.0/16" with Size 24 will define the set of 256\n10.10.[0-255].0/24 address pools.\n', ) - Warnings: list[str] | None = Field( - None, + warnings: list[str] | None = Field( + default=None, + alias="Warnings", description="List of warnings / informational messages about missing features, or\nissues related to the daemon configuration.\n\nThese messages can be printed by the client as information to the user.\n", examples=[ [ diff --git a/scripts/openapi-pydantic-models-generator.bash b/scripts/openapi-pydantic-models-generator.bash index 82cf503f1d6..788cb90e792 100755 --- a/scripts/openapi-pydantic-models-generator.bash +++ b/scripts/openapi-pydantic-models-generator.bash @@ -18,17 +18,24 @@ Build() { --load \ - < FastAPI: for name in _NOISY_LOGGERS: logging.getLogger(name).setLevel(quiet_level) - logger.info("app settings: %s", settings.json(indent=1)) + logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.AUTOSCALING_DEBUG, diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/errors.py b/services/autoscaling/src/simcore_service_autoscaling/core/errors.py index 398b1278806..e4294631224 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/errors.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/errors.py @@ -1,12 +1,7 @@ -from typing import Any - -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class AutoscalingRuntimeError(OsparcErrorMixin, RuntimeError): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) - msg_template: str = "Autoscaling unexpected error" diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index 600a8cd507c..d0763fd8070 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -1,6 +1,6 @@ import datetime from functools import cached_property -from typing import Any, ClassVar, Final, cast +from typing import Final, Self, cast from aws_library.ec2 import EC2InstanceBootSpecific, EC2Tags from fastapi import FastAPI @@ -14,14 +14,16 @@ from models_library.clusters import InternalClusterAuthentication from models_library.docker import DockerLabelKey from pydantic import ( + AliasChoices, AnyUrl, Field, NonNegativeInt, PositiveInt, - parse_obj_as, - root_validator, - validator, + TypeAdapter, + field_validator, + model_validator, ) +from pydantic_settings import SettingsConfigDict from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings @@ -42,10 +44,9 @@ class AutoscalingSSMSettings(SSMSettings): class AutoscalingEC2Settings(EC2Settings): - class Config(EC2Settings.Config): - env_prefix = AUTOSCALING_ENV_PREFIX - - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + env_prefix=AUTOSCALING_ENV_PREFIX, + json_schema_extra={ "examples": [ { f"{AUTOSCALING_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", @@ -54,7 +55,8 @@ class Config(EC2Settings.Config): f"{AUTOSCALING_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", } ], - } + }, + ) class EC2InstancesSettings(BaseCustomSettings): @@ -95,7 +97,7 @@ class EC2InstancesSettings(BaseCustomSettings): EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., - min_items=1, + min_length=1, description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " " this is required to start a new EC2 instance", @@ -132,7 +134,7 @@ class EC2InstancesSettings(BaseCustomSettings): description="ARN the EC2 instance should be attached to (example: arn:aws:iam::XXXXX:role/NAME), to disable pass an empty string", ) - @validator("EC2_INSTANCES_TIME_BEFORE_DRAINING") + @field_validator("EC2_INSTANCES_TIME_BEFORE_DRAINING") @classmethod def _ensure_draining_delay_time_is_in_range( cls, value: datetime.timedelta @@ -143,7 +145,7 @@ def _ensure_draining_delay_time_is_in_range( value = datetime.timedelta(minutes=1) return value - @validator("EC2_INSTANCES_TIME_BEFORE_TERMINATION") + @field_validator("EC2_INSTANCES_TIME_BEFORE_TERMINATION") @classmethod def _ensure_termination_delay_time_is_in_range( cls, value: datetime.timedelta @@ -154,14 +156,14 @@ def _ensure_termination_delay_time_is_in_range( value = datetime.timedelta(minutes=59) return value - @validator("EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("EC2_INSTANCES_ALLOWED_TYPES") @classmethod def _check_valid_instance_names_and_not_empty( cls, value: dict[str, EC2InstanceBootSpecific] ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(list[InstanceTypeType], list(value)) + TypeAdapter(list[InstanceTypeType]).validate_python(list(value)) if not value: # NOTE: Field( ... , min_items=...) cannot be used to contraint number of iterms in a dict @@ -225,36 +227,39 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # RUNTIME ----------------------------------------------------------- AUTOSCALING_DEBUG: bool = Field( - default=False, description="Debug mode", env=["AUTOSCALING_DEBUG", "DEBUG"] + default=False, + description="Debug mode", + validation_alias=AliasChoices("AUTOSCALING_DEBUG", "DEBUG"), ) - AUTOSCALING_REMOTE_DEBUG_PORT: PortInt = PortInt(3000) + AUTOSCALING_REMOTE_DEBUG_PORT: PortInt = 3000 AUTOSCALING_LOGLEVEL: LogLevel = Field( - LogLevel.INFO, env=["AUTOSCALING_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + LogLevel.INFO, + validation_alias=AliasChoices("AUTOSCALING_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) AUTOSCALING_EC2_ACCESS: AutoscalingEC2Settings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) AUTOSCALING_SSM_ACCESS: AutoscalingSSMSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) AUTOSCALING_EC2_INSTANCES: EC2InstancesSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) AUTOSCALING_NODES_MONITORING: NodesMonitoringSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) AUTOSCALING_POLL_INTERVAL: datetime.timedelta = Field( @@ -263,13 +268,21 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", ) - AUTOSCALING_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True) + AUTOSCALING_RABBITMQ: RabbitSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - AUTOSCALING_REDIS: RedisSettings = Field(auto_default_from_env=True) + AUTOSCALING_REDIS: RedisSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - AUTOSCALING_REGISTRY: RegistrySettings | None = Field(auto_default_from_env=True) + AUTOSCALING_REGISTRY: RegistrySettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - AUTOSCALING_DASK: DaskMonitoringSettings | None = Field(auto_default_from_env=True) + AUTOSCALING_DASK: DaskMonitoringSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) AUTOSCALING_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True @@ -280,7 +293,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): "but a docker node label named osparc-services-ready is attached", ) AUTOSCALING_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + description="settings for opentelemetry tracing", + json_schema_extra={"auto_default_from_env": True}, ) AUTOSCALING_DOCKER_JOIN_DRAINED: bool = Field( @@ -298,21 +312,20 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def LOG_LEVEL(self): # noqa: N802 return self.AUTOSCALING_LOGLEVEL - @validator("AUTOSCALING_LOGLEVEL", pre=True) + @field_validator("AUTOSCALING_LOGLEVEL", mode="before") @classmethod def _valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - @root_validator() - @classmethod - def _exclude_both_dynamic_computational_mode(cls, values): + @model_validator(mode="after") + def exclude_both_dynamic_computational_mode(self) -> Self: if ( - values.get("AUTOSCALING_DASK") is not None - and values.get("AUTOSCALING_NODES_MONITORING") is not None + self.AUTOSCALING_DASK is not None + and self.AUTOSCALING_NODES_MONITORING is not None ): msg = "Autoscaling cannot be set to monitor both computational and dynamic services (both AUTOSCALING_DASK and AUTOSCALING_NODES_MONITORING are currently set!)" raise ValueError(msg) - return values + return self def get_application_settings(app: FastAPI) -> ApplicationSettings: diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index e7fc947cba5..8d5ff16dd9a 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -63,8 +63,8 @@ def _node_not_ready(node: Node) -> bool: - assert node.Status # nosec - return bool(node.Status.State != NodeState.ready) + assert node.status # nosec + return bool(node.status.state != NodeState.ready) async def _analyze_current_cluster( @@ -177,9 +177,9 @@ async def _cleanup_disconnected_nodes(app: FastAPI, cluster: Cluster) -> Cluster removeable_nodes = [ node for node in cluster.disconnected_nodes - if node.UpdatedAt + if node.updated_at and ( - (utc_now - arrow.get(node.UpdatedAt).datetime).total_seconds() + (utc_now - arrow.get(node.updated_at).datetime).total_seconds() > _DELAY_FOR_REMOVING_DISCONNECTED_NODES_S ) ] @@ -886,7 +886,7 @@ async def _find_drainable_nodes( if drainable_nodes: _logger.info( "the following nodes were found to be drainable: '%s'", - f"{[instance.node.Description.Hostname for instance in drainable_nodes if instance.node.Description]}", + f"{[instance.node.description.hostname for instance in drainable_nodes if instance.node.description]}", ) return drainable_nodes @@ -914,7 +914,7 @@ async def _deactivate_empty_nodes(app: FastAPI, cluster: Cluster) -> Cluster: if updated_nodes: _logger.info( "following nodes were set to drain: '%s'", - f"{[node.Description.Hostname for node in updated_nodes if node.Description]}", + f"{[node.description.hostname for node in updated_nodes if node.description]}", ) newly_drained_instances = [ AssociatedInstance(node=node, ec2_instance=instance.ec2_instance) @@ -964,7 +964,7 @@ async def _find_terminateable_instances( if terminateable_nodes: _logger.info( "the following nodes were found to be terminateable: '%s'", - f"{[instance.node.Description.Hostname for instance in terminateable_nodes if instance.node.Description]}", + f"{[instance.node.description.hostname for instance in terminateable_nodes if instance.node.description]}", ) return terminateable_nodes @@ -975,11 +975,11 @@ async def _try_scale_down_cluster(app: FastAPI, cluster: Cluster) -> Cluster: # instances found to be terminateable will now start the termination process. new_terminating_instances = [] for instance in await _find_terminateable_instances(app, cluster): - assert instance.node.Description is not None # nosec + assert instance.node.description is not None # nosec with log_context( _logger, logging.INFO, - msg=f"termination process for {instance.node.Description.Hostname}:{instance.ec2_instance.id}", + msg=f"termination process for {instance.node.description.hostname}:{instance.ec2_instance.id}", ), log_catch(_logger, reraise=False): await utils_docker.set_node_begin_termination_process( get_docker_client(app), instance.node @@ -999,7 +999,7 @@ async def _try_scale_down_cluster(app: FastAPI, cluster: Cluster) -> Cluster: with log_context( _logger, logging.INFO, - msg=f"definitely terminate '{[i.node.Description.Hostname for i in instances_to_terminate if i.node.Description]}'", + msg=f"definitely terminate '{[i.node.description.hostname for i in instances_to_terminate if i.node.description]}'", ): await get_ec2_client(app).terminate_instances( [i.ec2_instance for i in instances_to_terminate] @@ -1103,7 +1103,7 @@ async def _drain_retired_nodes( if updated_nodes: _logger.info( "following nodes were set to drain: '%s'", - f"{[node.Description.Hostname for node in updated_nodes if node.Description]}", + f"{[node.description.hostname for node in updated_nodes if node.description]}", ) newly_drained_instances = [ AssociatedInstance(node=node, ec2_instance=instance.ec2_instance) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py index ecddfc5e8ec..cc6dcef68a4 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py @@ -145,16 +145,16 @@ async def compute_node_used_resources( async def compute_cluster_used_resources( app: FastAPI, instances: list[AssociatedInstance] ) -> Resources: - list_of_used_resources = await logged_gather( + list_of_used_resources: list[Resources] = await logged_gather( *( ComputationalAutoscaling.compute_node_used_resources(app, i) for i in instances ) ) - counter = collections.Counter({k: 0 for k in Resources.__fields__}) + counter = collections.Counter({k: 0 for k in Resources.model_fields}) for result in list_of_used_resources: - counter.update(result.dict()) - return Resources.parse_obj(dict(counter)) + counter.update(result.model_dump()) + return Resources.model_validate(dict(counter)) @staticmethod async def compute_cluster_total_resources( diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py index 5e1c7e2f0c7..b547ce2bbd4 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py @@ -13,7 +13,7 @@ from dask_task_models_library.resource_constraints import DaskTaskResources from distributed.core import Status from models_library.clusters import InternalClusterAuthentication, TLSAuthentication -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from ..core.errors import ( DaskNoWorkersError, @@ -59,7 +59,7 @@ async def _scheduler_client( require_encryption=True, ) async with distributed.Client( - url, + f"{url}", asynchronous=True, timeout=f"{_DASK_SCHEDULER_CONNECT_TIMEOUT_S}", security=security, @@ -173,9 +173,9 @@ def _list_tasks( } async with _scheduler_client(scheduler_url, authentication) as client: - list_of_tasks: dict[dask.typing.Key, DaskTaskResources] = ( - await _wrap_client_async_routine(client.run_on_scheduler(_list_tasks)) - ) + list_of_tasks: dict[ + dask.typing.Key, DaskTaskResources + ] = await _wrap_client_async_routine(client.run_on_scheduler(_list_tasks)) _logger.debug("found unrunnable tasks: %s", list_of_tasks) return [ DaskTask( @@ -207,10 +207,10 @@ def _list_processing_tasks( return worker_to_processing_tasks async with _scheduler_client(scheduler_url, authentication) as client: - worker_to_tasks: dict[str, list[tuple[dask.typing.Key, DaskTaskResources]]] = ( - await _wrap_client_async_routine( - client.run_on_scheduler(_list_processing_tasks) - ) + worker_to_tasks: dict[ + str, list[tuple[dask.typing.Key, DaskTaskResources]] + ] = await _wrap_client_async_routine( + client.run_on_scheduler(_list_processing_tasks) ) _logger.debug("found processing tasks: %s", worker_to_tasks) tasks_per_worker = defaultdict(list) @@ -276,12 +276,12 @@ def _list_processing_tasks_on_worker( _logger.debug("looking for processing tasksfor %s", f"{worker_url=}") # now get the used resources - worker_processing_tasks: list[tuple[dask.typing.Key, DaskTaskResources]] = ( - await _wrap_client_async_routine( - client.run_on_scheduler( - _list_processing_tasks_on_worker, worker_url=worker_url - ), - ) + worker_processing_tasks: list[ + tuple[dask.typing.Key, DaskTaskResources] + ] = await _wrap_client_async_routine( + client.run_on_scheduler( + _list_processing_tasks_on_worker, worker_url=worker_url + ), ) total_resources_used: collections.Counter[str] = collections.Counter() @@ -291,7 +291,9 @@ def _list_processing_tasks_on_worker( _logger.debug("found %s for %s", f"{total_resources_used=}", f"{worker_url=}") return Resources( cpus=total_resources_used.get("CPU", 0), - ram=parse_obj_as(ByteSize, total_resources_used.get("RAM", 0)), + ram=TypeAdapter(ByteSize).validate_python( + total_resources_used.get("RAM", 0) + ), ) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py index d5fca4c3bb6..d7f69d50b54 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py @@ -56,8 +56,8 @@ async def associate_ec2_instances_with_nodes( non_associated_instances: list[EC2InstanceData] = [] def _find_node_with_name(node: Node) -> bool: - assert node.Description # nosec - return bool(node.Description.Hostname == docker_node_name) + assert node.description # nosec + return bool(node.description.hostname == docker_node_name) for instance_data in ec2_instances: try: diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py index 133708001ae..dcbba97015d 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py @@ -1,11 +1,12 @@ from collections.abc import Iterable from operator import itemgetter +from typing import Final -from aws_library.ec2 import AWSTagKey, AWSTagValue, EC2Tags +from aws_library.ec2 import AWS_TAG_VALUE_MAX_LENGTH, AWSTagKey, AWSTagValue, EC2Tags from fastapi import FastAPI from models_library.docker import DockerGenericTag from models_library.utils.json_serialization import json_dumps -from pydantic import parse_obj_as, parse_raw_as +from pydantic import TypeAdapter from ..constants import ( ACTIVATED_BUFFER_MACHINE_EC2_TAGS, @@ -16,6 +17,8 @@ ) from ..modules.auto_scaling_mode_base import BaseAutoscaling +_NAME_EC2_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python("Name") + def get_activated_buffer_ec2_tags( app: FastAPI, auto_scaling_mode: BaseAutoscaling @@ -29,8 +32,8 @@ def get_deactivated_buffer_ec2_tags( base_ec2_tags = ( auto_scaling_mode.get_ec2_tags(app) | DEACTIVATED_BUFFER_MACHINE_EC2_TAGS ) - base_ec2_tags[AWSTagKey("Name")] = AWSTagValue( - f"{base_ec2_tags[AWSTagKey('Name')]}-buffer" + base_ec2_tags[_NAME_EC2_TAG_KEY] = AWSTagValue( + f"{base_ec2_tags[_NAME_EC2_TAG_KEY]}-buffer" ) return base_ec2_tags @@ -43,20 +46,26 @@ def dump_pre_pulled_images_as_tags(images: Iterable[DockerGenericTag]) -> EC2Tag # AWS Tag Values are limited to 256 characaters so we chunk the images # into smaller chunks jsonized_images = json_dumps(images) - assert AWSTagValue.max_length # nosec - if len(jsonized_images) > AWSTagValue.max_length: + assert AWS_TAG_VALUE_MAX_LENGTH # nosec + if len(jsonized_images) > AWS_TAG_VALUE_MAX_LENGTH: # let's chunk the string - chunk_size = AWSTagValue.max_length + chunk_size = AWS_TAG_VALUE_MAX_LENGTH chunks = [ jsonized_images[i : i + chunk_size] for i in range(0, len(jsonized_images), chunk_size) ] return { - AWSTagKey(f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_({i})"): AWSTagValue(c) + TypeAdapter(AWSTagKey) + .validate_python(f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_{i}"): TypeAdapter( + AWSTagValue + ) + .validate_python(c) for i, c in enumerate(chunks) } return { - PRE_PULLED_IMAGES_EC2_TAG_KEY: parse_obj_as(AWSTagValue, json_dumps(images)) + PRE_PULLED_IMAGES_EC2_TAG_KEY: TypeAdapter(AWSTagValue).validate_python( + json_dumps(images) + ) } @@ -64,7 +73,9 @@ def load_pre_pulled_images_from_tags(tags: EC2Tags) -> list[DockerGenericTag]: # AWS Tag values are limited to 256 characters so we chunk the images if PRE_PULLED_IMAGES_EC2_TAG_KEY in tags: # read directly - return parse_raw_as(list[DockerGenericTag], tags[PRE_PULLED_IMAGES_EC2_TAG_KEY]) + return TypeAdapter(list[DockerGenericTag]).validate_json( + tags[PRE_PULLED_IMAGES_EC2_TAG_KEY] + ) assembled_json = "".join( map( @@ -80,5 +91,5 @@ def load_pre_pulled_images_from_tags(tags: EC2Tags) -> list[DockerGenericTag]: ) ) if assembled_json: - return parse_raw_as(list[DockerGenericTag], assembled_json) + return TypeAdapter(list[DockerGenericTag]).validate_json(assembled_json) return [] diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py b/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py index cb1623cb476..81781fb5346 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py @@ -42,7 +42,7 @@ async def progress_tasks_message( async def post_task_progress_message(app: FastAPI, task: Task, progress: float) -> None: with log_catch(logger, reraise=False): simcore_label_keys = StandardSimcoreDockerLabels.from_docker_task(task) - message = ProgressRabbitMessageNode.construct( + message = ProgressRabbitMessageNode.model_construct( node_id=simcore_label_keys.node_id, user_id=simcore_label_keys.user_id, project_id=simcore_label_keys.project_id, @@ -55,7 +55,7 @@ async def post_task_progress_message(app: FastAPI, task: Task, progress: float) async def post_task_log_message(app: FastAPI, task: Task, log: str, level: int) -> None: with log_catch(logger, reraise=False): simcore_label_keys = StandardSimcoreDockerLabels.from_docker_task(task) - message = LoggerRabbitMessage.construct( + message = LoggerRabbitMessage.model_construct( node_id=simcore_label_keys.node_id, user_id=simcore_label_keys.user_id, project_id=simcore_label_keys.project_id, @@ -79,15 +79,15 @@ async def create_autoscaling_status_message( origin = f"dynamic:node_labels={app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_NODE_LABELS}" elif app_settings.AUTOSCALING_DASK: origin = f"computational:scheduler_url={app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL}" - return RabbitAutoscalingStatusMessage.construct( + return RabbitAutoscalingStatusMessage.model_construct( origin=origin, nodes_total=len(cluster.active_nodes) + len(cluster.drained_nodes) + len(cluster.buffer_drained_nodes), nodes_active=len(cluster.active_nodes), nodes_drained=len(cluster.drained_nodes) + len(cluster.buffer_drained_nodes), - cluster_total_resources=cluster_total_resources.dict(), - cluster_used_resources=cluster_used_resources.dict(), + cluster_total_resources=cluster_total_resources.model_dump(), + cluster_used_resources=cluster_used_resources.model_dump(), instances_pending=len(cluster.pending_ec2s), instances_running=len(cluster.active_nodes) + len(cluster.drained_nodes) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/redis.py b/services/autoscaling/src/simcore_service_autoscaling/utils/redis.py index 02644975952..b5ad337c872 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/redis.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/redis.py @@ -22,10 +22,10 @@ def create_lock_key_and_value(app: FastAPI) -> tuple[str, str]: elif app_settings.AUTOSCALING_DASK: lock_key_parts += [ "computational", - app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL, + f"{app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL}", ] lock_value = json.dumps( - {"scheduler_url": app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL} + {"scheduler_url": f"{app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL}"} ) lock_key = ":".join(f"{k}" for k in lock_key_parts) return lock_key, lock_value diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py index 6449952decd..4758c91a12f 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py @@ -29,7 +29,7 @@ Task, TaskState, ) -from pydantic import ByteSize, ValidationError, parse_obj_as +from pydantic import ByteSize, TypeAdapter, ValidationError from servicelib.docker_utils import to_datetime from servicelib.logging_utils import log_context from servicelib.utils import logged_gather @@ -59,25 +59,27 @@ _PENDING_DOCKER_TASK_MESSAGE: Final[str] = "pending task scheduling" _INSUFFICIENT_RESOURCES_DOCKER_TASK_ERR: Final[str] = "insufficient resources on" _NOT_SATISFIED_SCHEDULING_CONSTRAINTS_TASK_ERR: Final[str] = "no suitable node" -_OSPARC_SERVICE_READY_LABEL_KEY: Final[DockerLabelKey] = parse_obj_as( - DockerLabelKey, "io.simcore.osparc-services-ready" -) -_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY: Final[DockerLabelKey] = parse_obj_as( - DockerLabelKey, f"{_OSPARC_SERVICE_READY_LABEL_KEY}-last-changed" +_OSPARC_SERVICE_READY_LABEL_KEY: Final[DockerLabelKey] = TypeAdapter( + DockerLabelKey +).validate_python( + "io.simcore.osparc-services-ready", ) +_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY: Final[DockerLabelKey] = TypeAdapter( + DockerLabelKey +).validate_python(f"{_OSPARC_SERVICE_READY_LABEL_KEY}-last-changed") _OSPARC_SERVICE_READY_LABEL_KEYS: Final[list[DockerLabelKey]] = [ _OSPARC_SERVICE_READY_LABEL_KEY, _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY, ] -_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY: Final[DockerLabelKey] = parse_obj_as( - DockerLabelKey, "io.simcore.osparc-node-found-empty" -) +_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY: Final[DockerLabelKey] = TypeAdapter( + DockerLabelKey +).validate_python("io.simcore.osparc-node-found-empty") -_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY: Final[DockerLabelKey] = parse_obj_as( - DockerLabelKey, "io.simcore.osparc-node-termination-started" -) +_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY: Final[DockerLabelKey] = TypeAdapter( + DockerLabelKey +).validate_python("io.simcore.osparc-node-termination-started") async def get_monitored_nodes( @@ -86,15 +88,13 @@ async def get_monitored_nodes( node_label_filters = [f"{label}=true" for label in node_labels] + [ f"{label}" for label in _OSPARC_SERVICE_READY_LABEL_KEYS ] - return parse_obj_as( - list[Node], - await docker_client.nodes.list(filters={"node.label": node_label_filters}), + return TypeAdapter(list[Node]).validate_python( + await docker_client.nodes.list(filters={"node.label": node_label_filters}) ) async def get_worker_nodes(docker_client: AutoscalingDocker) -> list[Node]: - return parse_obj_as( - list[Node], + return TypeAdapter(list[Node]).validate_python( await docker_client.nodes.list( filters={ "role": ["worker"], @@ -102,7 +102,7 @@ async def get_worker_nodes(docker_client: AutoscalingDocker) -> list[Node]: f"{label}" for label in _OSPARC_SERVICE_READY_LABEL_KEYS ], } - ), + ) ) @@ -112,8 +112,8 @@ async def remove_nodes( """removes docker nodes that are in the down state (unless force is used and they will be forcibly removed)""" def _check_if_node_is_removable(node: Node) -> bool: - if node.Status and node.Status.State: - return node.Status.State in [ + if node.status and node.status.state: + return node.status.state in [ NodeState.down, NodeState.disconnected, NodeState.unknown, @@ -129,30 +129,30 @@ def _check_if_node_is_removable(node: Node) -> bool: n for n in nodes if (force is True) or _check_if_node_is_removable(n) ] for node in nodes_that_need_removal: - assert node.ID # nosec - with log_context(logger, logging.INFO, msg=f"remove {node.ID=}"): - await docker_client.nodes.remove(node_id=node.ID, force=force) + assert node.id # nosec + with log_context(logger, logging.INFO, msg=f"remove {node.id=}"): + await docker_client.nodes.remove(node_id=node.id, force=force) return nodes_that_need_removal def _is_task_waiting_for_resources(task: Task) -> bool: # NOTE: https://docs.docker.com/engine/swarm/how-swarm-mode-works/swarm-task-states/ with log_context( - logger, level=logging.DEBUG, msg=f"_is_task_waiting_for_resources: {task.ID}" + logger, level=logging.DEBUG, msg=f"_is_task_waiting_for_resources: {task.id}" ): if ( - not task.Status - or not task.Status.State - or not task.Status.Message - or not task.Status.Err + not task.status + or not task.status.state + or not task.status.message + or not task.status.err ): return False return ( - task.Status.State == TaskState.pending - and task.Status.Message == _PENDING_DOCKER_TASK_MESSAGE + task.status.state == TaskState.pending + and task.status.message == _PENDING_DOCKER_TASK_MESSAGE and ( - _INSUFFICIENT_RESOURCES_DOCKER_TASK_ERR in task.Status.Err - or _NOT_SATISFIED_SCHEDULING_CONSTRAINTS_TASK_ERR in task.Status.Err + _INSUFFICIENT_RESOURCES_DOCKER_TASK_ERR in task.status.err + or _NOT_SATISFIED_SCHEDULING_CONSTRAINTS_TASK_ERR in task.status.err ) ) @@ -160,21 +160,21 @@ def _is_task_waiting_for_resources(task: Task) -> bool: async def _associated_service_has_no_node_placement_contraints( docker_client: AutoscalingDocker, task: Task ) -> bool: - assert task.ServiceID # nosec - service_inspect = parse_obj_as( - Service, await docker_client.services.inspect(task.ServiceID) + assert task.service_id # nosec + service_inspect = TypeAdapter(Service).validate_python( + await docker_client.services.inspect(task.service_id) ) - assert service_inspect.Spec # nosec - assert service_inspect.Spec.TaskTemplate # nosec + assert service_inspect.spec # nosec + assert service_inspect.spec.task_template # nosec if ( - not service_inspect.Spec.TaskTemplate.Placement - or not service_inspect.Spec.TaskTemplate.Placement.Constraints + not service_inspect.spec.task_template.placement + or not service_inspect.spec.task_template.placement.constraints ): return True # parse the placement contraints service_placement_constraints = ( - service_inspect.Spec.TaskTemplate.Placement.Constraints + service_inspect.spec.task_template.placement.constraints ) for constraint in service_placement_constraints: # is of type node.id==alskjladskjs or node.hostname==thiscomputerhostname or node.role==manager, sometimes with spaces... @@ -186,15 +186,13 @@ async def _associated_service_has_no_node_placement_contraints( def _by_created_dt(task: Task) -> datetime.datetime: - # NOTE: SAFE implementation to extract task.CreatedAt as datetime for comparison - if task.CreatedAt: + # NOTE: SAFE implementation to extract task.created_at as datetime for comparison + if task.created_at: with suppress(ValueError): - created_at = to_datetime(task.CreatedAt) - created_at_utc: datetime.datetime = created_at.replace( - tzinfo=datetime.timezone.utc - ) + created_at = to_datetime(task.created_at) + created_at_utc: datetime.datetime = created_at.replace(tzinfo=datetime.UTC) return created_at_utc - return datetime.datetime.now(datetime.timezone.utc) + return datetime.datetime.now(datetime.UTC) async def pending_service_tasks_with_insufficient_resources( @@ -209,20 +207,19 @@ async def pending_service_tasks_with_insufficient_resources( - have an error message with "insufficient resources" - are not scheduled on any node """ - tasks = parse_obj_as( - list[Task], + tasks = TypeAdapter(list[Task]).validate_python( await docker_client.tasks.list( filters={ "desired-state": "running", "label": service_labels, } - ), + ) ) sorted_tasks = sorted(tasks, key=_by_created_dt) logger.debug( "found following tasks that might trigger autoscaling: %s", - [task.ID for task in tasks], + [task.id for task in tasks], ) return [ @@ -238,13 +235,13 @@ async def pending_service_tasks_with_insufficient_resources( def get_node_total_resources(node: Node) -> Resources: - assert node.Description # nosec - assert node.Description.Resources # nosec - assert node.Description.Resources.NanoCPUs # nosec - assert node.Description.Resources.MemoryBytes # nosec + assert node.description # nosec + assert node.description.resources # nosec + assert node.description.resources.nano_cp_us # nosec + assert node.description.resources.memory_bytes # nosec return Resources( - cpus=node.Description.Resources.NanoCPUs / _NANO_CPU, - ram=ByteSize(node.Description.Resources.MemoryBytes), + cpus=node.description.resources.nano_cp_us / _NANO_CPU, + ram=ByteSize(node.description.resources.memory_bytes), ) @@ -254,47 +251,46 @@ async def compute_cluster_total_resources(nodes: list[Node]) -> Resources: """ cluster_resources_counter = collections.Counter({"ram": 0, "cpus": 0}) for node in nodes: - assert node.Description # nosec - assert node.Description.Resources # nosec - assert node.Description.Resources.NanoCPUs # nosec + assert node.description # nosec + assert node.description.resources # nosec + assert node.description.resources.nano_cp_us # nosec cluster_resources_counter.update( { - "ram": node.Description.Resources.MemoryBytes, - "cpus": node.Description.Resources.NanoCPUs / _NANO_CPU, + "ram": node.description.resources.memory_bytes, + "cpus": node.description.resources.nano_cp_us / _NANO_CPU, } ) - return Resources.parse_obj(dict(cluster_resources_counter)) + return Resources.model_validate(dict(cluster_resources_counter)) def get_max_resources_from_docker_task(task: Task) -> Resources: """returns the highest values for resources based on both docker reservations and limits""" - assert task.Spec # nosec - if task.Spec.Resources: + assert task.spec # nosec + if task.spec.resources: return Resources( cpus=max( ( - task.Spec.Resources.Reservations - and task.Spec.Resources.Reservations.NanoCPUs + task.spec.resources.reservations + and task.spec.resources.reservations.nano_cp_us or 0 ), ( - task.Spec.Resources.Limits - and task.Spec.Resources.Limits.NanoCPUs + task.spec.resources.limits + and task.spec.resources.limits.nano_cp_us or 0 ), ) / _NANO_CPU, - ram=parse_obj_as( - ByteSize, + ram=TypeAdapter(ByteSize).validate_python( max( - task.Spec.Resources.Reservations - and task.Spec.Resources.Reservations.MemoryBytes + task.spec.resources.reservations + and task.spec.resources.reservations.memory_bytes or 0, - task.Spec.Resources.Limits - and task.Spec.Resources.Limits.MemoryBytes + task.spec.resources.limits + and task.spec.resources.limits.memory_bytes or 0, - ), + ) ), ) return Resources(cpus=0, ram=ByteSize(0)) @@ -304,21 +300,21 @@ async def get_task_instance_restriction( docker_client: AutoscalingDocker, task: Task ) -> InstanceTypeType | None: with contextlib.suppress(ValidationError): - assert task.ServiceID # nosec - service_inspect = parse_obj_as( - Service, await docker_client.services.inspect(task.ServiceID) + assert task.service_id # nosec + service_inspect = TypeAdapter(Service).validate_python( + await docker_client.services.inspect(task.service_id) ) - assert service_inspect.Spec # nosec - assert service_inspect.Spec.TaskTemplate # nosec + assert service_inspect.spec # nosec + assert service_inspect.spec.task_template # nosec if ( - not service_inspect.Spec.TaskTemplate.Placement - or not service_inspect.Spec.TaskTemplate.Placement.Constraints + not service_inspect.spec.task_template.placement + or not service_inspect.spec.task_template.placement.constraints ): return None # parse the placement contraints service_placement_constraints = ( - service_inspect.Spec.TaskTemplate.Placement.Constraints + service_inspect.spec.task_template.placement.constraints ) # should be node.labels.{} node_label_to_find = ( @@ -326,8 +322,8 @@ async def get_task_instance_restriction( ) for constraint in service_placement_constraints: if constraint.startswith(node_label_to_find): - return parse_obj_as( - InstanceTypeType, constraint.removeprefix(node_label_to_find) # type: ignore[arg-type] + return TypeAdapter(InstanceTypeType).validate_python( + constraint.removeprefix(node_label_to_find) ) return None @@ -347,30 +343,29 @@ async def compute_node_used_resources( service_labels: list[DockerLabelKey] | None = None, ) -> Resources: cluster_resources_counter = collections.Counter({"ram": 0, "cpus": 0}) - assert node.ID # nosec - task_filters: dict[str, str | list[DockerLabelKey]] = {"node": node.ID} + assert node.id # nosec + task_filters: dict[str, str | list[DockerLabelKey]] = {"node": node.id} if service_labels is not None: task_filters |= {"label": service_labels} - all_tasks_on_node = parse_obj_as( - list[Task], - await docker_client.tasks.list(filters=task_filters), + all_tasks_on_node = TypeAdapter(list[Task]).validate_python( + await docker_client.tasks.list(filters=task_filters) ) for task in all_tasks_on_node: - assert task.Status # nosec + assert task.status # nosec if ( - task.Status.State in _TASK_STATUS_WITH_ASSIGNED_RESOURCES - and task.Spec - and task.Spec.Resources - and task.Spec.Resources.Reservations + task.status.state in _TASK_STATUS_WITH_ASSIGNED_RESOURCES + and task.spec + and task.spec.resources + and task.spec.resources.reservations ): - task_reservations = task.Spec.Resources.Reservations.dict(exclude_none=True) cluster_resources_counter.update( { - "ram": task_reservations.get("MemoryBytes", 0), - "cpus": task_reservations.get("NanoCPUs", 0) / _NANO_CPU, + "ram": task.spec.resources.reservations.memory_bytes or 0, + "cpus": (task.spec.resources.reservations.nano_cp_us or 0) + / _NANO_CPU, } ) - return Resources.parse_obj(dict(cluster_resources_counter)) + return Resources.model_validate(dict(cluster_resources_counter)) async def compute_cluster_used_resources( @@ -380,11 +375,11 @@ async def compute_cluster_used_resources( list_of_used_resources = await logged_gather( *(compute_node_used_resources(docker_client, node) for node in nodes) ) - counter = collections.Counter({k: 0 for k in Resources.__fields__}) + counter = collections.Counter({k: 0 for k in Resources.model_fields}) for result in list_of_used_resources: - counter.update(result.dict()) + counter.update(result.model_dump()) - return Resources.parse_obj(dict(counter)) + return Resources.model_validate(dict(counter)) _COMMAND_TIMEOUT_S = 10 @@ -446,10 +441,7 @@ def write_compose_file_command( }, } compose_yaml = yaml.safe_dump(compose) - write_compose_file_cmd = " ".join( - ["echo", f'"{compose_yaml}"', ">", f"{_PRE_PULL_COMPOSE_PATH}"] - ) - return write_compose_file_cmd + return " ".join(["echo", f'"{compose_yaml}"', ">", f"{_PRE_PULL_COMPOSE_PATH}"]) def get_docker_pull_images_on_start_bash_command( @@ -504,10 +496,10 @@ async def find_node_with_name( if not list_of_nodes: return None # note that there might be several nodes with a common_prefixed name. so now we want exact matching - parsed_list_of_nodes = parse_obj_as(list[Node], list_of_nodes) + parsed_list_of_nodes = TypeAdapter(list[Node]).validate_python(list_of_nodes) for node in parsed_list_of_nodes: - assert node.Description # nosec - if node.Description.Hostname == name: + assert node.description # nosec + if node.description.hostname == name: return node return None @@ -521,39 +513,41 @@ async def tag_node( available: bool, ) -> Node: with log_context( - logger, logging.DEBUG, msg=f"tagging {node.ID=} with {tags=} and {available=}" + logger, logging.DEBUG, msg=f"tagging {node.id=} with {tags=} and {available=}" ): - assert node.ID # nosec + assert node.id # nosec - latest_version_node = parse_obj_as( - Node, await docker_client.nodes.inspect(node_id=node.ID) + latest_version_node = TypeAdapter(Node).validate_python( + await docker_client.nodes.inspect(node_id=node.id) ) - assert latest_version_node.Version # nosec - assert latest_version_node.Version.Index # nosec - assert latest_version_node.Spec # nosec - assert latest_version_node.Spec.Role # nosec + assert latest_version_node.version # nosec + assert latest_version_node.version.index # nosec + assert latest_version_node.spec # nosec + assert latest_version_node.spec.role # nosec # updating now should work nicely await docker_client.nodes.update( - node_id=node.ID, - version=latest_version_node.Version.Index, + node_id=node.id, + version=latest_version_node.version.index, spec={ "Availability": "active" if available else "drain", "Labels": tags, - "Role": latest_version_node.Spec.Role.value, + "Role": latest_version_node.spec.role.value, }, ) - return parse_obj_as(Node, await docker_client.nodes.inspect(node_id=node.ID)) + return TypeAdapter(Node).validate_python( + await docker_client.nodes.inspect(node_id=node.id) + ) async def set_node_availability( docker_client: AutoscalingDocker, node: Node, *, available: bool ) -> Node: - assert node.Spec # nosec + assert node.spec # nosec return await tag_node( docker_client, node, - tags=cast(dict[DockerLabelKey, str], node.Spec.Labels), + tags=cast(dict[DockerLabelKey, str], node.spec.labels), available=available, ) @@ -576,21 +570,21 @@ def get_new_node_docker_tags( def is_node_ready_and_available(node: Node, *, availability: Availability) -> bool: - assert node.Status # nosec - assert node.Spec # nosec + assert node.status # nosec + assert node.spec # nosec return bool( - node.Status.State == NodeState.ready and node.Spec.Availability == availability + node.status.state == NodeState.ready and node.spec.availability == availability ) def is_node_osparc_ready(node: Node) -> bool: if not is_node_ready_and_available(node, availability=Availability.active): return False - assert node.Spec # nosec + assert node.spec # nosec return bool( - node.Spec.Labels - and _OSPARC_SERVICE_READY_LABEL_KEY in node.Spec.Labels - and node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] == "true" + node.spec.labels + and _OSPARC_SERVICE_READY_LABEL_KEY in node.spec.labels + and node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] == "true" ) @@ -601,8 +595,8 @@ async def set_node_osparc_ready( *, ready: bool, ) -> Node: - assert node.Spec # nosec - new_tags = deepcopy(cast(dict[DockerLabelKey, str], node.Spec.Labels)) + assert node.spec # nosec + new_tags = deepcopy(cast(dict[DockerLabelKey, str], node.spec.labels)) new_tags[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" if ready else "false" new_tags[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] = arrow.utcnow().isoformat() # NOTE: docker drain sometimes impeed on performance when undraining see https://github.com/ITISFoundation/osparc-simcore/issues/5339 @@ -616,10 +610,10 @@ async def set_node_osparc_ready( def get_node_last_readyness_update(node: Node) -> datetime.datetime: - assert node.Spec # nosec - assert node.Spec.Labels # nosec + assert node.spec # nosec + assert node.spec.labels # nosec return arrow.get( - node.Spec.Labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] + node.spec.labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] ).datetime @@ -629,8 +623,8 @@ async def set_node_found_empty( *, empty: bool, ) -> Node: - assert node.Spec # nosec - new_tags = deepcopy(cast(dict[DockerLabelKey, str], node.Spec.Labels)) + assert node.spec # nosec + new_tags = deepcopy(cast(dict[DockerLabelKey, str], node.spec.labels)) if empty: new_tags[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY] = arrow.utcnow().isoformat() else: @@ -639,25 +633,25 @@ async def set_node_found_empty( docker_client, node, tags=new_tags, - available=bool(node.Spec.Availability is Availability.active), + available=bool(node.spec.availability is Availability.active), ) async def get_node_empty_since(node: Node) -> datetime.datetime | None: """returns the last time when the node was found empty or None if it was not empty""" - assert node.Spec # nosec - assert node.Spec.Labels # nosec - if _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY not in node.Spec.Labels: + assert node.spec # nosec + assert node.spec.labels # nosec + if _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY not in node.spec.labels: return None - return arrow.get(node.Spec.Labels[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY]).datetime + return arrow.get(node.spec.labels[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY]).datetime async def set_node_begin_termination_process( docker_client: AutoscalingDocker, node: Node ) -> Node: """sets the node to drain and adds a docker label with the time""" - assert node.Spec # nosec - new_tags = deepcopy(cast(dict[DockerLabelKey, str], node.Spec.Labels)) + assert node.spec # nosec + new_tags = deepcopy(cast(dict[DockerLabelKey, str], node.spec.labels)) new_tags[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] = arrow.utcnow().isoformat() return await tag_node( @@ -669,12 +663,12 @@ async def set_node_begin_termination_process( def get_node_termination_started_since(node: Node) -> datetime.datetime | None: - assert node.Spec # nosec - assert node.Spec.Labels # nosec - if _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY not in node.Spec.Labels: + assert node.spec # nosec + assert node.spec.labels # nosec + if _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY not in node.spec.labels: return None return arrow.get( - node.Spec.Labels[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] + node.spec.labels[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] ).datetime @@ -685,8 +679,8 @@ async def attach_node( *, tags: dict[DockerLabelKey, str], ) -> Node: - assert node.Spec # nosec - current_tags = cast(dict[DockerLabelKey, str], node.Spec.Labels or {}) + assert node.spec # nosec + current_tags = cast(dict[DockerLabelKey, str], node.spec.labels or {}) new_tags = current_tags | tags | {_OSPARC_SERVICE_READY_LABEL_KEY: "false"} new_tags[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] = arrow.utcnow().isoformat() return await tag_node( diff --git a/services/autoscaling/tests/manual/.env-devel b/services/autoscaling/tests/manual/.env-devel index a7069054e6a..e654a4df523 100644 --- a/services/autoscaling/tests/manual/.env-devel +++ b/services/autoscaling/tests/manual/.env-devel @@ -3,7 +3,7 @@ AUTOSCALING_DRAIN_NODES_WITH_LABELS=False AUTOSCALING_DOCKER_JOIN_DRAINED=True AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION=False AUTOSCALING_LOGLEVEL=INFO -AUTOSCALING_POLL_INTERVAL=10 +AUTOSCALING_POLL_INTERVAL="00:00:10" AUTOSCALING_EC2_ACCESS_KEY_ID=XXXXXXXXXX AUTOSCALING_EC2_SECRET_ACCESS_KEY=XXXXXXXXXX AUTOSCALING_EC2_ENDPOINT=null diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index b705ea85b78..5492abb86e3 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -43,8 +43,9 @@ ObjectVersion, ResourceObject, Service, + TaskSpec, ) -from pydantic import ByteSize, PositiveInt, parse_obj_as +from pydantic import ByteSize, PositiveInt, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.logging_tools import log_context @@ -118,7 +119,7 @@ def mocked_ec2_server_envs( # NOTE: overrides the EC2Settings with what autoscaling expects changed_envs: EnvVarsDict = { f"{AUTOSCALING_ENV_PREFIX}{k}": v - for k, v in mocked_ec2_server_settings.dict().items() + for k, v in mocked_ec2_server_settings.model_dump().items() } return setenvs_from_dict(monkeypatch, changed_envs) # type: ignore @@ -173,7 +174,8 @@ def app_with_docker_join_drained( @pytest.fixture(scope="session") def fake_ssm_settings() -> SSMSettings: - return SSMSettings(**SSMSettings.Config.schema_extra["examples"][0]) + assert "json_schema_extra" in SSMSettings.model_config + return SSMSettings(**SSMSettings.model_config["json_schema_extra"]["examples"][0]) @pytest.fixture @@ -213,7 +215,6 @@ def app_environment( external_envfile_dict: EnvVarsDict, ) -> EnvVarsDict: # SEE https://faker.readthedocs.io/en/master/providers/faker.providers.internet.html?highlight=internet#faker-providers-internet - if external_envfile_dict: delenvs_from_dict(monkeypatch, mock_env_devel_environment, raising=False) return setenvs_from_dict(monkeypatch, {**external_envfile_dict}) @@ -236,7 +237,9 @@ def app_environment( "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( { ec2_type_name: random.choice( # noqa: S311 - EC2InstanceBootSpecific.Config.schema_extra["examples"] + EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ] ) for ec2_type_name in aws_allowed_ec2_instance_type_names } @@ -267,7 +270,9 @@ def mocked_ec2_instances_envs( "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( { ec2_type_name: random.choice( # noqa: S311 - EC2InstanceBootSpecific.Config.schema_extra["examples"] + EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ] ) | {"ami_id": aws_ami_id} for ec2_type_name in aws_allowed_ec2_instance_type_names @@ -436,49 +441,51 @@ async def host_node( docker_swarm: None, async_docker_client: aiodocker.Docker, ) -> AsyncIterator[DockerNode]: - nodes = parse_obj_as(list[DockerNode], await async_docker_client.nodes.list()) + nodes = TypeAdapter(list[DockerNode]).validate_python( + await async_docker_client.nodes.list() + ) assert len(nodes) == 1 # keep state of node for later revert old_node = deepcopy(nodes[0]) - assert old_node.ID - assert old_node.Spec - assert old_node.Spec.Role - assert old_node.Spec.Availability - assert old_node.Version - assert old_node.Version.Index - labels = old_node.Spec.Labels or {} + assert old_node.id + assert old_node.spec + assert old_node.spec.role + assert old_node.spec.availability + assert old_node.version + assert old_node.version.index + labels = old_node.spec.labels or {} # ensure we have the necessary labels await async_docker_client.nodes.update( - node_id=old_node.ID, - version=old_node.Version.Index, + node_id=old_node.id, + version=old_node.version.index, spec={ - "Availability": old_node.Spec.Availability.value, + "Availability": old_node.spec.availability.value, "Labels": labels | { _OSPARC_SERVICE_READY_LABEL_KEY: "true", _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY: arrow.utcnow().isoformat(), }, - "Role": old_node.Spec.Role.value, + "Role": old_node.spec.role.value, }, ) - modified_host_node = parse_obj_as( - DockerNode, await async_docker_client.nodes.inspect(node_id=old_node.ID) + modified_host_node = TypeAdapter(DockerNode).validate_python( + await async_docker_client.nodes.inspect(node_id=old_node.id) ) yield modified_host_node # revert state - current_node = parse_obj_as( - DockerNode, await async_docker_client.nodes.inspect(node_id=old_node.ID) + current_node = TypeAdapter(DockerNode).validate_python( + await async_docker_client.nodes.inspect(node_id=old_node.id) ) - assert current_node.ID - assert current_node.Version - assert current_node.Version.Index + assert current_node.id + assert current_node.version + assert current_node.version.index await async_docker_client.nodes.update( - node_id=current_node.ID, - version=current_node.Version.Index, + node_id=current_node.id, + version=current_node.version.index, spec={ - "Availability": old_node.Spec.Availability.value, - "Labels": old_node.Spec.Labels, - "Role": old_node.Spec.Role.value, + "Availability": old_node.spec.availability.value, + "Labels": old_node.spec.labels, + "Role": old_node.spec.role.value, }, ) @@ -596,46 +603,54 @@ async def _creator( labels=base_labels, # type: ignore ) assert service - service = parse_obj_as( - Service, await async_docker_client.services.inspect(service["ID"]) + service = TypeAdapter(Service).validate_python( + await async_docker_client.services.inspect(service["ID"]) ) - assert service.Spec + assert service.spec ctx.logger.info( "%s", - f"service {service.ID} with {service.Spec.Name} created", + f"service {service.id} with {service.spec.name} created", ) - assert service.Spec.Labels == base_labels + assert service.spec.labels == base_labels created_services.append(service) # get more info on that service - assert service.Spec.Name == service_name + assert service.spec.name == service_name + + original_task_template_model = TypeAdapter(TaskSpec).validate_python( + task_template + ) + excluded_paths = { - "ForceUpdate", - "Runtime", - "root['ContainerSpec']['Isolation']", + "force_update", + "runtime", + "root['container_spec']['isolation']", } if not base_labels: - excluded_paths.add("root['ContainerSpec']['Labels']") - for reservation in ["MemoryBytes", "NanoCPUs"]: + excluded_paths.add("root['container_spec']['labels']") + for reservation in ["memory_bytes", "nano_cp_us"]: if ( - task_template.get("Resources", {}) - .get("Reservations", {}) - .get(reservation, 0) + original_task_template_model.resources + and original_task_template_model.resources.reservations + and getattr( + original_task_template_model.resources.reservations, reservation + ) == 0 ): # NOTE: if a 0 memory reservation is done, docker removes it from the task inspection excluded_paths.add( - f"root['Resources']['Reservations']['{reservation}']" + f"root['resources']['reservations']['{reservation}']" ) - assert service.Spec.TaskTemplate + + assert service.spec.task_template diff = DeepDiff( - task_template, - service.Spec.TaskTemplate.dict(exclude_unset=True), + original_task_template_model.model_dump(exclude_unset=True), + service.spec.task_template.model_dump(exclude_unset=True), exclude_paths=list(excluded_paths), ) assert not diff, f"{diff}" - assert service.Spec.Labels == base_labels + assert service.spec.labels == base_labels await _assert_wait_for_service_state( async_docker_client, service, [wait_for_service_state] ) @@ -644,7 +659,7 @@ async def _creator( yield _creator await asyncio.gather( - *(async_docker_client.services.delete(s.ID) for s in created_services), + *(async_docker_client.services.delete(s.id) for s in created_services), return_exceptions=True, ) @@ -656,15 +671,15 @@ async def _creator( stop=stop_after_delay(30), ) async def _check_service_task_gone(service: Service) -> None: - assert service.Spec + assert service.spec with log_context( logging.INFO, - msg=f"check service {service.ID}:{service.Spec.Name} is really gone", + msg=f"check service {service.id}:{service.spec.name} is really gone", ): assert not await async_docker_client.containers.list( all=True, filters={ - "label": [f"com.docker.swarm.service.id={service.ID}"], + "label": [f"com.docker.swarm.service.id={service.id}"], }, ) @@ -680,7 +695,7 @@ async def _assert_wait_for_service_state( async_docker_client: aiodocker.Docker, service: Service, expected_states: list[str] ) -> None: with log_context( - logging.INFO, msg=f"wait for service {service.ID} to become {expected_states}" + logging.INFO, msg=f"wait for service {service.id} to become {expected_states}" ) as ctx: number_of_success = {"count": 0} @@ -694,9 +709,9 @@ async def _assert_wait_for_service_state( ) async def _() -> None: services = await async_docker_client.services.list( - filters={"id": service.ID} + filters={"id": service.id} ) - assert services, f"no service with {service.ID}!" + assert services, f"no service with {service.id}!" assert len(services) == 1 found_service = services[0] @@ -762,7 +777,7 @@ def host_memory_total() -> ByteSize: def osparc_docker_label_keys( faker: Faker, ) -> StandardSimcoreDockerLabels: - return StandardSimcoreDockerLabels.parse_obj( + return StandardSimcoreDockerLabels.model_validate( { "user_id": faker.pyint(), "project_id": faker.uuid4(), @@ -838,11 +853,11 @@ async def _fake_set_node_availability( docker_client: AutoscalingDocker, node: DockerNode, *, available: bool ) -> DockerNode: returned_node = deepcopy(node) - assert returned_node.Spec - returned_node.Spec.Availability = ( + assert returned_node.spec + returned_node.spec.availability = ( Availability.active if available else Availability.drain ) - returned_node.UpdatedAt = datetime.datetime.now( + returned_node.updated_at = datetime.datetime.now( tz=datetime.timezone.utc ).isoformat() return returned_node @@ -864,9 +879,9 @@ async def fake_tag_node( available: bool, ) -> DockerNode: updated_node = deepcopy(node) - assert updated_node.Spec - updated_node.Spec.Labels = deepcopy(cast(dict[str, str], tags)) - updated_node.Spec.Availability = ( + assert updated_node.spec + updated_node.spec.labels = deepcopy(cast(dict[str, str], tags)) + updated_node.spec.availability = ( Availability.active if available else Availability.drain ) return updated_node diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index 353aabf31a4..e3c22afddac 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -42,7 +42,7 @@ async def test_status_no_rabbit( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is False @@ -66,7 +66,7 @@ async def test_status_no_ssm( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is False @@ -94,7 +94,7 @@ async def test_status( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is True @@ -114,7 +114,7 @@ async def test_status( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is True diff --git a/services/autoscaling/tests/unit/test_models.py b/services/autoscaling/tests/unit/test_models.py index f859ff591d6..f2271889ddb 100644 --- a/services/autoscaling/tests/unit/test_models.py +++ b/services/autoscaling/tests/unit/test_models.py @@ -10,7 +10,7 @@ import pytest from models_library.docker import DockerLabelKey, StandardSimcoreDockerLabels from models_library.generated_models.docker_rest_api import Service, Task -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError async def test_get_simcore_service_docker_labels_from_task_with_missing_labels_raises( @@ -19,12 +19,11 @@ async def test_get_simcore_service_docker_labels_from_task_with_missing_labels_r task_template: dict[str, Any], ): service_missing_osparc_labels = await create_service(task_template, {}, "running") - assert service_missing_osparc_labels.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_missing_osparc_labels.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await async_docker_client.tasks.list( - filters={"service": service_missing_osparc_labels.Spec.Name} - ), + filters={"service": service_missing_osparc_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -45,12 +44,11 @@ async def test_get_simcore_service_docker_labels( osparc_docker_label_keys.to_simcore_runtime_docker_labels(), "running", ) - assert service_with_labels.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_labels.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await async_docker_client.tasks.list( - filters={"service": service_with_labels.Spec.Name} - ), + filters={"service": service_with_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 5811b43b2f0..f9e0e4c416d 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -31,7 +31,7 @@ from models_library.generated_models.docker_rest_api import Node as DockerNode from models_library.generated_models.docker_rest_api import NodeState, NodeStatus from models_library.rabbitmq_messages import RabbitAutoscalingStatusMessage -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.aws_ec2 import assert_autoscaled_computational_ec2_instances from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict @@ -109,12 +109,12 @@ def _assert_rabbit_autoscaling_message_sent( nodes_total=0, nodes_active=0, nodes_drained=0, - cluster_total_resources=Resources.create_as_empty().dict(), - cluster_used_resources=Resources.create_as_empty().dict(), + cluster_total_resources=Resources.create_as_empty().model_dump(), + cluster_used_resources=Resources.create_as_empty().model_dump(), instances_pending=0, instances_running=0, ) - expected_message = default_message.copy(update=message_update_kwargs) + expected_message = default_message.model_copy(update=message_update_kwargs) mock_rabbitmq_post_message.assert_called_once_with( app, expected_message, @@ -241,7 +241,9 @@ async def test_cluster_scaling_with_task_with_too_much_resources_starts_nothing( dask_spec_local_cluster: distributed.SpecCluster, ): # create a task that needs too much power - dask_future = create_dask_task({"RAM": int(parse_obj_as(ByteSize, "12800GiB"))}) + dask_future = create_dask_task( + {"RAM": int(TypeAdapter(ByteSize).validate_python("12800GiB"))} + ) assert dask_future await auto_scale_cluster( @@ -317,8 +319,7 @@ async def _create_task_with_resources( assert instance_types["InstanceTypes"] assert "MemoryInfo" in instance_types["InstanceTypes"][0] assert "SizeInMiB" in instance_types["InstanceTypes"][0]["MemoryInfo"] - dask_ram = parse_obj_as( - ByteSize, + dask_ram = TypeAdapter(ByteSize).validate_python( f"{instance_types['InstanceTypes'][0]['MemoryInfo']['SizeInMiB']}MiB", ) dask_task_resources = create_dask_task_resources( @@ -335,7 +336,7 @@ async def _create_task_with_resources( [ pytest.param( None, - parse_obj_as(ByteSize, "128Gib"), + TypeAdapter(ByteSize).validate_python("128Gib"), "r5n.4xlarge", id="No explicit instance defined", ), @@ -347,7 +348,7 @@ async def _create_task_with_resources( ), pytest.param( "r5n.8xlarge", - parse_obj_as(ByteSize, "116Gib"), + TypeAdapter(ByteSize).validate_python("116Gib"), "r5n.8xlarge", id="Explicitely ask for r5n.8xlarge and set the resources", ), @@ -458,22 +459,22 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY: expected_ec2_type } assert mock_docker_tag_node.call_count == 2 - assert fake_node.Spec - assert fake_node.Spec.Labels + assert fake_node.spec + assert fake_node.spec.labels fake_attached_node = deepcopy(fake_node) - assert fake_attached_node.Spec - fake_attached_node.Spec.Availability = ( + assert fake_attached_node.spec + fake_attached_node.spec.availability = ( Availability.active if with_drain_nodes_labelled else Availability.drain ) - assert fake_attached_node.Spec.Labels - fake_attached_node.Spec.Labels |= expected_docker_node_tags | { + assert fake_attached_node.spec.labels + fake_attached_node.spec.labels |= expected_docker_node_tags | { _OSPARC_SERVICE_READY_LABEL_KEY: "false", } # check attach call assert mock_docker_tag_node.call_args_list[0] == mock.call( get_docker_client(initialized_app), fake_node, - tags=fake_node.Spec.Labels + tags=fake_node.spec.labels | expected_docker_node_tags | { _OSPARC_SERVICE_READY_LABEL_KEY: "false", @@ -482,7 +483,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 available=with_drain_nodes_labelled, ) # update our fake node - fake_attached_node.Spec.Labels[ + fake_attached_node.spec.labels[ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] = mock_docker_tag_node.call_args_list[0][1]["tags"][ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY @@ -502,7 +503,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 assert mock_docker_tag_node.call_args_list[1] == mock.call( get_docker_client(initialized_app), fake_attached_node, - tags=fake_node.Spec.Labels + tags=fake_node.spec.labels | expected_docker_node_tags | { _OSPARC_SERVICE_READY_LABEL_KEY: "true", @@ -511,7 +512,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 available=True, ) # update our fake node - fake_attached_node.Spec.Labels[ + fake_attached_node.spec.labels[ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] = mock_docker_tag_node.call_args_list[1][1]["tags"][ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY @@ -522,13 +523,13 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 mock_rabbitmq_post_message.reset_mock() # now we have 1 monitored node that needs to be mocked - fake_attached_node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" - fake_attached_node.Status = NodeStatus( + fake_attached_node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" + fake_attached_node.status = NodeStatus( State=NodeState.ready, Message=None, Addr=None ) - fake_attached_node.Spec.Availability = Availability.active - assert fake_attached_node.Description - fake_attached_node.Description.Hostname = internal_dns_name + fake_attached_node.spec.availability = Availability.active + assert fake_attached_node.description + fake_attached_node.description.hostname = internal_dns_name auto_scaling_mode = ComputationalAutoscaling() mocker.patch.object( @@ -591,7 +592,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY: mock.ANY, }, @@ -601,7 +602,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 # now update the fake node to have the required label as expected assert app_settings.AUTOSCALING_EC2_INSTANCES - fake_attached_node.Spec.Labels[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY] = ( + fake_attached_node.spec.labels[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY] = ( arrow.utcnow() .shift( seconds=-app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_DRAINING.total_seconds() @@ -625,7 +626,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY: mock.ANY, _OSPARC_SERVICE_READY_LABEL_KEY: "false", @@ -639,7 +640,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] ) > arrow.get( - fake_attached_node.Spec.Labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] + fake_attached_node.spec.labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] ) mock_docker_tag_node.reset_mock() @@ -653,9 +654,9 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 ) # we artifically set the node to drain - fake_attached_node.Spec.Availability = Availability.drain - fake_attached_node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "false" - fake_attached_node.Spec.Labels[ + fake_attached_node.spec.availability = Availability.drain + fake_attached_node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "false" + fake_attached_node.spec.labels[ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] = datetime.datetime.now(tz=datetime.timezone.utc).isoformat() @@ -682,7 +683,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 ) # now changing the last update timepoint will trigger the node removal and shutdown the ec2 instance - fake_attached_node.Spec.Labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] = ( + fake_attached_node.spec.labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] = ( datetime.datetime.now(tz=datetime.timezone.utc) - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - datetime.timedelta(seconds=1) @@ -701,7 +702,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY: mock.ANY, }, @@ -709,8 +710,8 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 ) mock_docker_tag_node.reset_mock() # set the fake node to drain - fake_attached_node.Spec.Availability = Availability.drain - fake_attached_node.Spec.Labels[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] = ( + fake_attached_node.spec.availability = Availability.drain + fake_attached_node.spec.labels[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] = ( arrow.utcnow() .shift( seconds=-app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_FINAL_TERMINATION.total_seconds() @@ -751,7 +752,7 @@ async def test_cluster_does_not_scale_up_if_defined_instance_is_not_allowed( # create a task that needs more power dask_task_resources = create_dask_task_resources( - faker.pystr(), parse_obj_as(ByteSize, "128GiB") + faker.pystr(), TypeAdapter(ByteSize).validate_python("128GiB") ) dask_future = create_dask_task(dask_task_resources) assert dask_future @@ -787,7 +788,7 @@ async def test_cluster_does_not_scale_up_if_defined_instance_is_not_fitting_reso # create a task that needs more power dask_task_resources = create_dask_task_resources( - "t2.xlarge", parse_obj_as(ByteSize, "128GiB") + "t2.xlarge", TypeAdapter(ByteSize).validate_python("128GiB") ) dask_future = create_dask_task(dask_task_resources) assert dask_future @@ -817,7 +818,8 @@ class _ScaleUpParams: def _dask_task_resources_from_resources(resources: Resources) -> DaskTaskResources: return { - res_key.upper(): res_value for res_key, res_value in resources.dict().items() + res_key.upper(): res_value + for res_key, res_value in resources.model_dump().items() } @@ -847,7 +849,9 @@ async def _change_parameters(*args, **kwargs) -> list[EC2InstanceData]: [ pytest.param( _ScaleUpParams( - task_resources=Resources(cpus=5, ram=parse_obj_as(ByteSize, "36Gib")), + task_resources=Resources( + cpus=5, ram=TypeAdapter(ByteSize).validate_python("36Gib") + ), num_tasks=10, expected_instance_type="g3.4xlarge", expected_num_instances=4, @@ -1106,7 +1110,7 @@ async def test_cluster_scaling_up_more_than_allowed_with_multiple_types_max_star [ pytest.param( None, - parse_obj_as(ByteSize, "128Gib"), + TypeAdapter(ByteSize).validate_python("128Gib"), "r5n.4xlarge", id="No explicit instance defined", ), diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index 3a79a11c853..461baee21fa 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -35,7 +35,7 @@ Task, ) from models_library.rabbitmq_messages import RabbitAutoscalingStatusMessage -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockType from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.aws_ec2 import assert_autoscaled_dynamic_ec2_instances @@ -130,51 +130,51 @@ def with_valid_time_before_termination( ) -> datetime.timedelta: time = "00:11:00" monkeypatch.setenv("EC2_INSTANCES_TIME_BEFORE_TERMINATION", time) - return parse_obj_as(datetime.timedelta, time) + return TypeAdapter(datetime.timedelta).validate_python(time) @pytest.fixture async def drained_host_node( host_node: Node, async_docker_client: aiodocker.Docker ) -> AsyncIterator[Node]: - assert host_node.ID - assert host_node.Version - assert host_node.Version.Index - assert host_node.Spec - assert host_node.Spec.Availability - assert host_node.Spec.Role - - old_availability = host_node.Spec.Availability + assert host_node.id + assert host_node.version + assert host_node.version.index + assert host_node.spec + assert host_node.spec.availability + assert host_node.spec.role + + old_availability = host_node.spec.availability await async_docker_client.nodes.update( - node_id=host_node.ID, - version=host_node.Version.Index, + node_id=host_node.id, + version=host_node.version.index, spec={ "Availability": "drain", - "Labels": host_node.Spec.Labels, - "Role": host_node.Spec.Role.value, + "Labels": host_node.spec.labels, + "Role": host_node.spec.role.value, }, ) - drained_node = parse_obj_as( - Node, await async_docker_client.nodes.inspect(node_id=host_node.ID) + drained_node = TypeAdapter(Node).validate_python( + await async_docker_client.nodes.inspect(node_id=host_node.id) ) yield drained_node # revert # NOTE: getting the node again as the version might have changed - drained_node = parse_obj_as( - Node, await async_docker_client.nodes.inspect(node_id=host_node.ID) - ) - assert drained_node.ID - assert drained_node.Version - assert drained_node.Version.Index - assert drained_node.Spec - assert drained_node.Spec.Role + drained_node = TypeAdapter(Node).validate_python( + await async_docker_client.nodes.inspect(node_id=host_node.id) + ) + assert drained_node.id + assert drained_node.version + assert drained_node.version.index + assert drained_node.spec + assert drained_node.spec.role await async_docker_client.nodes.update( - node_id=drained_node.ID, - version=drained_node.Version.Index, + node_id=drained_node.id, + version=drained_node.version.index, spec={ "Availability": old_availability.value, - "Labels": drained_node.Spec.Labels, - "Role": drained_node.Spec.Role.value, + "Labels": drained_node.spec.labels, + "Role": drained_node.spec.role.value, }, ) @@ -208,12 +208,12 @@ def _assert_rabbit_autoscaling_message_sent( nodes_total=0, nodes_active=0, nodes_drained=0, - cluster_total_resources=Resources.create_as_empty().dict(), - cluster_used_resources=Resources.create_as_empty().dict(), + cluster_total_resources=Resources.create_as_empty().model_dump(), + cluster_used_resources=Resources.create_as_empty().model_dump(), instances_pending=0, instances_running=0, ) - expected_message = default_message.copy(update=message_update_kwargs) + expected_message = default_message.model_copy(update=message_update_kwargs) assert mock_rabbitmq_post_message.call_args == mock.call(app, expected_message) @@ -322,10 +322,10 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect expected_additional_tag_keys=list(ec2_instance_custom_tags), instance_filters=instance_type_filters, ) - assert fake_node.Description - assert fake_node.Description.Resources - assert fake_node.Description.Resources.NanoCPUs - assert fake_node.Description.Resources.MemoryBytes + assert fake_node.description + assert fake_node.description.resources + assert fake_node.description.resources.nano_cp_us + assert fake_node.description.resources.memory_bytes _assert_rabbit_autoscaling_message_sent( mock_rabbitmq_post_message, app_settings, @@ -335,9 +335,9 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect instances_running=mock_machines_buffer, cluster_total_resources={ "cpus": mock_machines_buffer - * fake_node.Description.Resources.NanoCPUs + * fake_node.description.resources.nano_cp_us / 1e9, - "ram": mock_machines_buffer * fake_node.Description.Resources.MemoryBytes, + "ram": mock_machines_buffer * fake_node.description.resources.memory_bytes, }, ) @@ -533,11 +533,11 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: ) fake_attached_node = deepcopy(fake_node) - assert fake_attached_node.Spec - fake_attached_node.Spec.Availability = ( + assert fake_attached_node.spec + fake_attached_node.spec.availability = ( Availability.active if with_drain_nodes_labelled else Availability.drain ) - assert fake_attached_node.Spec.Labels + assert fake_attached_node.spec.labels assert app_settings.AUTOSCALING_NODES_MONITORING expected_docker_node_tags = { tag_key: "true" @@ -548,7 +548,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: } | { DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY: scale_up_params.expected_instance_type } - fake_attached_node.Spec.Labels |= expected_docker_node_tags | { + fake_attached_node.spec.labels |= expected_docker_node_tags | { _OSPARC_SERVICE_READY_LABEL_KEY: "false" } @@ -557,13 +557,13 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: mock_find_node_with_name_returns_fake_node.reset_mock() assert mock_docker_tag_node.call_count == 2 - assert fake_node.Spec - assert fake_node.Spec.Labels + assert fake_node.spec + assert fake_node.spec.labels # check attach call assert mock_docker_tag_node.call_args_list[0] == mock.call( get_docker_client(initialized_app), fake_node, - tags=fake_node.Spec.Labels + tags=fake_node.spec.labels | expected_docker_node_tags | { _OSPARC_SERVICE_READY_LABEL_KEY: "false", @@ -572,7 +572,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: available=with_drain_nodes_labelled, ) # update our fake node - fake_attached_node.Spec.Labels[ + fake_attached_node.spec.labels[ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] = mock_docker_tag_node.call_args_list[0][1]["tags"][ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY @@ -596,7 +596,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: assert mock_docker_tag_node.call_args_list[1] == mock.call( get_docker_client(initialized_app), fake_attached_node, - tags=fake_node.Spec.Labels + tags=fake_node.spec.labels | expected_docker_node_tags | { _OSPARC_SERVICE_READY_LABEL_KEY: "true", @@ -605,7 +605,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: available=True, ) # update our fake node - fake_attached_node.Spec.Labels[ + fake_attached_node.spec.labels[ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] = mock_docker_tag_node.call_args_list[1][1]["tags"][ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY @@ -629,9 +629,9 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: internal_dns_name = instances[0]["PrivateDnsName"].removesuffix(".ec2.internal") # check rabbit messages were sent, we do have worker - assert fake_attached_node.Description - assert fake_attached_node.Description.Resources - assert fake_attached_node.Description.Resources.NanoCPUs + assert fake_attached_node.description + assert fake_attached_node.description.resources + assert fake_attached_node.description.resources.nano_cp_us _assert_rabbit_autoscaling_message_sent( mock_rabbitmq_post_message, app_settings, @@ -639,8 +639,8 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: nodes_total=scale_up_params.expected_num_instances, nodes_active=scale_up_params.expected_num_instances, cluster_total_resources={ - "cpus": fake_attached_node.Description.Resources.NanoCPUs / 1e9, - "ram": fake_attached_node.Description.Resources.MemoryBytes, + "cpus": fake_attached_node.description.resources.nano_cp_us / 1e9, + "ram": fake_attached_node.description.resources.memory_bytes, }, cluster_used_resources={ "cpus": float(0), @@ -651,12 +651,12 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: mock_rabbitmq_post_message.reset_mock() # now we have 1 monitored node that needs to be mocked - fake_attached_node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" - fake_attached_node.Status = NodeStatus( + fake_attached_node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" + fake_attached_node.status = NodeStatus( State=NodeState.ready, Message=None, Addr=None ) - fake_attached_node.Spec.Availability = Availability.active - fake_attached_node.Description.Hostname = internal_dns_name + fake_attached_node.spec.availability = Availability.active + fake_attached_node.description.hostname = internal_dns_name auto_scaling_mode = DynamicAutoscaling() mocker.patch.object( @@ -700,9 +700,9 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: # await asyncio.gather( *( - async_docker_client.services.delete(d.ID) + async_docker_client.services.delete(d.id) for d in created_docker_services - if d.ID + if d.id ) ) @@ -723,7 +723,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY: mock.ANY, }, @@ -733,7 +733,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: # now update the fake node to have the required label as expected assert app_settings.AUTOSCALING_EC2_INSTANCES - fake_attached_node.Spec.Labels[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY] = ( + fake_attached_node.spec.labels[_OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY] = ( arrow.utcnow() .shift( seconds=-app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_DRAINING.total_seconds() @@ -748,7 +748,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_SERVICE_READY_LABEL_KEY: "false", _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY: mock.ANY, @@ -761,7 +761,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] ) > arrow.get( - fake_attached_node.Spec.Labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] + fake_attached_node.spec.labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] ) mock_docker_tag_node.reset_mock() @@ -771,7 +771,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_SERVICE_READY_LABEL_KEY: "false", _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY: mock.ANY, @@ -793,9 +793,9 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: # we artifically set the node to drain if not with_drain_nodes_labelled: - fake_attached_node.Spec.Availability = Availability.drain - fake_attached_node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "false" - fake_attached_node.Spec.Labels[ + fake_attached_node.spec.availability = Availability.drain + fake_attached_node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "false" + fake_attached_node.spec.labels[ _OSPARC_SERVICES_READY_DATETIME_LABEL_KEY ] = datetime.datetime.now(tz=datetime.UTC).isoformat() @@ -824,7 +824,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: assert created_instances == instances # now changing the last update timepoint will trigger the node removal process - fake_attached_node.Spec.Labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] = ( + fake_attached_node.spec.labels[_OSPARC_SERVICES_READY_DATETIME_LABEL_KEY] = ( datetime.datetime.now(tz=datetime.UTC) - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - datetime.timedelta(seconds=1) @@ -845,7 +845,7 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_attached_node, - tags=fake_attached_node.Spec.Labels + tags=fake_attached_node.spec.labels | { _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY: mock.ANY, }, @@ -853,8 +853,8 @@ async def _assert_wait_for_ec2_instances_running() -> list[InstanceTypeDef]: ) mock_docker_tag_node.reset_mock() # set the fake node to drain - fake_attached_node.Spec.Availability = Availability.drain - fake_attached_node.Spec.Labels[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] = ( + fake_attached_node.spec.availability = Availability.drain + fake_attached_node.spec.labels[_OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY] = ( arrow.utcnow() .shift( seconds=-app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_FINAL_TERMINATION.total_seconds() @@ -907,7 +907,7 @@ async def _assert_wait_for_ec2_instances_terminated() -> None: _ScaleUpParams( imposed_instance_type=None, service_resources=Resources( - cpus=4, ram=parse_obj_as(ByteSize, "128Gib") + cpus=4, ram=TypeAdapter(ByteSize).validate_python("128Gib") ), num_services=1, expected_instance_type="r5n.4xlarge", @@ -918,7 +918,9 @@ async def _assert_wait_for_ec2_instances_terminated() -> None: pytest.param( _ScaleUpParams( imposed_instance_type="t2.xlarge", - service_resources=Resources(cpus=4, ram=parse_obj_as(ByteSize, "4Gib")), + service_resources=Resources( + cpus=4, ram=TypeAdapter(ByteSize).validate_python("4Gib") + ), num_services=1, expected_instance_type="t2.xlarge", expected_num_instances=1, @@ -929,7 +931,7 @@ async def _assert_wait_for_ec2_instances_terminated() -> None: _ScaleUpParams( imposed_instance_type="r5n.8xlarge", service_resources=Resources( - cpus=4, ram=parse_obj_as(ByteSize, "128Gib") + cpus=4, ram=TypeAdapter(ByteSize).validate_python("128Gib") ), num_services=1, expected_instance_type="r5n.8xlarge", @@ -998,7 +1000,7 @@ async def test_cluster_scaling_up_and_down( _ScaleUpParams( imposed_instance_type=None, service_resources=Resources( - cpus=4, ram=parse_obj_as(ByteSize, "62Gib") + cpus=4, ram=TypeAdapter(ByteSize).validate_python("62Gib") ), num_services=1, expected_instance_type="r6a.2xlarge", @@ -1084,7 +1086,7 @@ async def test_cluster_scaling_up_and_down_against_aws( _ScaleUpParams( imposed_instance_type=None, service_resources=Resources( - cpus=5, ram=parse_obj_as(ByteSize, "36Gib") + cpus=5, ram=TypeAdapter(ByteSize).validate_python("36Gib") ), num_services=10, expected_instance_type="g3.4xlarge", # 1 GPU, 16 CPUs, 122GiB @@ -1096,7 +1098,7 @@ async def test_cluster_scaling_up_and_down_against_aws( _ScaleUpParams( imposed_instance_type="g4dn.8xlarge", service_resources=Resources( - cpus=5, ram=parse_obj_as(ByteSize, "20480MB") + cpus=5, ram=TypeAdapter(ByteSize).validate_python("20480MB") ), num_services=7, expected_instance_type="g4dn.8xlarge", # 1 GPU, 32 CPUs, 128GiB @@ -1190,7 +1192,7 @@ async def test_cluster_scaling_up_starts_multiple_instances( [ pytest.param( None, - parse_obj_as(ByteSize, "128Gib"), + TypeAdapter(ByteSize).validate_python("128Gib"), "r5n.4xlarge", id="No explicit instance defined", ), @@ -1451,12 +1453,11 @@ async def test__activate_drained_nodes_with_no_drained_nodes( service_with_no_reservations = await create_service( task_template_that_runs, {}, "running" ) - assert service_with_no_reservations.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_no_reservations.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await autoscaling_docker.tasks.list( - filters={"service": service_with_no_reservations.Spec.Name} - ), + filters={"service": service_with_no_reservations.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -1495,12 +1496,11 @@ async def test__activate_drained_nodes_with_drained_node( service_with_no_reservations = await create_service( task_template_that_runs, {}, "pending" ) - assert service_with_no_reservations.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_no_reservations.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await autoscaling_docker.tasks.list( - filters={"service": service_with_no_reservations.Spec.Name} - ), + filters={"service": service_with_no_reservations.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -1518,7 +1518,7 @@ async def test__activate_drained_nodes_with_drained_node( initialized_app, cluster_with_drained_nodes, DynamicAutoscaling() ) assert updated_cluster.active_nodes == cluster_with_drained_nodes.drained_nodes - assert drained_host_node.Spec + assert drained_host_node.spec mock_docker_tag_node.assert_called_once_with( mock.ANY, drained_host_node, diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py index 4c4037e51ab..4a3d3e85bae 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py @@ -5,15 +5,18 @@ import asyncio +import datetime +from typing import Final from unittest import mock import pytest from fastapi import FastAPI +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_autoscaling.core.settings import ApplicationSettings -_FAST_POLL_INTERVAL = 1 +_FAST_POLL_INTERVAL: Final[int] = 1 @pytest.fixture @@ -26,7 +29,10 @@ def app_environment( monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: # fast interval - monkeypatch.setenv("AUTOSCALING_POLL_INTERVAL", f"{_FAST_POLL_INTERVAL}") + monkeypatch.setenv( + "AUTOSCALING_POLL_INTERVAL", + f"{TypeAdapter(datetime.timedelta).validate_python(_FAST_POLL_INTERVAL)}", + ) app_environment["AUTOSCALING_POLL_INTERVAL"] = f"{_FAST_POLL_INTERVAL}" return app_environment diff --git a/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py b/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py index 26ac271db29..8135ac9d2c4 100644 --- a/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py +++ b/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py @@ -22,7 +22,7 @@ from fastapi.encoders import jsonable_encoder from models_library.docker import DockerGenericTag from models_library.utils.json_serialization import json_dumps -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.aws_ec2 import ( assert_autoscaled_dynamic_warm_pools_ec2_instances, @@ -47,14 +47,13 @@ @pytest.fixture def fake_pre_pull_images() -> list[DockerGenericTag]: - return parse_obj_as( - list[DockerGenericTag], + return TypeAdapter(list[DockerGenericTag]).validate_python( [ "nginx:latest", "itisfoundation/my-very-nice-service:latest", "simcore/services/dynamic/another-nice-one:2.4.5", "asd", - ], + ] ) @@ -90,7 +89,7 @@ def ec2_instances_allowed_types_with_only_1_buffered( len(allowed_ec2_types_with_buffer_defined) == 1 ), "more than one type with buffer is disallowed in this test!" return { - parse_obj_as(InstanceTypeType, k): v + TypeAdapter(InstanceTypeType).validate_python(k): v for k, v in allowed_ec2_types_with_buffer_defined.items() } @@ -450,7 +449,11 @@ class _BufferMachineParams: _BufferMachineParams( "stopped", [], - [parse_obj_as(AWSTagKey, "io.simcore.autoscaling.pre_pulled_images")], + [ + TypeAdapter(AWSTagKey).validate_python( + "io.simcore.autoscaling.pre_pulled_images" + ) + ], ), ], ) @@ -589,7 +592,11 @@ def unneeded_instance_type( _BufferMachineParams( "stopped", [], - [parse_obj_as(AWSTagKey, "io.simcore.autoscaling.pre_pulled_images")], + [ + TypeAdapter(AWSTagKey).validate_python( + "io.simcore.autoscaling.pre_pulled_images" + ) + ], ), ], ) diff --git a/services/autoscaling/tests/unit/test_modules_dask.py b/services/autoscaling/tests/unit/test_modules_dask.py index 76dab6883e0..ae2ed0c5f15 100644 --- a/services/autoscaling/tests/unit/test_modules_dask.py +++ b/services/autoscaling/tests/unit/test_modules_dask.py @@ -17,7 +17,7 @@ NoAuthentication, TLSAuthentication, ) -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_simcore.helpers.host import get_localhost_ip from simcore_service_autoscaling.core.errors import ( DaskNoWorkersError, @@ -42,7 +42,9 @@ _authentication_types = [ NoAuthentication(), - TLSAuthentication.construct(**TLSAuthentication.Config.schema_extra["examples"][0]), + TLSAuthentication.model_construct( + **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + ), ] @@ -54,7 +56,9 @@ async def test__scheduler_client_with_wrong_url( ): with pytest.raises(DaskSchedulerNotFoundError): async with _scheduler_client( - parse_obj_as(AnyUrl, f"tcp://{faker.ipv4()}:{faker.port_number()}"), + TypeAdapter(AnyUrl).validate_python( + f"tcp://{faker.ipv4()}:{faker.port_number()}" + ), authentication, ): ... @@ -62,7 +66,9 @@ async def test__scheduler_client_with_wrong_url( @pytest.fixture def scheduler_url(dask_spec_local_cluster: distributed.SpecCluster) -> AnyUrl: - return parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address) + return TypeAdapter(AnyUrl).validate_python( + dask_spec_local_cluster.scheduler_address + ) @pytest.fixture @@ -95,8 +101,8 @@ async def test__scheduler_client( async def test_list_unrunnable_tasks_with_no_workers( dask_local_cluster_without_workers: distributed.SpecCluster, ): - scheduler_url = parse_obj_as( - AnyUrl, dask_local_cluster_without_workers.scheduler_address + scheduler_url = TypeAdapter(AnyUrl).validate_python( + dask_local_cluster_without_workers.scheduler_address ) assert await list_unrunnable_tasks(scheduler_url, NoAuthentication()) == [] @@ -199,8 +205,8 @@ async def test_get_worker_still_has_results_in_memory_with_no_workers_raises( dask_local_cluster_without_workers: distributed.SpecCluster, fake_localhost_ec2_instance_data: EC2InstanceData, ): - scheduler_url = parse_obj_as( - AnyUrl, dask_local_cluster_without_workers.scheduler_address + scheduler_url = TypeAdapter(AnyUrl).validate_python( + dask_local_cluster_without_workers.scheduler_address ) with pytest.raises(DaskNoWorkersError): await get_worker_still_has_results_in_memory( @@ -300,8 +306,8 @@ async def test_worker_used_resources_with_no_workers_raises( dask_local_cluster_without_workers: distributed.SpecCluster, fake_localhost_ec2_instance_data: EC2InstanceData, ): - scheduler_url = parse_obj_as( - AnyUrl, dask_local_cluster_without_workers.scheduler_address + scheduler_url = TypeAdapter(AnyUrl).validate_python( + dask_local_cluster_without_workers.scheduler_address ) with pytest.raises(DaskNoWorkersError): await get_worker_used_resources( diff --git a/services/autoscaling/tests/unit/test_modules_rabbitmq.py b/services/autoscaling/tests/unit/test_modules_rabbitmq.py index 2cc76d1465f..9aab8d68e34 100644 --- a/services/autoscaling/tests/unit/test_modules_rabbitmq.py +++ b/services/autoscaling/tests/unit/test_modules_rabbitmq.py @@ -131,7 +131,7 @@ async def test_post_message( f"--> checking for message in rabbit exchange {rabbit_message.channel_name}, {attempt.retry_state.retry_object.statistics}" ) mocked_message_handler.assert_called_once_with( - rabbit_message.json().encode() + rabbit_message.model_dump_json().encode() ) print("... message received") diff --git a/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py b/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py index dd0983f3986..f576292ec6b 100644 --- a/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py +++ b/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py @@ -16,7 +16,7 @@ from faker import Faker from models_library.docker import DockerGenericTag from models_library.generated_models.docker_rest_api import Node as DockerNode -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_autoscaling.core.errors import Ec2InvalidDnsNameError @@ -129,10 +129,10 @@ async def test_associate_ec2_instances_with_corresponding_nodes( assert len(associated_instances) == len(ec2_instances) assert len(associated_instances) == len(nodes) for associated_instance in associated_instances: - assert associated_instance.node.Description - assert associated_instance.node.Description.Hostname + assert associated_instance.node.description + assert associated_instance.node.description.hostname assert ( - associated_instance.node.Description.Hostname + associated_instance.node.description.hostname in associated_instance.ec2_instance.aws_private_dns ) @@ -211,14 +211,13 @@ def ec2_instances_boot_ami_scripts( def ec2_instances_boot_ami_pre_pull( app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker ) -> EnvVarsDict: - images = parse_obj_as( - list[DockerGenericTag], + images = TypeAdapter(list[DockerGenericTag]).validate_python( [ "nginx:latest", "itisfoundation/my-very-nice-service:latest", "simcore/services/dynamic/another-nice-one:2.4.5", "asd", - ], + ] ) envs = setenvs_from_dict( monkeypatch, @@ -364,9 +363,9 @@ def test_sort_drained_nodes( for _ in range(_NUM_NODES_TERMINATING): fake_node = create_fake_node() - assert fake_node.Spec - assert fake_node.Spec.Labels - fake_node.Spec.Labels[ + assert fake_node.spec + assert fake_node.spec.labels + fake_node.spec.labels[ _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY ] = arrow.utcnow().datetime.isoformat() fake_associated_instance = create_associated_instance( @@ -404,6 +403,6 @@ def test_sort_drained_nodes( ) assert len(terminating_nodes) == _NUM_NODES_TERMINATING for n in terminating_nodes: - assert n.node.Spec - assert n.node.Spec.Labels - assert _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY in n.node.Spec.Labels + assert n.node.spec + assert n.node.spec.labels + assert _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY in n.node.spec.labels diff --git a/services/autoscaling/tests/unit/test_utils_buffer_machines_pool_core.py b/services/autoscaling/tests/unit/test_utils_buffer_machines_pool_core.py index 84ca4baa474..19cc33c2575 100644 --- a/services/autoscaling/tests/unit/test_utils_buffer_machines_pool_core.py +++ b/services/autoscaling/tests/unit/test_utils_buffer_machines_pool_core.py @@ -6,7 +6,7 @@ from faker import Faker from fastapi import FastAPI from models_library.docker import DockerGenericTag -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_autoscaling.constants import ( ACTIVATED_BUFFER_MACHINE_EC2_TAGS, @@ -65,8 +65,8 @@ def test_get_deactivated_buffer_ec2_tags_dynamic( | DEACTIVATED_BUFFER_MACHINE_EC2_TAGS ) assert "Name" in expected_tags - expected_tags[AWSTagKey("Name")] = parse_obj_as( - AWSTagValue, str(expected_tags[AWSTagKey("Name")]) + "-buffer" + expected_tags[AWSTagKey("Name")] = TypeAdapter(AWSTagValue).validate_python( + str(expected_tags[AWSTagKey("Name")]) + "-buffer" ) assert expected_tags == deactivated_buffer_tags @@ -107,8 +107,8 @@ def test_get_deactivated_buffer_ec2_tags_computational( | DEACTIVATED_BUFFER_MACHINE_EC2_TAGS ) assert "Name" in expected_tags - expected_tags[AWSTagKey("Name")] = parse_obj_as( - AWSTagValue, str(expected_tags[AWSTagKey("Name")]) + "-buffer" + expected_tags[AWSTagKey("Name")] = TypeAdapter(AWSTagValue).validate_python( + str(expected_tags[AWSTagKey("Name")]) + "-buffer" ) assert expected_tags == deactivated_buffer_tags @@ -144,10 +144,10 @@ def test_is_buffer_machine(tags: EC2Tags, expected_is_buffer: bool): "registry.pytest.com/simcore/services/dynamic/sym-server-8-0-0-dy:3.2.34", ], { - f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_(0)": '["itisfoundation/dynamic-sidecar:latest","itisfoundation/agent:latest","registry.pytest.com/simcore/services/dynamic/ti-postpro:2.0.34","registry.pytest.com/simcore/services/dynamic/ti-simu:1.0.12","registry.pytest.com/simcore/services/dynamic/ti-pers:1.0.', - f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_(1)": '19","registry.pytest.com/simcore/services/dynamic/sim4life-postpro:2.0.106","registry.pytest.com/simcore/services/dynamic/s4l-core-postpro:2.0.106","registry.pytest.com/simcore/services/dynamic/s4l-core-stream:2.0.106","registry.pytest.com/simcore/services', - f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_(2)": '/dynamic/sym-server-8-0-0-dy:2.0.106","registry.pytest.com/simcore/services/dynamic/sim4life-8-0-0-modeling:3.2.34","registry.pytest.com/simcore/services/dynamic/s4l-core-8-0-0-modeling:3.2.34","registry.pytest.com/simcore/services/dynamic/s4l-stream-8-0-0', - f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_(3)": '-dy:3.2.34","registry.pytest.com/simcore/services/dynamic/sym-server-8-0-0-dy:3.2.34"]', + f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_0": '["itisfoundation/dynamic-sidecar:latest","itisfoundation/agent:latest","registry.pytest.com/simcore/services/dynamic/ti-postpro:2.0.34","registry.pytest.com/simcore/services/dynamic/ti-simu:1.0.12","registry.pytest.com/simcore/services/dynamic/ti-pers:1.0.', + f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_1": '19","registry.pytest.com/simcore/services/dynamic/sim4life-postpro:2.0.106","registry.pytest.com/simcore/services/dynamic/s4l-core-postpro:2.0.106","registry.pytest.com/simcore/services/dynamic/s4l-core-stream:2.0.106","registry.pytest.com/simcore/services', + f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_2": '/dynamic/sym-server-8-0-0-dy:2.0.106","registry.pytest.com/simcore/services/dynamic/sim4life-8-0-0-modeling:3.2.34","registry.pytest.com/simcore/services/dynamic/s4l-core-8-0-0-modeling:3.2.34","registry.pytest.com/simcore/services/dynamic/s4l-stream-8-0-0', + f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_3": '-dy:3.2.34","registry.pytest.com/simcore/services/dynamic/sym-server-8-0-0-dy:3.2.34"]', }, id="many images that get chunked to AWS Tag max length", ), diff --git a/services/autoscaling/tests/unit/test_utils_computational_scaling.py b/services/autoscaling/tests/unit/test_utils_computational_scaling.py index 97cf493b6b9..b5744f17053 100644 --- a/services/autoscaling/tests/unit/test_utils_computational_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_computational_scaling.py @@ -6,7 +6,7 @@ import pytest from aws_library.ec2 import Resources -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from simcore_service_autoscaling.models import DaskTask, DaskTaskResources from simcore_service_autoscaling.utils.computational_scaling import ( _DEFAULT_MAX_CPU, @@ -21,13 +21,16 @@ pytest.param( DaskTask(task_id="fake", required_resources=DaskTaskResources()), Resources( - cpus=_DEFAULT_MAX_CPU, ram=parse_obj_as(ByteSize, _DEFAULT_MAX_RAM) + cpus=_DEFAULT_MAX_CPU, + ram=TypeAdapter(ByteSize).validate_python(_DEFAULT_MAX_RAM), ), id="missing resources returns defaults", ), pytest.param( DaskTask(task_id="fake", required_resources={"CPU": 2.5}), - Resources(cpus=2.5, ram=parse_obj_as(ByteSize, _DEFAULT_MAX_RAM)), + Resources( + cpus=2.5, ram=TypeAdapter(ByteSize).validate_python(_DEFAULT_MAX_RAM) + ), id="only cpus defined", ), pytest.param( @@ -35,7 +38,7 @@ task_id="fake", required_resources={"CPU": 2.5, "RAM": 2 * 1024 * 1024 * 1024}, ), - Resources(cpus=2.5, ram=parse_obj_as(ByteSize, "2GiB")), + Resources(cpus=2.5, ram=TypeAdapter(ByteSize).validate_python("2GiB")), id="cpu and ram defined", ), pytest.param( @@ -43,7 +46,9 @@ task_id="fake", required_resources={"CPU": 2.5, "ram": 2 * 1024 * 1024 * 1024}, ), - Resources(cpus=2.5, ram=parse_obj_as(ByteSize, _DEFAULT_MAX_RAM)), + Resources( + cpus=2.5, ram=TypeAdapter(ByteSize).validate_python(_DEFAULT_MAX_RAM) + ), id="invalid naming", ), ], diff --git a/services/autoscaling/tests/unit/test_utils_docker.py b/services/autoscaling/tests/unit/test_utils_docker.py index 8e5b8cd90a8..a99f6939d1a 100644 --- a/services/autoscaling/tests/unit/test_utils_docker.py +++ b/services/autoscaling/tests/unit/test_utils_docker.py @@ -30,7 +30,7 @@ Service, Task, ) -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from servicelib.docker_utils import to_datetime @@ -81,23 +81,23 @@ async def create_node_labels( host_node: Node, async_docker_client: aiodocker.Docker, ) -> AsyncIterator[Callable[[list[str]], Awaitable[None]]]: - assert host_node.Spec - old_labels = deepcopy(host_node.Spec.Labels) + assert host_node.spec + old_labels = deepcopy(host_node.spec.labels) async def _creator(labels: list[str]) -> None: - assert host_node.ID - assert host_node.Version - assert host_node.Version.Index - assert host_node.Spec - assert host_node.Spec.Role - assert host_node.Spec.Availability + assert host_node.id + assert host_node.version + assert host_node.version.index + assert host_node.spec + assert host_node.spec.role + assert host_node.spec.availability await async_docker_client.nodes.update( - node_id=host_node.ID, - version=host_node.Version.Index, + node_id=host_node.id, + version=host_node.version.index, spec={ "Name": "foo", - "Availability": host_node.Spec.Availability.value, - "Role": host_node.Spec.Role.value, + "Availability": host_node.spec.availability.value, + "Role": host_node.spec.role.value, "Labels": {f"{label}": "true" for label in labels}, }, ) @@ -158,12 +158,12 @@ async def test_get_monitored_nodes_with_valid_label( # this is the host node with some keys slightly changed EXCLUDED_KEYS = { - "Index": True, - "UpdatedAt": True, - "Version": True, - "Spec": {"Labels", "Name"}, + "index": True, + "updated_at": True, + "version": True, + "spec": {"labels", "name"}, } - assert host_node.dict(exclude=EXCLUDED_KEYS) == monitored_nodes[0].dict( + assert host_node.model_dump(exclude=EXCLUDED_KEYS) == monitored_nodes[0].model_dump( exclude=EXCLUDED_KEYS ) @@ -191,10 +191,10 @@ async def test_remove_monitored_down_nodes_of_non_down_node_does_nothing( @pytest.fixture def fake_docker_node(host_node: Node, faker: Faker) -> Node: - fake_node = host_node.copy(deep=True) - fake_node.ID = faker.uuid4() + fake_node = host_node.model_copy(deep=True) + fake_node.id = faker.uuid4(cast_to=str) assert ( - host_node.ID != fake_node.ID + host_node.id != fake_node.id ), "this should never happen, or you are really unlucky" return fake_node @@ -205,15 +205,15 @@ async def test_remove_monitored_down_nodes_of_down_node( mocker: MockerFixture, ): mocked_aiodocker = mocker.patch.object(autoscaling_docker, "nodes", autospec=True) - assert fake_docker_node.Status - fake_docker_node.Status.State = NodeState.down - assert fake_docker_node.Status.State == NodeState.down + assert fake_docker_node.status + fake_docker_node.status.state = NodeState.down + assert fake_docker_node.status.state == NodeState.down assert await remove_nodes(autoscaling_docker, nodes=[fake_docker_node]) == [ fake_docker_node ] # NOTE: this is the same as calling with aiodocker.Docker() as docker: docker.nodes.remove() mocked_aiodocker.remove.assert_called_once_with( - node_id=fake_docker_node.ID, force=False + node_id=fake_docker_node.id, force=False ) @@ -221,9 +221,9 @@ async def test_remove_monitored_down_node_with_unexpected_state_does_nothing( autoscaling_docker: AutoscalingDocker, fake_docker_node: Node, ): - assert fake_docker_node.Status - fake_docker_node.Status = None - assert not fake_docker_node.Status + assert fake_docker_node.status + fake_docker_node.status = None + assert not fake_docker_node.status assert await remove_nodes(autoscaling_docker, nodes=[fake_docker_node]) == [] @@ -276,7 +276,7 @@ async def test_pending_service_task_with_placement_constrain_is_skipped( service_with_too_many_resources = await create_service( task_template_with_too_many_resource, {}, "pending" ) - assert service_with_too_many_resources.Spec + assert service_with_too_many_resources.spec pending_tasks = await pending_service_tasks_with_insufficient_resources( autoscaling_docker, service_labels=[] @@ -312,13 +312,12 @@ async def test_pending_service_task_with_insufficient_resources_with_service_lac service_with_too_many_resources = await create_service( task_template_with_too_many_resource, {}, "pending" ) - assert service_with_too_many_resources.Spec + assert service_with_too_many_resources.spec - service_tasks = parse_obj_as( - list[Task], + service_tasks = TypeAdapter(list[Task]).validate_python( await autoscaling_docker.tasks.list( - filters={"service": service_with_too_many_resources.Spec.Name} - ), + filters={"service": service_with_too_many_resources.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -382,16 +381,15 @@ async def test_pending_service_task_with_insufficient_resources_with_labelled_se service_with_labels = await create_service( task_template_with_too_many_resource, service_labels, "pending" ) - assert service_with_labels.Spec + assert service_with_labels.spec pending_tasks = await pending_service_tasks_with_insufficient_resources( autoscaling_docker, service_labels=list(service_labels) ) - service_tasks = parse_obj_as( - list[Task], + service_tasks = TypeAdapter(list[Task]).validate_python( await autoscaling_docker.tasks.list( - filters={"service": service_with_labels.Spec.Name} - ), + filters={"service": service_with_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -438,21 +436,16 @@ async def test_pending_service_task_with_insufficient_resources_properly_sorts_t assert len(pending_tasks) == len(services) # check sorting is done by creation date - last_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( - days=1 - ) + last_date = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=1) for task in pending_tasks: - assert task.CreatedAt # NOTE: in this case they are but they might be None - assert ( - to_datetime(task.CreatedAt).replace(tzinfo=datetime.timezone.utc) - > last_date - ) - last_date = to_datetime(task.CreatedAt).replace(tzinfo=datetime.timezone.utc) + assert task.created_at # NOTE: in this case they are but they might be None + assert to_datetime(task.created_at).replace(tzinfo=datetime.UTC) > last_date + last_date = to_datetime(task.created_at).replace(tzinfo=datetime.UTC) def test_safe_sort_key_callback(): tasks_with_faulty_timestamp = [ - Task(ID=n, CreatedAt=value) # type: ignore + Task(ID=f"{n}", CreatedAt=value) for n, value in enumerate( [ # SEE test_to_datetime_conversion_known_errors @@ -460,7 +453,7 @@ def test_safe_sort_key_callback(): "2023-03-15 09:20:58.123456", "2023-03-15T09:20:58.123456", "2023-03-15T09:20:58.123456Z", - f"{datetime.datetime.now(datetime.timezone.utc)}", + f"{datetime.datetime.now(datetime.UTC)}", "corrupted string", ] ) @@ -468,16 +461,16 @@ def test_safe_sort_key_callback(): sorted_tasks = sorted(tasks_with_faulty_timestamp, key=_by_created_dt) assert len(sorted_tasks) == len(tasks_with_faulty_timestamp) - assert {t.ID for t in sorted_tasks} == {t.ID for t in tasks_with_faulty_timestamp} + assert {t.id for t in sorted_tasks} == {t.id for t in tasks_with_faulty_timestamp} def test_get_node_total_resources(host_node: Node): resources = get_node_total_resources(host_node) - assert host_node.Description - assert host_node.Description.Resources - assert host_node.Description.Resources.NanoCPUs - assert resources.cpus == (host_node.Description.Resources.NanoCPUs / 10**9) - assert resources.ram == host_node.Description.Resources.MemoryBytes + assert host_node.description + assert host_node.description.resources + assert host_node.description.resources.nano_cp_us + assert resources.cpus == (host_node.description.resources.nano_cp_us / 10**9) + assert resources.ram == host_node.description.resources.memory_bytes async def test_compute_cluster_total_resources_with_no_nodes_returns_0( @@ -502,12 +495,11 @@ async def test_get_resources_from_docker_task_with_no_reservation_returns_0( task_template: dict[str, Any], ): service_with_no_resources = await create_service(task_template, {}, "running") - assert service_with_no_resources.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_no_resources.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await autoscaling_docker.tasks.list( - filters={"service": service_with_no_resources.Spec.Name} - ), + filters={"service": service_with_no_resources.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -531,10 +523,9 @@ async def test_get_resources_from_docker_task_with_reservations( NUM_CPUS, 0 ) service = await create_service(task_template_with_reservations, {}, "running") - assert service.Spec - service_tasks = parse_obj_as( - list[Task], - await async_docker_client.tasks.list(filters={"service": service.Spec.Name}), + assert service.spec + service_tasks = TypeAdapter(list[Task]).validate_python( + await async_docker_client.tasks.list(filters={"service": service.spec.name}) ) assert service_tasks assert len(service_tasks) == 1 @@ -559,19 +550,18 @@ async def test_get_resources_from_docker_task_with_reservations_and_limits_retur NUM_CPUS, 0 ) task_template_with_reservations["Resources"] |= create_task_limits( - host_cpu_count, parse_obj_as(ByteSize, "100Mib") + host_cpu_count, TypeAdapter(ByteSize).validate_python("100Mib") )["Resources"] service = await create_service(task_template_with_reservations, {}, "running") - assert service.Spec - service_tasks = parse_obj_as( - list[Task], - await async_docker_client.tasks.list(filters={"service": service.Spec.Name}), + assert service.spec + service_tasks = TypeAdapter(list[Task]).validate_python( + await async_docker_client.tasks.list(filters={"service": service.spec.name}) ) assert service_tasks assert len(service_tasks) == 1 assert get_max_resources_from_docker_task(service_tasks[0]) == Resources( - cpus=host_cpu_count, ram=parse_obj_as(ByteSize, "100Mib") + cpus=host_cpu_count, ram=TypeAdapter(ByteSize).validate_python("100Mib") ) @@ -619,10 +609,9 @@ async def test_get_task_instance_restriction( "pending" if placement_constraints else "running", placement_constraints, ) - assert service.Spec - service_tasks = parse_obj_as( - list[Task], - await autoscaling_docker.tasks.list(filters={"service": service.Spec.Name}), + assert service.spec + service_tasks = TypeAdapter(list[Task]).validate_python( + await autoscaling_docker.tasks.list(filters={"service": service.spec.name}) ) instance_type_or_none = await get_task_instance_restriction( autoscaling_docker, service_tasks[0] @@ -642,12 +631,11 @@ async def test_compute_tasks_needed_resources( faker: Faker, ): service_with_no_resources = await create_service(task_template, {}, "running") - assert service_with_no_resources.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_no_resources.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await autoscaling_docker.tasks.list( - filters={"service": service_with_no_resources.Spec.Name} - ), + filters={"service": service_with_no_resources.spec.name} + ) ) assert compute_tasks_needed_resources(service_tasks) == Resources.create_as_empty() @@ -662,10 +650,9 @@ async def test_compute_tasks_needed_resources( ) all_tasks = service_tasks for s in services: - assert s.Spec - service_tasks = parse_obj_as( - list[Task], - await autoscaling_docker.tasks.list(filters={"service": s.Spec.Name}), + assert s.spec + service_tasks = TypeAdapter(list[Task]).validate_python( + await autoscaling_docker.tasks.list(filters={"service": s.spec.name}) ) assert compute_tasks_needed_resources(service_tasks) == Resources( cpus=1, ram=ByteSize(0) @@ -872,7 +859,7 @@ async def test_get_docker_swarm_join_script_returning_unexpected_command_raises( def test_get_docker_login_on_start_bash_command(): registry_settings = RegistrySettings( - **RegistrySettings.Config.schema_extra["examples"][0] + **RegistrySettings.model_config["json_schema_extra"]["examples"][0] ) returned_command = get_docker_login_on_start_bash_command(registry_settings) assert ( @@ -884,11 +871,11 @@ def test_get_docker_login_on_start_bash_command(): async def test_try_get_node_with_name( autoscaling_docker: AutoscalingDocker, host_node: Node ): - assert host_node.Description - assert host_node.Description.Hostname + assert host_node.description + assert host_node.description.hostname received_node = await find_node_with_name( - autoscaling_docker, host_node.Description.Hostname + autoscaling_docker, host_node.description.hostname ) assert received_node == host_node @@ -896,11 +883,11 @@ async def test_try_get_node_with_name( async def test_try_get_node_with_name_fake( autoscaling_docker: AutoscalingDocker, fake_node: Node ): - assert fake_node.Description - assert fake_node.Description.Hostname + assert fake_node.description + assert fake_node.description.hostname received_node = await find_node_with_name( - autoscaling_docker, fake_node.Description.Hostname + autoscaling_docker, fake_node.description.hostname ) assert received_node is None @@ -921,8 +908,8 @@ async def test_find_node_with_name_with_common_prefixed_nodes( needed_host_name = f"{common_prefix}11" found_node = await find_node_with_name(autoscaling_docker, needed_host_name) assert found_node - assert found_node.Description - assert found_node.Description.Hostname == needed_host_name + assert found_node.description + assert found_node.description.hostname == needed_host_name async def test_find_node_with_smaller_name_with_common_prefixed_nodes_returns_none( @@ -946,53 +933,53 @@ async def test_find_node_with_smaller_name_with_common_prefixed_nodes_returns_no async def test_tag_node( autoscaling_docker: AutoscalingDocker, host_node: Node, faker: Faker ): - assert host_node.Description - assert host_node.Description.Hostname + assert host_node.description + assert host_node.description.hostname tags = faker.pydict(allowed_types=(str,)) await tag_node(autoscaling_docker, host_node, tags=tags, available=False) updated_node = await find_node_with_name( - autoscaling_docker, host_node.Description.Hostname + autoscaling_docker, host_node.description.hostname ) assert updated_node - assert updated_node.Spec - assert updated_node.Spec.Availability == Availability.drain - assert updated_node.Spec.Labels == tags + assert updated_node.spec + assert updated_node.spec.availability == Availability.drain + assert updated_node.spec.labels == tags await tag_node(autoscaling_docker, updated_node, tags={}, available=True) updated_node = await find_node_with_name( - autoscaling_docker, host_node.Description.Hostname + autoscaling_docker, host_node.description.hostname ) assert updated_node - assert updated_node.Spec - assert updated_node.Spec.Availability == Availability.active - assert updated_node.Spec.Labels == {} + assert updated_node.spec + assert updated_node.spec.availability == Availability.active + assert updated_node.spec.labels == {} async def test_tag_node_out_of_sequence_error( autoscaling_docker: AutoscalingDocker, host_node: Node, faker: Faker ): - assert host_node.Description - assert host_node.Description.Hostname + assert host_node.description + assert host_node.description.hostname tags = faker.pydict(allowed_types=(str,)) # this works updated_node = await tag_node( autoscaling_docker, host_node, tags=tags, available=False ) assert updated_node - assert host_node.Version - assert host_node.Version.Index - assert updated_node.Version - assert updated_node.Version.Index - assert host_node.Version.Index < updated_node.Version.Index + assert host_node.version + assert host_node.version.index + assert updated_node.version + assert updated_node.version.index + assert host_node.version.index < updated_node.version.index # running the same call with the old node should not raise an out of sequence error updated_node2 = await tag_node( autoscaling_docker, host_node, tags=tags, available=True ) assert updated_node2 - assert updated_node2.Version - assert updated_node2.Version.Index - assert updated_node2.Version.Index > updated_node.Version.Index + assert updated_node2.version + assert updated_node2.version.index + assert updated_node2.version.index > updated_node.version.index async def test_set_node_availability( @@ -1132,25 +1119,25 @@ def test_is_node_ready_and_available(create_fake_node: Callable[..., Node]): def test_is_node_osparc_ready(create_fake_node: Callable[..., Node], faker: Faker): fake_node = create_fake_node() - assert fake_node.Spec - assert fake_node.Spec.Availability is Availability.drain + assert fake_node.spec + assert fake_node.spec.availability is Availability.drain # no labels, not ready and drained assert not is_node_osparc_ready(fake_node) # no labels, not ready, but active - fake_node.Spec.Availability = Availability.active + fake_node.spec.availability = Availability.active assert not is_node_osparc_ready(fake_node) # no labels, ready and active - fake_node.Status = NodeStatus(State=NodeState.ready, Message=None, Addr=None) + fake_node.status = NodeStatus(State=NodeState.ready, Message=None, Addr=None) assert not is_node_osparc_ready(fake_node) # add some random labels - assert fake_node.Spec - fake_node.Spec.Labels = faker.pydict(allowed_types=(str,)) + assert fake_node.spec + fake_node.spec.labels = faker.pydict(allowed_types=(str,)) assert not is_node_osparc_ready(fake_node) # add the expected label - fake_node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "false" + fake_node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "false" assert not is_node_osparc_ready(fake_node) # make it ready - fake_node.Spec.Labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" + fake_node.spec.labels[_OSPARC_SERVICE_READY_LABEL_KEY] = "true" assert is_node_osparc_ready(fake_node) @@ -1209,9 +1196,9 @@ async def test_set_node_found_empty( ): # initial state assert is_node_ready_and_available(host_node, availability=Availability.active) - assert host_node.Spec - assert host_node.Spec.Labels - assert _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY not in host_node.Spec.Labels + assert host_node.spec + assert host_node.spec.labels + assert _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY not in host_node.spec.labels # the date does not exist as nothing was done node_empty_since = await get_node_empty_since(host_node) @@ -1219,9 +1206,9 @@ async def test_set_node_found_empty( # now we set it to empty updated_node = await set_node_found_empty(autoscaling_docker, host_node, empty=True) - assert updated_node.Spec - assert updated_node.Spec.Labels - assert _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY in updated_node.Spec.Labels + assert updated_node.spec + assert updated_node.spec.labels + assert _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY in updated_node.spec.labels # we can get that empty date back node_empty_since = await get_node_empty_since(updated_node) @@ -1232,9 +1219,9 @@ async def test_set_node_found_empty( updated_node = await set_node_found_empty( autoscaling_docker, host_node, empty=False ) - assert updated_node.Spec - assert updated_node.Spec.Labels - assert _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY not in updated_node.Spec.Labels + assert updated_node.spec + assert updated_node.spec.labels + assert _OSPARC_NODE_EMPTY_DATETIME_LABEL_KEY not in updated_node.spec.labels # we can't get a date anymore node_empty_since = await get_node_empty_since(updated_node) @@ -1253,9 +1240,9 @@ async def test_set_node_begin_termination_process( ): # initial state assert is_node_ready_and_available(host_node, availability=Availability.active) - assert host_node.Spec - assert host_node.Spec.Labels - assert _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY not in host_node.Spec.Labels + assert host_node.spec + assert host_node.spec.labels + assert _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY not in host_node.spec.labels # the termination was not started, therefore no date assert get_node_termination_started_since(host_node) is None @@ -1263,9 +1250,9 @@ async def test_set_node_begin_termination_process( updated_node = await set_node_begin_termination_process( autoscaling_docker, host_node ) - assert updated_node.Spec - assert updated_node.Spec.Labels - assert _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY in updated_node.Spec.Labels + assert updated_node.spec + assert updated_node.spec.labels + assert _OSPARC_NODE_TERMINATION_PROCESS_LABEL_KEY in updated_node.spec.labels await asyncio.sleep(1) diff --git a/services/autoscaling/tests/unit/test_utils_rabbitmq.py b/services/autoscaling/tests/unit/test_utils_rabbitmq.py index 1c5920f9dc7..93924bf9a5a 100644 --- a/services/autoscaling/tests/unit/test_utils_rabbitmq.py +++ b/services/autoscaling/tests/unit/test_utils_rabbitmq.py @@ -19,7 +19,7 @@ ProgressRabbitMessageNode, ProgressType, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from servicelib.rabbitmq import BIND_TO_ALL_TOPICS, RabbitMQClient from settings_library.rabbit import RabbitSettings @@ -78,12 +78,11 @@ async def test_post_task_log_message( osparc_docker_label_keys.to_simcore_runtime_docker_labels(), "running", ) - assert service_with_labels.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_labels.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await async_docker_client.tasks.list( - filters={"service": service_with_labels.Spec.Name} - ), + filters={"service": service_with_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -104,7 +103,7 @@ async def test_post_task_log_message( messages=[f"[cluster] {log_message}"], log_level=0, ) - .json() + .model_dump_json() .encode() ) print("... message received") @@ -125,12 +124,11 @@ async def test_post_task_log_message_does_not_raise_if_service_has_no_labels( faker: Faker, ): service_without_labels = await create_service(task_template, {}, "running") - assert service_without_labels.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_without_labels.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await async_docker_client.tasks.list( - filters={"service": service_without_labels.Spec.Name} - ), + filters={"service": service_without_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -170,12 +168,11 @@ async def test_post_task_progress_message( osparc_docker_label_keys.to_simcore_runtime_docker_labels(), "running", ) - assert service_with_labels.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_with_labels.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await async_docker_client.tasks.list( - filters={"service": service_with_labels.Spec.Name} - ), + filters={"service": service_with_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 @@ -196,7 +193,7 @@ async def test_post_task_progress_message( progress_type=ProgressType.CLUSTER_UP_SCALING, report=ProgressReport(actual_value=progress_value, total=1), ) - .json() + .model_dump_json() .encode() ) print("... message received") @@ -217,12 +214,11 @@ async def test_post_task_progress_does_not_raise_if_service_has_no_labels( faker: Faker, ): service_without_labels = await create_service(task_template, {}, "running") - assert service_without_labels.Spec - service_tasks = parse_obj_as( - list[Task], + assert service_without_labels.spec + service_tasks = TypeAdapter(list[Task]).validate_python( await async_docker_client.tasks.list( - filters={"service": service_without_labels.Spec.Name} - ), + filters={"service": service_without_labels.spec.name} + ) ) assert service_tasks assert len(service_tasks) == 1 diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml index dc76ded446f..d0e829c151f 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml @@ -90,7 +90,7 @@ services: AUTOSCALING_EC2_REGION_NAME: ${CLUSTERS_KEEPER_EC2_REGION_NAME} AUTOSCALING_EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} AUTOSCALING_NODES_MONITORING: null - AUTOSCALING_POLL_INTERVAL: 10 + AUTOSCALING_POLL_INTERVAL: 00:00:10 DASK_MONITORING_URL: tls://dask-scheduler:8786 DASK_SCHEDULER_AUTH: '{"type":"tls","tls_ca_file":"${DASK_TLS_CA_FILE}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}' EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} From e97258e50c97ad3a966f7524c39bb6ce1778ebdc Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Fri, 18 Oct 2024 08:06:12 +0200 Subject: [PATCH 09/53] =?UTF-8?q?=F0=9F=8E=A8=20pydantic2=20migration:=20f?= =?UTF-8?q?ixed=20unit-tests=20for=20dynamic-sidecar=20(#6534)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../aws-library/src/aws_library/s3/_client.py | 8 +- packages/aws-library/tests/test_s3_client.py | 58 ++-- packages/aws-library/tests/test_s3_utils.py | 46 ++- .../common_library/pydantic_type_adapters.py | 8 - .../src/common_library/pydantic_validators.py | 35 +++ .../tests/test_pydantic_validators.py | 40 +++ .../src/models_library/aiodocker_api.py | 2 +- .../src/models_library/rest_pagination.py | 6 +- .../models_library/rest_pagination_utils.py | 5 +- .../src/models_library/services_creation.py | 11 +- .../src/models_library/services_io.py | 2 - .../src/models_library/services_resources.py | 4 - .../src/models_library/services_types.py | 28 +- .../helpers/playwright_sim4life.py | 3 +- .../aiohttp/long_running_tasks/_server.py | 10 +- .../src/servicelib/fastapi/docker_utils.py | 9 +- .../fastapi/long_running_tasks/_client.py | 6 +- .../src/settings_library/ssm.py | 16 +- .../simcore_service_agent/core/settings.py | 20 +- .../services/volumes_manager.py | 4 +- .../core/application.py | 2 +- .../dynamic_services_settings/scheduler.py | 10 + .../core/settings.py | 5 + ...modules_dynamic_sidecar_client_api_thin.py | 9 +- .../core/settings.py | 5 + services/dynamic-sidecar/.env-devel | 2 +- services/dynamic-sidecar/openapi.json | 294 +++++++++++------- .../dynamic-sidecar/requirements/_base.txt | 214 +++++++++++-- .../dynamic-sidecar/requirements/_test.txt | 4 +- .../api/rest/containers.py | 6 +- .../core/application.py | 14 +- .../core/docker_utils.py | 2 +- .../core/errors.py | 6 +- .../core/external_dependencies.py | 4 +- .../core/reserved_space.py | 4 +- .../core/settings.py | 62 +++- .../core/storage.py | 8 +- .../models/shared_store.py | 16 +- .../modules/long_running_tasks.py | 2 +- .../modules/outputs/_directory_utils.py | 6 +- .../modules/outputs/_event_filter.py | 6 +- .../modules/outputs/_manager.py | 32 +- .../modules/resource_tracking/_models.py | 6 +- .../modules/service_liveness.py | 4 +- .../modules/user_services_preferences/_db.py | 6 +- .../user_services_preferences/_errors.py | 6 +- .../user_services_preferences/_packaging.py | 6 +- services/dynamic-sidecar/tests/conftest.py | 9 +- .../test_modules_long_running_tasks.py | 7 +- .../test_modules_user_services_preferences.py | 10 +- .../tests/unit/test_api_rest_containers.py | 31 +- ..._api_rest_containers_long_running_tasks.py | 11 +- .../unit/test_api_rest_prometheus_metrics.py | 13 +- .../tests/unit/test_api_rest_volumes.py | 4 +- .../test_api_rest_workflow_service_metrics.py | 17 +- .../tests/unit/test_core_reserved_space.py | 6 +- .../tests/unit/test_core_settings.py | 11 +- .../tests/unit/test_core_stroage.py | 3 +- .../tests/unit/test_models_shared_store.py | 50 ++- .../tests/unit/test_modules_notifier.py | 10 +- .../unit/test_modules_outputs_event_filter.py | 11 +- .../unit/test_modules_outputs_manager.py | 4 +- .../unit/test_modules_outputs_watcher.py | 18 +- ...test_modules_system_monitor__disk_usage.py | 4 +- ...er_services_preferences_user_preference.py | 6 +- .../core/settings.py | 7 + .../projects/settings.py | 5 + .../studies_dispatcher/settings.py | 5 + 68 files changed, 903 insertions(+), 401 deletions(-) delete mode 100644 packages/common-library/src/common_library/pydantic_type_adapters.py create mode 100644 packages/common-library/src/common_library/pydantic_validators.py create mode 100644 packages/common-library/tests/test_pydantic_validators.py diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py index 816208c22c8..acd9402fda3 100644 --- a/packages/aws-library/src/aws_library/s3/_client.py +++ b/packages/aws-library/src/aws_library/s3/_client.py @@ -13,7 +13,7 @@ from boto3.s3.transfer import TransferConfig from botocore import exceptions as botocore_exc from botocore.client import Config -from common_library.pydantic_type_adapters import AnyUrlLegacyAdapter +from common_library.pydantic_networks_extension import AnyUrlLegacy from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart from models_library.basic_types import SHA256Str from pydantic import AnyUrl, ByteSize, TypeAdapter @@ -264,7 +264,7 @@ async def create_single_presigned_download_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - return f"{AnyUrlLegacyAdapter.validate_python(generated_link)}" + return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}" @s3_exception_handler(_logger) async def create_single_presigned_upload_link( @@ -277,7 +277,7 @@ async def create_single_presigned_upload_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - return f"{AnyUrlLegacyAdapter.validate_python(generated_link)}" + return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}" @s3_exception_handler(_logger) async def create_multipart_upload_links( @@ -474,6 +474,6 @@ def is_multipart(file_size: ByteSize) -> bool: @staticmethod def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl: - return AnyUrlLegacyAdapter.validate_python( + return TypeAdapter(AnyUrlLegacy).validate_python( f"s3://{bucket}/{urllib.parse.quote(object_key)}" ) diff --git a/packages/aws-library/tests/test_s3_client.py b/packages/aws-library/tests/test_s3_client.py index af98fd6523f..d07075aed30 100644 --- a/packages/aws-library/tests/test_s3_client.py +++ b/packages/aws-library/tests/test_s3_client.py @@ -14,7 +14,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass from pathlib import Path -from typing import Any, Final +from typing import Any import botocore.exceptions import pytest @@ -49,8 +49,6 @@ from types_aiobotocore_s3 import S3Client from types_aiobotocore_s3.literals import BucketLocationConstraintType -_BYTE_SIZE_ADAPTER: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize) - @pytest.fixture async def simcore_s3_api( @@ -113,7 +111,7 @@ async def _( file, MultiPartUploadLinks( upload_id="fake", - chunk_size=_BYTE_SIZE_ADAPTER.validate_python(file.stat().st_size), + chunk_size=TypeAdapter(ByteSize).validate_python(file.stat().st_size), urls=[presigned_url], ), ) @@ -137,7 +135,7 @@ async def with_uploaded_file_on_s3( s3_client: S3Client, with_s3_bucket: S3BucketName, ) -> AsyncIterator[UploadedFile]: - test_file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("10Kib")) + test_file = create_file_of_size(TypeAdapter(ByteSize).validate_python("10Kib")) await s3_client.upload_file( Filename=f"{test_file}", Bucket=with_s3_bucket, @@ -592,7 +590,7 @@ async def test_undelete_file( assert file_metadata.size == with_uploaded_file_on_s3.local_path.stat().st_size # upload another file on top of the existing one - new_file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("5Kib")) + new_file = create_file_of_size(TypeAdapter(ByteSize).validate_python("5Kib")) await s3_client.upload_file( Filename=f"{new_file}", Bucket=with_s3_bucket, @@ -747,7 +745,7 @@ async def test_create_single_presigned_upload_link( [Path, str, S3BucketName, S3ObjectKey], Awaitable[None] ], ): - file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib")) + file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1Mib")) s3_object_key = file.name presigned_url = await simcore_s3_api.create_single_presigned_upload_link( bucket=with_s3_bucket, @@ -775,7 +773,7 @@ async def test_create_single_presigned_upload_link_with_non_existing_bucket_rais create_file_of_size: Callable[[ByteSize], Path], default_expiration_time_seconds: int, ): - file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib")) + file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1Mib")) s3_object_key = file.name with pytest.raises(S3BucketInvalidError): await simcore_s3_api.create_single_presigned_upload_link( @@ -1082,7 +1080,7 @@ async def test_copy_file_invalid_raises( create_file_of_size: Callable[[ByteSize], Path], faker: Faker, ): - file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1MiB")) + file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1MiB")) uploaded_file = await upload_file(file) dst_object_key = faker.file_name() # NOTE: since aioboto3 13.1.0 this raises S3KeyNotFoundError instead of S3BucketInvalidError @@ -1107,9 +1105,9 @@ async def test_copy_file_invalid_raises( "directory_size, min_file_size, max_file_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("1Mib"), - _BYTE_SIZE_ADAPTER.validate_python("1B"), - _BYTE_SIZE_ADAPTER.validate_python("10Kib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("1B"), + TypeAdapter(ByteSize).validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1133,9 +1131,9 @@ async def test_get_directory_metadata( "directory_size, min_file_size, max_file_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("1Mib"), - _BYTE_SIZE_ADAPTER.validate_python("1B"), - _BYTE_SIZE_ADAPTER.validate_python("10Kib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("1B"), + TypeAdapter(ByteSize).validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1165,9 +1163,9 @@ async def test_get_directory_metadata_raises( "directory_size, min_file_size, max_file_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("1Mib"), - _BYTE_SIZE_ADAPTER.validate_python("1B"), - _BYTE_SIZE_ADAPTER.validate_python("10Kib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("1B"), + TypeAdapter(ByteSize).validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1201,9 +1199,9 @@ async def test_delete_file_recursively( "directory_size, min_file_size, max_file_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("1Mib"), - _BYTE_SIZE_ADAPTER.validate_python("1B"), - _BYTE_SIZE_ADAPTER.validate_python("10Kib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("1B"), + TypeAdapter(ByteSize).validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1239,9 +1237,9 @@ async def test_delete_file_recursively_raises( "directory_size, min_file_size, max_file_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("1Mib"), - _BYTE_SIZE_ADAPTER.validate_python("1B"), - _BYTE_SIZE_ADAPTER.validate_python("10Kib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("1B"), + TypeAdapter(ByteSize).validate_python("10Kib"), ) ], ids=byte_size_ids, @@ -1339,14 +1337,14 @@ def run_async_test(*args, **kwargs) -> None: "directory_size, min_file_size, max_file_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("1Mib"), - _BYTE_SIZE_ADAPTER.validate_python("1B"), - _BYTE_SIZE_ADAPTER.validate_python("10Kib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("1B"), + TypeAdapter(ByteSize).validate_python("10Kib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("500Mib"), - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), - _BYTE_SIZE_ADAPTER.validate_python("50Mib"), + TypeAdapter(ByteSize).validate_python("500Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("50Mib"), ), ], ids=byte_size_ids, diff --git a/packages/aws-library/tests/test_s3_utils.py b/packages/aws-library/tests/test_s3_utils.py index bf021978585..cfba1634943 100644 --- a/packages/aws-library/tests/test_s3_utils.py +++ b/packages/aws-library/tests/test_s3_utils.py @@ -4,8 +4,6 @@ # pylint: disable=unused-variable -from typing import Final - import pytest from aws_library.s3._utils import ( _MULTIPART_MAX_NUMBER_OF_PARTS, @@ -15,61 +13,59 @@ from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.parametrizations import byte_size_ids -_BYTE_SIZE_ADAPTER: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize) - @pytest.mark.parametrize( "file_size, expected_num_chunks, expected_chunk_size", [ ( - _BYTE_SIZE_ADAPTER.validate_python("5Mib"), + TypeAdapter(ByteSize).validate_python("5Mib"), 1, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), 1, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("20Mib"), + TypeAdapter(ByteSize).validate_python("20Mib"), 2, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("50Mib"), + TypeAdapter(ByteSize).validate_python("50Mib"), 5, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("150Mib"), + TypeAdapter(ByteSize).validate_python("150Mib"), 15, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("550Mib"), + TypeAdapter(ByteSize).validate_python("550Mib"), 55, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("560Gib"), + TypeAdapter(ByteSize).validate_python("560Gib"), 5735, - _BYTE_SIZE_ADAPTER.validate_python("100Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("5Tib"), + TypeAdapter(ByteSize).validate_python("5Tib"), 8739, - _BYTE_SIZE_ADAPTER.validate_python("600Mib"), + TypeAdapter(ByteSize).validate_python("600Mib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("15Tib"), + TypeAdapter(ByteSize).validate_python("15Tib"), 7680, - _BYTE_SIZE_ADAPTER.validate_python("2Gib"), + TypeAdapter(ByteSize).validate_python("2Gib"), ), ( - _BYTE_SIZE_ADAPTER.validate_python("9431773844"), + TypeAdapter(ByteSize).validate_python("9431773844"), 900, - _BYTE_SIZE_ADAPTER.validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), ), ], ids=byte_size_ids, @@ -83,7 +79,7 @@ def test_compute_num_file_chunks( def test_enormous_file_size_raises_value_error(): - enormous_file_size = _BYTE_SIZE_ADAPTER.validate_python( + enormous_file_size = TypeAdapter(ByteSize).validate_python( ( max(_MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE) * _MULTIPART_MAX_NUMBER_OF_PARTS diff --git a/packages/common-library/src/common_library/pydantic_type_adapters.py b/packages/common-library/src/common_library/pydantic_type_adapters.py deleted file mode 100644 index 883100c4717..00000000000 --- a/packages/common-library/src/common_library/pydantic_type_adapters.py +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Final - -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy, AnyUrlLegacy -from pydantic import TypeAdapter - -AnyUrlLegacyAdapter: Final[TypeAdapter[AnyUrlLegacy]] = TypeAdapter(AnyUrlLegacy) - -AnyHttpUrlLegacyAdapter: Final[TypeAdapter[AnyHttpUrlLegacy]] = TypeAdapter(AnyHttpUrlLegacy) diff --git a/packages/common-library/src/common_library/pydantic_validators.py b/packages/common-library/src/common_library/pydantic_validators.py new file mode 100644 index 00000000000..33f693a4ba5 --- /dev/null +++ b/packages/common-library/src/common_library/pydantic_validators.py @@ -0,0 +1,35 @@ +import datetime +import warnings +from datetime import timedelta + +from pydantic import TypeAdapter, field_validator + + +def timedelta_try_convert_str_to_float(field: str): + """Transforms a float/int number into a valid datetime as it used to work in the past""" + + def _try_convert_str_to_float_or_return( + v: datetime.timedelta | str | float, + ) -> datetime.timedelta | str | float: + if isinstance(v, str): + try: + converted_value = float(v) + + iso8601_format = TypeAdapter(timedelta).dump_python( + timedelta(seconds=converted_value), mode="json" + ) + warnings.warn( + f"{field}='{v}' -should be set to-> {field}='{iso8601_format}' (ISO8601 datetime format). " + "Please also convert the value in the >>OPS REPOSITORY<<. " + "For details: https://docs.pydantic.dev/1.10/usage/types/#datetime-types.", + DeprecationWarning, + stacklevel=8, + ) + + return converted_value + except ValueError: + # returns format like "1:00:00" + return v + return v + + return field_validator(field, mode="before")(_try_convert_str_to_float_or_return) diff --git a/packages/common-library/tests/test_pydantic_validators.py b/packages/common-library/tests/test_pydantic_validators.py new file mode 100644 index 00000000000..825f986918a --- /dev/null +++ b/packages/common-library/tests/test_pydantic_validators.py @@ -0,0 +1,40 @@ +from datetime import timedelta + +import pytest +from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from faker import Faker +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict + + +def test_validate_timedelta_in_legacy_mode( + monkeypatch: pytest.MonkeyPatch, faker: Faker +): + class Settings(BaseSettings): + APP_NAME: str + REQUEST_TIMEOUT: timedelta = Field(default=timedelta(seconds=40)) + + _try_convert_request_timeout = timedelta_try_convert_str_to_float( + "REQUEST_TIMEOUT" + ) + + model_config = SettingsConfigDict() + + app_name = faker.pystr() + env_vars: dict[str, str | bool] = {"APP_NAME": app_name} + + # without timedelta + setenvs_from_dict(monkeypatch, env_vars) + settings = Settings() + print(settings.model_dump()) + assert app_name == settings.APP_NAME + assert timedelta(seconds=40) == settings.REQUEST_TIMEOUT + + # with timedelta in seconds + env_vars["REQUEST_TIMEOUT"] = "5555" + setenvs_from_dict(monkeypatch, env_vars) + settings = Settings() + print(settings.model_dump()) + assert app_name == settings.APP_NAME + assert timedelta(seconds=5555) == settings.REQUEST_TIMEOUT diff --git a/packages/models-library/src/models_library/aiodocker_api.py b/packages/models-library/src/models_library/aiodocker_api.py index c5c3b3333a0..865570fbe36 100644 --- a/packages/models-library/src/models_library/aiodocker_api.py +++ b/packages/models-library/src/models_library/aiodocker_api.py @@ -11,7 +11,7 @@ class AioDockerContainerSpec(ContainerSpec): - Env: dict[str, str | None] | None = Field( # type: ignore + Env: dict[str, str | None] | None = Field( default=None, description="aiodocker expects here a dictionary and re-convert it back internally`.\n", ) diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index 63b4e4948b0..28556d50303 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -1,6 +1,6 @@ from typing import Annotated, Final, Generic, TypeAlias, TypeVar -from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from pydantic import ( BaseModel, BeforeValidator, @@ -99,7 +99,9 @@ class PageLinks( PageRefs[ Annotated[ str, - BeforeValidator(lambda x: str(AnyHttpUrlLegacyAdapter.validate_python(x))), + BeforeValidator( + lambda x: str(TypeAdapter(AnyHttpUrlLegacy).validate_python(x)) + ), ] ] ): diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index 1adfc5625c6..41899acd8cf 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -1,7 +1,8 @@ from math import ceil from typing import Any, Protocol, TypedDict, Union, runtime_checkable -from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy +from pydantic import TypeAdapter from .rest_pagination import PageLinks, PageMetaInfoLimitOffset @@ -40,7 +41,7 @@ def _replace_query(url: _URLType, query: dict[str, Any]) -> str: new_url = url.replace_query_params(**query) new_url_str = f"{new_url}" - return f"{AnyHttpUrlLegacyAdapter.validate_python(new_url_str)}" + return f"{TypeAdapter(AnyHttpUrlLegacy).validate_python(new_url_str)}" class PageDict(TypedDict): diff --git a/packages/models-library/src/models_library/services_creation.py b/packages/models-library/src/models_library/services_creation.py index c1c2c5172fc..5abb8c9e4d2 100644 --- a/packages/models-library/src/models_library/services_creation.py +++ b/packages/models-library/src/models_library/services_creation.py @@ -1,9 +1,9 @@ from typing import Any -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, TypeAdapter -from .services import ServiceKey, ServiceVersion from .services_resources import ServiceResourcesDict +from .services_types import ServiceKey, ServiceVersion from .wallets import WalletID @@ -36,10 +36,13 @@ class CreateServiceMetricsAdditionalParams(BaseModel): "user_email": "test@test.com", "project_name": "_!New Study", "node_name": "the service of a lifetime _ *!", - "service_key": ServiceKey("simcore/services/dynamic/test"), - "service_version": ServiceVersion("0.0.1"), + "service_key": TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/test" + ), + "service_version": TypeAdapter(ServiceVersion).validate_python("0.0.1"), "service_resources": {}, "service_additional_metadata": {}, + "pricing_unit_cost_id": None, } } ) diff --git a/packages/models-library/src/models_library/services_io.py b/packages/models-library/src/models_library/services_io.py index 6f1acd3d494..db43ee6eb6c 100644 --- a/packages/models-library/src/models_library/services_io.py +++ b/packages/models-library/src/models_library/services_io.py @@ -149,7 +149,6 @@ class ServiceInput(BaseServiceIOModel): ) model_config = ConfigDict( - **BaseServiceIOModel.model_config, json_schema_extra={ "examples": [ # file-wo-widget: @@ -222,7 +221,6 @@ class ServiceOutput(BaseServiceIOModel): ) model_config = ConfigDict( - **BaseServiceIOModel.model_config, json_schema_extra={ "examples": [ { diff --git a/packages/models-library/src/models_library/services_resources.py b/packages/models-library/src/models_library/services_resources.py index ac7a6862754..7a2b65456a4 100644 --- a/packages/models-library/src/models_library/services_resources.py +++ b/packages/models-library/src/models_library/services_resources.py @@ -1,4 +1,3 @@ -import logging from enum import auto from typing import Any, Final, TypeAlias @@ -17,9 +16,6 @@ from .utils.enums import StrAutoEnum from .utils.fastapi_encoders import jsonable_encoder -_logger = logging.getLogger(__name__) - - ResourceName = str # NOTE: replace hard coded `container` with function which can diff --git a/packages/models-library/src/models_library/services_types.py b/packages/models-library/src/models_library/services_types.py index 87559eff993..e882c8dae09 100644 --- a/packages/models-library/src/models_library/services_types.py +++ b/packages/models-library/src/models_library/services_types.py @@ -1,8 +1,9 @@ -from typing import Annotated, TypeAlias +from typing import Annotated, Any, TypeAlias from uuid import uuid4 import arrow -from pydantic import StringConstraints +from pydantic import GetCoreSchemaHandler, StringConstraints, ValidationInfo +from pydantic_core import CoreSchema, core_schema from .basic_regex import PROPERTY_KEY_RE, SIMPLE_VERSION_RE from .services_regex import ( @@ -19,9 +20,13 @@ ServiceKey: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_KEY_RE)] -ServiceKeyEncoded: TypeAlias = Annotated[str, StringConstraints(pattern=SERVICE_ENCODED_KEY_RE)] +ServiceKeyEncoded: TypeAlias = Annotated[ + str, StringConstraints(pattern=SERVICE_ENCODED_KEY_RE) +] -DynamicServiceKey: TypeAlias = Annotated[str, StringConstraints(pattern=DYNAMIC_SERVICE_KEY_RE)] +DynamicServiceKey: TypeAlias = Annotated[ + str, StringConstraints(pattern=DYNAMIC_SERVICE_KEY_RE) +] ComputationalServiceKey: TypeAlias = Annotated[ str, StringConstraints(pattern=COMPUTATIONAL_SERVICE_KEY_RE) @@ -53,3 +58,18 @@ def create(cls) -> "RunID": utc_int_timestamp: int = arrow.utcnow().int_timestamp run_id_format = f"{utc_int_timestamp}_{uuid4()}" return cls(run_id_format) + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_after_validator_function(cls, handler(str)) + + @classmethod + def validate(cls, v: "RunID | str", _: ValidationInfo) -> "RunID": + if isinstance(v, cls): + return v + if isinstance(v, str): + return cls(v) + msg = f"Invalid value for RunID: {v}" + raise TypeError(msg) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/playwright_sim4life.py b/packages/pytest-simcore/src/pytest_simcore/helpers/playwright_sim4life.py index d2349d6fa7c..57f3b699377 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/playwright_sim4life.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/playwright_sim4life.py @@ -6,8 +6,7 @@ import arrow from playwright.sync_api import FrameLocator, Page, WebSocket, expect -from pydantic import TypeAdapter # pylint: disable=no-name-in-module -from pydantic import ByteSize +from pydantic import ByteSize, TypeAdapter from .logging_tools import log_context from .playwright import ( diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index bce905e9cd6..47640c491f2 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -5,9 +5,9 @@ from typing import Any from aiohttp import web -from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from models_library.utils.json_serialization import json_dumps -from pydantic import PositiveFloat +from pydantic import PositiveFloat, TypeAdapter from ...aiohttp import status from ...long_running_tasks._models import TaskGet @@ -68,13 +68,13 @@ async def start_long_running_task( ip_addr, port = request_.transport.get_extra_info( "sockname" ) # https://docs.python.org/3/library/asyncio-protocol.html#asyncio.BaseTransport.get_extra_info - status_url = AnyHttpUrlLegacyAdapter.validate_python( + status_url = TypeAdapter(AnyHttpUrlLegacy).validate_python( f"http://{ip_addr}:{port}{request_.app.router['get_task_status'].url_for(task_id=task_id)}" # NOSONAR ) - result_url = AnyHttpUrlLegacyAdapter.validate_python( + result_url = TypeAdapter(AnyHttpUrlLegacy).validate_python( f"http://{ip_addr}:{port}{request_.app.router['get_task_result'].url_for(task_id=task_id)}" # NOSONAR ) - abort_url = AnyHttpUrlLegacyAdapter.validate_python( + abort_url = TypeAdapter(AnyHttpUrlLegacy).validate_python( f"http://{ip_addr}:{port}{request_.app.router['cancel_and_delete_task'].url_for(task_id=task_id)}" # NOSONAR ) task_get = TaskGet( diff --git a/packages/service-library/src/servicelib/fastapi/docker_utils.py b/packages/service-library/src/servicelib/fastapi/docker_utils.py index c7db4c1cf2d..420c1418873 100644 --- a/packages/service-library/src/servicelib/fastapi/docker_utils.py +++ b/packages/service-library/src/servicelib/fastapi/docker_utils.py @@ -22,6 +22,10 @@ from ..logging_utils import log_catch from ..progress_bar import AsyncReportCB, ProgressBarData +_DEFAULT_MIN_IMAGE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( + "200MiB" +) + _logger = logging.getLogger(__name__) @@ -104,11 +108,6 @@ async def retrieve_image_layer_information( return None -_DEFAULT_MIN_IMAGE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( - "200MiB" -) - - async def pull_images( images: set[DockerGenericTag], registry_settings: RegistrySettings, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index 36458031ff8..3aa82e76a46 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -4,10 +4,10 @@ import warnings from typing import Any, Awaitable, Callable, Final -from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI, status from httpx import AsyncClient, HTTPError -from pydantic import PositiveFloat +from pydantic import PositiveFloat, TypeAdapter from tenacity import RetryCallState from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -132,7 +132,7 @@ def _client_configuration(self) -> ClientConfiguration: def _get_url(self, path: str) -> str: url = f"{self._base_url}{self._client_configuration.router_prefix}{path}" - return f"{AnyHttpUrlLegacyAdapter.validate_python(url)}" + return f"{TypeAdapter(AnyHttpUrlLegacy).validate_python(url)}" @retry_on_http_errors async def get_task_status( diff --git a/packages/settings-library/src/settings_library/ssm.py b/packages/settings-library/src/settings_library/ssm.py index 73bee409464..823ac57c1a2 100644 --- a/packages/settings-library/src/settings_library/ssm.py +++ b/packages/settings-library/src/settings_library/ssm.py @@ -1,7 +1,7 @@ from typing import Annotated -from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter -from pydantic import BeforeValidator, Field, SecretStr +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy +from pydantic import BeforeValidator, Field, SecretStr, TypeAdapter from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings @@ -9,9 +9,15 @@ class SSMSettings(BaseCustomSettings): SSM_ACCESS_KEY_ID: SecretStr - SSM_ENDPOINT: Annotated[ - str, BeforeValidator(lambda x: str(AnyHttpUrlLegacyAdapter.validate_python(x))) - ] | None = Field(default=None, description="do not define if using standard AWS") + SSM_ENDPOINT: ( + Annotated[ + str, + BeforeValidator( + lambda x: str(TypeAdapter(AnyHttpUrlLegacy).validate_python(x)) + ), + ] + | None + ) = Field(default=None, description="do not define if using standard AWS") SSM_REGION_NAME: str = "us-east-1" SSM_SECRET_ACCESS_KEY: SecretStr diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index 7c5f453f2d7..da2abf506d3 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -1,5 +1,6 @@ from datetime import timedelta +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from models_library.basic_types import BootModeEnum, LogLevel from pydantic import AliasChoices, AnyHttpUrl, Field, field_validator from settings_library.base import BaseCustomSettings @@ -52,14 +53,14 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): AGENT_VOLUMES_CLEANUP_INTERVAL: timedelta = Field( timedelta(minutes=1), description="interval for running volumes removal" ) - AGENT_VOLUMES_CLENUP_BOOK_KEEPING_INTERVAL: timedelta = Field( + AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL: timedelta = Field( timedelta(minutes=1), description=( "interval at which to scan for unsued volumes and keep track since " "they were detected as being unused" ), ) - AGENT_VOLUMES_CLENUP_REMOVE_VOLUMES_INACTIVE_FOR: timedelta = Field( + AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR: timedelta = Field( timedelta(minutes=65), description=( "if a volume is unused for more than this interval it can be removed. " @@ -77,6 +78,21 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): json_schema_extra={"auto_default_from_env": True}, ) + _try_convert_agent_volumes_cleanup_interval = timedelta_try_convert_str_to_float( + "AGENT_VOLUMES_CLEANUP_INTERVAL" + ) + + _try_convert_agent_volumes_cleanup_book_keeping_interval = ( + timedelta_try_convert_str_to_float( + "AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL" + ) + ) + _try_convert_agent_volumes_cleanup_remove_volumes_inactive_for = ( + timedelta_try_convert_str_to_float( + "AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR" + ) + ) + @field_validator("LOGLEVEL") @classmethod def valid_log_level(cls, value) -> LogLevel: diff --git a/services/agent/src/simcore_service_agent/services/volumes_manager.py b/services/agent/src/simcore_service_agent/services/volumes_manager.py index 526589a2c9c..fa5a0cd1b17 100644 --- a/services/agent/src/simcore_service_agent/services/volumes_manager.py +++ b/services/agent/src/simcore_service_agent/services/volumes_manager.py @@ -174,9 +174,9 @@ async def _on_startup() -> None: volumes_manager = VolumesManager( app=app, - book_keeping_interval=settings.AGENT_VOLUMES_CLENUP_BOOK_KEEPING_INTERVAL, + book_keeping_interval=settings.AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL, volume_cleanup_interval=settings.AGENT_VOLUMES_CLEANUP_INTERVAL, - remove_volumes_inactive_for=settings.AGENT_VOLUMES_CLENUP_REMOVE_VOLUMES_INACTIVE_FOR.total_seconds(), + remove_volumes_inactive_for=settings.AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR.total_seconds(), ) volumes_manager.set_to_app_state(app) await volumes_manager.setup() diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 330717e6062..3f904448696 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -115,7 +115,7 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: config_all_loggers( log_format_local_dev_enabled=settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED ) - _logger.debug(settings.json(indent=2)) + _logger.debug(settings.model_dump_json(indent=2)) # keep mostly quiet noisy loggers quiet_level: int = max( diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py index 831c7df2f18..fe7fa7db653 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py @@ -1,6 +1,7 @@ from datetime import timedelta from typing import Final +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from models_library.projects_networks import DockerNetworkName from pydantic import Field, NonNegativeInt, PositiveFloat from settings_library.base import BaseCustomSettings @@ -166,3 +167,12 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings): DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL: timedelta = Field( timedelta(0), description="time to sleep before removing a container" ) + + _try_convert_director_v2_dynamic_scheduler_interval = ( + timedelta_try_convert_str_to_float("DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL") + ) + _try_convert_director_v2_dynamic_sidecar_sleep_after_container_removal = ( + timedelta_try_convert_str_to_float( + "DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL" + ) + ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index d495dd4aeef..bf5006a228f 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -5,6 +5,7 @@ import datetime from functools import cached_property +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from models_library.basic_types import ( BootModeEnum, BuildTargetEnum, @@ -234,3 +235,7 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) return log_level + + _try_convert_service_tracking_heartbeat = timedelta_try_convert_str_to_float( + "SERVICE_TRACKING_HEARTBEAT" + ) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py index bd75682dec2..6403be5a78e 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py @@ -282,9 +282,12 @@ async def test_put_volumes( "post_containers_tasks", "/containers", { - "metrics_params": parse_obj_as( - CreateServiceMetricsAdditionalParams, - CreateServiceMetricsAdditionalParams.Config.schema_extra["example"], + "metrics_params": TypeAdapter( + CreateServiceMetricsAdditionalParams + ).validate_python( + CreateServiceMetricsAdditionalParams.model_config[ + "json_schema_extra" + ]["example"], ) }, id="post_containers_tasks", diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index a60ccc504cd..e7441665a30 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,5 +1,6 @@ import datetime +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from pydantic import Field, parse_obj_as, validator from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -48,6 +49,10 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def _validate_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + _try_convert_dynamic_scheduler_stop_service_timeout = ( + timedelta_try_convert_str_to_float("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT") + ) + class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables diff --git a/services/dynamic-sidecar/.env-devel b/services/dynamic-sidecar/.env-devel index 39b904af4d3..9bccac9239e 100644 --- a/services/dynamic-sidecar/.env-devel +++ b/services/dynamic-sidecar/.env-devel @@ -24,7 +24,7 @@ DY_SIDECAR_USER_SERVICES_HAVE_INTERNET_ACCESS=false # DOCKER REGISTRY DY_DEPLOYMENT_REGISTRY_SETTINGS='{"REGISTRY_AUTH":"false","REGISTRY_USER":"test","REGISTRY_PW":"test","REGISTRY_SSL":"false"}' -S3_ENDPOINT=http://145.456.25.54:12345 +S3_ENDPOINT=http://111.111.111.111:12345 S3_ACCESS_KEY=mocked S3_REGION=mocked S3_SECRET_KEY=mocked diff --git a/services/dynamic-sidecar/openapi.json b/services/dynamic-sidecar/openapi.json index d3bfc8ab243..cccb9924cdc 100644 --- a/services/dynamic-sidecar/openapi.json +++ b/services/dynamic-sidecar/openapi.json @@ -120,16 +120,16 @@ "operationId": "containers_docker_inspect_v1_containers_get", "parameters": [ { - "description": "if True only show the status of the container", + "name": "only_status", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Only Status", "description": "if True only show the status of the container", - "default": false + "default": false, + "title": "Only Status" }, - "name": "only_status", - "in": "query" + "description": "if True only show the status of the container" } ], "responses": { @@ -166,14 +166,14 @@ "summary": "Starts the containers as defined in ContainerCreate by:\n- cleaning up resources from previous runs if any\n- starting the containers\n\nProgress may be obtained through URL\nProcess may be cancelled through URL", "operationId": "create_service_containers_task_v1_containers_post", "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ContainersCreate" } } - }, - "required": true + } }, "responses": { "202": { @@ -213,7 +213,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ActivityInfo" + "anyOf": [ + { + "$ref": "#/components/schemas/ActivityInfo" + }, + { + "type": "null" + } + ], + "title": "Response Get Containers Activity V1 Containers Activity Get" } } } @@ -231,16 +239,17 @@ "operationId": "get_container_logs_v1_containers__id__logs_get", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } }, { - "description": "Only return logs since this time, as a UNIX timestamp", + "name": "since", + "in": "query", "required": false, "schema": { "type": "integer", @@ -248,11 +257,11 @@ "description": "Only return logs since this time, as a UNIX timestamp", "default": 0 }, - "name": "since", - "in": "query" + "description": "Only return logs since this time, as a UNIX timestamp" }, { - "description": "Only return logs before this time, as a UNIX timestamp", + "name": "until", + "in": "query", "required": false, "schema": { "type": "integer", @@ -260,11 +269,11 @@ "description": "Only return logs before this time, as a UNIX timestamp", "default": 0 }, - "name": "until", - "in": "query" + "description": "Only return logs before this time, as a UNIX timestamp" }, { - "description": "Enabling this parameter will include timestamps in logs", + "name": "timestamps", + "in": "query", "required": false, "schema": { "type": "boolean", @@ -272,8 +281,7 @@ "description": "Enabling this parameter will include timestamps in logs", "default": false }, - "name": "timestamps", - "in": "query" + "description": "Enabling this parameter will include timestamps in logs" } ], "responses": { @@ -282,10 +290,10 @@ "content": { "application/json": { "schema": { + "type": "array", "items": { "type": "string" }, - "type": "array", "title": "Response Get Container Logs V1 Containers Id Logs Get" } } @@ -320,15 +328,15 @@ "operationId": "get_containers_name_v1_containers_name_get", "parameters": [ { - "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters", + "name": "filters", + "in": "query", "required": true, "schema": { "type": "string", - "title": "Filters", - "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters" + "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters", + "title": "Filters" }, - "name": "filters", - "in": "query" + "description": "JSON encoded dictionary. FastAPI does not allow for dict as type in query parameters" } ], "responses": { @@ -369,13 +377,13 @@ "operationId": "inspect_container_v1_containers__id__get", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } } ], "responses": { @@ -486,24 +494,24 @@ "operationId": "attach_container_to_network_v1_containers__id__networks_attach_post", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AttachContainerToNetworkItem" } } - }, - "required": true + } }, "responses": { "204": { @@ -531,24 +539,24 @@ "operationId": "detach_container_from_network_v1_containers__id__networks_detach_post", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Id" - }, - "name": "id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DetachContainerFromNetworkItem" } } - }, - "required": true + } }, "responses": { "204": { @@ -666,10 +674,17 @@ "content": { "application/json": { "schema": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Port Keys" } } @@ -711,10 +726,17 @@ "content": { "application/json": { "schema": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Port Keys" } } @@ -798,23 +820,23 @@ "operationId": "put_volume_state_v1_volumes__id__put", "parameters": [ { + "name": "id", + "in": "path", "required": true, "schema": { "$ref": "#/components/schemas/VolumeCategory" - }, - "name": "id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PutVolumeItem" } } - }, - "required": true + } }, "responses": { "204": { @@ -873,7 +895,14 @@ "default": true }, "error_message": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Error Message", "description": "in case of error this gets set" } @@ -909,8 +938,7 @@ "GPU", "MPI" ], - "title": "BootMode", - "description": "An enumeration." + "title": "BootMode" }, "ContainersComposeSpec": { "properties": { @@ -956,25 +984,60 @@ "CreateServiceMetricsAdditionalParams": { "properties": { "wallet_id": { - "type": "integer", - "exclusiveMinimum": true, - "title": "Wallet Id", - "minimum": 0 + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Wallet Id" }, "wallet_name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Wallet Name" }, "pricing_plan_id": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pricing Plan Id" }, "pricing_unit_id": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pricing Unit Id" }, "pricing_unit_cost_id": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pricing Unit Cost Id" }, "product_name": { @@ -1008,9 +1071,6 @@ "title": "Service Version" }, "service_resources": { - "additionalProperties": { - "$ref": "#/components/schemas/ImageResources" - }, "type": "object", "title": "Service Resources" }, @@ -1021,6 +1081,11 @@ }, "type": "object", "required": [ + "wallet_id", + "wallet_name", + "pricing_plan_id", + "pricing_unit_id", + "pricing_unit_cost_id", "product_name", "simcore_user_agent", "user_email", @@ -1033,20 +1098,20 @@ ], "title": "CreateServiceMetricsAdditionalParams", "example": { - "wallet_id": 1, - "wallet_name": "a private wallet for me", + "node_name": "the service of a lifetime _ *!", "pricing_plan_id": 1, - "pricing_unit_id": 1, "pricing_unit_detail_id": 1, + "pricing_unit_id": 1, "product_name": "osparc", - "simcore_user_agent": "undefined", - "user_email": "test@test.com", "project_name": "_!New Study", - "node_name": "the service of a lifetime _ *!", + "service_additional_metadata": {}, "service_key": "simcore/services/dynamic/test", - "service_version": "0.0.1", "service_resources": {}, - "service_additional_metadata": {} + "service_version": "0.0.1", + "simcore_user_agent": "undefined", + "user_email": "test@test.com", + "wallet_id": 1, + "wallet_name": "a private wallet for me" } }, "DetachContainerFromNetworkItem": { @@ -1095,6 +1160,7 @@ "$ref": "#/components/schemas/BootMode" }, "type": "array", + "title": "Boot Modes", "description": "describe how a service shall be booted, using CPU, MPI, openMP or GPU", "default": [ "CPU" @@ -1110,6 +1176,14 @@ "example": { "image": "simcore/service/dynamic/pretty-intense:1.0.0", "resources": { + "AIRAM": { + "limit": 1, + "reservation": 1 + }, + "ANY_resource": { + "limit": "some_value", + "reservation": "some_value" + }, "CPU": { "limit": 4, "reservation": 0.1 @@ -1121,14 +1195,6 @@ "VRAM": { "limit": 1, "reservation": 1 - }, - "AIRAM": { - "limit": 1, - "reservation": 1 - }, - "ANY_resource": { - "limit": "some_value", - "reservation": "some_value" } } } @@ -1222,7 +1288,14 @@ "ServiceOutput": { "properties": { "displayOrder": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], "title": "Displayorder", "description": "DEPRECATED: new display order is taken from the item position. This will be removed.", "deprecated": true @@ -1230,14 +1303,12 @@ "label": { "type": "string", "title": "Label", - "description": "short name for the property", - "example": "Age" + "description": "short name for the property" }, "description": { "type": "string", "title": "Description", - "description": "description of the property", - "example": "Age in seconds since 1970" + "description": "description of the property" }, "type": { "type": "string", @@ -1246,32 +1317,51 @@ "description": "data type expected on this input glob matching for data type is allowed" }, "contentSchema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Contentschema", "description": "jsonschema of this input/output. Required when type='ref_contentSchema'" }, "fileToKeyMap": { - "additionalProperties": { - "type": "string", - "pattern": "^[-_a-zA-Z0-9]+$" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Filetokeymap", "description": "Place the data associated with the named keys in files" }, "unit": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Unit", "description": "Units, when it refers to a physical quantity", "deprecated": true }, "widget": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Widget" + }, + { + "type": "null" } ], - "title": "Widget", "description": "custom widget to use instead of the default one determined from the data-type", "deprecated": true } @@ -1283,8 +1373,7 @@ "description", "type" ], - "title": "ServiceOutput", - "description": "Base class for service input/outputs" + "title": "ServiceOutput" }, "Structure": { "properties": { @@ -1389,11 +1478,7 @@ "Widget": { "properties": { "type": { - "allOf": [ - { - "$ref": "#/components/schemas/WidgetType" - } - ], + "$ref": "#/components/schemas/WidgetType", "description": "type of the property" }, "details": { @@ -1422,8 +1507,7 @@ "TextArea", "SelectBox" ], - "title": "WidgetType", - "description": "An enumeration." + "title": "WidgetType" } } } diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 49082d6f508..3d1836e8ae7 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -25,17 +25,31 @@ aiofiles==23.2.1 # -r requirements/_base.in aiohttp==3.9.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -53,6 +67,8 @@ alembic==1.13.1 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -87,17 +103,31 @@ bidict==0.23.1 # via python-socketio certifi==2024.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -122,22 +152,8 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.2 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -170,17 +186,31 @@ httpcore==1.0.5 # via httpx httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -205,17 +235,31 @@ jsonschema-specifications==2023.7.1 # via jsonschema mako==1.3.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -291,17 +335,31 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -334,39 +392,86 @@ psycopg2-binary==2.9.9 # via # aiopg # sqlalchemy -pydantic==1.10.15 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -375,6 +480,8 @@ pyinstrument==4.6.2 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via arrow +python-dotenv==1.0.1 + # via pydantic-settings python-engineio==4.9.0 # via python-socketio python-magic==0.4.27 @@ -383,17 +490,31 @@ python-socketio==5.11.2 # via -r requirements/_base.in pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -402,17 +523,31 @@ pyyaml==6.0.1 # -r requirements/_base.in redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -455,17 +590,31 @@ sniffio==1.3.1 # httpx sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -473,19 +622,33 @@ sqlalchemy==1.4.52 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # aiopg # alembic -starlette==0.27.0 +starlette==0.39.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -525,22 +688,37 @@ typing-extensions==4.11.0 # opentelemetry-sdk # pint # pydantic + # pydantic-core # typer u-msgpack-python==2.8.0 # via -r requirements/_base.in urllib3==2.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt diff --git a/services/dynamic-sidecar/requirements/_test.txt b/services/dynamic-sidecar/requirements/_test.txt index 7bcc6a8243f..3a87310dcdb 100644 --- a/services/dynamic-sidecar/requirements/_test.txt +++ b/services/dynamic-sidecar/requirements/_test.txt @@ -105,7 +105,9 @@ python-dateutil==2.9.0.post0 # botocore # faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in requests==2.32.3 # via # -c requirements/_base.txt diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py index 4269646e9bb..a07d5db2fef 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py @@ -12,7 +12,7 @@ ActivityInfo, ActivityInfoOrNone, ) -from pydantic import parse_raw_as +from pydantic import TypeAdapter, ValidationError from servicelib.fastapi.requests_decorators import cancel_on_disconnect from ...core.docker_utils import docker_client @@ -174,8 +174,8 @@ async def get_containers_activity( return ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME) try: - return parse_raw_as(ActivityInfo, inactivity_response) - except json.JSONDecodeError: + return TypeAdapter(ActivityInfo).validate_json(inactivity_response) + except ValidationError: _logger.warning( "Could not parse command result '%s' as '%s'", inactivity_response, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 8592afd2440..b669198a715 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -131,16 +131,18 @@ def create_base_app() -> FastAPI: # settings settings = ApplicationSettings.create_from_envs() setup_logger(settings) - logger.debug(settings.json(indent=2)) + logger.debug(settings.model_dump_json(indent=2)) # minimal app = FastAPI( - debug=settings.SC_BOOT_MODE.is_devel_mode(), + debug=settings.SC_BOOT_MODE.is_devel_mode(), # pylint: disable=no-member title=PROJECT_NAME, description=SUMMARY, version=API_VERSION, openapi_url=f"/api/{API_VTAG}/openapi.json", - **get_common_oas_options(settings.SC_BOOT_MODE.is_devel_mode()), + **get_common_oas_options( + settings.SC_BOOT_MODE.is_devel_mode() # pylint: disable=no-member + ), ) override_fastapi_openapi_method(app) app.state.settings = settings @@ -190,8 +192,10 @@ def create_app(): setup_prometheus_metrics(app) # ERROR HANDLERS ------------ - app.add_exception_handler(NodeNotFound, node_not_found_error_handler) - app.add_exception_handler(BaseDynamicSidecarError, http_error_handler) + app.add_exception_handler( + NodeNotFound, node_not_found_error_handler # type: ignore[arg-type] + ) + app.add_exception_handler(BaseDynamicSidecarError, http_error_handler) # type: ignore[arg-type] # EVENTS --------------------- diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py index 7804d3de35c..43959d5fba5 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py @@ -110,7 +110,7 @@ def are_all_containers_in_expected_states( states: Iterable[ContainerState | None], ) -> bool: return all( - s is not None and s.Status in _ACCEPTED_CONTAINER_STATUSES for s in states + s is not None and s.status in _ACCEPTED_CONTAINER_STATUSES for s in states ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py index b0fd128a942..636260814a1 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py @@ -1,8 +1,8 @@ from typing import Any +from common_library.errors_classes import OsparcErrorMixin from fastapi import status from models_library.services import RunID -from pydantic.errors import PydanticErrorMixin class BaseDynamicSidecarError(Exception): @@ -35,8 +35,8 @@ def __init__(self, message: str, status_code: int) -> None: ) -class BaseError(PydanticErrorMixin, BaseDynamicSidecarError): - code = "dy_sidecar.error" +class BaseError(OsparcErrorMixin, BaseDynamicSidecarError): + code = "dy_sidecar.error" # type: ignore[assignment] class ContainerExecContainerNotFoundError(BaseError): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py index 12696fe13f0..278f29e7ad1 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/external_dependencies.py @@ -1,5 +1,5 @@ +from common_library.errors_classes import OsparcErrorMixin from fastapi import FastAPI -from pydantic.errors import PydanticErrorMixin from servicelib.utils import logged_gather from .postgres import wait_for_postgres_liveness @@ -8,7 +8,7 @@ from .storage import wait_for_storage_liveness -class CouldNotReachExternalDependenciesError(PydanticErrorMixin, Exception): +class CouldNotReachExternalDependenciesError(OsparcErrorMixin, Exception): msg_template: str = ( "Could not start because the following external dependencies failed: {failed}" ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py index 945aaccb8e5..e43946f5375 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/reserved_space.py @@ -3,14 +3,14 @@ from typing import Final from fastapi import FastAPI -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from .settings import ApplicationSettings _RESERVED_DISK_SPACE_NAME: Final[Path] = Path( "/tmp/reserved_disk_space" # nosec # noqa: S108 ) -_DEFAULT_CHUNK_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "8k") +_DEFAULT_CHUNK_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("8k") def _write_random_binary_file( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index 4e151e29c00..e8c75cc9e89 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import cast +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from models_library.basic_types import BootModeEnum, PortInt from models_library.callbacks_mapping import CallbacksMapping from models_library.products import ProductName @@ -11,7 +12,14 @@ from models_library.projects_nodes_io import NodeID from models_library.services import DynamicServiceKey, RunID, ServiceVersion from models_library.users import UserID -from pydantic import ByteSize, Field, PositiveInt, parse_obj_as, validator +from pydantic import ( + AliasChoices, + ByteSize, + Field, + PositiveInt, + TypeAdapter, + field_validator, +) from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings @@ -31,6 +39,10 @@ class ResourceTrackingSettings(BaseCustomSettings): description="each time the status of the service is propagated", ) + _try_convert_resource_tracking_heartbeat_interval = ( + timedelta_try_convert_str_to_float("RESOURCE_TRACKING_HEARTBEAT_INTERVAL") + ) + class SystemMonitorSettings(BaseCustomSettings): DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE: bool = Field( @@ -61,7 +73,10 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # LOGGING LOG_LEVEL: str = Field( - default="WARNING", env=["DYNAMIC_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL"] + default="WARNING", + validation_alias=AliasChoices( + "DYNAMIC_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) # SERVICE SERVER (see : https://www.uvicorn.org/settings/) @@ -100,7 +115,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) DYNAMIC_SIDECAR_RESERVED_SPACE_SIZE: ByteSize = Field( - parse_obj_as(ByteSize, "10Mib"), + TypeAdapter(ByteSize).validate_python("10Mib"), description=( "Disk space reserve when the dy-sidecar is started. Can be freed at " "any time via an API call. Main reason to free this disk space is " @@ -130,7 +145,10 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=["DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + validation_alias=AliasChoices( + "DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DY_SIDECAR_USER_ID: UserID @@ -144,32 +162,50 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): DY_SIDECAR_PRODUCT_NAME: ProductName | None = None NODE_PORTS_STORAGE_AUTH: StorageAuthSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} + ) + DY_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field( + json_schema_extra={"auto_default_from_env": True} ) - DY_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) DY_SIDECAR_AWS_S3_CLI_SETTINGS: AwsS3CliSettings | None = Field( None, description="AWS S3 settings are used for the AWS S3 CLI. If these settings are filled, the AWS S3 CLI is used instead of RClone.", ) - POSTGRES_SETTINGS: PostgresSettings = Field(auto_default_from_env=True) - RABBIT_SETTINGS: RabbitSettings = Field(auto_default_from_env=True) + POSTGRES_SETTINGS: PostgresSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) + RABBIT_SETTINGS: RabbitSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) DY_DEPLOYMENT_REGISTRY_SETTINGS: RegistrySettings = Field() - DY_DOCKER_HUB_REGISTRY_SETTINGS: RegistrySettings | None = Field() + DY_DOCKER_HUB_REGISTRY_SETTINGS: RegistrySettings | None = Field(default=None) - RESOURCE_TRACKING: ResourceTrackingSettings = Field(auto_default_from_env=True) + RESOURCE_TRACKING: ResourceTrackingSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - SYSTEM_MONITOR_SETTINGS: SystemMonitorSettings = Field(auto_default_from_env=True) + SYSTEM_MONITOR_SETTINGS: SystemMonitorSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) @property def are_prometheus_metrics_enabled(self) -> bool: - return self.DY_SIDECAR_CALLBACKS_MAPPING.metrics is not None + return ( # pylint: disable=no-member + self.DY_SIDECAR_CALLBACKS_MAPPING.metrics is not None + ) - @validator("LOG_LEVEL", pre=True) + @field_validator("LOG_LEVEL", mode="before") @classmethod def _check_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + _try_convert_dynamic_sidecar_telemetry_disk_usage_monitor_interval = ( + timedelta_try_convert_str_to_float( + "DYNAMIC_SIDECAR_TELEMETRY_DISK_USAGE_MONITOR_INTERVAL" + ) + ) + @lru_cache def get_settings() -> ApplicationSettings: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py index 9118711a573..639ee0dd810 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/storage.py @@ -4,7 +4,7 @@ from fastapi import FastAPI, status from httpx import AsyncClient -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from servicelib.logging_utils import log_context from settings_library.node_ports import StorageAuthSettings @@ -33,8 +33,10 @@ def _get_auth_or_none(storage_auth_settings: StorageAuthSettings) -> _AuthTuple def _get_url(storage_auth_settings: StorageAuthSettings) -> str: - url: str = parse_obj_as(AnyUrl, f"{storage_auth_settings.api_base_url}/") - return url + url: AnyUrl = TypeAdapter(AnyUrl).validate_python( + f"{storage_auth_settings.api_base_url}/" + ) + return f"{url}" async def _is_storage_responsive(storage_auth_settings: StorageAuthSettings) -> bool: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py index 0745f595fa8..0ca422a3390 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py @@ -30,7 +30,7 @@ async def _persist_to_disk(self) -> None: async with aiofiles.open( self._shared_store_dir / STORE_FILE_NAME, "w" ) as data_file: - await data_file.write(self.json()) + await data_file.write(self.model_dump_json()) def post_init(self, shared_store_dir: Path): self._shared_store_dir = shared_store_dir @@ -66,6 +66,17 @@ class SharedStore(_StoreMixin): default_factory=dict, description="persist the state of each volume" ) + def __eq__(self, other: object) -> bool: + return all( + getattr(self, n, None) == getattr(other, n, None) + for n in ( + "compose_spec", + "container_names", + "original_to_container_names", + "volume_states", + ) + ) + async def _setup_initial_volume_states(self) -> None: async with self: for category, status in [ @@ -74,6 +85,7 @@ async def _setup_initial_volume_states(self) -> None: (VolumeCategory.OUTPUTS, VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED), (VolumeCategory.STATES, VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED), ]: + # pylint: disable=unsupported-assignment-operation self.volume_states[category] = VolumeState(status=status) @classmethod @@ -93,7 +105,7 @@ async def init_from_disk( async with aiofiles.open(shared_store_dir / store_file_name) as data_file: file_content = await data_file.read() - obj = cls.parse_raw(file_content) + obj = cls.model_validate_json(file_content) obj.post_init(shared_store_dir) return obj diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 0134d481f78..6e7a7a19009 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -263,7 +263,7 @@ async def _send_resource_tracking_stop(platform_status: SimcorePlatformStatus): simcore_platform_status = platform_status if not containers_were_ok: any_container_oom_killed = any( - c.OOMKilled is True + c.oom_killed is True for c in container_states.values() if c is not None ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py index 7cc13922244..21f07bf1523 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_directory_utils.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter def get_directory_total_size(path: Path) -> ByteSize: @@ -10,7 +10,7 @@ def get_directory_total_size(path: Path) -> ByteSize: # until we do not hit 1 million it can be ignored # NOTE: file size has no impact on performance if not path.exists(): - return parse_obj_as(ByteSize, 0) + return TypeAdapter(ByteSize).validate_python(0) total = 0 for entry in os.scandir(path): @@ -18,4 +18,4 @@ def get_directory_total_size(path: Path) -> ByteSize: total += entry.stat().st_size elif entry.is_dir(): total += get_directory_total_size(Path(entry.path)) - return parse_obj_as(ByteSize, total) + return TypeAdapter(ByteSize).validate_python(total) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py index 8490f9cd72e..227358f4960 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_filter.py @@ -14,7 +14,7 @@ NonNegativeInt, PositiveFloat, PositiveInt, - parse_obj_as, + TypeAdapter, ) from servicelib.logging_utils import log_context from watchdog.observers.api import DEFAULT_OBSERVER_TIMEOUT @@ -27,8 +27,8 @@ logger = logging.getLogger(__name__) -_1_MB: Final[PositiveInt] = parse_obj_as(ByteSize, "1mib") -_500_MB: Final[PositiveInt] = parse_obj_as(ByteSize, "500mib") +_1_MB: Final[PositiveInt] = TypeAdapter(ByteSize).validate_python("1mib") +_500_MB: Final[PositiveInt] = TypeAdapter(ByteSize).validate_python("500mib") class BaseDelayPolicy(ABC): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py index d4a8ac8d07a..fc33fe660ac 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py @@ -10,7 +10,6 @@ from models_library.basic_types import IDStr from models_library.rabbitmq_messages import ProgressType from pydantic import PositiveFloat -from pydantic.errors import PydanticErrorMixin from servicelib import progress_bar from servicelib.background_task import start_periodic_task, stop_periodic_task from servicelib.logging_utils import log_catch, log_context @@ -22,18 +21,17 @@ from ..nodeports import upload_outputs from ._context import OutputsContext -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) async def _cancel_task(task: Task, task_cancellation_timeout_s: PositiveFloat) -> None: task.cancel() - with suppress(CancelledError): - with log_catch(logger, reraise=False): - await wait((task,), timeout=task_cancellation_timeout_s) + with suppress(CancelledError), log_catch(_logger, reraise=False): + await wait((task,), timeout=task_cancellation_timeout_s) -class UploadPortsFailed(PydanticErrorMixin, RuntimeError): - code: str = "dynamic_sidecar.outputs_manager.failed_while_uploading" +class UploadPortsFailedError(OsparcErrorMixin, RuntimeError): + code: str = "dynamic_sidecar.outputs_manager.failed_while_uploading" # type: ignore[assignment] msg_template: str = "Failed while uploading: failures={failures}" @@ -130,7 +128,9 @@ async def _uploading_task_start(self) -> None: assert len(port_keys) > 0 # nosec async def _upload_ports() -> None: - with log_context(logger, logging.INFO, f"Uploading port keys: {port_keys}"): + with log_context( + _logger, logging.INFO, f"Uploading port keys: {port_keys}" + ): async with progress_bar.ProgressBarData( num_steps=1, progress_report_cb=self.task_progress_cb, @@ -155,7 +155,7 @@ def _remove_downloads(future: Future) -> None: if future._exception.__traceback__ else "" ) - logger.warning( + _logger.warning( "%s ended with exception: %s%s", task_name, future._exception, @@ -200,7 +200,7 @@ async def start(self) -> None: ) async def shutdown(self) -> None: - with log_context(logger, logging.INFO, f"{OutputsManager.__name__} shutdown"): + with log_context(_logger, logging.INFO, f"{OutputsManager.__name__} shutdown"): await self._uploading_task_cancel() if self._task_scheduler_worker is not None: await stop_periodic_task( @@ -222,7 +222,7 @@ async def wait_for_all_uploads_to_finish(self) -> None: # always scheduling non file based ports for upload # there is no auto detection when these change for non_file_port_key in self.outputs_context.non_file_type_port_keys: - logger.info("Adding non file port key %s", non_file_port_key) + _logger.info("Adding non file port key %s", non_file_port_key) await self.port_key_content_changed(non_file_port_key) # NOTE: the file system watchdog was found unhealthy and to make @@ -232,7 +232,7 @@ async def wait_for_all_uploads_to_finish(self) -> None: # is missed. if self._schedule_all_ports_for_upload: self._schedule_all_ports_for_upload = False - logger.warning( + _logger.warning( "Scheduled %s for upload. The watchdog was rebooted. " "This is a safety measure to make sure no data is lost. ", self.outputs_context.outputs_path, @@ -240,10 +240,10 @@ async def wait_for_all_uploads_to_finish(self) -> None: for file_port_key in self.outputs_context.file_type_port_keys: await self.port_key_content_changed(file_port_key) - logger.info("Port status before waiting %s", f"{self._port_key_tracker}") + _logger.info("Port status before waiting %s", f"{self._port_key_tracker}") while not await self._port_key_tracker.no_tracked_ports(): await asyncio.sleep(self.task_monitor_interval_s) - logger.info("Port status after waiting %s", f"{self._port_key_tracker}") + _logger.info("Port status after waiting %s", f"{self._port_key_tracker}") # NOTE: checking if there were any errors during the last port upload, # for each port. If any error is detected this will raise. @@ -251,7 +251,7 @@ async def wait_for_all_uploads_to_finish(self) -> None: True for v in self._last_upload_error_tracker.values() if v is not None ) if any_failed_upload: - raise UploadPortsFailed(failures=self._last_upload_error_tracker) + raise UploadPortsFailedError(failures=self._last_upload_error_tracker) def setup_outputs_manager(app: FastAPI) -> None: @@ -264,7 +264,7 @@ async def on_startup() -> None: io_log_redirect_cb: LogRedirectCB | None = None if settings.RABBIT_SETTINGS: io_log_redirect_cb = partial(post_log_message, app, log_level=logging.INFO) - logger.debug( + _logger.debug( "setting up outputs manager %s", "with redirection of logs..." if io_log_redirect_cb else "...", ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_models.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_models.py index 4708e9c291f..f87fa415a74 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_models.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_models.py @@ -1,10 +1,8 @@ from asyncio import Task -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class ResourceTrackingState(BaseModel): heart_beat_task: Task | None = None - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py index 78976b53a86..33c0083861f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/service_liveness.py @@ -4,7 +4,7 @@ from datetime import timedelta from typing import Final -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin from tenacity import AsyncRetrying, RetryCallState, TryAgain from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed @@ -16,7 +16,7 @@ _DEFAULT_TIMEOUT_INTERVAL: Final[timedelta] = timedelta(seconds=30) -class CouldNotReachServiceError(PydanticErrorMixin, Exception): +class CouldNotReachServiceError(OsparcErrorMixin, Exception): msg_template: str = "Could not contact service '{service_name}' at '{endpoint}'. Look above for details." diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py index f2cc53a0d9d..0d010794e23 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py @@ -6,7 +6,7 @@ from models_library.user_preferences import PreferenceName from models_library.users import UserID from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.utils_user_preferences import ( UserServicesUserPreferencesRepo, ) @@ -73,5 +73,7 @@ async def load_preferences( if payload is None: return - preference = parse_obj_as(preference_class, umsgpack.unpackb(payload)) + preference = TypeAdapter(preference_class).validate_python( + umsgpack.unpackb(payload) + ) await dir_from_bytes(preference.value, user_preferences_path) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py index d1b373ac1f5..b2c4a327c83 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py @@ -1,8 +1,8 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class BaseServicesPreferencesError(PydanticErrorMixin, Exception): - code = "dynamic_sidecar.user_service_preferences" +class BaseServicesPreferencesError(OsparcErrorMixin, Exception): + code = "dynamic_sidecar.user_service_preferences" # type: ignore[assignment] class DestinationIsNotADirectoryError(BaseServicesPreferencesError): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_packaging.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_packaging.py index 38f2eab7111..bdffd81a4a9 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_packaging.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_packaging.py @@ -2,13 +2,15 @@ from typing import Final import aiofiles -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.archiving_utils import archive_dir, unarchive_dir from servicelib.file_utils import remove_directory from ._errors import DestinationIsNotADirectoryError, PreferencesAreTooBigError -_MAX_PREFERENCES_TOTAL_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "128kib") +_MAX_PREFERENCES_TOTAL_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( + "128kib" +) async def dir_to_bytes(source: Path) -> bytes: diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 8b4760b26dd..0dc55477b47 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -21,7 +21,7 @@ from models_library.services_creation import CreateServiceMetricsAdditionalParams from models_library.users import UserID from models_library.utils.json_serialization import json_dumps -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import ( EnvVarsDict, @@ -347,9 +347,10 @@ def mock_stop_heart_beat_task(mocker: MockerFixture) -> AsyncMock: @pytest.fixture def mock_metrics_params(faker: Faker) -> CreateServiceMetricsAdditionalParams: - return parse_obj_as( - CreateServiceMetricsAdditionalParams, - CreateServiceMetricsAdditionalParams.Config.schema_extra["example"], + return TypeAdapter(CreateServiceMetricsAdditionalParams).validate_python( + CreateServiceMetricsAdditionalParams.model_config["json_schema_extra"][ + "example" + ], ) diff --git a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py index 5e70b0a6f79..a1b64635e58 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py @@ -23,7 +23,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_project from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict @@ -199,7 +199,7 @@ async def restore_legacy_state_archives( user_id=user_id, store_id=SIMCORE_LOCATION, store_name=None, - s3_object=parse_obj_as(SimcoreS3FileID, s3_path), + s3_object=TypeAdapter(SimcoreS3FileID).validate_python(s3_path), path_to_upload=legacy_archive_zip, io_log_redirect_cb=None, ) @@ -303,8 +303,7 @@ def s3_settings(app_state: AppState) -> S3Settings: @pytest.fixture def bucket_name(app_state: AppState) -> S3BucketName: - return parse_obj_as( - S3BucketName, + return TypeAdapter(S3BucketName).validate_python( app_state.settings.DY_SIDECAR_R_CLONE_SETTINGS.R_CLONE_S3.S3_BUCKET_NAME, ) diff --git a/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py b/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py index 094b3014404..9be0bbdebbf 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_user_services_preferences.py @@ -14,7 +14,7 @@ from models_library.projects import ProjectID from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.postgres_tools import PostgresTestConfig from simcore_service_dynamic_sidecar.core.application import create_app @@ -46,17 +46,19 @@ def dy_sidecar_user_preferences_path(tmp_path: Path) -> Path: @pytest.fixture def service_key() -> ServiceKey: - return parse_obj_as(ServiceKey, "simcore/services/dynamic/test-service-34") + return TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/test-service-34" + ) @pytest.fixture def service_version() -> ServiceVersion: - return parse_obj_as(ServiceVersion, "1.0.0") + return TypeAdapter(ServiceVersion).validate_python("1.0.0") @pytest.fixture def product_name() -> ProductName: - return parse_obj_as(ProductName, "osparc") + return TypeAdapter(ProductName).validate_python("osparc") @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index f16b883de15..0877e7ab0fe 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -23,8 +23,9 @@ from faker import Faker from fastapi import FastAPI, status from models_library.api_schemas_dynamic_sidecar.containers import ActivityInfo -from models_library.services import ServiceOutput from models_library.services_creation import CreateServiceMetricsAdditionalParams +from models_library.services_io import ServiceOutput +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.docker_constants import SUFFIX_EGRESS_PROXY_NAME @@ -266,11 +267,11 @@ def not_started_containers() -> list[str]: @pytest.fixture def mock_outputs_labels() -> dict[str, ServiceOutput]: return { - "output_port_1": ServiceOutput.parse_obj( - ServiceOutput.Config.schema_extra["examples"][3] + "output_port_1": TypeAdapter(ServiceOutput).validate_python( + ServiceOutput.model_config["json_schema_extra"]["examples"][3] ), - "output_port_2": ServiceOutput.parse_obj( - ServiceOutput.Config.schema_extra["examples"][3] + "output_port_2": TypeAdapter(ServiceOutput).validate_python( + ServiceOutput.model_config["json_schema_extra"]["examples"][3] ), } @@ -366,12 +367,12 @@ def test_ensure_api_vtag_is_v1(): async def test_start_same_space_twice(compose_spec: str, test_client: TestClient): settings = test_client.application.state.settings - settings_1 = settings.copy( + settings_1 = settings.model_copy( update={"DYNAMIC_SIDECAR_COMPOSE_NAMESPACE": "test_name_space_1"}, deep=True ) await _assert_compose_spec_pulled(compose_spec, settings_1) - settings_2 = settings.copy( + settings_2 = settings.model_copy( update={"DYNAMIC_SIDECAR_COMPOSE_NAMESPACE": "test_name_space_2"}, deep=True ) await _assert_compose_spec_pulled(compose_spec, settings_2) @@ -750,7 +751,10 @@ async def test_containers_activity_command_failed( ): response = await test_client.get(f"/{API_VTAG}/containers/activity") assert response.status_code == 200, response.text - assert response.json() == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME) + assert ( + response.json() + == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME).model_dump() + ) async def test_containers_activity_no_inactivity_defined( @@ -772,8 +776,8 @@ def mock_inactive_since_command_response( activity_response: ActivityInfo, ) -> None: mocker.patch( - "simcore_service_dynamic_sidecar.api.rest.containers.run_command_in_container", - return_value=activity_response.json(), + "simcore_service_dynamic_sidecar.api.containers.run_command_in_container", + return_value=activity_response.model_dump_json(), ) @@ -786,7 +790,7 @@ async def test_containers_activity_inactive_since( ): response = await test_client.get(f"/{API_VTAG}/containers/activity") assert response.status_code == 200, response.text - assert response.json() == activity_response + assert response.json() == activity_response.model_dump() @pytest.fixture @@ -805,4 +809,7 @@ async def test_containers_activity_unexpected_response( ): response = await test_client.get(f"/{API_VTAG}/containers/activity") assert response.status_code == 200, response.text - assert response.json() == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME) + assert ( + response.json() + == ActivityInfo(seconds_inactive=_INACTIVE_FOR_LONG_TIME).model_dump() + ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 9ea4de06dbb..66a31f1c13c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -16,6 +16,7 @@ from aiodocker.containers import DockerContainer from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI from fastapi.routing import APIRoute from httpx import AsyncClient @@ -24,7 +25,7 @@ ProgressPercent, ) from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from servicelib.fastapi.long_running_tasks.client import ( @@ -157,7 +158,9 @@ def compose_spec(request: pytest.FixtureRequest) -> str: @pytest.fixture def backend_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io" + ) @pytest.fixture @@ -187,7 +190,7 @@ async def httpx_async_client( # crete dir here async with AsyncClient( app=app, - base_url=backend_url, + base_url=f"{backend_url}", headers={"Content-Type": "application/json"}, ) as client: yield client @@ -197,7 +200,7 @@ async def httpx_async_client( def client( app: FastAPI, httpx_async_client: AsyncClient, backend_url: AnyHttpUrl ) -> Client: - return Client(app=app, async_client=httpx_async_client, base_url=backend_url) + return Client(app=app, async_client=httpx_async_client, base_url=f"{backend_url}") @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index 1c56b71db2e..e8206dc47a9 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -10,11 +10,12 @@ import pytest from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI, status from httpx import AsyncClient from models_library.callbacks_mapping import CallbacksMapping from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.long_running_tasks.client import ( Client, @@ -44,7 +45,7 @@ async def enable_prometheus_metrics( monkeypatch, { "DY_SIDECAR_CALLBACKS_MAPPING": json.dumps( - CallbacksMapping.Config.schema_extra["examples"][2] + CallbacksMapping.model_config["json_schema_extra"]["examples"][2] ) }, ) @@ -59,7 +60,9 @@ async def app(mock_rabbitmq_envs: EnvVarsDict, app: FastAPI) -> AsyncIterable[Fa @pytest.fixture def backend_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io" + ) @pytest.fixture @@ -71,7 +74,7 @@ async def httpx_async_client( ) -> AsyncIterable[AsyncClient]: async with AsyncClient( app=app, - base_url=backend_url, + base_url=f"{backend_url}", headers={"Content-Type": "application/json"}, ) as client: yield client @@ -81,7 +84,7 @@ async def httpx_async_client( def client( app: FastAPI, httpx_async_client: AsyncClient, backend_url: AnyHttpUrl ) -> Client: - return Client(app=app, async_client=httpx_async_client, base_url=backend_url) + return Client(app=app, async_client=httpx_async_client, base_url=f"{backend_url}") @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py b/services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py index fe396d002ad..40eab12336a 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py @@ -56,6 +56,4 @@ async def test_volumes_state_saved_error( ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text json_response = response.json() - assert ( - invalid_volume_category not in json_response["detail"][0]["ctx"]["enum_values"] - ) + assert invalid_volume_category not in json_response["detail"][0]["ctx"]["expected"] diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index 22400cb0e80..ddb5b18df1a 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -16,6 +16,7 @@ from aiodocker.utils import clean_filters from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI from httpx import AsyncClient from models_library.generated_models.docker_rest_api import ContainerState @@ -28,7 +29,7 @@ SimcorePlatformStatus, ) from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.long_running_tasks.client import ( @@ -79,7 +80,9 @@ def compose_spec(raw_compose_spec: dict[str, Any]) -> str: @pytest.fixture def backend_url() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://backgroud.testserver.io") + return TypeAdapter(AnyHttpUrlLegacy).validate_python( + "http://backgroud.testserver.io" + ) @pytest.fixture @@ -113,7 +116,7 @@ async def httpx_async_client( ) -> AsyncIterable[AsyncClient]: # crete dir here async with AsyncClient( - app=app, base_url=backend_url, headers={"Content-Type": "application/json"} + app=app, base_url=f"{backend_url}", headers={"Content-Type": "application/json"} ) as client: yield client @@ -122,7 +125,7 @@ async def httpx_async_client( def client( app: FastAPI, httpx_async_client: AsyncClient, backend_url: AnyHttpUrl ) -> Client: - return Client(app=app, async_client=httpx_async_client, base_url=backend_url) + return Client(app=app, async_client=httpx_async_client, base_url=f"{backend_url}") @pytest.fixture @@ -189,7 +192,7 @@ async def _wait_for_containers_to_be_running(app: FastAPI) -> None: running_container_statuses = [ x for x in containers_statuses.values() - if x is not None and x.Status == ContainerStatus.running + if x is not None and x.status == ContainerStatus.running ] if len(running_container_statuses) != len(shared_store.container_names): @@ -361,8 +364,8 @@ async def _mocked_get_container_states( results = await get_container_states(container_names) for result in results.values(): if result: - result.OOMKilled = True - result.Status = ContainerStatus.exited + result.oom_killed = True + result.status = ContainerStatus.exited break return results diff --git a/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py b/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py index 123f21864a0..c78b800ce5a 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py +++ b/services/dynamic-sidecar/tests/unit/test_core_reserved_space.py @@ -2,7 +2,7 @@ # pylint:disable=unused-argument -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_dynamic_sidecar.core.application import create_base_app from simcore_service_dynamic_sidecar.core.reserved_space import ( @@ -18,7 +18,9 @@ def test_reserved_disk_space_workflow( create_base_app() assert _RESERVED_DISK_SPACE_NAME.exists() - assert _RESERVED_DISK_SPACE_NAME.stat().st_size == parse_obj_as(ByteSize, "10MiB") + assert _RESERVED_DISK_SPACE_NAME.stat().st_size == TypeAdapter( + ByteSize + ).validate_python("10MiB") remove_reserved_disk_space() assert not _RESERVED_DISK_SPACE_NAME.exists() diff --git a/services/dynamic-sidecar/tests/unit/test_core_settings.py b/services/dynamic-sidecar/tests/unit/test_core_settings.py index 4512abf71d6..9e581d90999 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_settings.py +++ b/services/dynamic-sidecar/tests/unit/test_core_settings.py @@ -61,6 +61,7 @@ def test_settings_with_node_ports_storage_auth( settings = ApplicationSettings.create_from_envs() assert settings.NODE_PORTS_STORAGE_AUTH + # pylint:disable=no-member assert settings.NODE_PORTS_STORAGE_AUTH.STORAGE_SECURE is True assert settings.NODE_PORTS_STORAGE_AUTH.STORAGE_HOST == "host" assert settings.NODE_PORTS_STORAGE_AUTH.STORAGE_PORT == 42 @@ -72,15 +73,10 @@ def test_settings_with_node_ports_storage_auth( assert ( settings.NODE_PORTS_STORAGE_AUTH.STORAGE_PASSWORD.get_secret_value() == "passwd" ) - assert "passwd" not in settings.NODE_PORTS_STORAGE_AUTH.json() + assert "passwd" not in settings.NODE_PORTS_STORAGE_AUTH.model_dump_json() -@pytest.mark.parametrize( - "envs", - [ - {}, - ], -) +@pytest.mark.parametrize("envs", [{}]) def test_settings_with_node_ports_storage_auth_as_missing( mock_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, envs: dict[str, str] ): @@ -88,6 +84,7 @@ def test_settings_with_node_ports_storage_auth_as_missing( settings = ApplicationSettings.create_from_envs() assert settings.NODE_PORTS_STORAGE_AUTH is not None + # pylint:disable=no-member assert settings.NODE_PORTS_STORAGE_AUTH.auth_required is False assert settings.NODE_PORTS_STORAGE_AUTH.STORAGE_USERNAME is None assert settings.NODE_PORTS_STORAGE_AUTH.STORAGE_PASSWORD is None diff --git a/services/dynamic-sidecar/tests/unit/test_core_stroage.py b/services/dynamic-sidecar/tests/unit/test_core_stroage.py index c8f1d19405d..0fdf000f7c0 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_stroage.py +++ b/services/dynamic-sidecar/tests/unit/test_core_stroage.py @@ -12,6 +12,7 @@ import uvicorn from fastapi import Depends, FastAPI, HTTPException, status from fastapi.security import HTTPBasic, HTTPBasicCredentials +from pydantic import TypeAdapter from pytest_mock import MockerFixture from settings_library.node_ports import StorageAuthSettings from simcore_service_dynamic_sidecar.core.storage import ( @@ -60,7 +61,7 @@ async def unprotected_route(): def storage_auth_settings( username: str | None, password: str | None ) -> StorageAuthSettings: - return StorageAuthSettings.parse_obj( + return TypeAdapter(StorageAuthSettings).validate_python( { "STORAGE_HOST": "localhost", "STORAGE_PORT": 44332, diff --git a/services/dynamic-sidecar/tests/unit/test_models_shared_store.py b/services/dynamic-sidecar/tests/unit/test_models_shared_store.py index c72a8bdb85a..2c2b474a029 100644 --- a/services/dynamic-sidecar/tests/unit/test_models_shared_store.py +++ b/services/dynamic-sidecar/tests/unit/test_models_shared_store.py @@ -6,11 +6,12 @@ from pathlib import Path from typing import Any +import arrow import pytest from async_asgi_testclient import TestClient from fastapi import FastAPI from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from servicelib.utils import logged_gather from simcore_service_dynamic_sidecar.core import application @@ -53,17 +54,17 @@ def mock_docker_compose(mocker: MockerFixture) -> None: {"volume_states": {}}, { "volume_states": { - VolumeCategory.OUTPUTS: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} + VolumeCategory.OUTPUTS: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} ), - VolumeCategory.INPUTS: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED} + VolumeCategory.INPUTS: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED} ), - VolumeCategory.STATES: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_WAS_SAVED} + VolumeCategory.STATES: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_WAS_SAVED} ), - VolumeCategory.SHARED_STORE: parse_obj_as( - VolumeState, {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} + VolumeCategory.SHARED_STORE: TypeAdapter(VolumeState).validate_python( + {"status": VolumeStatus.CONTENT_NO_SAVE_REQUIRED} ), } }, @@ -88,7 +89,18 @@ async def test_shared_store_updates( # check the contes of the file should be the same as the shared_store's assert store_file_path.exists() is True - assert shared_store == SharedStore.parse_raw(store_file_path.read_text()) + + def _normalize_datetimes(shared_store: SharedStore) -> None: + for state in shared_store.volume_states.values(): + state.last_changed = arrow.get(state.last_changed.isoformat()).datetime + + shared_store_from_file = SharedStore.model_validate_json( + store_file_path.read_text() + ) + _normalize_datetimes(shared_store) + _normalize_datetimes(shared_store_from_file) + + assert shared_store == shared_store_from_file async def test_no_concurrency_with_parallel_writes( @@ -119,12 +131,20 @@ async def test_init_from_disk_with_legacy_data_format(project_tests_dir: Path): # if file is missing it correctly loaded the storage_file assert (MOCKS_DIR / STORE_FILE_NAME).exists() is False - # ensure object objects are compatible - parsed_legacy_format = json.loads(disk_shared_store.json()) + def _normalize_datetimes(data: dict[str, Any]) -> None: + for state in data["volume_states"].values(): + state["last_changed"] = arrow.get( + state["last_changed"] + ).datetime.isoformat() - assert parsed_legacy_format == json.loads( - (MOCKS_DIR / LEGACY_SHARED_STORE).read_text() - ) + # ensure objects are compatible + parsed_legacy_format = json.loads(disk_shared_store.model_dump_json()) + load_raw_from_disk = json.loads((MOCKS_DIR / LEGACY_SHARED_STORE).read_text()) + + _normalize_datetimes(parsed_legacy_format) + _normalize_datetimes(load_raw_from_disk) + + assert parsed_legacy_format == load_raw_from_disk async def test_init_from_disk_no_file_present(tmp_path: Path): diff --git a/services/dynamic-sidecar/tests/unit/test_modules_notifier.py b/services/dynamic-sidecar/tests/unit/test_modules_notifier.py index eabd1114083..51855ffd20f 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_notifier.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_notifier.py @@ -33,7 +33,7 @@ from models_library.projects_nodes_io import NodeID from models_library.services_types import ServicePortKey from models_library.users import UserID -from pydantic import ByteSize, NonNegativeInt, parse_obj_as +from pydantic import ByteSize, NonNegativeInt, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.utils import logged_gather @@ -133,7 +133,7 @@ def _get_mocked_disk_usage(byte_size_str: str) -> DiskUsage: return DiskUsage( total=ByteSize.validate(byte_size_str), used=ByteSize(0), - free=ByteSize.validate(byte_size_str), + free=TypeAdapter(ByteSize).validate_python(byte_size_str), used_percent=0, ) @@ -144,7 +144,7 @@ def _get_on_service_disk_usage_spy( # emulates front-end receiving message async def on_service_status(data): - assert parse_obj_as(ServiceDiskUsage, data) is not None + assert TypeAdapter(ServiceDiskUsage).validate_python(data) is not None on_event_spy = AsyncMock(wraps=on_service_status) socketio_client.on(SOCKET_IO_SERVICE_DISK_USAGE_EVENT, on_event_spy) @@ -231,7 +231,7 @@ def _get_on_input_port_spy( # emulates front-end receiving message async def on_service_status(data): - assert parse_obj_as(ServiceDiskUsage, data) is not None + assert TypeAdapter(ServiceDiskUsage).validate_python(data) is not None on_event_spy = AsyncMock(wraps=on_service_status) socketio_client.on(SOCKET_IO_STATE_INPUT_PORTS_EVENT, on_event_spy) @@ -320,7 +320,7 @@ def _get_on_output_port_spy( # emulates front-end receiving message async def on_service_status(data): - assert parse_obj_as(ServiceDiskUsage, data) is not None + assert TypeAdapter(ServiceDiskUsage).validate_python(data) is not None on_event_spy = AsyncMock(wraps=on_service_status) socketio_client.on(SOCKET_IO_STATE_OUTPUT_PORTS_EVENT, on_event_spy) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py index 38b217bab8f..0a275d1d70b 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_event_filter.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock import pytest -from pydantic import ByteSize, NonNegativeFloat, NonNegativeInt, parse_obj_as +from pydantic import ByteSize, NonNegativeFloat, NonNegativeInt, TypeAdapter from pytest_mock.plugin import MockerFixture from simcore_service_dynamic_sidecar.modules.notifications._notifications_ports import ( PortNotifier, @@ -230,8 +230,8 @@ def test_default_delay_policy(): wait_policy = DefaultDelayPolicy() # below items are defined by the default policy - LOWER_BOUND = parse_obj_as(ByteSize, "1mib") - UPPER_BOUND = parse_obj_as(ByteSize, "500mib") + LOWER_BOUND = TypeAdapter(ByteSize).validate_python("1mib") + UPPER_BOUND = TypeAdapter(ByteSize).validate_python("500mib") assert wait_policy.get_min_interval() == 1.0 @@ -243,4 +243,7 @@ def test_default_delay_policy(): assert wait_policy.get_wait_interval(UPPER_BOUND - 1) < 10.0 assert wait_policy.get_wait_interval(UPPER_BOUND) == 10.0 assert wait_policy.get_wait_interval(UPPER_BOUND + 1) == 10.0 - assert wait_policy.get_wait_interval(parse_obj_as(ByteSize, "1Tib")) == 10.0 + assert ( + wait_policy.get_wait_interval(TypeAdapter(ByteSize).validate_python("1Tib")) + == 10.0 + ) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py index 3bf17d09f92..a38658f222b 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_manager.py @@ -31,7 +31,7 @@ ) from simcore_service_dynamic_sidecar.modules.outputs._manager import ( OutputsManager, - UploadPortsFailed, + UploadPortsFailedError, _PortKeyTracker, setup_outputs_manager, ) @@ -234,7 +234,7 @@ async def test_recovers_after_raising_error( assert await outputs_manager._port_key_tracker.no_tracked_ports() is False await asyncio.sleep(outputs_manager.task_monitor_interval_s * 10) - with pytest.raises(UploadPortsFailed) as exec_info: + with pytest.raises(UploadPortsFailedError) as exec_info: await outputs_manager.wait_for_all_uploads_to_finish() assert set(exec_info.value.failures.keys()) == set(port_keys) | set( diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py index 7f9b81587c2..ffa4dfbef45 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py @@ -22,7 +22,7 @@ NonNegativeFloat, NonNegativeInt, PositiveFloat, - parse_obj_as, + TypeAdapter, ) from pytest_mock import MockerFixture from simcore_service_dynamic_sidecar.modules.mounted_fs import MountedVolumes @@ -172,20 +172,20 @@ class FileGenerationInfo: @pytest.fixture( params=[ FileGenerationInfo( - size=parse_obj_as(ByteSize, "100b"), - chunk_size=parse_obj_as(ByteSize, "1b"), + size=TypeAdapter(ByteSize).validate_python("100b"), + chunk_size=TypeAdapter(ByteSize).validate_python("1b"), ), FileGenerationInfo( - size=parse_obj_as(ByteSize, "100kib"), - chunk_size=parse_obj_as(ByteSize, "1kib"), + size=TypeAdapter(ByteSize).validate_python("100kib"), + chunk_size=TypeAdapter(ByteSize).validate_python("1kib"), ), FileGenerationInfo( - size=parse_obj_as(ByteSize, "100mib"), - chunk_size=parse_obj_as(ByteSize, "1mib"), + size=TypeAdapter(ByteSize).validate_python("100mib"), + chunk_size=TypeAdapter(ByteSize).validate_python("1mib"), ), FileGenerationInfo( - size=parse_obj_as(ByteSize, "100mib"), - chunk_size=parse_obj_as(ByteSize, "10mib"), + size=TypeAdapter(ByteSize).validate_python("100mib"), + chunk_size=TypeAdapter(ByteSize).validate_python("10mib"), ), ] ) diff --git a/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py b/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py index e423d588480..97a65d4553b 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py @@ -19,7 +19,7 @@ from models_library.users import UserID from models_library.utils.json_serialization import json_dumps from psutil._common import sdiskusage -from pydantic import ByteSize +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from simcore_service_dynamic_sidecar.modules.mounted_fs import MountedVolumes from simcore_service_dynamic_sidecar.modules.system_monitor._disk_usage import ( @@ -98,7 +98,7 @@ def _get_entry(mock: Mock, *, index: int) -> dict[Path, DiskUsage]: def _get_byte_size(byte_size_as_str: str) -> ByteSize: - return ByteSize.validate(byte_size_as_str) + return TypeAdapter(ByteSize).validate_python(byte_size_as_str) def _get_mocked_disk_usage(byte_size_as_str: str) -> DiskUsage: diff --git a/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py b/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py index 08b6f3da05e..9496bfceb78 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_user_services_preferences_user_preference.py @@ -3,7 +3,7 @@ import pytest from models_library.services import ServiceKey -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_service_dynamic_sidecar.modules.user_services_preferences._user_preference import ( get_model_class, ) @@ -11,7 +11,9 @@ @pytest.fixture def service_key() -> ServiceKey: - return parse_obj_as(ServiceKey, "simcore/services/dynamic/test-service-34") + return TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/test-service-34" + ) def test_get_model_class_only_defined_once(service_key: ServiceKey): diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 7658d1da0cd..48b9f76f89e 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -1,6 +1,7 @@ import datetime from functools import cached_property +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from models_library.basic_types import BootModeEnum from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings @@ -114,3 +115,9 @@ class ApplicationSettings(MinimalApplicationSettings): RESOURCE_USAGE_TRACKER_TRACING: TracingSettings | None = Field( auto_default_from_env=True, description="settings for opentelemetry tracing" ) + + _try_convert_resource_usage_tracker_missed_heartbeat_interval_sec = ( + timedelta_try_convert_str_to_float( + "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC" + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/settings.py b/services/web/server/src/simcore_service_webserver/projects/settings.py index 727dcb51b35..2ab2f4f7e30 100644 --- a/services/web/server/src/simcore_service_webserver/projects/settings.py +++ b/services/web/server/src/simcore_service_webserver/projects/settings.py @@ -1,6 +1,7 @@ from datetime import timedelta from aiohttp import web +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from pydantic import ByteSize, Field, NonNegativeInt, parse_obj_as from settings_library.base import BaseCustomSettings @@ -23,6 +24,10 @@ class ProjectsSettings(BaseCustomSettings): description="interval after which services need to be idle in order to be considered inactive", ) + _try_convert_projects_inactivity_interval = timedelta_try_convert_str_to_float( + "PROJECTS_INACTIVITY_INTERVAL" + ) + def get_plugin_settings(app: web.Application) -> ProjectsSettings: settings = app[APP_SETTINGS_KEY].WEBSERVER_PROJECTS diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index 3ef317631ed..b64d573fa90 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -2,6 +2,7 @@ from typing import Any, ClassVar from aiohttp import web +from common_library.pydantic_validators import timedelta_try_convert_str_to_float from pydantic import ByteSize, HttpUrl, parse_obj_as, validator from pydantic.fields import Field from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY @@ -50,6 +51,10 @@ def is_login_required(self): """ return not self.STUDIES_ACCESS_ANONYMOUS_ALLOWED + _try_convert_studies_guest_account_lifetime = timedelta_try_convert_str_to_float( + "STUDIES_GUEST_ACCOUNT_LIFETIME" + ) + class Config: schema_extra: ClassVar[dict[str, Any]] = { "example": { From 463488b22d79ef21b30cca6bd9945a7a1ff5f92b Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:26:55 +0200 Subject: [PATCH 10/53] =?UTF-8?q?=F0=9F=8E=A8=20pydantic2=20migration:=20f?= =?UTF-8?q?ixed=20unit-tests=20for=20dynamic-scheduler=20(#6547)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../src/common_library/pydantic_validators.py | 6 +- .../tests/test_pydantic_validators.py | 4 +- .../api_schemas_webserver/projects_nodes.py | 4 +- .../servicelib/fastapi/exceptions_utils.py | 4 +- .../simcore_service_agent/core/settings.py | 12 +- .../dynamic_services_settings/scheduler.py | 10 +- .../core/settings.py | 4 +- .../dynamic-scheduler/requirements/_base.txt | 265 ++++++++++++------ .../dynamic-scheduler/requirements/_test.txt | 26 +- .../dynamic-scheduler/requirements/_tools.txt | 29 +- .../core/errors.py | 6 +- .../core/settings.py | 32 ++- .../models/schemas/meta.py | 22 +- .../services/director_v2/_public_client.py | 9 +- .../services/service_tracker/_api.py | 2 +- .../unit/api_rest/test_api_rest__meta.py | 2 +- .../unit/api_rpc/test_api_rpc__services.py | 25 +- .../dynamic-scheduler/tests/unit/conftest.py | 13 +- .../tests/unit/service_tracker/test__api.py | 44 ++- .../unit/service_tracker/test__tracker.py | 10 + .../test_services_status_monitor__monitor.py | 28 +- .../tests/unit/test__model_examples.py | 4 +- .../dynamic-scheduler/tests/unit/test_cli.py | 4 +- .../core/settings.py | 10 +- .../core/settings.py | 6 +- .../projects/settings.py | 4 +- .../studies_dispatcher/settings.py | 4 +- 27 files changed, 370 insertions(+), 219 deletions(-) diff --git a/packages/common-library/src/common_library/pydantic_validators.py b/packages/common-library/src/common_library/pydantic_validators.py index 33f693a4ba5..60f6219fb13 100644 --- a/packages/common-library/src/common_library/pydantic_validators.py +++ b/packages/common-library/src/common_library/pydantic_validators.py @@ -5,10 +5,10 @@ from pydantic import TypeAdapter, field_validator -def timedelta_try_convert_str_to_float(field: str): +def validate_numeric_string_as_timedelta(field: str): """Transforms a float/int number into a valid datetime as it used to work in the past""" - def _try_convert_str_to_float_or_return( + def _numeric_string_as_timedelta( v: datetime.timedelta | str | float, ) -> datetime.timedelta | str | float: if isinstance(v, str): @@ -32,4 +32,4 @@ def _try_convert_str_to_float_or_return( return v return v - return field_validator(field, mode="before")(_try_convert_str_to_float_or_return) + return field_validator(field, mode="before")(_numeric_string_as_timedelta) diff --git a/packages/common-library/tests/test_pydantic_validators.py b/packages/common-library/tests/test_pydantic_validators.py index 825f986918a..da1ccf95adb 100644 --- a/packages/common-library/tests/test_pydantic_validators.py +++ b/packages/common-library/tests/test_pydantic_validators.py @@ -1,7 +1,7 @@ from datetime import timedelta import pytest -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from faker import Faker from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -15,7 +15,7 @@ class Settings(BaseSettings): APP_NAME: str REQUEST_TIMEOUT: timedelta = Field(default=timedelta(seconds=40)) - _try_convert_request_timeout = timedelta_try_convert_str_to_float( + _validate_request_timeout = validate_numeric_string_as_timedelta( "REQUEST_TIMEOUT" ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 772dd3e5373..0fddd5267b3 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -105,7 +105,7 @@ class NodeGet(OutputSchema): "service_basepath": "/x/E1O2E-LAH", "service_state": "pending", "service_message": "no suitable node (insufficient resources on 1 node)", - "user_id": 123, + "user_id": "123", }, # dynamic { @@ -119,7 +119,7 @@ class NodeGet(OutputSchema): "service_basepath": "/x/E1O2E-LAH", "service_state": "pending", "service_message": "no suitable node (insufficient resources on 1 node)", - "user_id": 123, + "user_id": "123", }, ] } diff --git a/packages/service-library/src/servicelib/fastapi/exceptions_utils.py b/packages/service-library/src/servicelib/fastapi/exceptions_utils.py index d55fc0e0a68..bd5f18448b1 100644 --- a/packages/service-library/src/servicelib/fastapi/exceptions_utils.py +++ b/packages/service-library/src/servicelib/fastapi/exceptions_utils.py @@ -15,9 +15,11 @@ async def http_exception_as_json_response( - request: Request, exc: HTTPException + request: Request, exc: Exception ) -> JSONResponse: + assert isinstance(exc, HTTPException) # nosec assert request # nosec + error = DefaultApiError.from_status_code(exc.status_code) error_detail = error.detail or "" diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index da2abf506d3..1b8959ea5ba 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -1,6 +1,6 @@ from datetime import timedelta -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum, LogLevel from pydantic import AliasChoices, AnyHttpUrl, Field, field_validator from settings_library.base import BaseCustomSettings @@ -78,17 +78,17 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): json_schema_extra={"auto_default_from_env": True}, ) - _try_convert_agent_volumes_cleanup_interval = timedelta_try_convert_str_to_float( + _validate_agent_volumes_cleanup_interval = validate_numeric_string_as_timedelta( "AGENT_VOLUMES_CLEANUP_INTERVAL" ) - _try_convert_agent_volumes_cleanup_book_keeping_interval = ( - timedelta_try_convert_str_to_float( + _validate_agent_volumes_cleanup_book_keeping_interval = ( + validate_numeric_string_as_timedelta( "AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL" ) ) - _try_convert_agent_volumes_cleanup_remove_volumes_inactive_for = ( - timedelta_try_convert_str_to_float( + _validate_agent_volumes_cleanup_remove_volumes_inactive_for = ( + validate_numeric_string_as_timedelta( "AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR" ) ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py index fe7fa7db653..74810cdd101 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py @@ -1,7 +1,7 @@ from datetime import timedelta from typing import Final -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.projects_networks import DockerNetworkName from pydantic import Field, NonNegativeInt, PositiveFloat from settings_library.base import BaseCustomSettings @@ -168,11 +168,11 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings): timedelta(0), description="time to sleep before removing a container" ) - _try_convert_director_v2_dynamic_scheduler_interval = ( - timedelta_try_convert_str_to_float("DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL") + _validate_director_v2_dynamic_scheduler_interval = ( + validate_numeric_string_as_timedelta("DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL") ) - _try_convert_director_v2_dynamic_sidecar_sleep_after_container_removal = ( - timedelta_try_convert_str_to_float( + _validate_director_v2_dynamic_sidecar_sleep_after_container_removal = ( + validate_numeric_string_as_timedelta( "DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL" ) ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index bf5006a228f..3c63028747b 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -5,7 +5,7 @@ import datetime from functools import cached_property -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import ( BootModeEnum, BuildTargetEnum, @@ -236,6 +236,6 @@ def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) return log_level - _try_convert_service_tracking_heartbeat = timedelta_try_convert_str_to_float( + _validate_service_tracking_heartbeat = validate_numeric_string_as_timedelta( "SERVICE_TRACKING_HEARTBEAT" ) diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 714f37a8b3e..8502347388d 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -1,30 +1,41 @@ -aio-pika==9.4.1 +aio-pika==9.4.3 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiocache==0.12.2 +aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.23.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.10.10 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp -alembic==1.13.1 +alembic==1.13.3 # via -r requirements/../../../packages/postgres-database/requirements/_base.in -anyio==4.3.0 +annotated-types==0.7.0 + # via pydantic +anyio==4.6.2.post1 # via # fast-depends # faststream @@ -43,26 +54,33 @@ async-timeout==4.0.3 # via asyncpg asyncpg==0.29.0 # via sqlalchemy -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing bidict==0.23.1 # via python-socketio -certifi==2024.2.2 +certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -74,25 +92,17 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic -fast-depends==2.4.2 +fast-depends==2.4.11 # via faststream -fastapi==0.99.1 +fastapi==0.115.2 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator -faststream==0.5.10 +faststream==0.5.27 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -102,66 +112,80 @@ googleapis-common-protos==1.65.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -grpcio==1.66.0 +grpcio==1.67.0 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn # wsproto -httpcore==1.0.5 +httpcore==1.0.6 # via httpx -httptools==0.6.1 +httptools==0.6.2 # via uvicorn -httpx==0.27.0 +httpx==0.27.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in -idna==3.6 +idna==3.10 # via # anyio # email-validator # httpx # requests # yarl -importlib-metadata==8.0.0 +importlib-metadata==8.4.0 # via opentelemetry-api -jsonschema==4.21.1 +jsonschema==4.23.0 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2023.7.1 # via jsonschema -mako==1.3.2 +mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.1 # via mako mdurl==0.1.2 # via markdown-it-py -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc @@ -172,125 +196,177 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -orjson==3.10.0 +orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -packaging==24.0 +packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 # via aiormq -prometheus-client==0.20.0 +prometheus-client==0.21.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 +prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +propcache==0.2.0 + # via yarl +protobuf==4.25.5 # via # googleapis-common-protos # opentelemetry-proto psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -psycopg2-binary==2.9.9 +psycopg2-binary==2.9.10 # via sqlalchemy -pydantic==1.10.15 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi -pygments==2.17.2 + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 - # via uvicorn -python-engineio==4.9.1 + # via + # pydantic-settings + # uvicorn +python-engineio==4.10.1 # via python-socketio -python-socketio==5.11.2 +python-socketio==5.11.4 # via -r requirements/_base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn -redis==5.0.4 +redis==5.1.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -303,20 +379,20 @@ repro-zipfile==0.3.1 # via -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.2 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing -setuptools==74.0.0 +setuptools==75.2.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -simple-websocket==1.0.0 +simple-websocket==1.1.0 # via python-engineio six==1.16.0 # via python-dateutil @@ -324,70 +400,91 @@ sniffio==1.3.1 # via # anyio # httpx -sqlalchemy==1.4.52 +sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.27.0 +starlette==0.40.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -tenacity==8.5.0 + # prometheus-fastapi-instrumentator +tenacity==9.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -tqdm==4.66.2 +tqdm==4.66.5 # via -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.12.5 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in - # faststream -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241003 # via arrow -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # aiodebug - # aiodocker # alembic # fastapi # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer -urllib3==2.2.2 +urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests -uvicorn==0.29.0 +uvicorn==0.32.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in -uvloop==0.19.0 +uvloop==0.21.0 # via uvicorn -watchfiles==0.21.0 +watchfiles==0.24.0 # via uvicorn -websockets==12.0 +websockets==13.1 # via uvicorn wrapt==1.16.0 # via @@ -395,11 +492,11 @@ wrapt==1.16.0 # opentelemetry-instrumentation wsproto==1.2.0 # via simple-websocket -yarl==1.9.4 +yarl==1.15.3 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # aio-pika # aiohttp # aiormq -zipp==3.20.1 +zipp==3.20.2 # via importlib-metadata diff --git a/services/dynamic-scheduler/requirements/_test.txt b/services/dynamic-scheduler/requirements/_test.txt index b48cff66d52..44dbc740669 100644 --- a/services/dynamic-scheduler/requirements/_test.txt +++ b/services/dynamic-scheduler/requirements/_test.txt @@ -1,44 +1,44 @@ -anyio==4.3.0 +anyio==4.6.2.post1 # via # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests -coverage==7.6.1 +coverage==7.6.3 # via # -r requirements/_test.in # pytest-cov docker==7.1.0 # via -r requirements/_test.in -faker==29.0.0 +faker==30.4.0 # via -r requirements/_test.in h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.5 +httpcore==1.0.6 # via # -c requirements/_base.txt # httpx -httpx==0.27.0 +httpx==0.27.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # respx icdiff==2.0.7 # via pytest-icdiff -idna==3.6 +idna==3.10 # via # -c requirements/_base.txt # anyio @@ -46,7 +46,7 @@ idna==3.6 # requests iniconfig==2.0.0 # via pytest -packaging==24.0 +packaging==24.1 # via # -c requirements/_base.txt # pytest @@ -101,9 +101,13 @@ sniffio==1.3.1 # anyio # asgi-lifespan # httpx -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar -urllib3==2.2.2 +typing-extensions==4.12.2 + # via + # -c requirements/_base.txt + # faker +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/dynamic-scheduler/requirements/_tools.txt b/services/dynamic-scheduler/requirements/_tools.txt index df53578298f..d15ef99dd1f 100644 --- a/services/dynamic-scheduler/requirements/_tools.txt +++ b/services/dynamic-scheduler/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -13,9 +13,9 @@ click==8.1.7 # -c requirements/_base.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv @@ -35,7 +35,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.1 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -52,32 +52,33 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.1 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # pre-commit -ruff==0.6.7 +ruff==0.6.9 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.2.0 # via # -c requirements/_base.txt # pip-tools tomlkit==0.13.2 # via pylint -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # -c requirements/_base.txt + # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.26.6 # via pre-commit wheel==0.44.0 # via pip-tools diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py index 260202d00f4..2677b7bc370 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py @@ -1,5 +1,5 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class BaseDynamicSchedulerError(PydanticErrorMixin, ValueError): - code = "simcore.service.dynamic.scheduler" +class BaseDynamicSchedulerError(OsparcErrorMixin, ValueError): + code = "simcore.service.dynamic.scheduler" # type:ignore[assignment] diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index e7441665a30..0c85785a596 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,7 +1,7 @@ import datetime -from common_library.pydantic_validators import timedelta_try_convert_str_to_float -from pydantic import Field, parse_obj_as, validator +from pydantic import AliasChoices, Field, TypeAdapter, field_validator +from pydantic_settings import SettingsConfigDict from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.director_v2 import DirectorV2Settings @@ -19,20 +19,22 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION APP_NAME: str = PROJECT_NAME - API_VTAG: VersionTag = parse_obj_as(VersionTag, API_VTAG) + API_VTAG: VersionTag = TypeAdapter(VersionTag).validate_python(API_VTAG) # RUNTIME ----------------------------------------------------------- DYNAMIC_SCHEDULER_LOGLEVEL: LogLevel = Field( default=LogLevel.INFO, - env=["DYNAMIC_SCHEDULER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "DYNAMIC_SCHEDULER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -44,14 +46,12 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ) - @validator("DYNAMIC_SCHEDULER_LOGLEVEL", pre=True) + @field_validator("DYNAMIC_SCHEDULER_LOGLEVEL", mode="before") @classmethod def _validate_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - _try_convert_dynamic_scheduler_stop_service_timeout = ( - timedelta_try_convert_str_to_float("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT") - ) + model_config = SettingsConfigDict(extra="allow") class ApplicationSettings(_BaseApplicationSettings): @@ -61,11 +61,13 @@ class ApplicationSettings(_BaseApplicationSettings): """ DYNAMIC_SCHEDULER_RABBITMQ: RabbitSettings = Field( - auto_default_from_env=True, description="settings for service/rabbitmq" + json_schema_extra={"auto_default_from_env": True}, + description="settings for service/rabbitmq", ) DYNAMIC_SCHEDULER_REDIS: RedisSettings = Field( - auto_default_from_env=True, description="settings for service/redis" + json_schema_extra={"auto_default_from_env": True}, + description="settings for service/redis", ) DYNAMIC_SCHEDULER_SWAGGER_API_DOC_ENABLED: bool = Field( @@ -73,12 +75,14 @@ class ApplicationSettings(_BaseApplicationSettings): ) DYNAMIC_SCHEDULER_DIRECTOR_V2_SETTINGS: DirectorV2Settings = Field( - auto_default_from_env=True, description="settings for director-v2 service" + json_schema_extra={"auto_default_from_env": True}, + description="settings for director-v2 service", ) DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True DYNAMIC_SCHEDULER_PROFILING: bool = False DYNAMIC_SCHEDULER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py index df9d3fa4315..ad73c58ac70 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py @@ -1,17 +1,17 @@ -from typing import Any, ClassVar - from models_library.api_schemas__common.meta import BaseMeta -from pydantic import HttpUrl +from pydantic import ConfigDict, HttpUrl class Meta(BaseMeta): docs_url: HttpUrl - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { - "example": { - "name": "simcore_service_dynamic_scheduler", - "version": "2.4.45", - "docs_url": "https://foo.io/doc", - } + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + { + "name": "simcore_service_dynamic_scheduler", + "version": "2.4.45", + "docs_url": "https://foo.io/doc", + } + ] } + ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py index fd5ce9a2cb2..5ee4ae3bcac 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py @@ -8,6 +8,7 @@ ) from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID +from pydantic import TypeAdapter from servicelib.fastapi.app_state import SingletonInAppStateMixin from servicelib.fastapi.http_client import AttachLifespanMixin, HasClientSetupInterface from servicelib.fastapi.http_client_thin import UnexpectedStatusError @@ -43,9 +44,9 @@ async def get_status( # in case of legacy version # we need to transfer the correct format! if "data" in dict_response: - return NodeGet.parse_obj(dict_response["data"]) + return TypeAdapter(NodeGet).validate_python(dict_response["data"]) - return DynamicServiceGet.parse_obj(dict_response) + return TypeAdapter(DynamicServiceGet).validate_python(dict_response) except UnexpectedStatusError as e: if ( e.response.status_code # type: ignore[attr-defined] # pylint:disable=no-member @@ -62,9 +63,9 @@ async def run_dynamic_service( # legacy services if "data" in dict_response: - return NodeGet.parse_obj(dict_response["data"]) + return TypeAdapter(NodeGet).validate_python(dict_response["data"]) - return DynamicServiceGet.parse_obj(dict_response) + return TypeAdapter(DynamicServiceGet).validate_python(dict_response) async def stop_dynamic_service( self, diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py index 1b1b4a0d9f8..99215c69123 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py @@ -150,7 +150,7 @@ async def set_if_status_changed_for_service( model.scheduled_to_run = False # check if model changed - json_status = status.json() + json_status = status.model_dump_json() if model.service_status != json_status: model.service_status = json_status model.current_state = _get_current_scheduler_service_state( diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py index 8d986dfe60e..ccf9aeab911 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py @@ -9,4 +9,4 @@ async def test_health(client: AsyncClient): response = await client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK - assert Meta.parse_raw(response.text) + assert Meta.model_validate_json(response.text) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index c484f722ff9..7ee876e9e4b 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -18,6 +18,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError @@ -52,14 +53,16 @@ def node_not_found(faker: Faker) -> NodeID: @pytest.fixture def service_status_new_style() -> DynamicServiceGet: - return DynamicServiceGet.parse_obj( - DynamicServiceGet.Config.schema_extra["examples"][1] + return TypeAdapter(DynamicServiceGet).validate_python( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] ) @pytest.fixture def service_status_legacy() -> NodeGet: - return NodeGet.parse_obj(NodeGet.Config.schema_extra["examples"][1]) + return TypeAdapter(NodeGet).validate_python( + NodeGet.model_config["json_schema_extra"]["examples"][1] + ) @pytest.fixture @@ -81,7 +84,9 @@ def mock_director_v0_service_state( ) as mock: mock.get(f"/fake-status/{node_id_legacy}").respond( status.HTTP_200_OK, - text=json.dumps(jsonable_encoder({"data": service_status_legacy.dict()})), + text=json.dumps( + jsonable_encoder({"data": service_status_legacy.model_dump()}) + ), ) # service was not found response @@ -104,7 +109,7 @@ def mock_director_v2_service_state( assert_all_mocked=True, # IMPORTANT: KEEP always True! ) as mock: mock.get(f"/dynamic_services/{node_id_new_style}").respond( - status.HTTP_200_OK, text=service_status_new_style.json() + status.HTTP_200_OK, text=service_status_new_style.model_dump_json() ) # emulate redirect response to director-v0 @@ -173,8 +178,8 @@ async def test_get_state( @pytest.fixture def dynamic_service_start() -> DynamicServiceStart: # one for legacy and one for new style? - return DynamicServiceStart.parse_obj( - DynamicServiceStart.Config.schema_extra["example"] + return TypeAdapter(DynamicServiceStart).validate_python( + DynamicServiceStart.model_config["json_schema_extra"]["example"] ) @@ -189,7 +194,9 @@ def mock_director_v0_service_run( ) as mock: mock.post("/fake-service-run").respond( status.HTTP_201_CREATED, - text=json.dumps(jsonable_encoder({"data": service_status_legacy.dict()})), + text=json.dumps( + jsonable_encoder({"data": service_status_legacy.model_dump()}) + ), ) yield None @@ -216,7 +223,7 @@ def mock_director_v2_service_run( else: request.respond( status.HTTP_201_CREATED, - text=service_status_new_style.json(), + text=service_status_new_style.model_dump_json(), ) yield None diff --git a/services/dynamic-scheduler/tests/unit/conftest.py b/services/dynamic-scheduler/tests/unit/conftest.py index 642ed2170ce..a25596bd4f2 100644 --- a/services/dynamic-scheduler/tests/unit/conftest.py +++ b/services/dynamic-scheduler/tests/unit/conftest.py @@ -7,14 +7,17 @@ DynamicServiceStop, ) from models_library.projects_nodes_io import NodeID +from pydantic import TypeAdapter @pytest.fixture def get_dynamic_service_start() -> Callable[[NodeID], DynamicServiceStart]: def _(node_id: NodeID) -> DynamicServiceStart: - dict_data = deepcopy(DynamicServiceStart.Config.schema_extra["example"]) + dict_data = deepcopy( + DynamicServiceStart.model_config["json_schema_extra"]["example"] + ) dict_data["service_uuid"] = f"{node_id}" - return DynamicServiceStart.parse_obj(dict_data) + return TypeAdapter(DynamicServiceStart).validate_python(dict_data) return _ @@ -22,8 +25,10 @@ def _(node_id: NodeID) -> DynamicServiceStart: @pytest.fixture def get_dynamic_service_stop() -> Callable[[NodeID], DynamicServiceStop]: def _(node_id: NodeID) -> DynamicServiceStop: - dict_data = deepcopy(DynamicServiceStop.Config.schema_extra["example"]) + dict_data = deepcopy( + DynamicServiceStop.model_config["json_schema_extra"]["example"] + ) dict_data["node_id"] = f"{node_id}" - return DynamicServiceStop.parse_obj(dict_data) + return TypeAdapter(DynamicServiceStop).validate_python(dict_data) return _ diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index 0755f7e5d78..489dd14f2b8 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -17,7 +17,7 @@ from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState -from pydantic import NonNegativeInt +from pydantic import NonNegativeInt, TypeAdapter from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.deferred_tasks import TaskUID from servicelib.utils import limited_gather @@ -115,12 +115,17 @@ async def test_services_tracer_workflow( @pytest.mark.parametrize( "status", [ - *[NodeGet.parse_obj(o) for o in NodeGet.Config.schema_extra["examples"]], *[ - DynamicServiceGet.parse_obj(o) - for o in DynamicServiceGet.Config.schema_extra["examples"] + NodeGet.model_validate(o) + for o in NodeGet.model_config["json_schema_extra"]["examples"] ], - NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]), + *[ + DynamicServiceGet.model_validate(o) + for o in DynamicServiceGet.model_config["json_schema_extra"]["examples"] + ], + NodeGetIdle.model_validate( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ), ], ) async def test_set_if_status_changed( @@ -138,7 +143,7 @@ async def test_set_if_status_changed( model = await get_tracked_service(app, node_id) assert model - assert model.service_status == status.json() + assert model.service_status == status.model_dump_json() async def test_set_service_status_task_uid( @@ -162,15 +167,22 @@ async def test_set_service_status_task_uid( "status, expected_poll_interval", [ ( - NodeGet.parse_obj(NodeGet.Config.schema_extra["examples"][1]), + TypeAdapter(NodeGet).validate_python( + NodeGet.model_config["json_schema_extra"]["examples"][1] + ), _LOW_RATE_POLL_INTERVAL, ), *[ - (DynamicServiceGet.parse_obj(o), NORMAL_RATE_POLL_INTERVAL) - for o in DynamicServiceGet.Config.schema_extra["examples"] + ( + TypeAdapter(DynamicServiceGet).validate_python(o), + NORMAL_RATE_POLL_INTERVAL, + ) + for o in DynamicServiceGet.model_config["json_schema_extra"]["examples"] ], ( - NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]), + TypeAdapter(NodeGetIdle).validate_python( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ), _LOW_RATE_POLL_INTERVAL, ), ], @@ -182,23 +194,25 @@ def test__get_poll_interval( def _get_node_get_from(service_state: ServiceState) -> NodeGet: - dict_data = NodeGet.Config.schema_extra["examples"][1] + dict_data = NodeGet.model_config["json_schema_extra"]["examples"][1] assert "service_state" in dict_data dict_data["service_state"] = service_state - return NodeGet.parse_obj(dict_data) + return TypeAdapter(NodeGet).validate_python(dict_data) def _get_dynamic_service_get_from( service_state: ServiceState, ) -> DynamicServiceGet: - dict_data = DynamicServiceGet.Config.schema_extra["examples"][1] + dict_data = DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] assert "state" in dict_data dict_data["state"] = service_state - return DynamicServiceGet.parse_obj(dict_data) + return TypeAdapter(DynamicServiceGet).validate_python(dict_data) def _get_node_get_idle() -> NodeGetIdle: - return NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]) + return TypeAdapter(NodeGetIdle).validate_python( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ) def __get_flat_list(nested_list: list[list[Any]]) -> list[Any]: diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py index 59739ddf8f6..20293f343b5 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py @@ -7,6 +7,7 @@ from fastapi import FastAPI from models_library.projects_nodes_io import NodeID from pydantic import NonNegativeInt +from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.utils import logged_gather from settings_library.redis import RedisSettings @@ -24,8 +25,17 @@ ] +@pytest.fixture +def disable_monitor_task(mocker: MockerFixture) -> None: + mocker.patch( + "simcore_service_dynamic_scheduler.services.status_monitor._monitor.Monitor._worker_start_get_status_requests", + autospec=True, + ) + + @pytest.fixture def app_environment( + disable_monitor_task: None, disable_rabbitmq_setup: None, disable_deferred_manager_setup: None, disable_notifier_setup: None, diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index 2dd5270b627..b1dfd7c0d1f 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -22,7 +22,7 @@ ) from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle from models_library.projects_nodes_io import NodeID -from pydantic import NonNegativeInt +from pydantic import NonNegativeInt, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.rabbit import RabbitSettings @@ -69,7 +69,7 @@ def _add_to_dict(dict_data: dict, entries: list[tuple[str, Any]]) -> None: def _get_node_get_with(state: str, node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGet: - dict_data = deepcopy(NodeGet.Config.schema_extra["examples"][1]) + dict_data = deepcopy(NodeGet.model_config["json_schema_extra"]["examples"][1]) _add_to_dict( dict_data, [ @@ -77,13 +77,15 @@ def _get_node_get_with(state: str, node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGe ("service_uuid", f"{node_id}"), ], ) - return NodeGet.parse_obj(dict_data) + return TypeAdapter(NodeGet).validate_python(dict_data) def _get_dynamic_service_get_legacy_with( state: str, node_id: NodeID = _DEFAULT_NODE_ID ) -> DynamicServiceGet: - dict_data = deepcopy(DynamicServiceGet.Config.schema_extra["examples"][0]) + dict_data = deepcopy( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][0] + ) _add_to_dict( dict_data, [ @@ -92,13 +94,15 @@ def _get_dynamic_service_get_legacy_with( ("node_uuid", f"{node_id}"), ], ) - return DynamicServiceGet.parse_obj(dict_data) + return TypeAdapter(DynamicServiceGet).validate_python(dict_data) def _get_dynamic_service_get_new_style_with( state: str, node_id: NodeID = _DEFAULT_NODE_ID ) -> DynamicServiceGet: - dict_data = deepcopy(DynamicServiceGet.Config.schema_extra["examples"][1]) + dict_data = deepcopy( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] + ) _add_to_dict( dict_data, [ @@ -107,18 +111,18 @@ def _get_dynamic_service_get_new_style_with( ("node_uuid", f"{node_id}"), ], ) - return DynamicServiceGet.parse_obj(dict_data) + return TypeAdapter(DynamicServiceGet).validate_python(dict_data) def _get_node_get_idle(node_id: NodeID = _DEFAULT_NODE_ID) -> NodeGetIdle: - dict_data = NodeGetIdle.Config.schema_extra["example"] + dict_data = NodeGetIdle.model_config["json_schema_extra"]["example"] _add_to_dict( dict_data, [ ("service_uuid", f"{node_id}"), ], ) - return NodeGetIdle.parse_obj(dict_data) + return TypeAdapter(NodeGetIdle).validate_python(dict_data) class _ResponseTimeline: @@ -209,10 +213,12 @@ def _side_effect_node_status_response(request: Request) -> Response: if isinstance(service_status, NodeGet): return Response( status.HTTP_200_OK, - text=json.dumps(jsonable_encoder({"data": service_status.dict()})), + text=json.dumps( + jsonable_encoder({"data": service_status.model_dump()}) + ), ) if isinstance(service_status, DynamicServiceGet): - return Response(status.HTTP_200_OK, text=service_status.json()) + return Response(status.HTTP_200_OK, text=service_status.model_dump_json()) if isinstance(service_status, NodeGetIdle): return Response(status.HTTP_404_NOT_FOUND) diff --git a/services/dynamic-scheduler/tests/unit/test__model_examples.py b/services/dynamic-scheduler/tests/unit/test__model_examples.py index 858bcc66a4d..e768927cfe4 100644 --- a/services/dynamic-scheduler/tests/unit/test__model_examples.py +++ b/services/dynamic-scheduler/tests/unit/test__model_examples.py @@ -3,7 +3,7 @@ import pytest import simcore_service_dynamic_scheduler.models -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from pytest_simcore.pydantic_models import walk_model_examples_in_package @@ -15,7 +15,7 @@ def test_api_server_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): try: - assert model_cls.parse_obj(example_data) is not None + assert TypeAdapter(model_cls).validate_python(example_data) is not None except ValidationError as err: pytest.fail( f"\n{example_name}: {json.dumps(example_data, indent=1)}\nError: {err}" diff --git a/services/dynamic-scheduler/tests/unit/test_cli.py b/services/dynamic-scheduler/tests/unit/test_cli.py index 2e812f7e118..85b2a5e2dcd 100644 --- a/services/dynamic-scheduler/tests/unit/test_cli.py +++ b/services/dynamic-scheduler/tests/unit/test_cli.py @@ -39,8 +39,8 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, result.output print(result.output) - settings = ApplicationSettings.parse_raw(result.output) - assert settings == ApplicationSettings.create_from_envs() + settings = ApplicationSettings.model_validate_json(result.output) + assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() def test_main(app_environment: EnvVarsDict): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index e8c75cc9e89..93a1f5e9cfb 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import cast -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum, PortInt from models_library.callbacks_mapping import CallbacksMapping from models_library.products import ProductName @@ -39,8 +39,8 @@ class ResourceTrackingSettings(BaseCustomSettings): description="each time the status of the service is propagated", ) - _try_convert_resource_tracking_heartbeat_interval = ( - timedelta_try_convert_str_to_float("RESOURCE_TRACKING_HEARTBEAT_INTERVAL") + _validate_resource_tracking_heartbeat_interval = ( + validate_numeric_string_as_timedelta("RESOURCE_TRACKING_HEARTBEAT_INTERVAL") ) @@ -200,8 +200,8 @@ def are_prometheus_metrics_enabled(self) -> bool: def _check_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - _try_convert_dynamic_sidecar_telemetry_disk_usage_monitor_interval = ( - timedelta_try_convert_str_to_float( + _validate_dynamic_sidecar_telemetry_disk_usage_monitor_interval = ( + validate_numeric_string_as_timedelta( "DYNAMIC_SIDECAR_TELEMETRY_DISK_USAGE_MONITOR_INTERVAL" ) ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 48b9f76f89e..cd23585d8b5 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -1,7 +1,7 @@ import datetime from functools import cached_property -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings @@ -116,8 +116,8 @@ class ApplicationSettings(MinimalApplicationSettings): auto_default_from_env=True, description="settings for opentelemetry tracing" ) - _try_convert_resource_usage_tracker_missed_heartbeat_interval_sec = ( - timedelta_try_convert_str_to_float( + _validate_resource_usage_tracker_missed_heartbeat_interval_sec = ( + validate_numeric_string_as_timedelta( "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC" ) ) diff --git a/services/web/server/src/simcore_service_webserver/projects/settings.py b/services/web/server/src/simcore_service_webserver/projects/settings.py index 2ab2f4f7e30..8a46b8def4f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/settings.py +++ b/services/web/server/src/simcore_service_webserver/projects/settings.py @@ -1,7 +1,7 @@ from datetime import timedelta from aiohttp import web -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from pydantic import ByteSize, Field, NonNegativeInt, parse_obj_as from settings_library.base import BaseCustomSettings @@ -24,7 +24,7 @@ class ProjectsSettings(BaseCustomSettings): description="interval after which services need to be idle in order to be considered inactive", ) - _try_convert_projects_inactivity_interval = timedelta_try_convert_str_to_float( + _validate_projects_inactivity_interval = validate_numeric_string_as_timedelta( "PROJECTS_INACTIVITY_INTERVAL" ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index b64d573fa90..a79c4865f12 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -2,7 +2,7 @@ from typing import Any, ClassVar from aiohttp import web -from common_library.pydantic_validators import timedelta_try_convert_str_to_float +from common_library.pydantic_validators import validate_numeric_string_as_timedelta from pydantic import ByteSize, HttpUrl, parse_obj_as, validator from pydantic.fields import Field from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY @@ -51,7 +51,7 @@ def is_login_required(self): """ return not self.STUDIES_ACCESS_ANONYMOUS_ALLOWED - _try_convert_studies_guest_account_lifetime = timedelta_try_convert_str_to_float( + _validate_studies_guest_account_lifetime = validate_numeric_string_as_timedelta( "STUDIES_GUEST_ACCOUNT_LIFETIME" ) From c2b4983dd3aa8f00cb2ccd1d12706129b9132218 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:39:32 +0200 Subject: [PATCH 11/53] =?UTF-8?q?=F0=9F=8E=A8=20pydantic2=20migration:=20f?= =?UTF-8?q?ixed=20unit-tests=20for=20agent=20(#6546)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../rabbitmq/rpc_interfaces/agent/errors.py | 4 +- services/agent/requirements/_base.txt | 115 ++++++++++++++---- services/agent/requirements/_test.txt | 59 ++++----- services/agent/requirements/_tools.txt | 20 +-- .../simcore_service_agent/core/settings.py | 5 +- .../simcore_service_agent/models/volumes.py | 10 +- .../services/docker_utils.py | 4 +- .../tests/unit/test_core_settings.py | 1 + .../tests/unit/test_utils_docker.py | 1 + .../constants.py | 24 ++-- .../core/settings.py | 23 ++-- .../clusters-keeper/tests/unit/conftest.py | 2 +- .../services/process_messages.py | 5 +- .../src/simcore_service_invitations/cli.py | 4 +- .../core/settings.py | 3 +- 15 files changed, 184 insertions(+), 96 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/errors.py index ae21b8f09a7..b297004e283 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class BaseAgentRPCError(PydanticErrorMixin, Exception): +class BaseAgentRPCError(OsparcErrorMixin, Exception): ... diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 2476acd1380..a6db5134bfb 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -10,14 +10,20 @@ aiodocker==0.23.0 # -r requirements/_base.in aiofiles==24.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiohappyeyeballs==2.4.0 +aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.6 +aiohttp==3.10.10 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -27,7 +33,7 @@ aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.6.0 +anyio==4.6.2.post1 # via # fast-depends # faststream @@ -47,16 +53,22 @@ attrs==24.2.0 # referencing certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -68,18 +80,17 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via pydantic fast-depends==2.4.11 # via faststream -fastapi==0.115.0 +fastapi==0.115.2 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator -faststream==0.5.25 +faststream==0.5.27 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 # via @@ -89,20 +100,26 @@ googleapis-common-protos==1.65.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.1 +grpcio==1.67.0 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.5 +httpcore==1.0.6 # via httpx httpx==0.27.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -184,10 +201,16 @@ opentelemetry-util-http==0.48b0 # opentelemetry-instrumentation-requests orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in @@ -200,8 +223,10 @@ prometheus-client==0.21.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 +prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +propcache==0.2.0 + # via yarl protobuf==4.25.5 # via # googleapis-common-protos @@ -210,16 +235,28 @@ psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends @@ -229,37 +266,50 @@ pydantic==2.9.2 pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 - # via -r requirements/../../../packages/models-library/requirements/_base.in + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in pydantic-settings==2.5.2 # via # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich -pyinstrument==4.7.3 +pyinstrument==5.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via arrow -python-dotenv==1.0.0 - # via - # -r requirements/_base.in - # pydantic-settings -rich==13.4.2 +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in -redis==5.0.4 +redis==5.1.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -272,7 +322,7 @@ repro-zipfile==0.3.1 # via -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.8.1 +rich==13.9.2 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -281,7 +331,7 @@ rpds-py==0.20.0 # via # jsonschema # referencing -setuptools==75.1.0 +setuptools==75.2.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer @@ -291,18 +341,25 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.38.6 +starlette==0.40.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi + # prometheus-fastapi-instrumentator tenacity==9.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in tqdm==4.66.5 # via -r requirements/../../../packages/service-library/requirements/_base.in @@ -310,7 +367,7 @@ typer==0.12.5 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -types-python-dateutil==2.9.0.20240906 +types-python-dateutil==2.9.0.20241003 # via arrow typing-extensions==4.12.2 # via @@ -323,14 +380,20 @@ typing-extensions==4.12.2 # typer urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests -uvicorn==0.30.6 +uvicorn==0.32.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -338,7 +401,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation -yarl==1.12.1 +yarl==1.15.3 # via # aio-pika # aiohttp diff --git a/services/agent/requirements/_test.txt b/services/agent/requirements/_test.txt index 01b05ccb4d8..b58735a4be8 100644 --- a/services/agent/requirements/_test.txt +++ b/services/agent/requirements/_test.txt @@ -1,16 +1,16 @@ -aioboto3==13.1.1 +aioboto3==13.2.0 # via -r requirements/_test.in -aiobotocore==2.13.1 +aiobotocore==2.15.2 # via aioboto3 aiofiles==24.1.0 # via # -c requirements/_base.txt # aioboto3 -aiohappyeyeballs==2.4.0 +aiohappyeyeballs==2.4.3 # via # -c requirements/_base.txt # aiohttp -aiohttp==3.10.6 +aiohttp==3.10.10 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -27,7 +27,7 @@ annotated-types==0.7.0 # pydantic antlr4-python3-runtime==4.13.2 # via moto -anyio==4.6.0 +anyio==4.6.2.post1 # via # -c requirements/_base.txt # httpx @@ -45,12 +45,12 @@ aws-xray-sdk==2.14.0 # via moto blinker==1.8.2 # via flask -boto3==1.34.131 +boto3==1.35.36 # via # aiobotocore # aws-sam-translator # moto -botocore==1.34.131 +botocore==1.35.36 # via # aiobotocore # aws-xray-sdk @@ -66,9 +66,9 @@ certifi==2024.8.30 # requests cffi==1.17.1 # via cryptography -cfn-lint==1.15.1 +cfn-lint==1.16.1 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests @@ -76,7 +76,7 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.1 +coverage==7.6.3 # via # -r requirements/_test.in # pytest-cov @@ -87,12 +87,7 @@ cryptography==43.0.1 # moto docker==7.1.0 # via moto -ecdsa==0.19.0 - # via - # moto - # python-jose - # sshpubkeys -faker==30.0.0 +faker==30.4.0 # via -r requirements/_test.in flask==3.0.3 # via @@ -105,13 +100,13 @@ frozenlist==1.4.1 # -c requirements/_base.txt # aiohttp # aiosignal -graphql-core==3.2.4 +graphql-core==3.2.5 # via moto h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.5 +httpcore==1.0.6 # via # -c requirements/_base.txt # httpx @@ -146,7 +141,7 @@ jsondiff==2.2.1 # via moto jsonpatch==1.33 # via cfn-lint -jsonpath-ng==1.6.1 +jsonpath-ng==1.7.0 # via moto jsonpointer==3.0.0 # via jsonpatch @@ -165,11 +160,11 @@ jsonschema-specifications==2023.7.1 # openapi-schema-validator lazy-object-proxy==1.10.0 # via openapi-spec-validator -markupsafe==2.1.5 +markupsafe==3.0.1 # via # jinja2 # werkzeug -moto==5.0.15 +moto==5.0.17 # via -r requirements/_test.in mpmath==1.3.0 # via sympy @@ -178,7 +173,7 @@ multidict==6.1.0 # -c requirements/_base.txt # aiohttp # yarl -networkx==3.3 +networkx==3.4.1 # via cfn-lint openapi-schema-validator==0.6.2 # via openapi-spec-validator @@ -194,6 +189,10 @@ pluggy==1.5.0 # via pytest ply==3.11 # via jsonpath-ng +propcache==0.2.0 + # via + # -c requirements/_base.txt + # yarl py-partiql-parser==0.5.6 # via moto pycparser==2.22 @@ -207,7 +206,7 @@ pydantic-core==2.23.4 # via # -c requirements/_base.txt # pydantic -pyparsing==3.1.4 +pyparsing==3.2.0 # via moto pytest==8.3.3 # via @@ -232,7 +231,9 @@ python-dateutil==2.9.0.post0 # faker # moto python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt @@ -266,9 +267,9 @@ rpds-py==0.20.0 # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.2 +s3transfer==0.10.3 # via boto3 -setuptools==75.1.0 +setuptools==75.2.0 # via # -c requirements/_base.txt # moto @@ -290,7 +291,9 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # aws-sam-translator # cfn-lint + # faker # pydantic + # pydantic-core urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt @@ -308,9 +311,9 @@ wrapt==1.16.0 # -c requirements/_base.txt # aiobotocore # aws-xray-sdk -xmltodict==0.13.0 +xmltodict==0.14.2 # via moto -yarl==1.12.1 +yarl==1.15.3 # via # -c requirements/_base.txt # aiohttp diff --git a/services/agent/requirements/_tools.txt b/services/agent/requirements/_tools.txt index 23facad9ab3..7b16cb6547b 100644 --- a/services/agent/requirements/_tools.txt +++ b/services/agent/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,9 +14,9 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv @@ -53,11 +53,11 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt pylint==3.3.1 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -67,9 +67,9 @@ pyyaml==6.0.2 # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit -ruff==0.6.7 +ruff==0.6.9 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.1.0 +setuptools==75.2.0 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -81,7 +81,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.26.6 # via pre-commit wheel==0.44.0 # via pip-tools diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index 1b8959ea5ba..f322e27163f 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -1,8 +1,9 @@ from datetime import timedelta +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum, LogLevel -from pydantic import AliasChoices, AnyHttpUrl, Field, field_validator +from pydantic import AliasChoices, Field, field_validator from settings_library.base import BaseCustomSettings from settings_library.r_clone import S3Provider from settings_library.rabbit import RabbitSettings @@ -34,7 +35,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field( ..., description="Exactly the same as director-v2's `SWARM_STACK_NAME` env var" ) - AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: AnyHttpUrl + AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: AnyHttpUrlLegacy AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY: str AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY: str AGENT_VOLUMES_CLEANUP_S3_BUCKET: str diff --git a/services/agent/src/simcore_service_agent/models/volumes.py b/services/agent/src/simcore_service_agent/models/volumes.py index ceb31048650..cf227bf69e9 100644 --- a/services/agent/src/simcore_service_agent/models/volumes.py +++ b/services/agent/src/simcore_service_agent/models/volumes.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Final from models_library.api_schemas_directorv2.services import ( CHARS_IN_VOLUME_NAME_BEFORE_DIR_NAME, @@ -7,7 +8,7 @@ from models_library.projects_nodes_io import NodeID from models_library.services_types import RunID from models_library.users import UserID -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter class DynamicServiceVolumeLabels(BaseModel): @@ -22,7 +23,14 @@ class DynamicServiceVolumeLabels(BaseModel): def directory_name(self) -> str: return self.source[CHARS_IN_VOLUME_NAME_BEFORE_DIR_NAME:][::-1].strip("_") + model_config = ConfigDict( + arbitrary_types_allowed=True, + ) + class VolumeDetails(BaseModel): mountpoint: Path = Field(alias="Mountpoint") labels: DynamicServiceVolumeLabels = Field(alias="Labels") + + +VolumeDetailsAdapter: Final[TypeAdapter[VolumeDetails]] = TypeAdapter(VolumeDetails) diff --git a/services/agent/src/simcore_service_agent/services/docker_utils.py b/services/agent/src/simcore_service_agent/services/docker_utils.py index 181fe13a275..83656783b55 100644 --- a/services/agent/src/simcore_service_agent/services/docker_utils.py +++ b/services/agent/src/simcore_service_agent/services/docker_utils.py @@ -15,7 +15,7 @@ from simcore_service_agent.core.settings import ApplicationSettings from starlette import status -from ..models.volumes import VolumeDetails +from ..models.volumes import VolumeDetails, VolumeDetailsAdapter from .backup import backup_volume from .instrumentation import get_instrumentation @@ -60,7 +60,7 @@ async def get_unused_dynamc_sidecar_volumes(docker: Docker) -> set[str]: async def get_volume_details(docker: Docker, *, volume_name: str) -> VolumeDetails: volume_details = await DockerVolume(docker, volume_name).show() - return VolumeDetails.parse_obj(volume_details) + return VolumeDetailsAdapter.validate_python(volume_details) @contextmanager diff --git a/services/autoscaling/tests/unit/test_core_settings.py b/services/autoscaling/tests/unit/test_core_settings.py index 9315c8fcfd1..10050a56594 100644 --- a/services/autoscaling/tests/unit/test_core_settings.py +++ b/services/autoscaling/tests/unit/test_core_settings.py @@ -1,3 +1,4 @@ +# pylint: disable=no-member # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable diff --git a/services/autoscaling/tests/unit/test_utils_docker.py b/services/autoscaling/tests/unit/test_utils_docker.py index a99f6939d1a..3f9677112bb 100644 --- a/services/autoscaling/tests/unit/test_utils_docker.py +++ b/services/autoscaling/tests/unit/test_utils_docker.py @@ -1,3 +1,4 @@ +# pylint: disable=no-member # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/constants.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/constants.py index 7f970665f25..a5d4f3636da 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/constants.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/constants.py @@ -1,15 +1,21 @@ from typing import Final from aws_library.ec2._models import AWSTagKey, AWSTagValue -from pydantic import parse_obj_as +from pydantic import TypeAdapter DOCKER_STACK_DEPLOY_COMMAND_NAME: Final[str] = "private cluster docker deploy" -DOCKER_STACK_DEPLOY_COMMAND_EC2_TAG_KEY: Final[AWSTagKey] = parse_obj_as( - AWSTagKey, "io.simcore.clusters-keeper.private_cluster_docker_deploy" -) +DOCKER_STACK_DEPLOY_COMMAND_EC2_TAG_KEY: Final[AWSTagKey] = TypeAdapter( + AWSTagKey +).validate_python("io.simcore.clusters-keeper.private_cluster_docker_deploy") -USER_ID_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "user_id") -WALLET_ID_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "wallet_id") -ROLE_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "role") -WORKER_ROLE_TAG_VALUE: Final[AWSTagValue] = parse_obj_as(AWSTagValue, "worker") -MANAGER_ROLE_TAG_VALUE: Final[AWSTagValue] = parse_obj_as(AWSTagValue, "manager") +USER_ID_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python("user_id") +WALLET_ID_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python( + "wallet_id" +) +ROLE_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python("role") +WORKER_ROLE_TAG_VALUE: Final[AWSTagValue] = TypeAdapter(AWSTagValue).validate_python( + "worker" +) +MANAGER_ROLE_TAG_VALUE: Final[AWSTagValue] = TypeAdapter(AWSTagValue).validate_python( + "manager" +) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 602406b6529..019d66cea9e 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -54,18 +54,22 @@ class ClustersKeeperEC2Settings(EC2Settings): class ClustersKeeperSSMSettings(SSMSettings): - class Config(SSMSettings.Config): - env_prefix = CLUSTERS_KEEPER_ENV_PREFIX - - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + env_prefix=CLUSTERS_KEEPER_ENV_PREFIX, + json_schema_extra={ "examples": [ { f"{CLUSTERS_KEEPER_ENV_PREFIX}{key}": var - for key, var in example.items() + for key, var in example.items() # type:ignore[union-attr] } - for example in SSMSettings.Config.schema_extra["examples"] + for example in SSMSettings.model_config[ # type:ignore[union-attr,index] + "json_schema_extra" + ][ + "examples" + ] ], - } + }, + ) class WorkersEC2InstancesSettings(BaseCustomSettings): @@ -280,7 +284,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) CLUSTERS_KEEPER_SSM_ACCESS: ClustersKeeperSSMSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: PrimaryEC2InstancesSettings | None = Field( @@ -350,7 +354,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ", see https://selectfrom.dev/deep-dive-into-dask-distributed-scheduler-9fdb3b36b7c7", ) CLUSTERS_KEEPER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) SWARM_STACK_NAME: str = Field( diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 14c5e0d93c1..da94a9886f3 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -98,7 +98,7 @@ def mocked_ssm_server_envs( f"{CLUSTERS_KEEPER_ENV_PREFIX}{k}": ( v.get_secret_value() if isinstance(v, SecretStr) else v ) - for k, v in mocked_ssm_server_settings.dict().items() + for k, v in mocked_ssm_server_settings.model_dump().items() } return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py index 11c7781bbae..d1e3b67353a 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py @@ -2,7 +2,6 @@ from fastapi import FastAPI from models_library.rabbitmq_messages import DynamicServiceRunningMessage -from pydantic import parse_raw_as from servicelib.logging_utils import log_context from simcore_service_efs_guardian.services.modules.redis import get_redis_lock_client @@ -14,8 +13,8 @@ async def process_dynamic_service_running_message(app: FastAPI, data: bytes) -> bool: assert app # nosec - rabbit_message: DynamicServiceRunningMessage = parse_raw_as( - DynamicServiceRunningMessage, data + rabbit_message: DynamicServiceRunningMessage = ( + DynamicServiceRunningMessage.model_validate_json(data) ) _logger.debug( "Process dynamic service running msg, project ID: %s node ID: %s, current user: %s", diff --git a/services/invitations/src/simcore_service_invitations/cli.py b/services/invitations/src/simcore_service_invitations/cli.py index cfb7cf2717a..67838b04615 100644 --- a/services/invitations/src/simcore_service_invitations/cli.py +++ b/services/invitations/src/simcore_service_invitations/cli.py @@ -125,7 +125,7 @@ def invite( invitation_link, _ = create_invitation_link_and_content( invitation_data=invitation_data, - secret_key=settings.INVITATIONS_SECRET_KEY.get_secret_value().encode(), + secret_key=settings.INVITATIONS_SECRET_KEY.get_secret_value().encode(), # pylint:disable=no-member base_url=settings.INVITATIONS_OSPARC_URL, default_product=settings.INVITATIONS_DEFAULT_PRODUCT, ) @@ -144,7 +144,7 @@ def extract(ctx: typer.Context, invitation_url: str): invitation_code=extract_invitation_code_from_query( TypeAdapter(HttpUrl).validate_python(invitation_url) ), - secret_key=settings.INVITATIONS_SECRET_KEY.get_secret_value().encode(), + secret_key=settings.INVITATIONS_SECRET_KEY.get_secret_value().encode(), # pylint:disable=no-member default_product=settings.INVITATIONS_DEFAULT_PRODUCT, ) assert invitation.product is not None # nosec diff --git a/services/invitations/src/simcore_service_invitations/core/settings.py b/services/invitations/src/simcore_service_invitations/core/settings.py index b4b776a393b..ea3b04c33ac 100644 --- a/services/invitations/src/simcore_service_invitations/core/settings.py +++ b/services/invitations/src/simcore_service_invitations/core/settings.py @@ -112,5 +112,6 @@ class ApplicationSettings(MinimalApplicationSettings): ) INVITATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True INVITATIONS_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) From 64293f0de301d7a5d4608954590ddcfbe26d68b4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:30:37 +0200 Subject: [PATCH 12/53] =?UTF-8?q?=F0=9F=8E=A8=20pydantic2=20migration:=20f?= =?UTF-8?q?ixed=20unit-tests=20for=20payments=20(#6553)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../api_schemas_payments/errors.py | 4 +- .../test_models_payments_transactions.py | 26 ++- .../pytest_simcore/helpers/faker_factories.py | 4 + .../src/settings_library/utils_cli.py | 14 +- .../settings-library/tests/test_utils_cli.py | 4 +- services/payments/openapi.json | 211 +++++++++++++----- services/payments/requirements/_base.txt | 140 +++++++++++- .../scripts/example_payment_gateway.py | 4 +- .../src/simcore_service_payments/cli.py | 5 +- .../simcore_service_payments/core/errors.py | 4 +- .../simcore_service_payments/core/settings.py | 42 ++-- .../db/auto_recharge_repo.py | 10 +- .../db/payments_methods_repo.py | 12 +- .../db/payments_transactions_repo.py | 17 +- .../src/simcore_service_payments/models/db.py | 31 ++- .../models/db_to_api.py | 2 +- .../models/payments_gateway.py | 14 +- .../models/schemas/acknowledgements.py | 29 +-- .../models/schemas/meta.py | 10 +- .../services/auto_recharge_process_message.py | 8 +- .../services/payments_gateway.py | 24 +- .../services/resource_usage_tracker.py | 2 +- .../services/stripe.py | 2 +- services/payments/tests/conftest.py | 4 +- .../api/test__one_time_payment_workflows.py | 11 +- .../api/test__payment_method_workflows.py | 11 +- .../unit/api/test_rest_acknowledgements.py | 13 +- .../payments/tests/unit/api/test_rest_auth.py | 4 +- .../payments/tests/unit/api/test_rest_meta.py | 4 +- services/payments/tests/unit/conftest.py | 14 +- .../tests/unit/test__model_examples.py | 2 +- services/payments/tests/unit/test_cli.py | 18 +- .../unit/test_db_payments_methods_repo.py | 4 +- .../test_db_payments_transactions_repo.py | 2 +- .../payments/tests/unit/test_rpc_payments.py | 10 +- .../tests/unit/test_rpc_payments_methods.py | 16 +- .../test_services_auto_recharge_listener.py | 10 +- .../tests/unit/test_services_notifier.py | 52 ++--- .../unit/test_services_notifier_email.py | 4 +- .../test_services_payments__get_invoice.py | 3 +- .../unit/test_services_payments_gateway.py | 7 +- .../test_services_resource_usage_tracker.py | 2 +- 42 files changed, 531 insertions(+), 279 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_payments/errors.py b/packages/models-library/src/models_library/api_schemas_payments/errors.py index eaeba92aab1..362482772f7 100644 --- a/packages/models-library/src/models_library/api_schemas_payments/errors.py +++ b/packages/models-library/src/models_library/api_schemas_payments/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class _BaseRpcApiError(PydanticErrorMixin, ValueError): +class _BaseRpcApiError(OsparcErrorMixin, ValueError): @classmethod def get_full_class_name(cls) -> str: # Can be used as unique code identifier diff --git a/packages/postgres-database/tests/test_models_payments_transactions.py b/packages/postgres-database/tests/test_models_payments_transactions.py index 9d4f748e094..6dde13b1abe 100644 --- a/packages/postgres-database/tests/test_models_payments_transactions.py +++ b/packages/postgres-database/tests/test_models_payments_transactions.py @@ -6,6 +6,7 @@ import decimal from collections.abc import Callable +from typing import Any import pytest import sqlalchemy as sa @@ -43,14 +44,25 @@ async def test_numerics_precission_and_scale(connection: SAConnection): assert float(got) == expected +def _remove_not_required(data: dict[str, Any]) -> dict[str, Any]: + for to_remove in ( + "completed_at", + "invoice_url", + "invoice_pdf_url", + "state", + "state_message", + "stripe_invoice_id", + ): + data.pop(to_remove) + return data + + @pytest.fixture def init_transaction(connection: SAConnection): async def _init(payment_id: str): # get payment_id from payment-gateway - values = random_payment_transaction(payment_id=payment_id) - # remove states - values.pop("state") - values.pop("completed_at") + values = _remove_not_required(random_payment_transaction(payment_id=payment_id)) + # init successful: set timestamp values["initiated_at"] = utcnow() @@ -180,10 +192,8 @@ def create_fake_user_transactions(connection: SAConnection, user_id: int) -> Cal async def _go(expected_total=5): payment_ids = [] for _ in range(expected_total): - values = random_payment_transaction(user_id=user_id) - # remove states - values.pop("state") - values.pop("completed_at") + values = _remove_not_required(random_payment_transaction(user_id=user_id)) + payment_id = await insert_init_payment_transaction(connection, **values) assert payment_id payment_ids.append(payment_id) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index 78da5521827..d4418a5ef81 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -301,6 +301,10 @@ def random_payment_transaction( "initiated_at": utcnow(), "state": PaymentTransactionState.PENDING, "completed_at": None, + "invoice_url": None, + "stripe_invoice_id": None, + "invoice_pdf_url": None, + "state_message": None, } # state is not added on purpose assert set(data.keys()).issubset({c.name for c in payments_transactions.columns}) diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index 2c1ab37116d..073a8ae2d5e 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -67,10 +67,15 @@ def print_as_envfile( def print_as_json( - settings_obj, *, compact=False, show_secrets, **pydantic_export_options + settings_obj, + *, + compact: bool = False, + show_secrets: bool, + json_serializer, + **pydantic_export_options, ): typer.echo( - json.dumps( + json_serializer( model_dump_with_secrets( settings_obj, show_secrets=show_secrets, **pydantic_export_options ), @@ -80,7 +85,9 @@ def print_as_json( def create_settings_command( - settings_cls: type[BaseCustomSettings], logger: logging.Logger | None = None + settings_cls: type[BaseCustomSettings], + logger: logging.Logger | None = None, + json_serializer=json.dumps, ) -> Callable: """Creates typer command function for settings""" @@ -145,6 +152,7 @@ def settings( settings_obj, compact=compact, show_secrets=show_secrets, + json_serializer=json_serializer, **pydantic_export_options, ) else: diff --git a/packages/settings-library/tests/test_utils_cli.py b/packages/settings-library/tests/test_utils_cli.py index b4e50902124..b3a0c10d899 100644 --- a/packages/settings-library/tests/test_utils_cli.py +++ b/packages/settings-library/tests/test_utils_cli.py @@ -434,7 +434,9 @@ class FakeSettings(BaseCustomSettings): assert "secret" not in captured.out assert "Some info" not in captured.out - print_as_json(settings_obj, compact=True, show_secrets=False) + print_as_json( + settings_obj, compact=True, show_secrets=False, json_serializer=json.dumps + ) captured = capsys.readouterr() assert "secret" not in captured.out assert "**" in captured.out diff --git a/services/payments/openapi.json b/services/payments/openapi.json index b4b49c630e3..47e509314ad 100644 --- a/services/payments/openapi.json +++ b/services/payments/openapi.json @@ -99,28 +99,33 @@ "summary": "Acknowledge Payment", "description": "completes (ie. ack) request initated by `/init` on the payments-gateway API", "operationId": "acknowledge_payment_v1_payments__payment_id__ack_post", + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], "parameters": [ { + "name": "payment_id", + "in": "path", "required": true, "schema": { "type": "string", - "maxLength": 100, "minLength": 1, + "maxLength": 100, "title": "Payment Id" - }, - "name": "payment_id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AckPayment" } } - }, - "required": true + } }, "responses": { "200": { @@ -141,12 +146,7 @@ } } } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ] + } } }, "/v1/payments-methods/{payment_method_id}:ack": { @@ -157,28 +157,33 @@ "summary": "Acknowledge Payment Method", "description": "completes (ie. ack) request initated by `/payments-methods:init` on the payments-gateway API", "operationId": "acknowledge_payment_method_v1_payments_methods__payment_method_id__ack_post", + "security": [ + { + "OAuth2PasswordBearer": [] + } + ], "parameters": [ { + "name": "payment_method_id", + "in": "path", "required": true, "schema": { "type": "string", - "maxLength": 100, "minLength": 1, + "maxLength": 100, "title": "Payment Method Id" - }, - "name": "payment_method_id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AckPaymentMethod" } } - }, - "required": true + } }, "responses": { "200": { @@ -199,12 +204,7 @@ } } } - }, - "security": [ - { - "OAuth2PasswordBearer": [] - } - ] + } } } }, @@ -217,53 +217,97 @@ "title": "Success" }, "message": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Message" }, "provider_payment_id": { - "type": "string", - "maxLength": 100, - "minLength": 1, + "anyOf": [ + { + "type": "string", + "maxLength": 100, + "minLength": 1 + }, + { + "type": "null" + } + ], "title": "Provider Payment Id", "description": "Payment ID from the provider (e.g. stripe payment ID)" }, "invoice_url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Invoice Url", "description": "Link to invoice is required when success=true" }, "invoice_pdf": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Invoice Pdf", "description": "Link to invoice PDF" }, "stripe_invoice_id": { - "type": "string", - "maxLength": 100, - "minLength": 1, + "anyOf": [ + { + "type": "string", + "maxLength": 100, + "minLength": 1 + }, + { + "type": "null" + } + ], "title": "Stripe Invoice Id", "description": "Stripe invoice ID" }, "stripe_customer_id": { - "type": "string", - "maxLength": 100, - "minLength": 1, + "anyOf": [ + { + "type": "string", + "maxLength": 100, + "minLength": 1 + }, + { + "type": "null" + } + ], "title": "Stripe Customer Id", "description": "Stripe customer ID" }, "saved": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/SavedPaymentMethod" + }, + { + "type": "null" } ], - "title": "Saved", "description": "Gets the payment-method if user opted to save it during payment.If used did not opt to save of payment-method was already saved, then it defaults to None" } }, @@ -273,13 +317,13 @@ ], "title": "AckPayment", "example": { - "success": true, - "provider_payment_id": "pi_123ABC", "invoice_url": "https://invoices.com/id=12345", + "provider_payment_id": "pi_123ABC", "saved": { - "success": true, - "payment_method_id": "3FA85F64-5717-4562-B3FC-2C963F66AFA6" - } + "payment_method_id": "3FA85F64-5717-4562-B3FC-2C963F66AFA6", + "success": true + }, + "success": true } }, "AckPaymentMethod": { @@ -289,7 +333,14 @@ "title": "Success" }, "message": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Message" } }, @@ -302,8 +353,15 @@ "Body_login_to_create_access_token": { "properties": { "grant_type": { - "type": "string", - "pattern": "password", + "anyOf": [ + { + "type": "string", + "pattern": "password" + }, + { + "type": "null" + } + ], "title": "Grant Type" }, "username": { @@ -320,11 +378,25 @@ "default": "" }, "client_id": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Client Id" }, "client_secret": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Client Secret" } }, @@ -360,11 +432,18 @@ "title": "Version" }, "released": { - "additionalProperties": { - "type": "string", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Released", "description": "Maps every route's path tag with a released version" }, @@ -384,9 +463,9 @@ ], "title": "Meta", "example": { + "docs_url": "https://foo.io/doc", "name": "simcore_service_payments", - "version": "2.4.45", - "docs_url": "https://foo.io/doc" + "version": "2.4.45" } }, "SavedPaymentMethod": { @@ -396,7 +475,14 @@ "title": "Success" }, "message": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Message" }, "payment_method_id": { @@ -424,6 +510,7 @@ "enum": [ "bearer" ], + "const": "bearer", "title": "Token Type" } }, diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index 9c6ed81bca4..3f8c2628ff9 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -12,11 +12,18 @@ aiohappyeyeballs==2.3.4 # via aiohttp aiohttp==3.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -28,6 +35,8 @@ aiosmtplib==3.0.2 # via -r requirements/_base.in alembic==1.13.2 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.4.0 # via # fast-depends @@ -55,11 +64,18 @@ bidict==0.23.1 # via python-socketio certifi==2024.7.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -75,11 +91,18 @@ click==8.1.7 # uvicorn cryptography==43.0.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -97,15 +120,8 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.7 # via faststream -fastapi==0.99.1 +fastapi==0.115.2 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -134,11 +150,18 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -154,11 +177,18 @@ importlib-metadata==8.0.0 # via opentelemetry-api jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -170,11 +200,18 @@ jsonschema-specifications==2023.7.1 # via jsonschema mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -245,11 +282,18 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.6 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in @@ -278,24 +322,51 @@ pyasn1==0.6.0 # rsa pycparser==2.22 # via cffi -pydantic==1.10.17 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.6.2 @@ -303,7 +374,9 @@ pyinstrument==4.6.2 python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 - # via uvicorn + # via + # pydantic-settings + # uvicorn python-engineio==4.9.1 # via python-socketio python-jose==3.3.0 @@ -314,22 +387,36 @@ python-socketio==5.11.3 # via -r requirements/_base.in pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -353,11 +440,18 @@ rpds-py==0.19.1 # referencing rsa==4.9 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # python-jose @@ -377,22 +471,36 @@ sniffio==1.3.1 # httpx sqlalchemy==1.4.53 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.27.0 +starlette==0.40.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -418,14 +526,22 @@ typing-extensions==4.12.2 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer urllib3==2.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests diff --git a/services/payments/scripts/example_payment_gateway.py b/services/payments/scripts/example_payment_gateway.py index aca2643a555..bc1e93bc59f 100755 --- a/services/payments/scripts/example_payment_gateway.py +++ b/services/payments/scripts/example_payment_gateway.py @@ -171,7 +171,7 @@ async def ack_payment(id_: PaymentID, acked: AckPayment, settings: Settings): async with httpx.AsyncClient() as client: await client.post( f"{settings.PAYMENTS_SERVICE_API_BASE_URL}/v1/payments/{id_}:ack", - json=acked.dict(), + json=acked.model_dump(), auth=PaymentsAuth( username=settings.PAYMENTS_USERNAME, password=settings.PAYMENTS_PASSWORD.get_secret_value(), @@ -185,7 +185,7 @@ async def ack_payment_method( async with httpx.AsyncClient() as client: await client.post( f"{settings.PAYMENTS_SERVICE_API_BASE_URL}/v1/payments-methods/{id_}:ack", - json=acked.dict(), + json=acked.model_dump(), auth=PaymentsAuth( username=settings.PAYMENTS_USERNAME, password=settings.PAYMENTS_PASSWORD.get_secret_value(), diff --git a/services/payments/src/simcore_service_payments/cli.py b/services/payments/src/simcore_service_payments/cli.py index 64c67d00e8f..db884e5f99f 100644 --- a/services/payments/src/simcore_service_payments/cli.py +++ b/services/payments/src/simcore_service_payments/cli.py @@ -3,6 +3,7 @@ import os import typer +from models_library.utils.json_serialization import json_dumps from servicelib.utils_secrets import generate_password, generate_token_secret_key from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings @@ -20,7 +21,9 @@ main = typer.Typer(name=PROJECT_NAME) main.command()( - create_settings_command(settings_cls=ApplicationSettings, logger=_logger) + create_settings_command( + settings_cls=ApplicationSettings, logger=_logger, json_serializer=json_dumps + ) ) main.callback()(create_version_callback(__version__)) diff --git a/services/payments/src/simcore_service_payments/core/errors.py b/services/payments/src/simcore_service_payments/core/errors.py index 5fce135e800..8b5113891fc 100644 --- a/services/payments/src/simcore_service_payments/core/errors.py +++ b/services/payments/src/simcore_service_payments/core/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class _BaseAppError(PydanticErrorMixin, ValueError): +class _BaseAppError(OsparcErrorMixin, ValueError): @classmethod def get_full_class_name(cls) -> str: # Can be used as unique code identifier diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index b56c79322cb..4b7965f4558 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -1,14 +1,16 @@ from functools import cached_property +from common_library.pydantic_networks_extension import HttpUrlLegacy from models_library.basic_types import NonNegativeDecimal from pydantic import ( + AliasChoices, + ConfigDict, EmailStr, Field, - HttpUrl, PositiveFloat, SecretStr, - parse_obj_as, - validator, + TypeAdapter, + field_validator, ) from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -28,19 +30,19 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION APP_NAME: str = PROJECT_NAME - API_VTAG: VersionTag = parse_obj_as(VersionTag, API_VTAG) + API_VTAG: VersionTag = TypeAdapter(VersionTag).validate_python(API_VTAG) # RUNTIME ----------------------------------------------------------- PAYMENTS_LOGLEVEL: LogLevel = Field( - default=LogLevel.INFO, env=["PAYMENTS_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + default=LogLevel.INFO, + validation_alias=AliasChoices("PAYMENTS_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ - "PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED", - "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + validation_alias=AliasChoices( + "LOG_FORMAT_LOCAL_DEV_ENABLED", "PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED" + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -48,11 +50,13 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def LOG_LEVEL(self): # noqa: N802 return self.PAYMENTS_LOGLEVEL - @validator("PAYMENTS_LOGLEVEL", pre=True) + @field_validator("PAYMENTS_LOGLEVEL", mode="before") @classmethod def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + model_config = ConfigDict(extra="allow") # type:ignore[assignment] + class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables @@ -60,7 +64,7 @@ class ApplicationSettings(_BaseApplicationSettings): These settings includes extra configuration for the http-API """ - PAYMENTS_GATEWAY_URL: HttpUrl = Field( + PAYMENTS_GATEWAY_URL: HttpUrlLegacy = Field( ..., description="Base url to the payment gateway" ) @@ -111,18 +115,21 @@ class ApplicationSettings(_BaseApplicationSettings): ) PAYMENTS_RABBITMQ: RabbitSettings = Field( - auto_default_from_env=True, description="settings for service/rabbitmq" + json_schema_extra={"auto_default_from_env": True}, + description="settings for service/rabbitmq", ) PAYMENTS_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) PAYMENTS_POSTGRES: PostgresSettings = Field( - auto_default_from_env=True, description="settings for postgres service" + json_schema_extra={"auto_default_from_env": True}, + description="settings for postgres service", ) - PAYMENTS_STRIPE_URL: HttpUrl = Field( + PAYMENTS_STRIPE_URL: HttpUrlLegacy = Field( ..., description="Base url to the payment Stripe" ) PAYMENTS_STRIPE_API_SECRET: SecretStr = Field( @@ -134,12 +141,13 @@ class ApplicationSettings(_BaseApplicationSettings): ) PAYMENTS_RESOURCE_USAGE_TRACKER: ResourceUsageTrackerSettings = Field( - auto_default_from_env=True, description="settings for RUT service" + json_schema_extra={"auto_default_from_env": True}, + description="settings for RUT service", ) PAYMENTS_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True PAYMENTS_EMAIL: SMTPSettings | None = Field( - auto_default_from_env=True, + json_schema_extra={"auto_default_from_env": True}, description="optional email (see notifier_email service)", ) diff --git a/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py index 4e7b25d228e..aa98896cf13 100644 --- a/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py +++ b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py @@ -5,7 +5,7 @@ from models_library.basic_types import NonNegativeDecimal from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, PositiveInt +from pydantic import BaseModel, ConfigDict, PositiveInt from simcore_postgres_database.utils_payments_autorecharge import AutoRechargeStmts from .base import BaseRepository @@ -19,9 +19,7 @@ class PaymentsAutorechargeDB(BaseModel): primary_payment_method_id: PaymentMethodID top_up_amount_in_usd: NonNegativeDecimal monthly_limit_in_usd: NonNegativeDecimal | None - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class AutoRechargeRepo(BaseRepository): @@ -38,7 +36,7 @@ async def get_wallet_autorecharge( stmt = AutoRechargeStmts.get_wallet_autorecharge(wallet_id) result = await conn.execute(stmt) row = result.first() - return PaymentsAutorechargeDB.from_orm(row) if row else None + return PaymentsAutorechargeDB.model_validate(row) if row else None async def replace_wallet_autorecharge( self, @@ -73,4 +71,4 @@ async def replace_wallet_autorecharge( result = await conn.execute(stmt) row = result.first() assert row # nosec - return PaymentsAutorechargeDB.from_orm(row) + return PaymentsAutorechargeDB.model_validate(row) diff --git a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py index 79e4b6d7ae4..4eb43b667b1 100644 --- a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py @@ -11,7 +11,7 @@ from models_library.api_schemas_webserver.wallets import PaymentMethodID from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.models.payments_methods import ( InitPromptAckFlowState, payments_methods, @@ -91,7 +91,7 @@ async def update_ack_payment_method( row = result.first() assert row, "execute above should have caught this" # nosec - return PaymentsMethodsDB.from_orm(row) + return PaymentsMethodsDB.model_validate(row) async def insert_payment_method( self, @@ -132,7 +132,7 @@ async def list_user_payment_methods( .order_by(payments_methods.c.created.desc()) ) # newest first rows = result.fetchall() or [] - return parse_obj_as(list[PaymentsMethodsDB], rows) + return TypeAdapter(list[PaymentsMethodsDB]).validate_python(rows) async def get_payment_method_by_id( self, @@ -149,7 +149,7 @@ async def get_payment_method_by_id( if row is None: raise PaymentMethodNotFoundError(payment_method_id=payment_method_id) - return PaymentsMethodsDB.from_orm(row) + return PaymentsMethodsDB.model_validate(row) async def get_payment_method( self, @@ -171,7 +171,7 @@ async def get_payment_method( if row is None: raise PaymentMethodNotFoundError(payment_method_id=payment_method_id) - return PaymentsMethodsDB.from_orm(row) + return PaymentsMethodsDB.model_validate(row) async def delete_payment_method( self, @@ -191,4 +191,4 @@ async def delete_payment_method( .returning(sa.literal_column("*")) ) row = result.first() - return row if row is None else PaymentsMethodsDB.from_orm(row) + return row if row is None else PaymentsMethodsDB.model_validate(row) diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index fc3ba5b3b10..8b2eef6f228 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -12,7 +12,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import HttpUrl, PositiveInt, parse_obj_as +from pydantic import HttpUrl, PositiveInt, TypeAdapter from simcore_postgres_database import errors as pg_errors from simcore_postgres_database.models.payments_transactions import ( PaymentTransactionState, @@ -114,9 +114,9 @@ async def update_ack_payment_transaction( .values( completed_at=sa.func.now(), state=completion_state, - invoice_url=invoice_url, + invoice_url=f"{invoice_url}" if invoice_url else None, stripe_invoice_id=stripe_invoice_id, - invoice_pdf_url=invoice_pdf_url, + invoice_pdf_url=f"{invoice_pdf_url}" if invoice_pdf_url else None, **optional, ) .where(payments_transactions.c.payment_id == f"{payment_id}") @@ -125,7 +125,7 @@ async def update_ack_payment_transaction( row = result.first() assert row, "execute above should have caught this" # nosec - return PaymentsTransactionsDB.from_orm(row) + return PaymentsTransactionsDB.model_validate(row) async def list_user_payment_transactions( self, @@ -171,8 +171,9 @@ async def list_user_payment_transactions( result = await connection.execute(stmt) rows = result.fetchall() - return total_number_of_items, parse_obj_as( - list[PaymentsTransactionsDB], rows + return ( + total_number_of_items, + TypeAdapter(list[PaymentsTransactionsDB]).validate_python(rows), ) async def get_payment_transaction( @@ -189,7 +190,7 @@ async def get_payment_transaction( ) ) row = result.fetchone() - return PaymentsTransactionsDB.from_orm(row) if row else None + return PaymentsTransactionsDB.model_validate(row) if row else None async def sum_current_month_dollars(self, *, wallet_id: WalletID) -> Decimal: _current_timestamp = datetime.now(tz=timezone.utc) @@ -229,4 +230,4 @@ async def get_last_payment_transaction_for_wallet( .limit(1) ) row = result.fetchone() - return PaymentsTransactionsDB.from_orm(row) if row else None + return PaymentsTransactionsDB.model_validate(row) if row else None diff --git a/services/payments/src/simcore_service_payments/models/db.py b/services/payments/src/simcore_service_payments/models/db.py index 8d69b8de70d..8811dbcc139 100644 --- a/services/payments/src/simcore_service_payments/models/db.py +++ b/services/payments/src/simcore_service_payments/models/db.py @@ -1,6 +1,5 @@ import datetime from decimal import Decimal -from typing import Any, ClassVar from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID from models_library.emails import LowerCaseEmailStr @@ -8,7 +7,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, HttpUrl +from pydantic import BaseModel, ConfigDict, HttpUrl from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState from simcore_postgres_database.models.payments_transactions import ( PaymentTransactionState, @@ -47,15 +46,14 @@ class PaymentsTransactionsDB(BaseModel): completed_at: datetime.datetime | None state: PaymentTransactionState state_message: str | None - - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "examples": [ - _EXAMPLE_AFTER_INIT, + _EXAMPLE_AFTER_INIT, # type:ignore[list-item] # successful completion { - **_EXAMPLE_AFTER_INIT, + **_EXAMPLE_AFTER_INIT, # type:ignore[dict-item] "invoice_url": "https://my-fake-pdf-link.com", "stripe_invoice_id": "12345", "invoice_pdf_url": "https://my-fake-pdf-link.com", @@ -64,7 +62,8 @@ class Config: "state_message": "Payment completed successfully", }, ] - } + }, + ) _EXAMPLE_AFTER_INIT_PAYMENT_METHOD = { @@ -86,18 +85,18 @@ class PaymentsMethodsDB(BaseModel): completed_at: datetime.datetime | None state: InitPromptAckFlowState state_message: str | None - - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "examples": [ - _EXAMPLE_AFTER_INIT_PAYMENT_METHOD, + _EXAMPLE_AFTER_INIT_PAYMENT_METHOD, # type:ignore[list-item] # successful completion { - **_EXAMPLE_AFTER_INIT_PAYMENT_METHOD, + **_EXAMPLE_AFTER_INIT_PAYMENT_METHOD, # type:ignore[dict-item] "completed_at": "2023-09-27T10:00:15", "state": "SUCCESS", "state_message": "Payment method completed successfully", }, ] - } + }, + ) diff --git a/services/payments/src/simcore_service_payments/models/db_to_api.py b/services/payments/src/simcore_service_payments/models/db_to_api.py index d3ffa832ed9..c6c79195383 100644 --- a/services/payments/src/simcore_service_payments/models/db_to_api.py +++ b/services/payments/src/simcore_service_payments/models/db_to_api.py @@ -15,7 +15,7 @@ def to_payments_api_model(transaction: PaymentsTransactionsDB) -> PaymentTransac "osparc_credits": transaction.osparc_credits, "wallet_id": transaction.wallet_id, "created_at": transaction.initiated_at, - "state": transaction.state, + "state": f"{transaction.state}", "completed_at": transaction.completed_at, } diff --git a/services/payments/src/simcore_service_payments/models/payments_gateway.py b/services/payments/src/simcore_service_payments/models/payments_gateway.py index e0d7481df58..dc1b3525633 100644 --- a/services/payments/src/simcore_service_payments/models/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/models/payments_gateway.py @@ -7,7 +7,7 @@ from models_library.basic_types import AmountDecimal, IDStr from models_library.payments import UserInvoiceAddress from models_library.products import StripePriceID, StripeTaxRateID -from pydantic import BaseModel, EmailStr, Extra, Field +from pydantic import BaseModel, ConfigDict, EmailStr, Field COUNTRIES_WITH_VAT = ["CH", "LI"] @@ -30,7 +30,9 @@ class InitPayment(BaseModel): amount_dollars: AmountDecimal # metadata to store for billing or reference credits_: AmountDecimal = Field( - ..., alias="credits", describe="This is equal to `quantity` field in Stripe" + ..., + alias="credits", + json_schema_extra={"describe": "This is equal to `quantity` field in Stripe"}, ) user_name: IDStr user_email: EmailStr @@ -39,9 +41,7 @@ class InitPayment(BaseModel): stripe_price_id: StripePriceID stripe_tax_rate_id: StripeTaxRateID stripe_tax_exempt_value: StripeTaxExempt - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class PaymentInitiated(BaseModel): @@ -58,9 +58,7 @@ class InitPaymentMethod(BaseModel): user_name: IDStr user_email: EmailStr wallet_name: IDStr - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class PaymentMethodInitiated(BaseModel): diff --git a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py index 5b73282cc3c..17e7cf3cf81 100644 --- a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py +++ b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py @@ -1,18 +1,19 @@ # mypy: disable-error-code=truthy-function -from typing import Any, ClassVar +from typing import Any from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID from models_library.basic_types import IDStr -from pydantic import BaseModel, Field, HttpUrl, validator +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator +from pydantic_core.core_schema import ValidationInfo class _BaseAck(BaseModel): success: bool - message: str = Field(default=None) + message: str | None = Field(default=None) class _BaseAckPayment(_BaseAck): - provider_payment_id: IDStr = Field( + provider_payment_id: IDStr | None = Field( default=None, description="Payment ID from the provider (e.g. stripe payment ID)", ) @@ -87,17 +88,17 @@ class AckPayment(_BaseAckPayment): description="Gets the payment-method if user opted to save it during payment." "If used did not opt to save of payment-method was already saved, then it defaults to None", ) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": _EXAMPLES[1].copy(), # shown in openapi.json - "examples": _EXAMPLES, + "examples": _EXAMPLES, # type:ignore[dict-item] } + ) - @validator("invoice_url") + @field_validator("invoice_url") @classmethod - def success_requires_invoice(cls, v, values): - success = values.get("success") + def success_requires_invoice(cls, v, info: ValidationInfo): + success = info.data.get("success") if success and not v: msg = "Invoice required on successful payments" raise ValueError(msg) @@ -112,14 +113,14 @@ class AckPaymentWithPaymentMethod(_BaseAckPayment): payment_id: PaymentID = Field( default=None, description="Payment ID from the gateway" ) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { **_ONE_TIME_SUCCESS, "payment_id": "D19EE68B-B007-4B61-A8BC-32B7115FB244", }, # shown in openapi.json } + ) assert PaymentID # nosec diff --git a/services/payments/src/simcore_service_payments/models/schemas/meta.py b/services/payments/src/simcore_service_payments/models/schemas/meta.py index 06352b54ba1..cf5e7c649a2 100644 --- a/services/payments/src/simcore_service_payments/models/schemas/meta.py +++ b/services/payments/src/simcore_service_payments/models/schemas/meta.py @@ -1,17 +1,15 @@ -from typing import Any, ClassVar - from models_library.api_schemas__common.meta import BaseMeta -from pydantic import HttpUrl +from pydantic import ConfigDict, HttpUrl class Meta(BaseMeta): docs_url: HttpUrl - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "simcore_service_payments", "version": "2.4.45", "docs_url": "https://foo.io/doc", } } + ) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 9507b32bf0a..d300bbf881b 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -14,7 +14,7 @@ from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage from models_library.wallets import WalletID -from pydantic import parse_obj_as, parse_raw_as +from pydantic import TypeAdapter from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo from simcore_service_payments.db.payments_transactions_repo import ( @@ -36,7 +36,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: - rabbit_message = parse_raw_as(WalletCreditsMessage, data) + rabbit_message = TypeAdapter(WalletCreditsMessage).validate_json(data) _logger.debug("Process msg: %s", rabbit_message) settings: ApplicationSettings = app.state.settings @@ -142,12 +142,12 @@ async def _perform_auto_recharge( result = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_invoice_data"), + TypeAdapter(RPCMethodName).validate_python("get_invoice_data"), user_id=payment_method_db.user_id, dollar_amount=wallet_auto_recharge.top_up_amount_in_usd, product_name=rabbit_message.product_name, ) - invoice_data_get = parse_obj_as(InvoiceDataGet, result) + invoice_data_get = TypeAdapter(InvoiceDataGet).validate_python(result) await pay_with_payment_method( gateway=PaymentsGatewayApi.get_from_app_state(app), diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 0b1097492c6..a1ac9d0b061 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -12,12 +12,12 @@ from contextlib import suppress import httpx +from common_library.errors_classes import OsparcErrorMixin from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from httpx import URL, HTTPStatusError from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID -from pydantic import ValidationError, parse_raw_as -from pydantic.errors import PydanticErrorMixin +from pydantic import TypeAdapter, ValidationError from servicelib.fastapi.app_state import SingletonInAppStateMixin from servicelib.fastapi.http_client import ( AttachLifespanMixin, @@ -48,11 +48,11 @@ def _parse_raw_as_or_none(cls: type, text: str | None): if text: with suppress(ValidationError): - return parse_raw_as(cls, text) + return TypeAdapter(cls).validate_python(text) return None -class PaymentsGatewayError(PydanticErrorMixin, ValueError): +class PaymentsGatewayError(OsparcErrorMixin, ValueError): msg_template = "{operation_id} error {status_code}: {reason}" @classmethod @@ -124,10 +124,10 @@ class PaymentsGatewayApi( async def init_payment(self, payment: InitPayment) -> PaymentInitiated: response = await self.client.post( "/init", - json=jsonable_encoder(payment.dict(exclude_none=True, by_alias=True)), + json=jsonable_encoder(payment.model_dump(exclude_none=True, by_alias=True)), ) response.raise_for_status() - return PaymentInitiated.parse_obj(response.json()) + return PaymentInitiated.model_validate(response.json()) def get_form_payment_url(self, id_: PaymentID) -> URL: return self.client.base_url.copy_with(path="/pay", params={"id": f"{id_}"}) @@ -141,7 +141,7 @@ async def cancel_payment( json=jsonable_encoder(payment_initiated), ) response.raise_for_status() - return PaymentCancelled.parse_obj(response.json()) + return PaymentCancelled.model_validate(response.json()) # # api: payment method workflows @@ -157,7 +157,7 @@ async def init_payment_method( json=jsonable_encoder(payment_method), ) response.raise_for_status() - return PaymentMethodInitiated.parse_obj(response.json()) + return PaymentMethodInitiated.model_validate(response.json()) def get_form_payment_method_url(self, id_: PaymentMethodID) -> URL: return self.client.base_url.copy_with( @@ -177,13 +177,13 @@ async def get_many_payment_methods( json=jsonable_encoder(BatchGetPaymentMethods(payment_methods_ids=ids_)), ) response.raise_for_status() - return PaymentMethodsBatch.parse_obj(response.json()).items + return PaymentMethodsBatch.model_validate(response.json()).items @_handle_status_errors async def get_payment_method(self, id_: PaymentMethodID) -> GetPaymentMethod: response = await self.client.get(f"/payment-methods/{id_}") response.raise_for_status() - return GetPaymentMethod.parse_obj(response.json()) + return GetPaymentMethod.model_validate(response.json()) @_handle_status_errors async def delete_payment_method(self, id_: PaymentMethodID) -> None: @@ -198,10 +198,10 @@ async def pay_with_payment_method( ) -> AckPaymentWithPaymentMethod: response = await self.client.post( f"/payment-methods/{id_}:pay", - json=jsonable_encoder(payment.dict(exclude_none=True, by_alias=True)), + json=jsonable_encoder(payment.model_dump(exclude_none=True, by_alias=True)), ) response.raise_for_status() - return AckPaymentWithPaymentMethod.parse_obj(response.json()) + return AckPaymentWithPaymentMethod.model_validate(response.json()) def setup_payments_gateway(app: FastAPI): diff --git a/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py b/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py index e66f650fe7b..0051bc6035f 100644 --- a/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py +++ b/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py @@ -63,7 +63,7 @@ async def create_credit_transaction( ) ), ) - credit_transaction = CreditTransactionCreated.parse_raw(response.text) + credit_transaction = CreditTransactionCreated.model_validate_json(response.text) return credit_transaction.credit_transaction_id diff --git a/services/payments/src/simcore_service_payments/services/stripe.py b/services/payments/src/simcore_service_payments/services/stripe.py index 38cc21fab0e..c640c10f716 100644 --- a/services/payments/src/simcore_service_payments/services/stripe.py +++ b/services/payments/src/simcore_service_payments/services/stripe.py @@ -81,7 +81,7 @@ async def get_invoice( response = await self.client.get(f"/v1/invoices/{stripe_invoice_id}") response.raise_for_status() - return InvoiceData.parse_raw(response.text) + return InvoiceData.model_validate_json(response.text) def setup_stripe(app: FastAPI): diff --git a/services/payments/tests/conftest.py b/services/payments/tests/conftest.py index 042ac85f968..3691f3c43d2 100644 --- a/services/payments/tests/conftest.py +++ b/services/payments/tests/conftest.py @@ -10,7 +10,7 @@ import simcore_service_payments from faker import Faker from models_library.users import GroupID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.utils_secrets import generate_token_secret_key @@ -89,4 +89,4 @@ def app_environment( @pytest.fixture def user_primary_group_id(faker: Faker) -> GroupID: - return parse_obj_as(GroupID, faker.pyint()) + return TypeAdapter(GroupID).validate_python(faker.pyint()) diff --git a/services/payments/tests/unit/api/test__one_time_payment_workflows.py b/services/payments/tests/unit/api/test__one_time_payment_workflows.py index 753432ac6d6..126116a5dc2 100644 --- a/services/payments/tests/unit/api/test__one_time_payment_workflows.py +++ b/services/payments/tests/unit/api/test__one_time_payment_workflows.py @@ -9,6 +9,7 @@ import pytest from faker import Faker from fastapi import FastAPI, status +from fastapi.encoders import jsonable_encoder from models_library.api_schemas_webserver.wallets import WalletPaymentInitiated from models_library.basic_types import IDStr from models_library.payments import UserInvoiceAddress @@ -16,7 +17,7 @@ from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -90,7 +91,7 @@ async def test_successful_one_time_payment_workflow( # ACK via api/rest inited = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment"), + TypeAdapter(RPCMethodName).validate_python("init_payment"), amount_dollars=1000, target_credits=10000, product_name="osparc", @@ -111,7 +112,9 @@ async def test_successful_one_time_payment_workflow( # ACK response = await client.post( f"/v1/payments/{inited.payment_id}:ack", - json=AckPayment(success=True, invoice_url=faker.url()).dict(), + json=jsonable_encoder( + AckPayment(success=True, invoice_url=faker.url()).model_dump() + ), headers=auth_headers, ) @@ -121,7 +124,7 @@ async def test_successful_one_time_payment_workflow( # LIST payments via api/rest got = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_payments_page"), + TypeAdapter(RPCMethodName).validate_python("get_payments_page"), user_id=user_id, product_name="osparc", timeout_s=None if is_pdb_enabled else RPC_REQUEST_DEFAULT_TIMEOUT_S, diff --git a/services/payments/tests/unit/api/test__payment_method_workflows.py b/services/payments/tests/unit/api/test__payment_method_workflows.py index 76640384f7b..697ddfd08f3 100644 --- a/services/payments/tests/unit/api/test__payment_method_workflows.py +++ b/services/payments/tests/unit/api/test__payment_method_workflows.py @@ -9,6 +9,7 @@ import pytest from faker import Faker from fastapi import FastAPI, status +from fastapi.encoders import jsonable_encoder from models_library.api_schemas_webserver.wallets import ( PaymentMethodGet, PaymentMethodInitiated, @@ -17,7 +18,7 @@ from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -89,7 +90,7 @@ async def test_successful_create_payment_method_workflow( # INIT via api/rpc inited = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_creation_of_payment_method"), + TypeAdapter(RPCMethodName).validate_python("init_creation_of_payment_method"), wallet_id=wallet_id, wallet_name=wallet_name, user_id=user_id, @@ -104,7 +105,9 @@ async def test_successful_create_payment_method_workflow( # ACK via api/rest response = await client.post( f"/v1/payments-methods/{inited.payment_method_id}:ack", - json=AckPayment(success=True, invoice_url=faker.url()).dict(), + json=jsonable_encoder( + AckPayment(success=True, invoice_url=faker.url()).model_dump() + ), headers=auth_headers, ) @@ -114,7 +117,7 @@ async def test_successful_create_payment_method_workflow( # GET via api/rpc got = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_payment_method"), + TypeAdapter(RPCMethodName).validate_python("get_payment_method"), payment_method_id=inited.payment_method_id, user_id=user_id, wallet_id=wallet_id, diff --git a/services/payments/tests/unit/api/test_rest_acknowledgements.py b/services/payments/tests/unit/api/test_rest_acknowledgements.py index 4df30829f93..b7254f22d42 100644 --- a/services/payments/tests/unit/api/test_rest_acknowledgements.py +++ b/services/payments/tests/unit/api/test_rest_acknowledgements.py @@ -11,6 +11,7 @@ import pytest from faker import Faker from fastapi import FastAPI, status +from fastapi.encoders import jsonable_encoder from models_library.api_schemas_payments.errors import ( PaymentMethodNotFoundError, PaymentNotFoundError, @@ -93,7 +94,9 @@ async def test_payments_api_authentication( auth_headers: dict[str, str], ): payments_id = faker.uuid4() - payment_ack = AckPayment(success=True, invoice_url=faker.url()).dict() + payment_ack = jsonable_encoder( + AckPayment(success=True, invoice_url=faker.url()).model_dump() + ) # w/o header response = await client.post( @@ -108,7 +111,7 @@ async def test_payments_api_authentication( ) assert response.status_code == status.HTTP_404_NOT_FOUND, response.json() - error = DefaultApiError.parse_obj(response.json()) + error = DefaultApiError.model_validate(response.json()) assert PaymentNotFoundError.msg_template.format(payment_id=payments_id) == str( error.detail ) @@ -121,7 +124,9 @@ async def test_payments_methods_api_authentication( auth_headers: dict[str, str], ): payment_method_id = faker.uuid4() - payment_method_ack = AckPaymentMethod(success=True, message=faker.word()).dict() + payment_method_ack = AckPaymentMethod( + success=True, message=faker.word() + ).model_dump() # w/o header response = await client.post( @@ -138,7 +143,7 @@ async def test_payments_methods_api_authentication( ) assert response.status_code == status.HTTP_404_NOT_FOUND, response.json() - error = DefaultApiError.parse_obj(response.json()) + error = DefaultApiError.model_validate(response.json()) assert PaymentMethodNotFoundError.msg_template.format( payment_method_id=payment_method_id ) == str(error.detail) diff --git a/services/payments/tests/unit/api/test_rest_auth.py b/services/payments/tests/unit/api/test_rest_auth.py index eef8ee23cd2..2139f99d233 100644 --- a/services/payments/tests/unit/api/test_rest_auth.py +++ b/services/payments/tests/unit/api/test_rest_auth.py @@ -17,7 +17,9 @@ async def test_bearer_token(httpbin_base_url: HttpUrl, faker: Faker): bearer_token = faker.word() headers = {"Authorization": f"Bearer {bearer_token}"} - async with httpx.AsyncClient(base_url=httpbin_base_url, headers=headers) as client: + async with httpx.AsyncClient( + base_url=f"{httpbin_base_url}", headers=headers + ) as client: response = await client.get("/bearer") assert response.json() == {"authenticated": True, "token": bearer_token} diff --git a/services/payments/tests/unit/api/test_rest_meta.py b/services/payments/tests/unit/api/test_rest_meta.py index 1ca81c9b0a2..3a6acf2b020 100644 --- a/services/payments/tests/unit/api/test_rest_meta.py +++ b/services/payments/tests/unit/api/test_rest_meta.py @@ -60,7 +60,7 @@ async def test_meta( ): response = await client.get(f"/{API_VTAG}/meta", headers=auth_headers) assert response.status_code == status.HTTP_200_OK - meta = Meta.parse_obj(response.json()) + meta = Meta.model_validate(response.json()) - response = await client.get(meta.docs_url) + response = await client.get(f"{meta.docs_url}") assert response.status_code == status.HTTP_200_OK diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index 63f4ed97bbd..be55736c6f6 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -23,7 +23,7 @@ from models_library.payments import StripeInvoiceID from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_payment_method_view from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -209,7 +209,7 @@ def mock_payments_gateway_service_api_base(app: FastAPI) -> Iterator[MockRouter] def mock_payments_routes(faker: Faker) -> Callable: def _mock(mock_router: MockRouter): def _init_200(request: httpx.Request): - assert InitPayment.parse_raw(request.content) is not None + assert InitPayment.model_validate_json(request.content) is not None assert "*" not in request.headers["X-Init-Api-Secret"] return httpx.Response( @@ -218,7 +218,7 @@ def _init_200(request: httpx.Request): ) def _cancel_200(request: httpx.Request): - assert PaymentInitiated.parse_raw(request.content) is not None + assert PaymentInitiated.model_validate_json(request.content) is not None assert "*" not in request.headers["X-Init-Api-Secret"] # responds with an empty authough it can also contain a message @@ -244,7 +244,7 @@ def no_funds_payment_method_id(faker: Faker) -> PaymentMethodID: USE create_fake_payment_method_in_db to inject this payment-method in DB Emulates https://stripe.com/docs/testing#declined-payments """ - return parse_obj_as(PaymentMethodID, "no_funds_payment_method_id") + return TypeAdapter(PaymentMethodID).validate_python("no_funds_payment_method_id") @pytest.fixture @@ -263,7 +263,7 @@ def _init(request: httpx.Request): pm_id = faker.uuid4() _payment_methods[pm_id] = PaymentMethodInfoTuple( - init=InitPaymentMethod.parse_raw(request.content), + init=InitPaymentMethod.model_validate_json(request.content), get=GetPaymentMethod(**random_payment_method_view(id=pm_id)), ) @@ -294,7 +294,7 @@ def _del(request: httpx.Request, pm_id: PaymentMethodID): def _batch_get(request: httpx.Request): assert "*" not in request.headers["X-Init-Api-Secret"] - batch = BatchGetPaymentMethods.parse_raw(request.content) + batch = BatchGetPaymentMethods.model_validate_json(request.content) try: items = [_payment_methods[pm].get for pm in batch.payment_methods_ids] @@ -308,7 +308,7 @@ def _batch_get(request: httpx.Request): def _pay(request: httpx.Request, pm_id: PaymentMethodID): assert "*" not in request.headers["X-Init-Api-Secret"] - assert InitPayment.parse_raw(request.content) is not None + assert InitPayment.model_validate_json(request.content) is not None # checks _get(request, pm_id) diff --git a/services/payments/tests/unit/test__model_examples.py b/services/payments/tests/unit/test__model_examples.py index beab80e794d..6e072aa658a 100644 --- a/services/payments/tests/unit/test__model_examples.py +++ b/services/payments/tests/unit/test__model_examples.py @@ -21,7 +21,7 @@ def test_api_server_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): try: - assert model_cls.parse_obj(example_data) is not None + assert model_cls.model_validate(example_data) is not None except ValidationError as err: pytest.fail( f"\n{example_name}: {json.dumps(example_data, indent=1)}\nError: {err}" diff --git a/services/payments/tests/unit/test_cli.py b/services/payments/tests/unit/test_cli.py index 2d01fd0fc31..84c654ee159 100644 --- a/services/payments/tests/unit/test_cli.py +++ b/services/payments/tests/unit/test_cli.py @@ -4,8 +4,10 @@ # pylint: disable=too-many-arguments import os +import traceback import pytest +from click.testing import Result from pytest_simcore.helpers.monkeypatch_envs import load_dotenv, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_payments._meta import API_VERSION @@ -14,10 +16,16 @@ from typer.testing import CliRunner +def _format_cli_error(result: Result) -> str: + assert result.exception + tb_message = "\n".join(traceback.format_tb(result.exception.__traceback__)) + return f"Below exception was raised by the cli:\n{tb_message}" + + def test_cli_help_and_version(cli_runner: CliRunner): # simcore-service-payments --help result = cli_runner.invoke(cli_main, "--help") - assert result.exit_code == os.EX_OK, result.output + assert result.exit_code == os.EX_OK, _format_cli_error(result) result = cli_runner.invoke(cli_main, "--version") assert result.exit_code == os.EX_OK, result.output @@ -27,7 +35,7 @@ def test_cli_help_and_version(cli_runner: CliRunner): def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): # simcore-service-payments echo-dotenv --auto-password result = cli_runner.invoke(cli_main, "echo-dotenv --auto-password") - assert result.exit_code == os.EX_OK, result.output + assert result.exit_code == os.EX_OK, _format_cli_error(result) environs = load_dotenv(result.stdout) @@ -39,11 +47,11 @@ def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): # simcore-service-payments settings --show-secrets --as-json result = cli_runner.invoke(cli_main, ["settings", "--show-secrets", "--as-json"]) - assert result.exit_code == os.EX_OK, result.output + assert result.exit_code == os.EX_OK, _format_cli_error(result) print(result.output) - settings = ApplicationSettings.parse_raw(result.output) - assert settings == ApplicationSettings.create_from_envs() + settings = ApplicationSettings.model_validate_json(result.output) + assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() def test_main(app_environment: EnvVarsDict): diff --git a/services/payments/tests/unit/test_db_payments_methods_repo.py b/services/payments/tests/unit/test_db_payments_methods_repo.py index f64570cf598..47595bb5557 100644 --- a/services/payments/tests/unit/test_db_payments_methods_repo.py +++ b/services/payments/tests/unit/test_db_payments_methods_repo.py @@ -43,7 +43,9 @@ def app_environment( async def test_create_payments_method_annotations_workflow(app: FastAPI): - fake = PaymentsMethodsDB(**PaymentsMethodsDB.Config.schema_extra["examples"][1]) + fake = PaymentsMethodsDB( + **PaymentsMethodsDB.model_config["json_schema_extra"]["examples"][1] + ) repo = PaymentsMethodsRepo(app.state.engine) diff --git a/services/payments/tests/unit/test_db_payments_transactions_repo.py b/services/payments/tests/unit/test_db_payments_transactions_repo.py index 62e217a9e7a..d4e728d14c5 100644 --- a/services/payments/tests/unit/test_db_payments_transactions_repo.py +++ b/services/payments/tests/unit/test_db_payments_transactions_repo.py @@ -49,7 +49,7 @@ def app_environment( async def test_one_time_payment_annotations_workflow(app: FastAPI): fake = PaymentsTransactionsDB( - **PaymentsTransactionsDB.Config.schema_extra["examples"][1] + **PaymentsTransactionsDB.model_config["json_schema_extra"]["examples"][1] ) repo = PaymentsTransactionsRepo(app.state.engine) diff --git a/services/payments/tests/unit/test_rpc_payments.py b/services/payments/tests/unit/test_rpc_payments.py index 3e5b3ad1c2a..b755acf7d08 100644 --- a/services/payments/tests/unit/test_rpc_payments.py +++ b/services/payments/tests/unit/test_rpc_payments.py @@ -13,7 +13,7 @@ from models_library.api_schemas_webserver.wallets import WalletPaymentInitiated from models_library.payments import UserInvoiceAddress from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx import MockRouter @@ -83,7 +83,7 @@ async def test_rpc_init_payment_fail( with pytest.raises(RPCServerError) as exc_info: await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment"), + TypeAdapter(RPCMethodName).validate_python("init_payment"), **init_payment_kwargs, ) @@ -107,7 +107,7 @@ async def test_webserver_one_time_payment_workflow( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment"), + TypeAdapter(RPCMethodName).validate_python("init_payment"), **init_payment_kwargs, ) @@ -118,7 +118,7 @@ async def test_webserver_one_time_payment_workflow( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "cancel_payment"), + TypeAdapter(RPCMethodName).validate_python("cancel_payment"), payment_id=result.payment_id, user_id=init_payment_kwargs["user_id"], wallet_id=init_payment_kwargs["wallet_id"], @@ -145,7 +145,7 @@ async def test_cancel_invalid_payment_id( with pytest.raises(PaymentNotFoundError) as exc_info: await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "cancel_payment"), + TypeAdapter(RPCMethodName).validate_python("cancel_payment"), payment_id=invalid_payment_id, user_id=init_payment_kwargs["user_id"], wallet_id=init_payment_kwargs["wallet_id"], diff --git a/services/payments/tests/unit/test_rpc_payments_methods.py b/services/payments/tests/unit/test_rpc_payments_methods.py index ef60bfa6c42..e3a6d377e27 100644 --- a/services/payments/tests/unit/test_rpc_payments_methods.py +++ b/services/payments/tests/unit/test_rpc_payments_methods.py @@ -19,7 +19,7 @@ from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx import MockRouter @@ -87,7 +87,7 @@ async def test_webserver_init_and_cancel_payment_method_workflow( initiated = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_creation_of_payment_method"), + TypeAdapter(RPCMethodName).validate_python("init_creation_of_payment_method"), wallet_id=wallet_id, wallet_name=wallet_name, user_id=user_id, @@ -104,7 +104,7 @@ async def test_webserver_init_and_cancel_payment_method_workflow( cancelled = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "cancel_creation_of_payment_method"), + TypeAdapter(RPCMethodName).validate_python("cancel_creation_of_payment_method"), payment_method_id=initiated.payment_method_id, user_id=user_id, wallet_id=wallet_id, @@ -135,7 +135,7 @@ async def test_webserver_crud_payment_method_workflow( inited = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_creation_of_payment_method"), + TypeAdapter(RPCMethodName).validate_python("init_creation_of_payment_method"), wallet_id=wallet_id, wallet_name=wallet_name, user_id=user_id, @@ -161,7 +161,7 @@ async def test_webserver_crud_payment_method_workflow( listed = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "list_payment_methods"), + TypeAdapter(RPCMethodName).validate_python("list_payment_methods"), user_id=user_id, wallet_id=wallet_id, timeout_s=None if is_pdb_enabled else RPC_REQUEST_DEFAULT_TIMEOUT_S, @@ -175,7 +175,7 @@ async def test_webserver_crud_payment_method_workflow( got = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_payment_method"), + TypeAdapter(RPCMethodName).validate_python("get_payment_method"), payment_method_id=inited.payment_method_id, user_id=user_id, wallet_id=wallet_id, @@ -187,7 +187,7 @@ async def test_webserver_crud_payment_method_workflow( await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "delete_payment_method"), + TypeAdapter(RPCMethodName).validate_python("delete_payment_method"), payment_method_id=inited.payment_method_id, user_id=user_id, wallet_id=wallet_id, @@ -230,7 +230,7 @@ async def test_webserver_pay_with_payment_method_workflow( transaction = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "pay_with_payment_method"), + TypeAdapter(RPCMethodName).validate_python("pay_with_payment_method"), payment_method_id=created.payment_method_id, amount_dollars=faker.pyint(), target_credits=faker.pyint(), diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 196c1cdcd98..aaf143ad7b7 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -161,8 +161,8 @@ def wallet_id(faker: Faker): async def mocked_pay_with_payment_method(mocker: MockerFixture) -> mock.AsyncMock: return mocker.patch( "simcore_service_payments.services.payments.PaymentsGatewayApi.pay_with_payment_method", - return_value=AckPaymentWithPaymentMethod.construct( - **AckPaymentWithPaymentMethod.Config.schema_extra["example"] + return_value=AckPaymentWithPaymentMethod.model_construct( + **AckPaymentWithPaymentMethod.model_config["json_schema_extra"]["example"] ), ) @@ -200,8 +200,8 @@ async def get_invoice_data( dollar_amount: Decimal, product_name: ProductName, ) -> InvoiceDataGet: - return InvoiceDataGet.parse_obj( - InvoiceDataGet.Config.schema_extra["examples"][0] + return InvoiceDataGet.model_validate( + InvoiceDataGet.model_config["json_schema_extra"]["examples"][0] ) await rpc_server.register_router(router, namespace=WEBSERVER_RPC_NAMESPACE) @@ -220,7 +220,7 @@ async def _assert_payments_transactions_db_row(postgres_db) -> PaymentsTransacti result = con.execute(sa.select(payments_transactions)) row = result.first() assert row - return PaymentsTransactionsDB.from_orm(row) + return PaymentsTransactionsDB.model_validate(row) async def test_process_message__whole_autorecharge_flow_success( diff --git a/services/payments/tests/unit/test_services_notifier.py b/services/payments/tests/unit/test_services_notifier.py index 5aab90f9f0f..ee55afa9be3 100644 --- a/services/payments/tests/unit/test_services_notifier.py +++ b/services/payments/tests/unit/test_services_notifier.py @@ -4,10 +4,9 @@ # pylint: disable=too-many-arguments -import asyncio -import threading from collections.abc import AsyncIterable, Callable from contextlib import _AsyncGeneratorContextManager +from typing import Awaitable from unittest.mock import AsyncMock import arrow @@ -20,7 +19,7 @@ from models_library.api_schemas_webserver.socketio import SocketIORoomStr from models_library.api_schemas_webserver.wallets import PaymentTransaction from models_library.users import GroupID, UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_payment_transaction from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -31,8 +30,7 @@ from simcore_service_payments.services.notifier import NotifierService from simcore_service_payments.services.rabbitmq import get_rabbitmq_settings from socketio import AsyncServer -from tenacity import AsyncRetrying -from tenacity.stop import stop_after_attempt +from tenacity import AsyncRetrying, stop_after_delay from tenacity.wait import wait_fixed pytest_simcore_core_services_selection = [ @@ -103,7 +101,7 @@ async def socketio_client_events( # emulates front-end receiving message async def on_payment(data): - assert parse_obj_as(PaymentTransaction, data) is not None + assert TypeAdapter(PaymentTransaction).validate_python(data) is not None on_event_spy = AsyncMock(wraps=on_payment) socketio_client.on(SOCKET_IO_PAYMENT_COMPLETED_EVENT, on_event_spy) @@ -112,8 +110,10 @@ async def on_payment(data): @pytest.fixture -async def notify_payment(app: FastAPI, user_id: UserID) -> Callable: - async def _(): +async def notify_payment( + app: FastAPI, user_id: UserID +) -> Callable[[], Awaitable[None]]: + async def _() -> None: transaction = PaymentsTransactionsDB( **random_payment_transaction( user_id=user_id, completed_at=arrow.utcnow().datetime @@ -127,23 +127,28 @@ async def _(): return _ +async def _assert_called_once(mock: AsyncMock) -> None: + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), stop=stop_after_delay(5), reraise=True + ): + with attempt: + assert mock.call_count == 1 + + async def test_emit_message_as_external_process_to_frontend_client( socketio_server_events: dict[str, AsyncMock], socketio_client: socketio.AsyncClient, socketio_client_events: dict[str, AsyncMock], - notify_payment: Callable, + notify_payment: Callable[[], Awaitable[None]], + socketio_client_factory: Callable[ + [], _AsyncGeneratorContextManager[socketio.AsyncClient] + ], ): """ front-end -> socketio client (many different clients) webserver -> socketio server (one/more replicas) payments -> Sends messages to clients from external processes (one/more replicas) """ - # Used iusntead of a fix asyncio.sleep - context_switch_retry_kwargs = { - "wait": wait_fixed(0.1), - "stop": stop_after_attempt(5), - "reraise": True, - } # web server spy events server_connect = socketio_server_events["connect"] @@ -160,20 +165,9 @@ async def test_emit_message_as_external_process_to_frontend_client( # client emits await socketio_client.emit("check", data="hoi") - async for attempt in AsyncRetrying(**context_switch_retry_kwargs): - with attempt: - assert server_on_check.called + await _assert_called_once(server_on_check) # payment server emits - def _(lp): - asyncio.run_coroutine_threadsafe(notify_payment(), lp) + await notify_payment() - threading.Thread( - target=_, - args=(asyncio.get_event_loop(),), - daemon=False, - ).start() - - async for attempt in AsyncRetrying(**context_switch_retry_kwargs): - with attempt: - assert client_on_payment.called + await _assert_called_once(client_on_payment) diff --git a/services/payments/tests/unit/test_services_notifier_email.py b/services/payments/tests/unit/test_services_notifier_email.py index c554c7a2c28..79edb79498c 100644 --- a/services/payments/tests/unit/test_services_notifier_email.py +++ b/services/payments/tests/unit/test_services_notifier_email.py @@ -86,7 +86,7 @@ def mocked_get_invoice_pdf_response( text=f"{request.fixturename} is set to '{request.param}'", ) - respx_mock.get(transaction.invoice_pdf_url).mock(return_value=response) + respx_mock.get(f"{transaction.invoice_pdf_url}").mock(return_value=response) return respx_mock @@ -97,7 +97,7 @@ def transaction( ) -> PaymentsTransactionsDB: kwargs = { k: successful_transaction[k] - for k in PaymentsTransactionsDB.__fields__ + for k in PaymentsTransactionsDB.model_fields if k in successful_transaction } return PaymentsTransactionsDB(**kwargs) diff --git a/services/payments/tests/unit/test_services_payments__get_invoice.py b/services/payments/tests/unit/test_services_payments__get_invoice.py index 7a391b22351..57c71945bf4 100644 --- a/services/payments/tests/unit/test_services_payments__get_invoice.py +++ b/services/payments/tests/unit/test_services_payments__get_invoice.py @@ -18,6 +18,7 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import HttpUrl +from pydantic_core import Url from pytest_simcore.helpers.faker_factories import random_payment_transaction from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -118,4 +119,4 @@ async def test_get_payment_invoice_url( payment_id=populate_payment_transaction_db, ) assert invoice_url - assert type(invoice_url) is HttpUrl + assert isinstance(invoice_url, Url) diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index f210d1e7258..e89c502894b 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -23,6 +23,7 @@ _raise_as_payments_gateway_error, setup_payments_gateway, ) +from yarl import URL async def test_setup_payment_gateway_api(app_environment: EnvVarsDict): @@ -120,7 +121,7 @@ async def test_one_time_payment_workflow( ) app_settings: ApplicationSettings = app.state.settings - assert submission_link.host == app_settings.PAYMENTS_GATEWAY_URL.host + assert submission_link.host == URL(app_settings.PAYMENTS_GATEWAY_URL).host # cancel payment_canceled = await payment_gateway_api.cancel_payment(payment_initiated) @@ -159,7 +160,7 @@ async def test_payment_methods_workflow( ) app_settings: ApplicationSettings = app.state.settings - assert form_link.host == app_settings.PAYMENTS_GATEWAY_URL.host + assert form_link.host == URL(app_settings.PAYMENTS_GATEWAY_URL).host # CRUD payment_method_id = initiated.payment_method_id @@ -169,7 +170,7 @@ async def test_payment_methods_workflow( payment_method_id ) assert got_payment_method.id == payment_method_id - print(got_payment_method.json(indent=2)) + print(got_payment_method.model_dump_json(indent=2)) # list payment-methods items = await payments_gateway_api.get_many_payment_methods([payment_method_id]) diff --git a/services/payments/tests/unit/test_services_resource_usage_tracker.py b/services/payments/tests/unit/test_services_resource_usage_tracker.py index 0959f535cc1..1010f3e3b00 100644 --- a/services/payments/tests/unit/test_services_resource_usage_tracker.py +++ b/services/payments/tests/unit/test_services_resource_usage_tracker.py @@ -72,7 +72,7 @@ async def test_add_credits_to_wallet( user_id=faker.pyint(), user_email=faker.email(), osparc_credits=100, - payment_transaction_id=faker.pyint(), + payment_transaction_id=faker.pystr(), created_at=datetime.now(tz=timezone.utc), ) > 0 From cf5546bc5bdbee049c0190e27f87f1bfc2e625e8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 15:37:13 +0200 Subject: [PATCH 13/53] Upgrade resource usage tracker (Pydantic v2) (#6517) --- .../credit_transactions.py | 14 ++-- .../pricing_plans.py | 7 +- .../src/settings_library/base.py | 5 +- services/resource-usage-tracker/openapi.json | 80 ++++++++++++------- .../requirements/_base.txt | 47 ++++++----- .../requirements/_test.txt | 15 ++-- .../_meta.py | 6 +- .../api/rpc/_resource_tracker.py | 3 +- .../core/application.py | 2 +- .../core/errors.py | 4 +- .../core/settings.py | 46 +++++++---- .../resource_tracker_credit_transactions.py | 6 +- .../models/resource_tracker_pricing_plans.py | 14 +--- .../resource_tracker_pricing_unit_costs.py | 6 +- .../models/resource_tracker_pricing_units.py | 8 +- .../models/resource_tracker_service_runs.py | 18 ++--- .../db/repositories/resource_tracker.py | 46 ++++++----- .../resource_tracker_process_messages.py | 8 +- .../resource_tracker_pricing_plans.py | 8 +- .../services/resource_tracker_service_runs.py | 6 +- .../tests/unit/api_rest/test_api_meta.py | 4 +- .../tests/unit/with_dbs/conftest.py | 13 ++- ...pi_resource_tracker_credit_transactions.py | 4 +- ...test_api_resource_tracker_pricing_plans.py | 16 +++- ..._api_resource_tracker_pricing_plans_rpc.py | 16 +++- ...i_resource_tracker_service_runs__export.py | 5 +- .../unit/with_dbs/test_background_task.py | 8 +- ...age_triggered_by_listening_with_billing.py | 12 ++- ...t_process_rabbitmq_message_with_billing.py | 12 ++- ...ss_rabbitmq_message_with_billing_cost_0.py | 4 +- 30 files changed, 259 insertions(+), 184 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index c24291ebbdb..db235ce8094 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -1,7 +1,8 @@ from datetime import datetime from decimal import Decimal +from typing import Annotated -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, BeforeValidator, PlainSerializer from ..products import ProductName from ..resource_tracker import CreditTransactionId @@ -11,12 +12,11 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID - available_osparc_credits: Decimal - - @field_validator("available_osparc_credits") - @classmethod - def ensure_rounded(cls, v): - return round(v, 2) + available_osparc_credits: Annotated[ + Decimal, + BeforeValidator(lambda x: round(x, 2)), + PlainSerializer(float, return_type=float, when_used="json"), + ] class CreditTransactionCreateBody(BaseModel): diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py index bbb5d52f906..294ea360e58 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py @@ -1,7 +1,8 @@ from datetime import datetime from decimal import Decimal +from typing import Annotated -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, PlainSerializer from ..resource_tracker import ( HardwareInfo, @@ -18,7 +19,9 @@ class PricingUnitGet(BaseModel): pricing_unit_id: PricingUnitId unit_name: str unit_extra_info: UnitExtraInfo - current_cost_per_unit: Decimal + current_cost_per_unit: Annotated[ + Decimal, PlainSerializer(float, return_type=float, when_used="json") + ] current_cost_per_unit_id: PricingUnitCostId default: bool specific_info: HardwareInfo diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index c3f0e103e7a..7bdd7d2cd00 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -5,7 +5,7 @@ from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable from pydantic import ValidationInfo, field_validator from pydantic.fields import FieldInfo -from pydantic_core import PydanticUndefined, ValidationError +from pydantic_core import ValidationError from pydantic_settings import BaseSettings, SettingsConfigDict _logger = logging.getLogger(__name__) @@ -102,9 +102,6 @@ def __pydantic_init_subclass__(cls, **kwargs: Any): and issubclass(field_type, BaseCustomSettings) ): if auto_default_from_env: - assert field.default is PydanticUndefined - assert field.default_factory is None - # Transform it into something like `Field(default_factory=create_settings_from_env(field))` field.default_factory = _create_settings_from_env(name, field) field.default = None diff --git a/services/resource-usage-tracker/openapi.json b/services/resource-usage-tracker/openapi.json index 0df986b36a5..6aa53c7118c 100644 --- a/services/resource-usage-tracker/openapi.json +++ b/services/resource-usage-tracker/openapi.json @@ -54,24 +54,24 @@ "operationId": "get_credit_transactions_sum_v1_credit_transactions_credits_sum_post", "parameters": [ { + "name": "product_name", + "in": "query", "required": true, "schema": { "type": "string", "title": "Product Name" - }, - "name": "product_name", - "in": "query" + } }, { + "name": "wallet_id", + "in": "query", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "Wallet Id", "minimum": 0 - }, - "name": "wallet_id", - "in": "query" + } } ], "responses": { @@ -149,33 +149,33 @@ "operationId": "get_service_default_pricing_plan", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "product_name", + "in": "query", "required": true, "schema": { "type": "string", "title": "Product Name" - }, - "name": "product_name", - "in": "query" + } } ], "responses": { @@ -212,35 +212,35 @@ "operationId": "list_service_pricing_plans", "parameters": [ { + "name": "pricing_plan_id", + "in": "path", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "Pricing Plan Id", "minimum": 0 - }, - "name": "pricing_plan_id", - "in": "path" + } }, { + "name": "pricing_unit_id", + "in": "path", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "Pricing Unit Id", "minimum": 0 - }, - "name": "pricing_unit_id", - "in": "path" + } }, { + "name": "product_name", + "in": "query", "required": true, "schema": { "type": "string", "title": "Product Name" - }, - "name": "product_name", - "in": "query" + } } ], "responses": { @@ -297,7 +297,14 @@ "title": "User Email" }, "osparc_credits": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ], "title": "Osparc Credits" }, "payment_transaction_id": { @@ -373,8 +380,8 @@ "enum": [ "TIER" ], - "title": "PricingPlanClassification", - "description": "An enumeration." + "const": "TIER", + "title": "PricingPlanClassification" }, "PricingPlanGet": { "properties": { @@ -405,10 +412,17 @@ "title": "Pricing Plan Key" }, "pricing_units": { - "items": { - "$ref": "#/components/schemas/PricingUnitGet" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PricingUnitGet" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Pricing Units" }, "is_active": { @@ -424,6 +438,7 @@ "classification", "created_at", "pricing_plan_key", + "pricing_units", "is_active" ], "title": "PricingPlanGet" @@ -482,13 +497,16 @@ }, "RAM": { "type": "integer", + "minimum": 0, "title": "Ram" }, "VRAM": { "type": "integer", + "minimum": 0, "title": "Vram" } }, + "additionalProperties": true, "type": "object", "required": [ "CPU", diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index d31e353e4c3..a5721f8254b 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -50,6 +50,8 @@ aiosignal==1.3.1 # via aiohttp alembic==1.13.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -128,21 +130,8 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -353,7 +342,7 @@ psutil==6.0.0 # -r requirements/../../../packages/service-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -365,7 +354,6 @@ pydantic==1.10.14 # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -383,6 +371,26 @@ pydantic==1.10.14 # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -399,7 +407,9 @@ python-dateutil==2.9.0.post0 # matplotlib # pandas python-dotenv==1.0.1 - # via uvicorn + # via + # pydantic-settings + # uvicorn pytz==2024.1 # via # dateparser @@ -500,7 +510,7 @@ sqlalchemy==1.4.52 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -557,6 +567,7 @@ typing-extensions==4.10.0 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 diff --git a/services/resource-usage-tracker/requirements/_test.txt b/services/resource-usage-tracker/requirements/_test.txt index 4db08363ded..6ddd9949286 100644 --- a/services/resource-usage-tracker/requirements/_test.txt +++ b/services/resource-usage-tracker/requirements/_test.txt @@ -2,6 +2,10 @@ alembic==1.13.1 # via # -c requirements/_base.txt # -r requirements/_test.in +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto anyio==4.3.0 @@ -77,10 +81,6 @@ flask-cors==5.0.0 # via moto graphql-core==3.2.4 # via moto -greenlet==3.0.3 - # via - # -c requirements/_base.txt - # sqlalchemy h11==0.14.0 # via # -c requirements/_base.txt @@ -182,11 +182,15 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via # -c requirements/_base.txt @@ -308,6 +312,7 @@ typing-extensions==4.10.0 # cfn-lint # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs urllib3==2.0.7 # via diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py index 63e86cce819..ab1dfa3467c 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py @@ -6,7 +6,7 @@ from models_library.basic_types import VersionStr from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.utils_meta import PackageInfo from settings_library.basic_types import VersionTag @@ -17,7 +17,9 @@ PROJECT_NAME: Final[str] = info.project_name VERSION: Final[Version] = info.version API_VERSION: Final[VersionStr] = info.__version__ -API_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, info.api_prefix_path_tag) +API_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + info.api_prefix_path_tag +) SUMMARY: Final[str] = info.get_summary() diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py index cae70b1152c..9da69bad6f3 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py @@ -24,7 +24,6 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import AnyUrl from servicelib.rabbitmq import RPCRouter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, @@ -79,7 +78,7 @@ async def export_service_runs( access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, filters: ServiceResourceUsagesFilters | None = None, -) -> AnyUrl: +) -> str: app_settings: ApplicationSettings = app.state.settings s3_settings = app_settings.RESOURCE_USAGE_TRACKER_S3 assert s3_settings # nosec diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index a97db0170ae..5c1ee907956 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -32,7 +32,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: - _logger.info("app settings: %s", settings.json(indent=1)) + _logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.RESOURCE_USAGE_TRACKER_DEBUG, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py index 298e63aef71..44557db0b02 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py @@ -12,8 +12,10 @@ class ConfigurationError(ResourceUsageTrackerRuntimeError): def http404_error_handler( request: Request, # pylint: disable=unused-argument - error: CustomResourceUsageTrackerError, + error: Exception, ) -> JSONResponse: + assert isinstance(error, CustomResourceUsageTrackerError) # nosec + return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={"message": f"{error.msg_template}"}, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index cd23585d8b5..b473eb11294 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -1,9 +1,8 @@ import datetime from functools import cached_property -from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import BootModeEnum -from pydantic import Field, PositiveInt, validator +from pydantic import AliasChoices, Field, PositiveInt, field_validator from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.postgres import PostgresSettings @@ -46,18 +45,23 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): RESOURCE_USAGE_TRACKER_DEBUG: bool = Field( default=False, description="Debug mode", - env=["RESOURCE_USAGE_TRACKER_DEBUG", "DEBUG"], + validation_alias=AliasChoices( + "RESOURCE_USAGE_TRACKER_DEBUG", + "DEBUG", + ), ) RESOURCE_USAGE_TRACKER_LOGLEVEL: LogLevel = Field( default=LogLevel.INFO, - env=["RESOURCE_USAGE_TRACKER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "RESOURCE_USAGE_TRACKER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -65,7 +69,7 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def LOG_LEVEL(self) -> LogLevel: # noqa: N802 return self.RESOURCE_USAGE_TRACKER_LOGLEVEL - @validator("RESOURCE_USAGE_TRACKER_LOGLEVEL", pre=True) + @field_validator("RESOURCE_USAGE_TRACKER_LOGLEVEL", mode="before") @classmethod def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) @@ -79,16 +83,18 @@ class MinimalApplicationSettings(_BaseApplicationSettings): """ RESOURCE_USAGE_TRACKER_PROMETHEUS: PrometheusSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) RESOURCE_USAGE_TRACKER_POSTGRES: PostgresSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True}, ) - RESOURCE_USAGE_TRACKER_REDIS: RedisSettings = Field(auto_default_from_env=True) + RESOURCE_USAGE_TRACKER_REDIS: RedisSettings = Field( + json_schema_extra={"auto_default_from_env": True}, + ) RESOURCE_USAGE_TRACKER_RABBITMQ: RabbitSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True}, ) @@ -111,13 +117,19 @@ class ApplicationSettings(MinimalApplicationSettings): description="Heartbeat couter limit when RUT considers service as unhealthy.", ) RESOURCE_USAGE_TRACKER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True - RESOURCE_USAGE_TRACKER_S3: S3Settings | None = Field(auto_default_from_env=True) + RESOURCE_USAGE_TRACKER_S3: S3Settings | None = Field( + json_schema_extra={"auto_default_from_env": True}, + ) RESOURCE_USAGE_TRACKER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + description="settings for opentelemetry tracing", + json_schema_extra={"auto_default_from_env": True}, ) - _validate_resource_usage_tracker_missed_heartbeat_interval_sec = ( - validate_numeric_string_as_timedelta( - "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC" - ) + @field_validator( + "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC", mode="before" ) + @classmethod + def _validate_interval(cls, v): + if isinstance(v, str) and v.isnumeric(): + return int(v) + return v diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py index a264f90d375..4cdf74b6429 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py @@ -13,7 +13,7 @@ ) from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class CreditTransactionCreate(BaseModel): @@ -64,6 +64,4 @@ class CreditTransactionDB(BaseModel): created: datetime last_heartbeat_at: datetime modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py index f946c92e5d9..7f27ef1096c 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py @@ -2,7 +2,7 @@ from models_library.resource_tracker import PricingPlanClassification, PricingPlanId from models_library.services import ServiceKey, ServiceVersion -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict ## DB Models @@ -15,16 +15,12 @@ class PricingPlansDB(BaseModel): is_active: bool created: datetime pricing_plan_key: str - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class PricingPlansWithServiceDefaultPlanDB(PricingPlansDB): service_default_plan: bool - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class PricingPlanToServiceDB(BaseModel): @@ -32,6 +28,4 @@ class PricingPlanToServiceDB(BaseModel): service_key: ServiceKey service_version: ServiceVersion created: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py index b5fa3daadf0..200419fbdca 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py @@ -6,7 +6,7 @@ PricingUnitCostId, PricingUnitId, ) -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class PricingUnitCostsDB(BaseModel): @@ -21,6 +21,4 @@ class PricingUnitCostsDB(BaseModel): created: datetime comment: str | None modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py index f0fed877d43..bffc25e951c 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py @@ -9,7 +9,7 @@ PricingUnitId, UnitExtraInfo, ) -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, field_validator class PricingUnitsDB(BaseModel): @@ -23,11 +23,9 @@ class PricingUnitsDB(BaseModel): modified: datetime current_cost_per_unit: Decimal current_cost_per_unit_id: PricingUnitCostId + model_config = ConfigDict(from_attributes=True) - class Config: - orm_mode = True - - @validator("specific_info", pre=True) + @field_validator("specific_info", mode="before") @classmethod def default_hardware_info_when_empty(cls, v) -> HardwareInfo | Any: if not v: diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py index 6bceaab4f8c..20314b8b065 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py @@ -16,7 +16,7 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, NonNegativeInt +from pydantic import BaseModel, ConfigDict, NonNegativeInt class ServiceRunCreate(BaseModel): @@ -93,17 +93,14 @@ class ServiceRunDB(BaseModel): last_heartbeat_at: datetime service_run_status_msg: str | None missed_heartbeat_counter: NonNegativeInt - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class ServiceRunWithCreditsDB(ServiceRunDB): - osparc_credits: Decimal | None + osparc_credits: Decimal | None = None transaction_status: CreditTransactionStatus | None - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class OsparcCreditsAggregatedByServiceKeyDB(BaseModel): @@ -111,8 +108,7 @@ class OsparcCreditsAggregatedByServiceKeyDB(BaseModel): service_key: ServiceKey running_time_in_hours: Decimal - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class ServiceRunForCheckDB(BaseModel): @@ -120,6 +116,4 @@ class ServiceRunForCheckDB(BaseModel): last_heartbeat_at: datetime missed_heartbeat_counter: NonNegativeInt modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py index 231c97502fb..4358d560396 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py @@ -161,7 +161,7 @@ async def update_service_run_last_heartbeat( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) async def update_service_run_stopped_at( self, data: ServiceRunStoppedAtUpdate @@ -191,7 +191,7 @@ async def update_service_run_stopped_at( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) async def get_service_run_by_id( self, service_run_id: ServiceRunId @@ -204,7 +204,7 @@ async def get_service_run_by_id( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) async def list_service_runs_by_product_and_user_and_wallet( self, @@ -309,7 +309,9 @@ async def list_service_runs_by_product_and_user_and_wallet( result = await conn.execute(query) - return [ServiceRunWithCreditsDB.from_orm(row) for row in result.fetchall()] + return [ + ServiceRunWithCreditsDB.model_validate(row) for row in result.fetchall() + ] async def get_osparc_credits_aggregated_by_service( self, @@ -405,7 +407,7 @@ async def get_osparc_credits_aggregated_by_service( return ( cast(int, count_result.scalar()), [ - OsparcCreditsAggregatedByServiceKeyDB.from_orm(row) + OsparcCreditsAggregatedByServiceKeyDB.model_validate(row) for row in list_result.fetchall() ], ) @@ -571,7 +573,7 @@ async def list_service_runs_with_running_status_across_all_products( ) result = await conn.execute(query) - return [ServiceRunForCheckDB.from_orm(row) for row in result.fetchall()] + return [ServiceRunForCheckDB.model_validate(row) for row in result.fetchall()] async def total_service_runs_with_running_status_across_all_products( self, @@ -620,7 +622,7 @@ async def update_service_missed_heartbeat_counter( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) ################################# # Credit transactions @@ -847,7 +849,7 @@ def _version(column_or_value): result = await conn.execute(query) return [ - PricingPlansWithServiceDefaultPlanDB.from_orm(row) + PricingPlansWithServiceDefaultPlanDB.model_validate(row) for row in result.fetchall() ] @@ -873,7 +875,7 @@ async def get_pricing_plan( raise CustomResourceUsageTrackerError( msg=f"Pricing plan does not exists: {pricing_plan_id}" ) - return PricingPlansDB.from_orm(row) + return PricingPlansDB.model_validate(row) async def list_pricing_plans_by_product( self, product_name: ProductName @@ -890,7 +892,7 @@ async def list_pricing_plans_by_product( ).where(resource_tracker_pricing_plans.c.product_name == product_name) result = await conn.execute(select_stmt) - return [PricingPlansDB.from_orm(row) for row in result.fetchall()] + return [PricingPlansDB.model_validate(row) for row in result.fetchall()] async def create_pricing_plan(self, data: PricingPlanCreate) -> PricingPlansDB: async with self.db_engine.begin() as conn: @@ -924,7 +926,7 @@ async def create_pricing_plan(self, data: PricingPlanCreate) -> PricingPlansDB: raise CustomResourceUsageTrackerError( msg=f"Pricing plan was not created: {data}" ) - return PricingPlansDB.from_orm(row) + return PricingPlansDB.model_validate(row) async def update_pricing_plan( self, product_name: ProductName, data: PricingPlanUpdate @@ -961,7 +963,7 @@ async def update_pricing_plan( row = result.first() if row is None: return None - return PricingPlansDB.from_orm(row) + return PricingPlansDB.model_validate(row) ################################# # Pricing plan to service @@ -1000,7 +1002,9 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ) result = await conn.execute(query) - return [PricingPlanToServiceDB.from_orm(row) for row in result.fetchall()] + return [ + PricingPlanToServiceDB.model_validate(row) for row in result.fetchall() + ] async def upsert_service_to_pricing_plan( self, @@ -1087,7 +1091,7 @@ async def upsert_service_to_pricing_plan( raise CustomResourceUsageTrackerError( msg="Pricing plan to service record was not created" ) - return PricingPlanToServiceDB.from_orm(row) + return PricingPlanToServiceDB.model_validate(row) ################################# # Pricing units @@ -1145,7 +1149,7 @@ async def list_pricing_units_by_pricing_plan( ) result = await conn.execute(query) - return [PricingUnitsDB.from_orm(row) for row in result.fetchall()] + return [PricingUnitsDB.model_validate(row) for row in result.fetchall()] async def get_valid_pricing_unit( self, @@ -1197,7 +1201,7 @@ async def get_valid_pricing_unit( raise CustomResourceUsageTrackerError( msg=f"Pricing plan {pricing_plan_id} and pricing unit {pricing_unit_id} for product {product_name} not found" ) - return PricingUnitsDB.from_orm(row) + return PricingUnitsDB.model_validate(row) async def create_pricing_unit_with_cost( self, data: PricingUnitWithCostCreate, pricing_plan_key: str @@ -1209,9 +1213,9 @@ async def create_pricing_unit_with_cost( .values( pricing_plan_id=data.pricing_plan_id, unit_name=data.unit_name, - unit_extra_info=data.unit_extra_info.dict(), + unit_extra_info=data.unit_extra_info.model_dump(), default=data.default, - specific_info=data.specific_info.dict(), + specific_info=data.specific_info.model_dump(), created=sa.func.now(), modified=sa.func.now(), ) @@ -1261,9 +1265,9 @@ async def update_pricing_unit_with_cost( resource_tracker_pricing_units.update() .values( unit_name=data.unit_name, - unit_extra_info=data.unit_extra_info.dict(), + unit_extra_info=data.unit_extra_info.model_dump(), default=data.default, - specific_info=data.specific_info.dict(), + specific_info=data.specific_info.model_dump(), modified=sa.func.now(), ) .where( @@ -1348,4 +1352,4 @@ async def get_pricing_unit_cost_by_id( raise CustomResourceUsageTrackerError( msg=f"Pricing unit cost id {pricing_unit_cost_id} not found in the resource_tracker_pricing_unit_costs table", ) - return PricingUnitCostsDB.from_orm(row) + return PricingUnitCostsDB.model_validate(row) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py index 4cb024b3b7c..7c925244f8d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py @@ -20,7 +20,7 @@ ServiceRunStatus, ) from models_library.services import ServiceType -from pydantic import parse_raw_as +from pydantic import TypeAdapter from .models.resource_tracker_credit_transactions import ( CreditTransactionCreate, @@ -45,9 +45,9 @@ async def process_message(app: FastAPI, data: bytes) -> bool: - rabbit_message: RabbitResourceTrackingMessages = parse_raw_as( - RabbitResourceTrackingMessages, data # type: ignore[arg-type] - ) + rabbit_message: RabbitResourceTrackingMessages = TypeAdapter( + RabbitResourceTrackingMessages + ).validate_json(data) _logger.info( "Process %s msg service_run_id: %s", rabbit_message.message_type, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py index d37f244dbc9..fe80ea2b443 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py @@ -13,6 +13,7 @@ PricingPlanUpdate, ) from models_library.services import ServiceKey, ServiceVersion +from pydantic import TypeAdapter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, ) @@ -96,7 +97,10 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ] = await resource_tracker_repo.list_connected_services_to_pricing_plan_by_pricing_plan( product_name=product_name, pricing_plan_id=pricing_plan_id ) - return [PricingPlanToServiceGet.parse_obj(item) for item in output_list] + return [ + TypeAdapter(PricingPlanToServiceGet).validate_python(item.model_dump()) + for item in output_list + ] async def connect_service_to_pricing_plan( @@ -116,7 +120,7 @@ async def connect_service_to_pricing_plan( service_version=service_version, ) ) - return PricingPlanToServiceGet.parse_obj(output) + return TypeAdapter(PricingPlanToServiceGet).validate_python(output.model_dump()) async def list_pricing_plans_by_product( diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index d3ac6f743b8..a02f4ecd646 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -18,7 +18,7 @@ from models_library.rest_ordering import OrderBy from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import AnyUrl, PositiveInt, TypeAdapter +from pydantic import PositiveInt, TypeAdapter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, ) @@ -153,7 +153,7 @@ async def export_service_runs( access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, filters: ServiceResourceUsagesFilters | None = None, -) -> AnyUrl: +) -> str: started_from = filters.started_at.from_ if filters else None started_until = filters.started_at.until if filters else None @@ -177,7 +177,7 @@ async def export_service_runs( ) # Create presigned S3 link - generated_url: AnyUrl = await s3_client.create_single_presigned_download_link( + generated_url: str = await s3_client.create_single_presigned_download_link( bucket=s3_bucket_name, object_key=s3_object_key, expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC, diff --git a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py index cffb606fae5..569ddd28db1 100644 --- a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py +++ b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py @@ -68,7 +68,7 @@ def test_meta( ): response = client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK - meta = _Meta.parse_obj(response.json()) + meta = _Meta.model_validate(response.json()) - response = client.get(meta.docs_url) + response = client.get(f"{meta.docs_url}") assert response.status_code == status.HTTP_200_OK diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index 6f240d658ee..28ea92020f8 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -19,6 +19,7 @@ RabbitResourceTrackingMessageType, RabbitResourceTrackingStartedMessage, ) +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient @@ -176,7 +177,7 @@ async def assert_service_runs_db_row( ) row = result.first() assert row - service_run_db = ServiceRunDB.from_orm(row) + service_run_db = ServiceRunDB.model_validate(row) if status: assert service_run_db.service_run_status == status return service_run_db @@ -201,7 +202,7 @@ async def assert_credit_transactions_db_row( ) row = result.first() assert row - credit_transaction_db = CreditTransactionDB.from_orm(row) + credit_transaction_db = CreditTransactionDB.model_validate(row) if modified_at: assert credit_transaction_db.modified > modified_at return credit_transaction_db @@ -215,7 +216,9 @@ def random_rabbit_message_heartbeat( def _creator(**kwargs: dict[str, Any]) -> RabbitResourceTrackingHeartbeatMessage: msg_config = {"service_run_id": faker.uuid4(), **kwargs} - return RabbitResourceTrackingHeartbeatMessage(**msg_config) + return TypeAdapter(RabbitResourceTrackingHeartbeatMessage).validate_python( + msg_config + ) return _creator @@ -265,7 +268,9 @@ def _creator(**kwargs: dict[str, Any]) -> RabbitResourceTrackingStartedMessage: **kwargs, } - return RabbitResourceTrackingStartedMessage(**msg_config) + return TypeAdapter(RabbitResourceTrackingStartedMessage).validate_python( + msg_config + ) return _creator diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py index 1e7098cecda..244a74c62d7 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py @@ -56,7 +56,7 @@ async def test_credit_transactions_workflow( ) assert response.status_code == status.HTTP_201_CREATED data = response.json() - data["credit_transaction_id"] == 1 + assert data["credit_transaction_id"] == 1 response = await async_client.post( url=f"{url}", @@ -73,7 +73,7 @@ async def test_credit_transactions_workflow( ) assert response.status_code == status.HTTP_201_CREATED data = response.json() - data["credit_transaction_id"] == 2 + assert data["credit_transaction_id"] == 2 response = await async_client.post( url=f"{url}", diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py index 609b0ebd54f..8aea2c291bf 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py @@ -76,7 +76,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -101,7 +103,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID, unit_name="M", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -126,7 +130,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID, unit_name="L", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -165,7 +171,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID_2, unit_name="XXL", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py index 4ec8d45bb72..721a17e05c7 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py @@ -195,7 +195,9 @@ async def test_rpc_pricing_plans_with_units_workflow( data=PricingUnitWithCostCreate( pricing_plan_id=_pricing_plan_id, unit_name="SMALL", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=True, specific_info=SpecificInfo(aws_ec2_instances=[]), cost_per_unit=Decimal(10), @@ -227,7 +229,9 @@ async def test_rpc_pricing_plans_with_units_workflow( pricing_plan_id=_pricing_plan_id, pricing_unit_id=_first_pricing_unit_id, unit_name=_unit_name, - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=True, specific_info=SpecificInfo(aws_ec2_instances=[]), pricing_unit_cost_update=None, @@ -246,7 +250,9 @@ async def test_rpc_pricing_plans_with_units_workflow( pricing_plan_id=_pricing_plan_id, pricing_unit_id=_first_pricing_unit_id, unit_name="MEDIUM", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=True, specific_info=SpecificInfo(aws_ec2_instances=[]), pricing_unit_cost_update=PricingUnitCostUpdate( @@ -277,7 +283,9 @@ async def test_rpc_pricing_plans_with_units_workflow( data=PricingUnitWithCostCreate( pricing_plan_id=_pricing_plan_id, unit_name="LARGE", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=False, specific_info=SpecificInfo(aws_ec2_instances=[]), cost_per_unit=Decimal(20), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py index c53c1accb90..e433d8c13e6 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py @@ -4,7 +4,7 @@ import pytest import sqlalchemy as sa from moto.server import ThreadedMotoServer -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient @@ -37,8 +37,7 @@ async def mocked_export(mocker: MockerFixture): async def mocked_presigned_link(mocker: MockerFixture): mock_presigned_link = mocker.patch( "simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.SimcoreS3API.create_single_presigned_download_link", - return_value=parse_obj_as( - AnyUrl, + return_value=TypeAdapter(AnyUrl).validate_python( "https://www.testing.com/", ), ) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py index 2b719326bc9..3bf73f32f6d 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py @@ -152,7 +152,7 @@ async def test_process_event_functions( # Check max acceptable missed heartbeats reached before considering them as unhealthy with postgres_db.connect() as con: result = con.execute(sa.select(resource_tracker_service_runs)) - service_run_db = [ServiceRunDB.from_orm(row) for row in result] + service_run_db = [ServiceRunDB.model_validate(row) for row in result] for service_run in service_run_db: if service_run.service_run_id in ( _SERVICE_RUN_ID_OSPARC_10_MIN_OLD, @@ -172,7 +172,7 @@ async def test_process_event_functions( with postgres_db.connect() as con: result = con.execute(sa.select(resource_tracker_service_runs)) - service_run_db = [ServiceRunDB.from_orm(row) for row in result] + service_run_db = [ServiceRunDB.model_validate(row) for row in result] for service_run in service_run_db: if service_run.service_run_id in ( _SERVICE_RUN_ID_OSPARC_10_MIN_OLD, @@ -186,7 +186,9 @@ async def test_process_event_functions( with postgres_db.connect() as con: result = con.execute(sa.select(resource_tracker_credit_transactions)) - credit_transaction_db = [CreditTransactionDB.from_orm(row) for row in result] + credit_transaction_db = [ + CreditTransactionDB.model_validate(row) for row in result + ] for transaction in credit_transaction_db: if transaction.service_run_id in ( _SERVICE_RUN_ID_OSPARC_10_MIN_OLD, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py index 7a5e2114c1d..6b0048edf61 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py @@ -58,7 +58,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -83,7 +85,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="M", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -108,7 +112,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="L", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py index 4b6c1a0dfac..0520a797af8 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py @@ -72,7 +72,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -97,7 +99,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="M", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -122,7 +126,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="L", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py index c1d62af5b23..5e7f9d323b5 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py @@ -68,7 +68,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), From 84f1e51866e8bd242cb97438257dd807d09ff3a6 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 22 Oct 2024 18:53:01 +0200 Subject: [PATCH 14/53] missing file from re-creating branch --- .../common_library/pydantic_basic_types.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 packages/common-library/src/common_library/pydantic_basic_types.py diff --git a/packages/common-library/src/common_library/pydantic_basic_types.py b/packages/common-library/src/common_library/pydantic_basic_types.py new file mode 100644 index 00000000000..452c118dae9 --- /dev/null +++ b/packages/common-library/src/common_library/pydantic_basic_types.py @@ -0,0 +1,79 @@ +from re import Pattern +from typing import Annotated, Final, TypeAlias + +from pydantic import Field +from pydantic_core import core_schema + +# https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Registered_ports +RegisteredPortInt: TypeAlias = Annotated[int, Field(gt=1024, lt=65535)] + +# non-empty bounded string used as identifier +# e.g. "123" or "name_123" or "fa327c73-52d8-462a-9267-84eeaf0f90e3" but NOT "" +_ELLIPSIS_CHAR: Final[str] = "..." + + +class ConstrainedStr(str): # noqa: SLOT000 + pattern: str | Pattern[str] | None = None + min_length: int | None = None + max_length: int | None = None + strip_whitespace: bool = False + curtail_length: int | None = None + + @classmethod + def _validate(cls, __input_value: str) -> str: + if cls.curtail_length and len(__input_value) > cls.curtail_length: + __input_value = __input_value[: cls.curtail_length] + return cls(__input_value) + + @classmethod + def __get_pydantic_core_schema__(cls, _source_type, _handler): + return core_schema.no_info_after_validator_function( + cls._validate, + core_schema.str_schema( + pattern=cls.pattern, + min_length=cls.min_length, + max_length=cls.max_length, + strip_whitespace=cls.strip_whitespace, + ), + ) + + +class IDStr(ConstrainedStr): + strip_whitespace = True + min_length = 1 + max_length = 100 + + @staticmethod + def concatenate(*args: "IDStr", link_char: str = " ") -> "IDStr": + result = link_char.join(args).strip() + assert IDStr.min_length # nosec + assert IDStr.max_length # nosec + if len(result) > IDStr.max_length: + if IDStr.max_length > len(_ELLIPSIS_CHAR): + result = ( + result[: IDStr.max_length - len(_ELLIPSIS_CHAR)].rstrip() + + _ELLIPSIS_CHAR + ) + else: + result = _ELLIPSIS_CHAR[0] * IDStr.max_length + if len(result) < IDStr.min_length: + msg = f"IDStr.concatenate: result is too short: {result}" + raise ValueError(msg) + return IDStr(result) + + +class ShortTruncatedStr(ConstrainedStr): + # NOTE: Use to input e.g. titles or display names + # A truncated string: + # - Strips whitespaces and truncate strings that exceed the specified characters limit (curtail_length). + # - Ensures that the **input** data length to the API is controlled and prevents exceeding large inputs silently, i.e. without raising errors. + # SEE https://github.com/ITISFoundation/osparc-simcore/pull/5989#discussion_r1650506583 + strip_whitespace = True + curtail_length = 600 + + +class LongTruncatedStr(ConstrainedStr): + # NOTE: Use to input e.g. descriptions or summaries + # Analogous to ShortTruncatedStr + strip_whitespace = True + curtail_length = 65536 # same as github descripton From cf95d8193aec1279fbf7d691ccd349eb63c99e71 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 22 Oct 2024 22:28:04 +0200 Subject: [PATCH 15/53] restored from old pydanitc_v2_migration branch --- .../src/common_library/error_codes.py | 31 +++++++++++++++++++ .../tests/test_errors_classes.py | 24 ++++++++------ 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 packages/common-library/src/common_library/error_codes.py diff --git a/packages/common-library/src/common_library/error_codes.py b/packages/common-library/src/common_library/error_codes.py new file mode 100644 index 00000000000..13b3b1566da --- /dev/null +++ b/packages/common-library/src/common_library/error_codes.py @@ -0,0 +1,31 @@ +""" osparc ERROR CODES (OEC) + Unique identifier of an exception instance + Intended to report a user about unexpected errors. + Unexpected exceptions can be traced by matching the + logged error code with that appeneded to the user-friendly message + +SEE test_error_codes for some use cases +""" + +import re +from typing import TYPE_CHECKING, Annotated + +from pydantic import StringConstraints, TypeAdapter + +_LABEL = "OEC:{}" +_PATTERN = r"OEC:\d+" + +if TYPE_CHECKING: + ErrorCodeStr = str +else: + ErrorCodeStr = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=_PATTERN) + ] + + +def create_error_code(exception: BaseException) -> ErrorCodeStr: + return TypeAdapter(ErrorCodeStr).validate_python(_LABEL.format(id(exception))) + + +def parse_error_code(obj) -> set[ErrorCodeStr]: + return set(re.findall(_PATTERN, f"{obj}")) diff --git a/packages/common-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py index 8f678e29914..3be2532f1ab 100644 --- a/packages/common-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -136,16 +136,10 @@ class MyError(OsparcErrorMixin, ValueError): def test_missing_keys_in_msg_template_does_not_raise(): - class MyErrorBefore(OsparcErrorMixin, ValueError): - msg_template = "{value} and {missing}" - - with pytest.raises(KeyError, match="missing"): - str(MyErrorBefore(value=42)) - - class MyErrorAfter(OsparcErrorMixin, ValueError): + class MyError(OsparcErrorMixin, ValueError): msg_template = "{value} and {missing}" - assert str(MyErrorAfter(value=42)) == "42 and 'missing=?'" + assert str(MyError(value=42)) == "42 and 'missing=?'" def test_exception_context(): @@ -153,7 +147,17 @@ class MyError(OsparcErrorMixin, ValueError): msg_template = "{value} and {missing}" exc = MyError(value=42, missing="foo", extra="bar") - assert exc.error_context() == {"value": 42, "missing": "foo", "extra": "bar"} + assert exc.error_context() == { + "code": "ValueError.MyError", + "message": "42 and foo", + "value": 42, + "missing": "foo", + "extra": "bar", + } exc = MyError(value=42) - assert exc.error_context() == {"value": 42} + assert exc.error_context() == { + "code": "ValueError.MyError", + "message": "42 and 'missing=?'", + "value": 42, + } From 7be9541a9a12831c290ecce3404087c22f06e4e9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 22 Oct 2024 23:04:53 +0200 Subject: [PATCH 16/53] fixes after merge --- .../api_schemas_dynamic_sidecar/telemetry.py | 4 ++-- packages/models-library/src/models_library/basic_types.py | 3 +-- packages/models-library/src/models_library/clusters.py | 3 +-- .../src/models_library/rabbitmq_messages.py | 2 +- .../src/models_library/utils/json_serialization.py | 8 ++++---- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py index 986b0c87215..b7d315a8aa6 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py @@ -57,8 +57,8 @@ class DiskUsage(BaseModel): total: ByteSize = Field(description="total space = free + used") used_percent: NonNegativeFloat = Field( - gte=0.00, - lte=100.00, + ge=0.00, + le=100.00, description="Percent of used space relative to the total space", ) diff --git a/packages/models-library/src/models_library/basic_types.py b/packages/models-library/src/models_library/basic_types.py index b56972da725..d47a60514da 100644 --- a/packages/models-library/src/models_library/basic_types.py +++ b/packages/models-library/src/models_library/basic_types.py @@ -28,8 +28,7 @@ # https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Registered_ports -class RegisteredPortInt: - TypeAlias = Annotated[int, Field(gt=1024, lt=65535)] +RegisteredPortInt: TypeAlias = Annotated[int, Field(gt=1024, lt=65535)] # e.g. 'v5' diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index 6a7eb3086e7..c98ea29757a 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -95,8 +95,7 @@ class JupyterHubTokenAuthentication(BaseAuthentication): class NoAuthentication(BaseAuthentication): type: Literal["none"] = "none" - class Config(BaseAuthentication.Config): - schema_extra: ClassVar[dict[str, Any]] = {"examples": [{"type": "none"}]} + model_config = ConfigDict(json_schema_extra={"examples": [{"type": "none"}]}) class TLSAuthentication(BaseAuthentication): diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 3fdea8786b0..dd891758603 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -192,7 +192,7 @@ def routing_key(self) -> str | None: class DynamicServiceRunningMessage(RabbitMessageBase): channel_name: Literal["io.simcore.service.dynamic-service-running"] = Field( - default="io.simcore.service.dynamic-service-running", const=True + default="io.simcore.service.dynamic-service-running" ) project_id: ProjectID diff --git a/packages/models-library/src/models_library/utils/json_serialization.py b/packages/models-library/src/models_library/utils/json_serialization.py index a2fee1295f1..9bd0abd5ef6 100644 --- a/packages/models-library/src/models_library/utils/json_serialization.py +++ b/packages/models-library/src/models_library/utils/json_serialization.py @@ -97,7 +97,8 @@ def pydantic_encoder(obj: Any) -> Any: return obj.model_dump() if is_dataclass(obj): - return asdict(obj) # type: ignore[call-overload] + assert not isinstance(obj, type) # nosec + return asdict(obj) # Check the class type and its superclasses for a matching encoder for base in obj.__class__.__mro__[:-1]: @@ -108,9 +109,8 @@ def pydantic_encoder(obj: Any) -> Any: return encoder(obj) # We have exited the for loop without finding a suitable encoder - raise TypeError( - f"Object of type '{obj.__class__.__name__}' is not JSON serializable" - ) + msg = f"Object of type '{obj.__class__.__name__}' is not JSON serializable" + raise TypeError(msg) def json_dumps( From 6199ed6e69fe139f9cc66a090b7576e321419293 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:33:22 +0200 Subject: [PATCH 17/53] =?UTF-8?q?=E2=AC=86=EF=B8=8FDask=20sidecar:=20migra?= =?UTF-8?q?tion=20to=20v2=20(#6591)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/common_library/serialization.py | 11 +- .../src/settings_library/utils_cli.py | 18 ++- services/dask-sidecar/requirements/_base.in | 2 +- services/dask-sidecar/requirements/_base.txt | 105 +++++++++++++++++- services/dask-sidecar/requirements/_test.txt | 11 +- .../computational_sidecar/core.py | 10 +- .../computational_sidecar/docker_utils.py | 8 +- .../computational_sidecar/errors.py | 4 +- .../computational_sidecar/models.py | 41 ++++--- .../dask_utils.py | 2 +- .../file_utils.py | 14 +-- .../simcore_service_dask_sidecar/settings.py | 13 ++- .../src/simcore_service_dask_sidecar/tasks.py | 4 +- .../src/simcore_service_dask_sidecar/utils.py | 12 +- services/dask-sidecar/tests/unit/conftest.py | 28 ++--- services/dask-sidecar/tests/unit/test_cli.py | 2 +- .../tests/unit/test_dask_utils.py | 5 +- .../tests/unit/test_docker_utils.py | 4 +- .../tests/unit/test_file_utils.py | 46 ++++---- .../dask-sidecar/tests/unit/test_models.py | 10 +- .../dask-sidecar/tests/unit/test_tasks.py | 36 +++--- .../dask-sidecar/tests/unit/test_utils.py | 10 +- 22 files changed, 272 insertions(+), 124 deletions(-) diff --git a/packages/common-library/src/common_library/serialization.py b/packages/common-library/src/common_library/serialization.py index 787e88123eb..964dfc01ef8 100644 --- a/packages/common-library/src/common_library/serialization.py +++ b/packages/common-library/src/common_library/serialization.py @@ -1,13 +1,14 @@ from datetime import timedelta from typing import Any -from common_library.pydantic_fields_extension import get_type from pydantic import BaseModel, SecretStr from pydantic_core import Url +from .pydantic_fields_extension import get_type + def model_dump_with_secrets( - settings_obj: BaseModel, show_secrets: bool, **pydantic_export_options + settings_obj: BaseModel, *, show_secrets: bool, **pydantic_export_options ) -> dict[str, Any]: data = settings_obj.model_dump(**pydantic_export_options) @@ -25,16 +26,16 @@ def model_dump_with_secrets( data[field_name] = field_data.get_secret_value() else: data[field_name] = str(field_data) - + elif isinstance(field_data, Url): data[field_name] = str(field_data) - + elif isinstance(field_data, dict): field_type = get_type(settings_obj.model_fields[field_name]) if issubclass(field_type, BaseModel): data[field_name] = model_dump_with_secrets( field_type.model_validate(field_data), - show_secrets, + show_secrets=show_secrets, **pydantic_export_options, ) diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index 073a8ae2d5e..0311ed28d7b 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -8,7 +8,9 @@ import rich import typer from common_library.serialization import model_dump_with_secrets +from models_library.utils.json_serialization import json_dumps from pydantic import ValidationError +from pydantic_core import to_jsonable_python from pydantic_settings import BaseSettings from ._constants import HEADER_STR @@ -87,7 +89,7 @@ def print_as_json( def create_settings_command( settings_cls: type[BaseCustomSettings], logger: logging.Logger | None = None, - json_serializer=json.dumps, + json_serializer=json_dumps, ) -> Callable: """Creates typer command function for settings""" @@ -112,14 +114,24 @@ def settings( """Resolves settings and prints envfile""" if as_json_schema: - typer.echo(settings_cls.schema_json(indent=0 if compact else 2)) + typer.echo( + json.dumps( + settings_cls.model_json_schema(), + default=to_jsonable_python, + indent=0 if compact else 2, + ) + ) return try: settings_obj = settings_cls.create_from_envs() except ValidationError as err: - settings_schema = settings_cls.schema_json(indent=2) + settings_schema = json.dumps( + settings_cls.model_json_schema(), + default=to_jsonable_python, + indent=2, + ) assert logger is not None # nosec logger.error( # noqa: TRY400 diff --git a/services/dask-sidecar/requirements/_base.in b/services/dask-sidecar/requirements/_base.in index 3bb1150d8e1..2352652e4a0 100644 --- a/services/dask-sidecar/requirements/_base.in +++ b/services/dask-sidecar/requirements/_base.in @@ -25,6 +25,6 @@ dask[distributed, diagnostics] dask-gateway # needed for the osparc-dask-gateway to preload the module fsspec[http, s3] # sub types needed as we acces http and s3 here lz4 # for compression -pydantic[email,dotenv] +pydantic prometheus_client repro-zipfile diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 6d776591b15..9c36e1d2e6e 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -16,13 +16,22 @@ aiofiles==23.2.1 # -r requirements/_base.in aiohttp==3.9.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore @@ -36,6 +45,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -59,13 +70,22 @@ botocore==1.34.106 # via aiobotocore certifi==2024.7.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -137,13 +157,22 @@ importlib-metadata==7.1.0 # opentelemetry-api jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # bokeh @@ -222,13 +251,22 @@ opentelemetry-util-http==0.47b0 # via opentelemetry-instrumentation-requests orjson==3.10.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in @@ -257,27 +295,62 @@ psutil==6.0.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # distributed -pydantic==1.10.15 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.6.0 + # via + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.6.2 @@ -288,18 +361,27 @@ python-dateutil==2.9.0.post0 # botocore # pandas python-dotenv==1.0.1 - # via pydantic + # via pydantic-settings pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -309,13 +391,22 @@ pyyaml==6.0.1 # distributed redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -384,18 +475,28 @@ typing-extensions==4.11.0 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer tzdata==2024.1 # via pandas urllib3==2.2.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/services/dask-sidecar/requirements/_test.txt b/services/dask-sidecar/requirements/_test.txt index 7f13a97ad89..92130c87b77 100644 --- a/services/dask-sidecar/requirements/_test.txt +++ b/services/dask-sidecar/requirements/_test.txt @@ -1,3 +1,7 @@ +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto attrs==23.2.0 @@ -141,11 +145,15 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyftpdlib==2.0.0 # via pytest-localftpserver pyopenssl==24.2.1 @@ -244,6 +252,7 @@ typing-extensions==4.11.0 # aws-sam-translator # cfn-lint # pydantic + # pydantic-core urllib3==2.2.1 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/core.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/core.py index 63e9bc97a1b..b6ae0b25611 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/core.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/core.py @@ -71,7 +71,7 @@ async def _write_input_data( if isinstance(input_params, FileUrl): file_name = ( input_params.file_mapping - or Path(URL(input_params.url).path.strip("/")).name + or Path(URL(f"{input_params.url}").path.strip("/")).name ) destination_path = task_volumes.inputs_folder / file_name @@ -114,7 +114,7 @@ async def _retrieve_output_data( ) _logger.debug( "following outputs will be searched for:\n%s", - self.task_parameters.output_data_keys.json(indent=1), + self.task_parameters.output_data_keys.model_dump_json(indent=1), ) output_data = TaskOutputData.from_task_output( @@ -132,7 +132,7 @@ async def _retrieve_output_data( if isinstance(output_params, FileUrl): assert ( # nosec output_params.file_mapping - ), f"{output_params.json(indent=1)} expected resolved in TaskOutputData.from_task_output" + ), f"{output_params.model_dump_json(indent=1)} expected resolved in TaskOutputData.from_task_output" src_path = task_volumes.outputs_folder / output_params.file_mapping upload_tasks.append( @@ -146,7 +146,9 @@ async def _retrieve_output_data( await asyncio.gather(*upload_tasks) await self._publish_sidecar_log("All the output data were uploaded.") - _logger.info("retrieved outputs data:\n%s", output_data.json(indent=1)) + _logger.info( + "retrieved outputs data:\n%s", output_data.model_dump_json(indent=1) + ) return output_data except (ValueError, ValidationError) as exc: diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/docker_utils.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/docker_utils.py index 0a2d9e3e9d3..289f5df9169 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/docker_utils.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/docker_utils.py @@ -26,7 +26,7 @@ from models_library.services_resources import BootMode from models_library.utils.labels_annotations import OSPARC_LABEL_PREFIXES, from_labels from packaging import version -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.logging_utils import ( LogLevelInt, LogMessageStr, @@ -95,7 +95,7 @@ async def create_container_config( NanoCPUs=nano_cpus_limit, ), ) - logger.debug("Container configuration: \n%s", pformat(config.dict())) + logger.debug("Container configuration: \n%s", pformat(config.model_dump())) return config @@ -109,7 +109,7 @@ async def managed_container( logger, logging.DEBUG, msg=f"managing container {name} for {config.image}" ): container = await docker_client.containers.create( - config.dict(by_alias=True), name=name + config.model_dump(by_alias=True), name=name ) yield container except asyncio.CancelledError: @@ -443,7 +443,7 @@ async def get_image_labels( data = from_labels( image_labels, prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False ) - return parse_obj_as(ImageLabels, data) + return TypeAdapter(ImageLabels).validate_python(data) return ImageLabels() diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/errors.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/errors.py index eabe5f00d03..8e5d1e8794f 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/errors.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class ComputationalSidecarRuntimeError(PydanticErrorMixin, RuntimeError): +class ComputationalSidecarRuntimeError(OsparcErrorMixin, RuntimeError): ... diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py index 691192716e9..ee270aeb888 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py @@ -3,7 +3,14 @@ from models_library.basic_regex import SIMPLE_VERSION_RE from models_library.services import ServiceMetaDataPublished from packaging import version -from pydantic import BaseModel, ByteSize, Extra, Field, validator +from pydantic import ( + BaseModel, + ByteSize, + ConfigDict, + Field, + field_validator, + model_validator, +) LEGACY_INTEGRATION_VERSION = version.Version("0") PROGRESS_REGEXP: re.Pattern[str] = re.compile( @@ -41,21 +48,15 @@ class ContainerHostConfig(BaseModel): ..., alias="NanoCPUs", description="CPU quota in units of 10-9 CPUs" ) - @validator("memory_swap", pre=True, always=True) - @classmethod - def ensure_no_memory_swap_means_no_swap(cls, v, values): - if v is None: - # if not set it will be the same value as memory to ensure swap is disabled - return values["memory"] - return v + @model_validator(mode="after") + def ensure_memory_swap_is_not_unlimited(self) -> "ContainerHostConfig": + if self.memory_swap is None: + self.memory_swap = self.memory - @validator("memory_swap") - @classmethod - def ensure_memory_swap_cannot_be_unlimited_nor_smaller_than_memory(cls, v, values): - if v < values["memory"]: + if self.memory_swap < self.memory: msg = "Memory swap cannot be set to a smaller value than memory" raise ValueError(msg) - return v + return self class DockerContainerConfig(BaseModel): @@ -71,7 +72,7 @@ class ImageLabels(BaseModel): default=str(LEGACY_INTEGRATION_VERSION), alias="integration-version", description="integration version number", - regex=SIMPLE_VERSION_RE, + pattern=SIMPLE_VERSION_RE, examples=["1.0.0"], ) progress_regexp: str = Field( @@ -79,18 +80,16 @@ class ImageLabels(BaseModel): alias="progress_regexp", description="regexp pattern for detecting computational service's progress", ) + model_config = ConfigDict(extra="ignore") - class Config: - extra = Extra.ignore - - @validator("integration_version", pre=True) + @field_validator("integration_version", mode="before") @classmethod def default_integration_version(cls, v): if v is None: return ImageLabels().integration_version return v - @validator("progress_regexp", pre=True) + @field_validator("progress_regexp", mode="before") @classmethod def default_progress_regexp(cls, v): if v is None: @@ -104,6 +103,6 @@ def get_progress_regexp(self) -> re.Pattern[str]: return re.compile(self.progress_regexp) -assert set(ImageLabels.__fields__).issubset( - ServiceMetaDataPublished.__fields__ +assert set(ImageLabels.model_fields).issubset( + ServiceMetaDataPublished.model_fields ), "ImageLabels must be compatible with ServiceDockerData" diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/dask_utils.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/dask_utils.py index e042c5c022a..d04682dac07 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/dask_utils.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/dask_utils.py @@ -153,4 +153,4 @@ async def periodicaly_check_if_aborted(task_name: str) -> None: def publish_event(dask_pub: distributed.Pub, event: BaseTaskEvent) -> None: """never reraises, only CancellationError""" with log_catch(_logger, reraise=False): - dask_pub.put(event.json()) + dask_pub.put(event.model_dump_json()) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py index b14b5db657f..e5b78bd286a 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py @@ -13,7 +13,7 @@ import aiofiles.tempfile import fsspec # type: ignore[import-untyped] import repro_zipfile # type: ignore[import-untyped] -from pydantic import ByteSize, FileUrl, parse_obj_as +from pydantic import ByteSize, FileUrl, TypeAdapter from pydantic.networks import AnyUrl from servicelib.logging_utils import LogLevelInt, LogMessageStr from settings_library.s3 import S3Settings @@ -96,9 +96,9 @@ async def _copy_file( ): src_storage_kwargs = src_storage_cfg or {} dst_storage_kwargs = dst_storage_cfg or {} - with fsspec.open(src_url, mode="rb", **src_storage_kwargs) as src_fp, fsspec.open( - dst_url, "wb", **dst_storage_kwargs - ) as dst_fp: + with fsspec.open( + f"{src_url}", mode="rb", **src_storage_kwargs + ) as src_fp, fsspec.open(f"{dst_url}", "wb", **dst_storage_kwargs) as dst_fp: assert isinstance(src_fp, IOBase) # nosec assert isinstance(dst_fp, IOBase) # nosec file_size = getattr(src_fp, "size", None) @@ -148,7 +148,7 @@ async def pull_file_from_remote( storage_kwargs = _s3fs_settings_from_s3_settings(s3_settings) await _copy_file( src_url, - parse_obj_as(FileUrl, dst_path.as_uri()), + TypeAdapter(FileUrl).validate_python(dst_path.as_uri()), src_storage_cfg=cast(dict[str, Any], storage_kwargs), log_publishing_cb=log_publishing_cb, text_prefix=f"Downloading '{src_url.path.strip('/')}':", @@ -218,7 +218,7 @@ async def _push_file_to_remote( storage_kwargs = _s3fs_settings_from_s3_settings(s3_settings) await _copy_file( - parse_obj_as(FileUrl, file_to_upload.as_uri()), + TypeAdapter(FileUrl).validate_python(file_to_upload.as_uri()), dst_url, dst_storage_cfg=cast(dict[str, Any], storage_kwargs), log_publishing_cb=log_publishing_cb, @@ -246,7 +246,7 @@ async def push_file_to_remote( src_mime_type, _ = mimetypes.guess_type(src_path) if dst_mime_type == _ZIP_MIME_TYPE and src_mime_type != _ZIP_MIME_TYPE: - archive_file_path = Path(tmp_dir) / Path(URL(dst_url).path).name + archive_file_path = Path(tmp_dir) / Path(URL(f"{dst_url}").path).name await log_publishing_cb( f"Compressing '{src_path.name}' to '{archive_file_path.name}'...", logging.INFO, diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py index 80661c7ecb2..7073f2a4caa 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py @@ -2,7 +2,7 @@ from typing import Any from models_library.basic_types import LogLevel -from pydantic import Field, validator +from pydantic import AliasChoices, Field, field_validator from settings_library.base import BaseCustomSettings from settings_library.utils_logging import MixinLoggingSettings @@ -14,7 +14,9 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): SC_BOOT_MODE: str | None = None LOG_LEVEL: LogLevel = Field( LogLevel.INFO.value, - env=["DASK_SIDECAR_LOGLEVEL", "SIDECAR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "DASK_SIDECAR_LOGLEVEL", "SIDECAR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) # sidecar config --- @@ -37,7 +39,10 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): DASK_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=["DASK_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + validation_alias=AliasChoices( + "DASK_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -50,7 +55,7 @@ def as_worker(self) -> bool: assert self.DASK_SCHEDULER_HOST is not None # nosec return as_worker - @validator("LOG_LEVEL", pre=True) + @field_validator("LOG_LEVEL", mode="before") @classmethod def _validate_loglevel(cls, value: Any) -> str: return cls.validate_log_level(f"{value}") diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py index 79dfd08cbdb..e818a3301b6 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py @@ -67,7 +67,7 @@ async def dask_setup(worker: distributed.Worker) -> None: ) logger.info("Setting up worker...") - logger.info("Settings: %s", pformat(settings.dict())) + logger.info("Settings: %s", pformat(settings.model_dump())) print_dask_sidecar_banner() @@ -94,7 +94,7 @@ async def _run_computational_sidecar_async( _logger.info( "run_computational_sidecar %s", - f"{task_parameters.dict()=}, {docker_auth=}, {log_file_url=}, {s3_settings=}", + f"{task_parameters.model_dump()=}, {docker_auth=}, {log_file_url=}, {s3_settings=}", ) current_task = asyncio.current_task() assert current_task # nosec diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils.py index 936d54a3377..61481d32c0a 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils.py @@ -6,7 +6,7 @@ import aiodocker from aiodocker.containers import DockerContainer -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter logger = logging.getLogger(__name__) @@ -57,7 +57,7 @@ async def async_num_available_gpus() -> int: if container_data.setdefault("StatusCode", 127) == 0 else 0 ) - except asyncio.TimeoutError as err: + except TimeoutError as err: logger.warning( "num_gpus timedout while check-run %s: %s", spec_config, err ) @@ -100,14 +100,14 @@ async def async_video_memory() -> int: Coroutine, container.log(stdout=True, stderr=True, follow=False), ) - video_ram = parse_obj_as(ByteSize, 0) + video_ram = TypeAdapter(ByteSize).validate_python(0) if container_data.setdefault("StatusCode", 127) == 0: for line in container_logs: - video_ram = parse_obj_as( - ByteSize, video_ram + parse_obj_as(ByteSize, line) + video_ram = TypeAdapter(ByteSize).validate_python( + video_ram + TypeAdapter(ByteSize).validate_python(line) ) - except asyncio.TimeoutError as err: + except TimeoutError as err: logger.warning( "num_gpus timedout while check-run %s: %s", spec_config, err ) diff --git a/services/dask-sidecar/tests/unit/conftest.py b/services/dask-sidecar/tests/unit/conftest.py index edc92c87969..4d4801752d9 100644 --- a/services/dask-sidecar/tests/unit/conftest.py +++ b/services/dask-sidecar/tests/unit/conftest.py @@ -6,6 +6,7 @@ from collections.abc import AsyncIterator, Callable, Iterator from pathlib import Path from pprint import pformat +from typing import cast import dask import dask.config @@ -19,7 +20,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_localftpserver.servers import ProcessFTPServer from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -169,8 +170,7 @@ def s3_settings(mocked_s3_server_envs: None) -> S3Settings: @pytest.fixture def s3_endpoint_url(s3_settings: S3Settings) -> AnyUrl: assert s3_settings.S3_ENDPOINT - return parse_obj_as( - AnyUrl, + return TypeAdapter(AnyUrl).validate_python( f"{s3_settings.S3_ENDPOINT}", ) @@ -203,9 +203,7 @@ async def bucket( response = await aiobotocore_s3_client.list_buckets() assert response["Buckets"] assert len(response["Buckets"]) == 1 - bucket_name = response["Buckets"][0]["Name"] - return bucket_name - # await _clean_bucket_content(aiobotocore_s3_client, bucket_name) + return response["Buckets"][0]["Name"] @pytest.fixture @@ -214,7 +212,7 @@ def creator(file_path: Path | None = None) -> AnyUrl: file_path_with_bucket = Path(s3_settings.S3_BUCKET_NAME) / ( file_path or faker.file_name() ) - return parse_obj_as(AnyUrl, f"s3://{file_path_with_bucket}") + return TypeAdapter(AnyUrl).validate_python(f"s3://{file_path_with_bucket}") return creator @@ -230,7 +228,7 @@ def file_on_s3_server( def creator() -> AnyUrl: new_remote_file = s3_remote_file_url() - open_file = fsspec.open(new_remote_file, mode="wt", **s3_storage_kwargs) + open_file = fsspec.open(f"{new_remote_file}", mode="wt", **s3_storage_kwargs) with open_file as fp: fp.write( # type: ignore f"This is the file contents of file #'{(len(list_of_created_files)+1):03}'\n" @@ -245,7 +243,7 @@ def creator() -> AnyUrl: # cleanup fs = fsspec.filesystem("s3", **s3_storage_kwargs) for file in list_of_created_files: - fs.delete(file.partition(f"{file.scheme}://")[2]) + fs.delete(f"{file}".partition(f"{file.scheme}://")[2]) @pytest.fixture @@ -255,12 +253,12 @@ def job_id() -> str: @pytest.fixture def project_id(faker: Faker) -> ProjectID: - return faker.uuid4(cast_to=None) + return cast(ProjectID, faker.uuid4(cast_to=None)) @pytest.fixture def node_id(faker: Faker) -> NodeID: - return faker.uuid4(cast_to=None) + return cast(NodeID, faker.uuid4(cast_to=None)) @pytest.fixture(params=["no_parent_node", "with_parent_node"]) @@ -276,9 +274,13 @@ def task_owner( project_id=project_id, node_id=node_id, parent_project_id=( - None if request.param == "no_parent_node" else faker.uuid4(cast_to=None) + None + if request.param == "no_parent_node" + else cast(ProjectID, faker.uuid4(cast_to=None)) ), parent_node_id=( - None if request.param == "no_parent_node" else faker.uuid4(cast_to=None) + None + if request.param == "no_parent_node" + else cast(NodeID, faker.uuid4(cast_to=None)) ), ) diff --git a/services/dask-sidecar/tests/unit/test_cli.py b/services/dask-sidecar/tests/unit/test_cli.py index 4af796ec69b..101b0e4bcdc 100644 --- a/services/dask-sidecar/tests/unit/test_cli.py +++ b/services/dask-sidecar/tests/unit/test_cli.py @@ -29,5 +29,5 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, result.output print(result.output) - settings = Settings.parse_raw(result.output) + settings = Settings.model_validate_json(result.output) assert settings == Settings.create_from_envs() diff --git a/services/dask-sidecar/tests/unit/test_dask_utils.py b/services/dask-sidecar/tests/unit/test_dask_utils.py index a12ee06e211..214a9550200 100644 --- a/services/dask-sidecar/tests/unit/test_dask_utils.py +++ b/services/dask-sidecar/tests/unit/test_dask_utils.py @@ -52,7 +52,8 @@ def test_publish_event( # hence the long time out message = dask_sub.get(timeout=DASK_TESTING_TIMEOUT_S) assert message is not None - received_task_log_event = TaskLogEvent.parse_raw(message) # type: ignore + assert isinstance(message, str) + received_task_log_event = TaskLogEvent.model_validate_json(message) assert received_task_log_event == event_to_publish @@ -73,7 +74,7 @@ async def test_publish_event_async( assert isinstance(message, Coroutine) message = await message assert message is not None - received_task_log_event = TaskLogEvent.parse_raw(message) # type: ignore + received_task_log_event = TaskLogEvent.model_validate_json(message) assert received_task_log_event == event_to_publish diff --git a/services/dask-sidecar/tests/unit/test_docker_utils.py b/services/dask-sidecar/tests/unit/test_docker_utils.py index 41e801b70bb..4bc154edd95 100644 --- a/services/dask-sidecar/tests/unit/test_docker_utils.py +++ b/services/dask-sidecar/tests/unit/test_docker_utils.py @@ -91,7 +91,7 @@ async def test_create_container_config( envs=task_envs, labels=task_labels, ) - assert container_config.dict(by_alias=True) == ( + assert container_config.model_dump(by_alias=True) == ( { "Env": [ "INPUT_FOLDER=/inputs", @@ -221,7 +221,7 @@ async def test_managed_container_always_removes_container( call() .__aenter__() .containers.create( - container_config.dict(by_alias=True), name=None + container_config.model_dump(by_alias=True), name=None ), ] ) diff --git a/services/dask-sidecar/tests/unit/test_file_utils.py b/services/dask-sidecar/tests/unit/test_file_utils.py index 5c51f5f5b00..b31980b46a5 100644 --- a/services/dask-sidecar/tests/unit/test_file_utils.py +++ b/services/dask-sidecar/tests/unit/test_file_utils.py @@ -15,7 +15,7 @@ import fsspec import pytest from faker import Faker -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_localftpserver.servers import ProcessFTPServer from pytest_mock.plugin import MockerFixture from settings_library.s3 import S3Settings @@ -28,7 +28,6 @@ @pytest.fixture() async def mocked_log_publishing_cb( - event_loop: asyncio.AbstractEventLoop, mocker: MockerFixture, ) -> AsyncIterable[mock.AsyncMock]: async with mocker.AsyncMock() as mocked_callback: @@ -46,8 +45,8 @@ def s3_presigned_link_storage_kwargs(s3_settings: S3Settings) -> dict[str, Any]: @pytest.fixture def ftp_remote_file_url(ftpserver: ProcessFTPServer, faker: Faker) -> AnyUrl: - return parse_obj_as( - AnyUrl, f"{ftpserver.get_login_data(style='url')}/{faker.file_name()}" + return TypeAdapter(AnyUrl).validate_python( + f"{ftpserver.get_login_data(style='url')}/{faker.file_name()}" ) @@ -57,8 +56,7 @@ async def s3_presigned_link_remote_file_url( aiobotocore_s3_client, faker: Faker, ) -> AnyUrl: - return parse_obj_as( - AnyUrl, + return TypeAdapter(AnyUrl).validate_python( await aiobotocore_s3_client.generate_presigned_url( "put_object", Params={"Bucket": s3_settings.S3_BUCKET_NAME, "Key": faker.file_name()}, @@ -69,7 +67,9 @@ async def s3_presigned_link_remote_file_url( @pytest.fixture def s3_remote_file_url(s3_settings: S3Settings, faker: Faker) -> AnyUrl: - return parse_obj_as(AnyUrl, f"s3://{s3_settings.S3_BUCKET_NAME}{faker.file_path()}") + return TypeAdapter(AnyUrl).validate_python( + f"s3://{s3_settings.S3_BUCKET_NAME}{faker.file_path()}" + ) @dataclass(frozen=True) @@ -122,7 +122,7 @@ async def test_push_file_to_remote( with cast( fsspec.core.OpenFile, fsspec.open( - remote_parameters.remote_file_url, + f"{remote_parameters.remote_file_url}", mode="rt", **storage_kwargs, ), @@ -153,15 +153,14 @@ async def test_push_file_to_remote_s3_http_presigned_link( ) # check the remote is actually having the file in, but we need s3 access now - s3_remote_file_url = parse_obj_as( - AnyUrl, + s3_remote_file_url = TypeAdapter(AnyUrl).validate_python( f"s3:/{s3_presigned_link_remote_file_url.path}", ) storage_kwargs = _s3fs_settings_from_s3_settings(s3_settings) with cast( fsspec.core.OpenFile, - fsspec.open(s3_remote_file_url, mode="rt", **storage_kwargs), + fsspec.open(f"{s3_remote_file_url}", mode="rt", **storage_kwargs), ) as fp: assert fp.read() == TEXT_IN_FILE mocked_log_publishing_cb.assert_called() @@ -173,7 +172,9 @@ async def test_push_file_to_remote_compresses_if_zip_destination( faker: Faker, mocked_log_publishing_cb: mock.AsyncMock, ): - destination_url = parse_obj_as(AnyUrl, f"{remote_parameters.remote_file_url}.zip") + destination_url = TypeAdapter(AnyUrl).validate_python( + f"{remote_parameters.remote_file_url}.zip" + ) src_path = tmp_path / faker.file_name() TEXT_IN_FILE = faker.text() src_path.write_text(TEXT_IN_FILE) @@ -214,7 +215,7 @@ async def test_pull_file_from_remote( with cast( fsspec.core.OpenFile, fsspec.open( - remote_parameters.remote_file_url, + f"{remote_parameters.remote_file_url}", mode="wt", **storage_kwargs, ), @@ -250,7 +251,7 @@ async def test_pull_file_from_remote_s3_presigned_link( with cast( fsspec.core.OpenFile, fsspec.open( - s3_remote_file_url, + f"{s3_remote_file_url}", mode="wt", **storage_kwargs, ), @@ -259,8 +260,7 @@ async def test_pull_file_from_remote_s3_presigned_link( # create a corresponding presigned get link assert s3_remote_file_url.path - remote_file_url = parse_obj_as( - AnyUrl, + remote_file_url = TypeAdapter(AnyUrl).validate_python( await aiobotocore_s3_client.generate_presigned_url( "get_object", Params={ @@ -303,7 +303,9 @@ async def test_pull_compressed_zip_file_from_remote( zfp.write(local_test_file, local_test_file.name) file_names_within_zip_file.add(local_test_file.name) - destination_url = parse_obj_as(AnyUrl, f"{remote_parameters.remote_file_url}.zip") + destination_url = TypeAdapter(AnyUrl).validate_python( + f"{remote_parameters.remote_file_url}.zip" + ) storage_kwargs = {} if remote_parameters.s3_settings: storage_kwargs = _s3fs_settings_from_s3_settings(remote_parameters.s3_settings) @@ -311,7 +313,7 @@ async def test_pull_compressed_zip_file_from_remote( with cast( fsspec.core.OpenFile, fsspec.open( - destination_url, + f"{destination_url}", mode="wb", **storage_kwargs, ), @@ -395,8 +397,12 @@ async def test_push_file_to_remote_creates_reproducible_zip_archive( faker: Faker, mocked_log_publishing_cb: mock.AsyncMock, ): - destination_url1 = parse_obj_as(AnyUrl, f"{remote_parameters.remote_file_url}1.zip") - destination_url2 = parse_obj_as(AnyUrl, f"{remote_parameters.remote_file_url}2.zip") + destination_url1 = TypeAdapter(AnyUrl).validate_python( + f"{remote_parameters.remote_file_url}1.zip" + ) + destination_url2 = TypeAdapter(AnyUrl).validate_python( + f"{remote_parameters.remote_file_url}2.zip" + ) src_path = tmp_path / faker.file_name() TEXT_IN_FILE = faker.text() src_path.write_text(TEXT_IN_FILE) diff --git a/services/dask-sidecar/tests/unit/test_models.py b/services/dask-sidecar/tests/unit/test_models.py index 65ec5304631..f9e80f67fa4 100644 --- a/services/dask-sidecar/tests/unit/test_models.py +++ b/services/dask-sidecar/tests/unit/test_models.py @@ -10,7 +10,7 @@ def test_container_host_config_sets_swap_same_as_memory_if_not_set(faker: Faker) instance = ContainerHostConfig( Binds=[faker.pystr() for _ in range(5)], Memory=ByteSize(faker.pyint()), - NanoCPUs=faker.pyfloat(min_value=0.1), + NanoCPUs=faker.pyint(min_value=1), ) assert instance.memory == instance.memory_swap @@ -22,7 +22,7 @@ def test_container_host_config_raises_if_set_negative( ContainerHostConfig( Binds=[faker.pystr() for _ in range(5)], Memory=ByteSize(faker.pyint(min_value=234)), - NanoCPUs=faker.pyfloat(min_value=0.1), + NanoCPUs=faker.pyint(min_value=1), MemorySwap=ByteSize(faker.pyint(min_value=-84654, max_value=-1)), ) @@ -34,14 +34,14 @@ def test_container_host_config_raises_if_set_smaller_than_memory( ContainerHostConfig( Binds=[faker.pystr() for _ in range(5)], Memory=ByteSize(faker.pyint(min_value=234)), - NanoCPUs=faker.pyfloat(min_value=0.1), + NanoCPUs=faker.pyint(min_value=1), MemorySwap=ByteSize(0), ) with pytest.raises(ValidationError): ContainerHostConfig( Binds=[faker.pystr() for _ in range(5)], Memory=ByteSize(faker.pyint(min_value=234)), - NanoCPUs=faker.pyfloat(min_value=0.1), + NanoCPUs=faker.pyint(min_value=1), MemorySwap=ByteSize(faker.pyint(min_value=1, max_value=233)), ) @@ -52,7 +52,7 @@ def test_container_host_config_sets_swap_if_set_bigger_than_memory( instance = ContainerHostConfig( Binds=[faker.pystr() for _ in range(5)], Memory=ByteSize(faker.pyint(min_value=234, max_value=434234)), - NanoCPUs=faker.pyfloat(min_value=0.1), + NanoCPUs=faker.pyint(min_value=1), MemorySwap=ByteSize(faker.pyint(min_value=434235, max_value=12343424234)), ) assert instance.memory_swap diff --git a/services/dask-sidecar/tests/unit/test_tasks.py b/services/dask-sidecar/tests/unit/test_tasks.py index 4aff3a1fd3d..5beebe2e37f 100644 --- a/services/dask-sidecar/tests/unit/test_tasks.py +++ b/services/dask-sidecar/tests/unit/test_tasks.py @@ -41,7 +41,7 @@ from models_library.services import ServiceMetaDataPublished from models_library.services_resources import BootMode from packaging import version -from pydantic import AnyUrl, SecretStr, parse_obj_as +from pydantic import AnyUrl, SecretStr, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.s3 import S3Settings @@ -178,7 +178,9 @@ def integration_version(request: pytest.FixtureRequest) -> version.Version: @pytest.fixture def additional_envs(faker: Faker) -> dict[EnvVarKey, str]: - return parse_obj_as(dict[EnvVarKey, str], faker.pydict(allowed_types=(str,))) + return TypeAdapter(dict[EnvVarKey, str]).validate_python( + faker.pydict(allowed_types=(str,)) + ) @pytest.fixture @@ -198,7 +200,7 @@ def sleeper_task( list_of_files = [file_on_s3_server() for _ in range(NUM_FILES)] # defines the inputs of the task - input_data = TaskInputData.parse_obj( + input_data = TaskInputData.model_validate( { "input_1": 23, "input_23": "a string input", @@ -276,7 +278,7 @@ def sleeper_task( "pytest_bool": False, } output_file_url = s3_remote_file_url(file_path="output_file") - expected_output_keys = TaskOutputDataSchema.parse_obj( + expected_output_keys = TaskOutputDataSchema.model_validate( { **( {k: {"required": True} for k in jsonable_outputs} @@ -295,7 +297,7 @@ def sleeper_task( ), } ) - expected_output_data = TaskOutputData.parse_obj( + expected_output_data = TaskOutputData.model_validate( { **( jsonable_outputs @@ -395,10 +397,10 @@ def _creator(command: list[str] | None = None) -> ServiceExampleParam: service_version="latest", command=command or ["/bin/bash", "-c", "echo 'hello I'm an empty ubuntu task!"], - input_data=TaskInputData.parse_obj({}), - output_data_keys=TaskOutputDataSchema.parse_obj({}), + input_data=TaskInputData.model_validate({}), + output_data_keys=TaskOutputDataSchema.model_validate({}), log_file_url=s3_remote_file_url(file_path="log.dat"), - expected_output_data=TaskOutputData.parse_obj({}), + expected_output_data=TaskOutputData.model_validate({}), expected_logs=[], integration_version=integration_version, task_envs={}, @@ -433,12 +435,16 @@ def caplog_info_level( yield caplog +# from pydantic.json_schema import JsonDict + + @pytest.fixture def mocked_get_image_labels( integration_version: version.Version, mocker: MockerFixture ) -> mock.Mock: - labels: ImageLabels = parse_obj_as( - ImageLabels, ServiceMetaDataPublished.Config.schema_extra["examples"][0] + assert "json_schema_extra" in ServiceMetaDataPublished.model_config + labels: ImageLabels = TypeAdapter(ImageLabels).validate_python( + ServiceMetaDataPublished.model_config["json_schema_extra"]["examples"][0], ) labels.integration_version = f"{integration_version}" return mocker.patch( @@ -580,7 +586,8 @@ async def test_run_computational_sidecar_dask( # check that the task produces expected logs worker_progresses = [ - TaskProgressEvent.parse_raw(msg).progress for msg in progress_sub.buffer + TaskProgressEvent.model_validate_json(msg).progress + for msg in progress_sub.buffer ] # check ordering assert worker_progresses == sorted( @@ -588,7 +595,7 @@ async def test_run_computational_sidecar_dask( ), "ordering of progress values incorrectly sorted!" assert worker_progresses[0] == 0, "missing/incorrect initial progress value" assert worker_progresses[-1] == 1, "missing/incorrect final progress value" - worker_logs = [TaskLogEvent.parse_raw(msg).log for msg in log_sub.buffer] + worker_logs = [TaskLogEvent.model_validate_json(msg).log for msg in log_sub.buffer] print(f"<-- we got {len(worker_logs)} lines of logs") for log in sleeper_task.expected_logs: @@ -649,7 +656,8 @@ async def test_run_computational_sidecar_dask_does_not_lose_messages_with_pubsub # check that the task produces expected logs worker_progresses = [ - TaskProgressEvent.parse_raw(msg).progress for msg in progress_sub.buffer + TaskProgressEvent.model_validate_json(msg).progress + for msg in progress_sub.buffer ] # check length assert len(worker_progresses) == len( @@ -659,7 +667,7 @@ async def test_run_computational_sidecar_dask_does_not_lose_messages_with_pubsub assert worker_progresses[0] == 0, "missing/incorrect initial progress value" assert worker_progresses[-1] == 1, "missing/incorrect final progress value" - worker_logs = [TaskLogEvent.parse_raw(msg).log for msg in log_sub.buffer] + worker_logs = [TaskLogEvent.model_validate_json(msg).log for msg in log_sub.buffer] # check all the awaited logs are in there filtered_worker_logs = filter(lambda log: "This is iteration" in log, worker_logs) assert len(list(filtered_worker_logs)) == NUMBER_OF_LOGS diff --git a/services/dask-sidecar/tests/unit/test_utils.py b/services/dask-sidecar/tests/unit/test_utils.py index 5ee6f9156e5..f3d162952ff 100644 --- a/services/dask-sidecar/tests/unit/test_utils.py +++ b/services/dask-sidecar/tests/unit/test_utils.py @@ -13,12 +13,11 @@ from simcore_service_dask_sidecar.utils import num_available_gpus -@pytest.fixture(scope="function") +@pytest.fixture def mock_aiodocker(mocker: MockerFixture) -> mock.MagicMock: - mock_docker = mocker.patch( + return mocker.patch( "simcore_service_dask_sidecar.utils.aiodocker.Docker", autospec=True ) - return mock_docker def test_num_available_gpus_returns_0_when_container_not_created( @@ -74,7 +73,7 @@ def test_num_available_gpus_returns_0_when_container_wait_timesout( mock_aiodocker: mock.MagicMock, ): mock_aiodocker.return_value.__aenter__.return_value.containers.run.return_value.wait.side_effect = ( - asyncio.TimeoutError() + TimeoutError() ) assert num_available_gpus() == 0 @@ -91,6 +90,9 @@ def test_num_available_gpus( mock_aiodocker: mock.MagicMock, ): # default with mock should return 0 gpus + mock_aiodocker.return_value.__aenter__.return_value.containers.run.return_value.wait.return_value = { + "StatusCode": 0 + } assert num_available_gpus() == 0 # add the correct log From cd6f5455ab1d0e9ae874842926dab86100df07c4 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:28:11 +0200 Subject: [PATCH 18/53] =?UTF-8?q?=E2=99=BB=EF=B8=8FUse=20correct=20import?= =?UTF-8?q?=20(#6597)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_error_codes.py | 2 +- .../src/models_library/error_codes.py | 32 ------------------- .../servicelib/aiohttp/rest_middlewares.py | 2 +- .../src/servicelib/logging_errors.py | 4 +-- .../tests/aiohttp/test_rest_middlewares.py | 4 +-- .../tests/test_logging_errors.py | 4 +-- .../exceptions/_base.py | 2 +- .../exceptions/handlers/_handlers_factory.py | 2 +- .../services/log_streaming.py | 4 +-- .../exceptions/errors.py | 2 +- .../models/dynamic_services_scheduler.py | 4 +-- .../scheduler/_core/_observer.py | 2 +- .../src/simcore_service_webserver/errors.py | 2 +- .../login/_registration.py | 2 +- .../login/handlers_confirmation.py | 2 +- .../login/handlers_registration.py | 2 +- .../studies_dispatcher/_redirects_handlers.py | 2 +- .../studies_dispatcher/_studies_access.py | 2 +- .../src/simcore_service_webserver/utils.py | 4 +-- .../wallets/_handlers.py | 2 +- 20 files changed, 25 insertions(+), 57 deletions(-) rename packages/{models-library => common-library}/tests/test_error_codes.py (95%) delete mode 100644 packages/models-library/src/models_library/error_codes.py diff --git a/packages/models-library/tests/test_error_codes.py b/packages/common-library/tests/test_error_codes.py similarity index 95% rename from packages/models-library/tests/test_error_codes.py rename to packages/common-library/tests/test_error_codes.py index 17f252a2539..5d4d78a5d2b 100644 --- a/packages/models-library/tests/test_error_codes.py +++ b/packages/common-library/tests/test_error_codes.py @@ -6,7 +6,7 @@ import logging import pytest -from models_library.error_codes import create_error_code, parse_error_code +from common_library.error_codes import create_error_code, parse_error_code logger = logging.getLogger(__name__) diff --git a/packages/models-library/src/models_library/error_codes.py b/packages/models-library/src/models_library/error_codes.py deleted file mode 100644 index 06cd14ac8bc..00000000000 --- a/packages/models-library/src/models_library/error_codes.py +++ /dev/null @@ -1,32 +0,0 @@ -""" osparc ERROR CODES (OEC) - Unique identifier of an exception instance - Intended to report a user about unexpected errors. - Unexpected exceptions can be traced by matching the - logged error code with that appeneded to the user-friendly message - -SEE test_error_codes for some use cases -""" - - -import re -from typing import TYPE_CHECKING, Annotated - -from pydantic import StringConstraints, TypeAdapter - -_LABEL = "OEC:{}" -_PATTERN = r"OEC:\d+" - -if TYPE_CHECKING: - ErrorCodeStr = str -else: - ErrorCodeStr = Annotated[ - str, StringConstraints(strip_whitespace=True, pattern=_PATTERN) - ] - - -def create_error_code(exception: BaseException) -> ErrorCodeStr: - return TypeAdapter(ErrorCodeStr).validate_python(_LABEL.format(id(exception))) - - -def parse_error_code(obj) -> set[ErrorCodeStr]: - return set(re.findall(_PATTERN, f"{obj}")) diff --git a/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py b/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py index e8d106885b6..d12ea2147d8 100644 --- a/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py +++ b/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py @@ -12,7 +12,7 @@ from aiohttp import web from aiohttp.web_request import Request from aiohttp.web_response import StreamResponse -from models_library.error_codes import create_error_code +from common_library.error_codes import create_error_code from models_library.utils.json_serialization import json_dumps from ..logging_errors import create_troubleshotting_log_kwargs diff --git a/packages/service-library/src/servicelib/logging_errors.py b/packages/service-library/src/servicelib/logging_errors.py index 926581a3f2c..f3b19a5ea4f 100644 --- a/packages/service-library/src/servicelib/logging_errors.py +++ b/packages/service-library/src/servicelib/logging_errors.py @@ -2,8 +2,8 @@ from pprint import pformat from typing import Any, TypedDict -from models_library.error_codes import ErrorCodeStr -from models_library.errors_classes import OsparcErrorMixin +from common_library.error_codes import ErrorCodeStr +from common_library.errors_classes import OsparcErrorMixin from .logging_utils import LogExtra, get_log_record_extra diff --git a/packages/service-library/tests/aiohttp/test_rest_middlewares.py b/packages/service-library/tests/aiohttp/test_rest_middlewares.py index 76fb8dc7c2b..554982e53cd 100644 --- a/packages/service-library/tests/aiohttp/test_rest_middlewares.py +++ b/packages/service-library/tests/aiohttp/test_rest_middlewares.py @@ -13,7 +13,7 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient -from models_library.error_codes import parse_error_code +from common_library.error_codes import parse_error_code from models_library.utils.json_serialization import json_dumps from servicelib.aiohttp import status from servicelib.aiohttp.rest_middlewares import ( @@ -99,7 +99,7 @@ async def raise_exception(cls, request: web.Request): case NotImplementedError.__name__: raise NotImplementedError case asyncio.TimeoutError.__name__: - raise asyncio.TimeoutError + raise TimeoutError case web.HTTPOk.__name__: raise web.HTTPOk # 2XX case web.HTTPUnauthorized.__name__: diff --git a/packages/service-library/tests/test_logging_errors.py b/packages/service-library/tests/test_logging_errors.py index 432d2421d17..b6b652a46d7 100644 --- a/packages/service-library/tests/test_logging_errors.py +++ b/packages/service-library/tests/test_logging_errors.py @@ -3,8 +3,8 @@ import logging import pytest -from models_library.error_codes import create_error_code -from models_library.errors_classes import OsparcErrorMixin +from common_library.error_codes import create_error_code +from common_library.errors_classes import OsparcErrorMixin from servicelib.logging_errors import ( create_troubleshotting_log_kwargs, create_troubleshotting_log_message, diff --git a/services/api-server/src/simcore_service_api_server/exceptions/_base.py b/services/api-server/src/simcore_service_api_server/exceptions/_base.py index 2e0b2e13c4f..5ea7664ec23 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/_base.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/_base.py @@ -1,6 +1,6 @@ from typing import Any -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class ApiServerBaseError(OsparcErrorMixin, Exception): diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py index 4b7eea72a40..fe2befdce63 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_factory.py @@ -1,8 +1,8 @@ import logging +from common_library.error_codes import create_error_code from fastapi.requests import Request from fastapi.responses import JSONResponse -from models_library.error_codes import create_error_code from servicelib.logging_errors import create_troubleshotting_log_kwargs from ._utils import ExceptionHandler, create_error_json_response diff --git a/services/api-server/src/simcore_service_api_server/services/log_streaming.py b/services/api-server/src/simcore_service_api_server/services/log_streaming.py index 5bb8cbd362e..dc031667fbb 100644 --- a/services/api-server/src/simcore_service_api_server/services/log_streaming.py +++ b/services/api-server/src/simcore_service_api_server/services/log_streaming.py @@ -4,7 +4,7 @@ from collections.abc import AsyncIterable from typing import Final -from models_library.error_codes import create_error_code +from common_library.error_codes import create_error_code from models_library.rabbitmq_messages import LoggerRabbitMessage from models_library.users import UserID from pydantic import NonNegativeInt @@ -123,7 +123,7 @@ async def log_generator(self) -> AsyncIterable[str]: self._queue.get(), timeout=self._log_check_timeout ) yield log.json() + _NEW_LINE - except asyncio.TimeoutError: + except TimeoutError: done = await self._project_done() except BaseBackEndError as exc: diff --git a/services/catalog/src/simcore_service_catalog/exceptions/errors.py b/services/catalog/src/simcore_service_catalog/exceptions/errors.py index 8729cb437f5..0384088d37c 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/errors.py +++ b/services/catalog/src/simcore_service_catalog/exceptions/errors.py @@ -1,6 +1,6 @@ from typing import Any -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class CatalogBaseError(OsparcErrorMixin, Exception): diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 3f112860fb4..17408a202ec 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -10,13 +10,13 @@ from uuid import UUID import arrow +from common_library.error_codes import ErrorCodeStr from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate from models_library.api_schemas_directorv2.dynamic_services_service import ( CommonServiceDetails, ) from models_library.basic_types import PortInt from models_library.callbacks_mapping import CallbacksMapping -from models_library.error_codes import ErrorCodeStr from models_library.generated_models.docker_rest_api import ContainerState, Status2 from models_library.projects_nodes_io import NodeID from models_library.resource_tracker import HardwareInfo, PricingInfo @@ -116,7 +116,7 @@ def __eq__(self, other: object) -> bool: @classmethod def create_as_initially_ok(cls) -> "Status": # the service is initially ok when started - initial_state: "Status" = cls(current=DynamicSidecarStatus.OK, info="") + initial_state: Status = cls(current=DynamicSidecarStatus.OK, info="") return initial_state diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py index 5ced0f1a591..ce3decc2ca6 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py @@ -5,8 +5,8 @@ from copy import deepcopy from math import floor +from common_library.error_codes import create_error_code from fastapi import FastAPI -from models_library.error_codes import create_error_code from servicelib.logging_errors import create_troubleshotting_log_kwargs from .....core.dynamic_services_settings.scheduler import ( diff --git a/services/web/server/src/simcore_service_webserver/errors.py b/services/web/server/src/simcore_service_webserver/errors.py index 173699f5888..1bc48eda031 100644 --- a/services/web/server/src/simcore_service_webserver/errors.py +++ b/services/web/server/src/simcore_service_webserver/errors.py @@ -1,6 +1,6 @@ from typing import Any -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class WebServerBaseError(OsparcErrorMixin, Exception): diff --git a/services/web/server/src/simcore_service_webserver/login/_registration.py b/services/web/server/src/simcore_service_webserver/login/_registration.py index 282256d1b16..1cfc53396d2 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration.py @@ -10,9 +10,9 @@ from datetime import datetime from aiohttp import web +from common_library.error_codes import create_error_code from models_library.basic_types import IdInt from models_library.emails import LowerCaseEmailStr -from models_library.error_codes import create_error_code from models_library.products import ProductName from pydantic import ( BaseModel, diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py b/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py index c627fb58358..63c4505c647 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py @@ -4,8 +4,8 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.error_codes import create_error_code from models_library.emails import LowerCaseEmailStr -from models_library.error_codes import create_error_code from models_library.products import ProductName from pydantic import ( BaseModel, diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_registration.py b/services/web/server/src/simcore_service_webserver/login/handlers_registration.py index 9ae8b1af582..7c140184767 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_registration.py @@ -4,8 +4,8 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.error_codes import create_error_code from models_library.emails import LowerCaseEmailStr -from models_library.error_codes import create_error_code from pydantic import BaseModel, Field, PositiveInt, SecretStr, validator from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_body_as diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py index 05757c80468..237aed0f7fd 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py @@ -8,7 +8,7 @@ from typing import TypeAlias from aiohttp import web -from models_library.error_codes import create_error_code +from common_library.error_codes import create_error_code from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services import ServiceKey, ServiceVersion diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py index 1e218e94c10..85d47f0dba8 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_studies_access.py @@ -20,7 +20,7 @@ from aiohttp import web from aiohttp_session import get_session -from models_library.error_codes import create_error_code +from common_library.error_codes import create_error_code from models_library.projects import ProjectID from servicelib.aiohttp import status from servicelib.aiohttp.typing_extension import Handler diff --git a/services/web/server/src/simcore_service_webserver/utils.py b/services/web/server/src/simcore_service_webserver/utils.py index 6e7e7fced67..657e347d1a6 100644 --- a/services/web/server/src/simcore_service_webserver/utils.py +++ b/services/web/server/src/simcore_service_webserver/utils.py @@ -1,6 +1,7 @@ """ General utilities and helper functions """ + import asyncio import hashlib import logging @@ -13,8 +14,8 @@ from typing import Any, TypedDict, cast import orjson +from common_library.error_codes import ErrorCodeStr from models_library.basic_types import SHA1Str -from models_library.error_codes import ErrorCodeStr _CURRENT_DIR = ( Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -80,7 +81,6 @@ def now_str() -> str: def to_datetime(snapshot: str) -> datetime: - # return datetime.strptime(snapshot, DATETIME_FORMAT) diff --git a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py index dc6855f2c01..3aa26158acb 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py @@ -2,13 +2,13 @@ import logging from aiohttp import web +from common_library.error_codes import create_error_code from models_library.api_schemas_webserver.wallets import ( CreateWalletBodyParams, PutWalletBodyParams, WalletGet, WalletGetWithAvailableCredits, ) -from models_library.error_codes import create_error_code from models_library.users import UserID from models_library.wallets import WalletID from pydantic import Field From 3cf68ba93f7d25b545103ddddee854b6f02fb494 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:44:20 +0200 Subject: [PATCH 19/53] =?UTF-8?q?=E2=AC=86=EF=B8=8FDatcore=20adapter:=20mi?= =?UTF-8?q?gration=20(#6594)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datcore-adapter/requirements/_base.in | 2 +- .../datcore-adapter/requirements/_base.txt | 97 ++++++++++++++++--- .../simcore_service_datcore_adapter/_meta.py | 4 +- .../api/errors/http_error.py | 7 +- .../api/errors/pennsieve_error.py | 5 +- .../api/errors/validation_error.py | 5 +- .../api/routes/files.py | 6 +- .../core/application.py | 2 +- .../core/settings.py | 29 +++--- .../models/schemas/datasets.py | 10 +- .../datcore-adapter/tests/unit/conftest.py | 16 +-- .../tests/unit/test_route_datasets.py | 14 ++- .../tests/unit/test_route_files.py | 4 +- .../tests/unit/test_route_health.py | 4 +- 14 files changed, 145 insertions(+), 60 deletions(-) diff --git a/services/datcore-adapter/requirements/_base.in b/services/datcore-adapter/requirements/_base.in index bcedb3cc8ed..791d139123a 100644 --- a/services/datcore-adapter/requirements/_base.in +++ b/services/datcore-adapter/requirements/_base.in @@ -18,6 +18,6 @@ aiofiles fastapi fastapi-pagination httpx[http2] -pydantic[email] +pydantic python-multipart # for fastapi multipart uploads uvicorn[standard] diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index 9cb77e4a359..b572351f2e2 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -14,10 +14,16 @@ aiofiles==23.2.1 # -r requirements/_base.in aiohttp==3.9.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -25,6 +31,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -52,10 +60,16 @@ botocore==1.34.75 # s3transfer certifi==2024.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -79,19 +93,12 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.99.1 +fastapi==0.115.3 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # fastapi-pagination # prometheus-fastapi-instrumentator -fastapi-pagination==0.12.21 +fastapi-pagination==0.12.31 # via -r requirements/_base.in faststream==0.5.10 # via -r requirements/../../../packages/service-library/requirements/_base.in @@ -119,10 +126,16 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -211,10 +224,16 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in @@ -233,24 +252,49 @@ protobuf==4.25.4 # opentelemetry-proto psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -pydantic==1.10.14 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends # fastapi # fastapi-pagination + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.6.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -260,25 +304,39 @@ python-dateutil==2.9.0.post0 # arrow # botocore python-dotenv==1.0.1 - # via uvicorn + # via + # pydantic-settings + # uvicorn python-multipart==0.0.9 # via -r requirements/_base.in pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -312,12 +370,18 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.27.0 +starlette==0.41.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -343,13 +407,20 @@ typing-extensions==4.10.0 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer urllib3==2.2.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/_meta.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/_meta.py index 673b3bec726..db004a8a9d3 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/_meta.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/_meta.py @@ -6,12 +6,12 @@ from typing import Final from models_library.basic_types import VersionStr -from pydantic import parse_obj_as +from pydantic import TypeAdapter current_distribution = distribution("simcore_service_datcore_adapter") __version__ = version("simcore_service_datcore_adapter") -API_VERSION: Final[VersionStr] = parse_obj_as(VersionStr, __version__) +API_VERSION: Final[VersionStr] = TypeAdapter(VersionStr).validate_python(__version__) MAJOR, MINOR, PATCH = __version__.split(".") API_VTAG: Final[str] = f"v{MAJOR}" APP_NAME: Final[str] = current_distribution.metadata["Name"] diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/http_error.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/http_error.py index 6b8dcd0796e..bcf8cdec9c6 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/http_error.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/http_error.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional +from typing import Callable from fastapi import HTTPException from fastapi.encoders import jsonable_encoder @@ -6,7 +6,8 @@ from starlette.responses import JSONResponse -async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse: +async def http_error_handler(_: Request, exc: Exception) -> JSONResponse: + assert isinstance(exc, HTTPException) # nosec return JSONResponse( content=jsonable_encoder({"errors": [exc.detail]}), status_code=exc.status_code ) @@ -16,7 +17,7 @@ def make_http_error_handler_for_exception( status_code: int, exception_cls: type[BaseException], *, - override_detail_message: Optional[str] = None, + override_detail_message: str | None = None, ) -> Callable: """ Produces a handler for BaseException-type exceptions which converts them diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/pennsieve_error.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/pennsieve_error.py index 79c16ebaa63..c1101961b34 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/pennsieve_error.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/pennsieve_error.py @@ -7,8 +7,11 @@ async def botocore_exceptions_handler( _: Request, - exc: ClientError, + exc: Exception, ) -> JSONResponse: + assert isinstance(exc, ClientError) # nosec + assert "Error" in exc.response # nosec + assert "Code" in exc.response["Error"] # nosec if exc.response["Error"]["Code"] == "NotAuthorizedException": return JSONResponse( content=jsonable_encoder({"errors": exc.response["Error"]}), diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/validation_error.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/validation_error.py index fb70f6791ac..3770d62cb23 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/validation_error.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/errors/validation_error.py @@ -1,5 +1,3 @@ -from typing import Union - from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from fastapi.openapi.constants import REF_PREFIX @@ -12,8 +10,9 @@ async def http422_error_handler( _: Request, - exc: Union[RequestValidationError, ValidationError], + exc: Exception, ) -> JSONResponse: + assert isinstance(exc, RequestValidationError | ValidationError) # nosec return JSONResponse( content=jsonable_encoder({"errors": exc.errors()}), status_code=HTTP_422_UNPROCESSABLE_ENTITY, diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/files.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/files.py index 44fdfb96599..2234c17d3dc 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/files.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/files.py @@ -2,7 +2,7 @@ from typing import Annotated, Any from fastapi import APIRouter, Depends, Header, Request -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from servicelib.fastapi.requests_decorators import cancel_on_disconnect from starlette import status @@ -34,7 +34,9 @@ async def download_file( api_secret=x_datcore_api_secret, package_id=file_id, ) - return FileDownloadOut(link=parse_obj_as(AnyUrl, f"{presigned_download_link}")) + return FileDownloadOut( + link=TypeAdapter(AnyUrl).validate_python(f"{presigned_download_link}") + ) @router.delete( diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py index 8c4fb44e8e9..7d308fbd3b1 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py @@ -50,7 +50,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: for name in NOISY_LOGGERS: logging.getLogger(name).setLevel(quiet_level) - logger.debug("App settings:\n%s", settings.json(indent=2)) + logger.debug("App settings:\n%s", settings.model_dump_json(indent=2)) app = FastAPI( debug=settings.debug, diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py index 68e879807ab..784ee5bc793 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py @@ -1,7 +1,7 @@ from functools import cached_property from models_library.basic_types import BootModeEnum, LogLevel -from pydantic import Field, parse_obj_as, validator +from pydantic import AliasChoices, Field, TypeAdapter, field_validator from pydantic.networks import AnyUrl from settings_library.base import BaseCustomSettings from settings_library.tracing import TracingSettings @@ -11,7 +11,9 @@ class PennsieveSettings(BaseCustomSettings): PENNSIEVE_ENABLED: bool = True - PENNSIEVE_API_URL: AnyUrl = parse_obj_as(AnyUrl, "https://api.pennsieve.io") + PENNSIEVE_API_URL: AnyUrl = TypeAdapter(AnyUrl).validate_python( + "https://api.pennsieve.io" + ) PENNSIEVE_API_GENERAL_TIMEOUT: float = 20.0 PENNSIEVE_HEALTCHCHECK_TIMEOUT: float = 1.0 @@ -21,28 +23,31 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): SC_BOOT_MODE: BootModeEnum | None LOG_LEVEL: LogLevel = Field( - LogLevel.INFO.value, - env=[ + default=LogLevel.INFO.value, + validation_alias=AliasChoices( "DATCORE_ADAPTER_LOGLEVEL", "DATCORE_ADAPTER_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL", - ], + ), ) - PENNSIEVE: PennsieveSettings = Field(auto_default_from_env=True) + PENNSIEVE: PennsieveSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - False, - env=[ + default=False, + validation_alias=AliasChoices( "DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DATCORE_ADAPTER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True DATCORE_ADAPTER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + description="settings for opentelemetry tracing", + json_schema_extra={"auto_default_from_env": True}, ) @cached_property @@ -54,7 +59,7 @@ def debug(self) -> bool: BootModeEnum.LOCAL, ] - @validator("LOG_LEVEL", pre=True) + @field_validator("LOG_LEVEL", mode="before") @classmethod - def _validate_loglevel(cls, value) -> str: + def _validate_loglevel(cls, value: str) -> str: return cls.validate_log_level(value) diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/models/schemas/datasets.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/models/schemas/datasets.py index 4d5190c5512..5a10a88dfcb 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/models/schemas/datasets.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/models/schemas/datasets.py @@ -45,14 +45,16 @@ def from_pennsieve_package( return cls( dataset_id=package["content"]["datasetNodeId"], package_id=package["content"]["nodeId"], - id=package["content"]["id"], + id=f"{package['content']['id']}", name=pck_name, path=base_path / pck_name, type=package["content"]["packageType"], size=file_size, created_at=package["content"]["createdAt"], last_modified_at=package["content"]["updatedAt"], - data_type=DataType.FOLDER - if package["content"]["packageType"] == "Collection" - else DataType.FILE, + data_type=( + DataType.FOLDER + if package["content"]["packageType"] == "Collection" + else DataType.FILE + ), ) diff --git a/services/datcore-adapter/tests/unit/conftest.py b/services/datcore-adapter/tests/unit/conftest.py index e4fa08204f5..0130fda027a 100644 --- a/services/datcore-adapter/tests/unit/conftest.py +++ b/services/datcore-adapter/tests/unit/conftest.py @@ -3,8 +3,9 @@ # pylint:disable=redefined-outer-name import json +from collections.abc import AsyncIterator, Callable from pathlib import Path -from typing import Any, AsyncIterator, Callable +from typing import Any from uuid import uuid4 import faker @@ -14,7 +15,9 @@ import simcore_service_datcore_adapter from asgi_lifespan import LifespanManager from fastapi.applications import FastAPI +from models_library.basic_types import BootModeEnum from pytest_mock import MockFixture +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from simcore_service_datcore_adapter.modules.pennsieve import ( PennsieveAuthorizationHeaders, ) @@ -61,7 +64,9 @@ def pennsieve_mock_dataset_packages(mocks_dir: Path) -> dict[str, Any]: @pytest.fixture() -def minimal_app() -> FastAPI: +def minimal_app( + app_envs: None, +) -> FastAPI: from simcore_service_datcore_adapter.main import the_app return the_app @@ -76,7 +81,7 @@ def client(minimal_app: FastAPI) -> TestClient: @pytest.fixture def app_envs(monkeypatch: pytest.MonkeyPatch): # disable tracing as together with LifespanManager, it does not remove itself nicely - ... + return setenvs_from_dict(monkeypatch, {"SC_BOOT_MODE": BootModeEnum.DEBUG}) @pytest.fixture() @@ -87,7 +92,7 @@ async def initialized_app( yield minimal_app -@pytest.fixture(scope="function") +@pytest.fixture async def async_client(initialized_app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: async with httpx.AsyncClient( app=initialized_app, @@ -215,14 +220,13 @@ def pennsieve_api_headers( def pennsieve_random_fake_datasets( create_pennsieve_fake_dataset_id: Callable, ) -> dict[str, Any]: - datasets = { + return { "datasets": [ {"content": {"id": create_pennsieve_fake_dataset_id(), "name": fake.text()}} for _ in range(10) ], "totalCount": 20, } - return datasets @pytest.fixture diff --git a/services/datcore-adapter/tests/unit/test_route_datasets.py b/services/datcore-adapter/tests/unit/test_route_datasets.py index 2c9c98b20f4..2a0d7dc85d6 100644 --- a/services/datcore-adapter/tests/unit/test_route_datasets.py +++ b/services/datcore-adapter/tests/unit/test_route_datasets.py @@ -3,12 +3,10 @@ # pylint:disable=redefined-outer-name -from typing import Optional - import httpx import respx from fastapi_pagination import Page -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_service_datcore_adapter.models.schemas.datasets import ( DatasetMetaData, FileMetaData, @@ -18,7 +16,7 @@ async def test_list_datasets_entrypoint( async_client: httpx.AsyncClient, - pennsieve_subsystem_mock: Optional[respx.MockRouter], + pennsieve_subsystem_mock: respx.MockRouter | None, pennsieve_api_headers: dict[str, str], ): response = await async_client.get( @@ -29,7 +27,7 @@ async def test_list_datasets_entrypoint( assert response.status_code == status.HTTP_200_OK data = response.json() assert data - parse_obj_as(Page[DatasetMetaData], data) + TypeAdapter(Page[DatasetMetaData]).validate_python(data) async def test_list_dataset_files_legacy_entrypoint( @@ -47,7 +45,7 @@ async def test_list_dataset_files_legacy_entrypoint( assert response.status_code == status.HTTP_200_OK data = response.json() assert data - parse_obj_as(list[FileMetaData], data) + TypeAdapter(list[FileMetaData]).validate_python(data) async def test_list_dataset_top_level_files_entrypoint( @@ -65,7 +63,7 @@ async def test_list_dataset_top_level_files_entrypoint( assert response.status_code == status.HTTP_200_OK data = response.json() assert data - parse_obj_as(Page[FileMetaData], data) + TypeAdapter(Page[FileMetaData]).validate_python(data) async def test_list_dataset_collection_files_entrypoint( @@ -85,4 +83,4 @@ async def test_list_dataset_collection_files_entrypoint( assert response.status_code == status.HTTP_200_OK data = response.json() assert data - parse_obj_as(Page[FileMetaData], data) + TypeAdapter(Page[FileMetaData]).validate_python(data) diff --git a/services/datcore-adapter/tests/unit/test_route_files.py b/services/datcore-adapter/tests/unit/test_route_files.py index 840a7edf79f..cbaa09704fa 100644 --- a/services/datcore-adapter/tests/unit/test_route_files.py +++ b/services/datcore-adapter/tests/unit/test_route_files.py @@ -5,7 +5,7 @@ from unittest.mock import Mock import httpx -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_service_datcore_adapter.models.domains.files import FileDownloadOut from starlette import status @@ -23,7 +23,7 @@ async def test_download_file_entrypoint( assert response.status_code == status.HTTP_200_OK data = response.json() assert data - parse_obj_as(FileDownloadOut, data) + TypeAdapter(FileDownloadOut).validate_python(data) async def test_delete_file_entrypoint( diff --git a/services/datcore-adapter/tests/unit/test_route_health.py b/services/datcore-adapter/tests/unit/test_route_health.py index 7ab697612c4..3f0b1712f7e 100644 --- a/services/datcore-adapter/tests/unit/test_route_health.py +++ b/services/datcore-adapter/tests/unit/test_route_health.py @@ -31,7 +31,7 @@ async def test_check_subsystem_health(async_client: httpx.AsyncClient): assert pennsieve_health_route.called assert response.status_code == status.HTTP_200_OK - app_status = AppStatusCheck.parse_obj(response.json()) + app_status = AppStatusCheck.model_validate(response.json()) assert app_status assert app_status.app_name == "simcore-service-datcore-adapter" assert app_status.services == {"pennsieve": True} @@ -43,7 +43,7 @@ async def test_check_subsystem_health(async_client: httpx.AsyncClient): assert pennsieve_health_route.called assert response.status_code == status.HTTP_200_OK - app_status = AppStatusCheck.parse_obj(response.json()) + app_status = AppStatusCheck.model_validate(response.json()) assert app_status assert app_status.app_name == "simcore-service-datcore-adapter" assert app_status.services == {"pennsieve": False} From 7410516a87455b5e1c307c1e4bf4429843401e52 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:13:31 +0200 Subject: [PATCH 20/53] =?UTF-8?q?=E2=AC=86=EF=B8=8FMigration:=20Storage=20?= =?UTF-8?q?(#6599)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aws-library/src/aws_library/s3/_client.py | 11 +- packages/aws-library/tests/test_s3_client.py | 12 +- .../pydantic_networks_extension.py | 7 +- .../src/models_library/api_schemas_storage.py | 25 +-- .../utils/json_serialization.py | 2 + .../src/settings_library/node_ports.py | 11 +- .../src/settings_library/utils_cli.py | 4 +- .../src/simcore_sdk/node_data/data_manager.py | 11 +- .../node_ports_common/filemanager.py | 21 +- scripts/release/monitor_release/settings.py | 4 +- .../computational_sidecar/models.py | 3 +- .../services/resource_tracker_service_runs.py | 4 +- services/storage/requirements/_base.txt | 184 +++++++++++++++++- services/storage/requirements/_test.txt | 15 +- .../simcore_service_storage/application.py | 2 +- .../datcore_adapter/datcore_adapter.py | 11 +- .../datcore_adapter_settings.py | 10 +- .../db_file_meta_data.py | 30 +-- .../simcore_service_storage/db_projects.py | 2 +- .../src/simcore_service_storage/exceptions.py | 10 +- .../handlers_datasets.py | 2 +- .../simcore_service_storage/handlers_files.py | 31 +-- .../handlers_health.py | 17 +- .../handlers_simcore_s3.py | 4 +- .../src/simcore_service_storage/models.py | 94 +++++---- .../src/simcore_service_storage/s3_utils.py | 6 +- .../src/simcore_service_storage/settings.py | 62 ++++-- .../simcore_service_storage/simcore_s3_dsm.py | 66 ++++--- .../simcore_s3_dsm_utils.py | 15 +- .../src/simcore_service_storage/utils.py | 4 +- services/storage/tests/conftest.py | 36 ++-- .../storage/tests/fixtures/data_models.py | 29 ++- .../test__legacy_storage_sdk_compatibility.py | 2 +- services/storage/tests/unit/test_cli.py | 6 +- services/storage/tests/unit/test_dsm.py | 4 +- .../storage/tests/unit/test_dsm_dsmcleaner.py | 34 ++-- .../storage/tests/unit/test_dsm_soft_links.py | 2 +- .../tests/unit/test_handlers_datasets.py | 6 +- .../storage/tests/unit/test_handlers_files.py | 155 ++++++++------- .../unit/test_handlers_files_metadata.py | 20 +- .../tests/unit/test_handlers_health.py | 14 +- .../tests/unit/test_handlers_simcore_s3.py | 58 +++--- services/storage/tests/unit/test_models.py | 4 +- .../storage/tests/unit/test_simcore_s3_dsm.py | 12 +- services/storage/tests/unit/test_utils.py | 23 ++- .../storage/tests/unit/test_utils_handlers.py | 8 +- 46 files changed, 682 insertions(+), 411 deletions(-) diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py index acd9402fda3..38e7c0f9636 100644 --- a/packages/aws-library/src/aws_library/s3/_client.py +++ b/packages/aws-library/src/aws_library/s3/_client.py @@ -13,7 +13,6 @@ from boto3.s3.transfer import TransferConfig from botocore import exceptions as botocore_exc from botocore.client import Config -from common_library.pydantic_networks_extension import AnyUrlLegacy from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart from models_library.basic_types import SHA256Str from pydantic import AnyUrl, ByteSize, TypeAdapter @@ -255,7 +254,7 @@ async def create_single_presigned_download_link( bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int, - ) -> str: + ) -> AnyUrl: # NOTE: ensure the bucket/object exists, this will raise if not await self._client.head_bucket(Bucket=bucket) await self._client.head_object(Bucket=bucket, Key=object_key) @@ -264,12 +263,12 @@ async def create_single_presigned_download_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}" + return TypeAdapter(AnyUrl).validate_python(generated_link) @s3_exception_handler(_logger) async def create_single_presigned_upload_link( self, *, bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int - ) -> str: + ) -> AnyUrl: # NOTE: ensure the bucket/object exists, this will raise if not await self._client.head_bucket(Bucket=bucket) generated_link = await self._client.generate_presigned_url( @@ -277,7 +276,7 @@ async def create_single_presigned_upload_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}" + return TypeAdapter(AnyUrl).validate_python(generated_link) @s3_exception_handler(_logger) async def create_multipart_upload_links( @@ -474,6 +473,6 @@ def is_multipart(file_size: ByteSize) -> bool: @staticmethod def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl: - return TypeAdapter(AnyUrlLegacy).validate_python( + return TypeAdapter(AnyUrl).validate_python( f"s3://{bucket}/{urllib.parse.quote(object_key)}" ) diff --git a/packages/aws-library/tests/test_s3_client.py b/packages/aws-library/tests/test_s3_client.py index d07075aed30..5e5efc962a5 100644 --- a/packages/aws-library/tests/test_s3_client.py +++ b/packages/aws-library/tests/test_s3_client.py @@ -742,7 +742,7 @@ async def test_create_single_presigned_upload_link( create_file_of_size: Callable[[ByteSize], Path], default_expiration_time_seconds: int, upload_to_presigned_link: Callable[ - [Path, str, S3BucketName, S3ObjectKey], Awaitable[None] + [Path, AnyUrl, S3BucketName, S3ObjectKey], Awaitable[None] ], ): file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1Mib")) @@ -1289,12 +1289,16 @@ def test_is_multipart(file_size: ByteSize, expected_multipart: bool): ( "some-bucket", "an/object/separate/by/slashes", - "s3://some-bucket/an/object/separate/by/slashes", + TypeAdapter(AnyUrl).validate_python( + "s3://some-bucket/an/object/separate/by/slashes" + ), ), ( "some-bucket", "an/object/separate/by/slashes-?/3#$", - r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24", + TypeAdapter(AnyUrl).validate_python( + r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24" + ), ), ], ) @@ -1302,7 +1306,7 @@ def test_compute_s3_url( bucket: S3BucketName, object_key: S3ObjectKey, expected_s3_url: AnyUrl ): assert ( - str(SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key)) + SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key) == expected_s3_url ) diff --git a/packages/common-library/src/common_library/pydantic_networks_extension.py b/packages/common-library/src/common_library/pydantic_networks_extension.py index 79c5da906b1..1e85a741760 100644 --- a/packages/common-library/src/common_library/pydantic_networks_extension.py +++ b/packages/common-library/src/common_library/pydantic_networks_extension.py @@ -1,6 +1,6 @@ from typing import Annotated, TypeAlias -from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, HttpUrl +from pydantic import AfterValidator, AnyHttpUrl, HttpUrl from pydantic_core import Url @@ -8,11 +8,6 @@ def _strip_last_slash(url: Url) -> str: return f"{url}".rstrip("/") -AnyUrlLegacy: TypeAlias = Annotated[ - AnyUrl, - AfterValidator(_strip_last_slash), -] - AnyHttpUrlLegacy: TypeAlias = Annotated[ AnyHttpUrl, AfterValidator(_strip_last_slash), diff --git a/packages/models-library/src/models_library/api_schemas_storage.py b/packages/models-library/src/models_library/api_schemas_storage.py index bd0185a9173..54c5e28d43d 100644 --- a/packages/models-library/src/models_library/api_schemas_storage.py +++ b/packages/models-library/src/models_library/api_schemas_storage.py @@ -8,7 +8,7 @@ from datetime import datetime from enum import Enum -from typing import Annotated, Any, TypeAlias +from typing import Annotated, Any, Literal, Self, TypeAlias from uuid import UUID from pydantic import ( @@ -107,6 +107,10 @@ class DatasetMetaDataGet(BaseModel): ) +UNDEFINED_SIZE_TYPE: TypeAlias = Literal[-1] +UNDEFINED_SIZE: UNDEFINED_SIZE_TYPE = -1 + + # /locations/{location_id}/files/metadata: # /locations/{location_id}/files/{file_id}/metadata: class FileMetaDataGet(BaseModel): @@ -130,8 +134,8 @@ class FileMetaDataGet(BaseModel): ) created_at: datetime last_modified: datetime - file_size: ByteSize | int = Field( - default=-1, description="File size in bytes (-1 means invalid)" + file_size: UNDEFINED_SIZE_TYPE | ByteSize = Field( + default=UNDEFINED_SIZE, description="File size in bytes (-1 means invalid)" ) entity_tag: ETag | None = Field( default=None, @@ -149,7 +153,7 @@ class FileMetaDataGet(BaseModel): ) model_config = ConfigDict( - extra="forbid", + extra="ignore", from_attributes=True, json_schema_extra={ "examples": [ @@ -312,19 +316,18 @@ class FoldersBody(BaseModel): nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict) @model_validator(mode="after") - @classmethod - def ensure_consistent_entries(cls, values): - source_node_keys = (NodeID(n) for n in values["source"].get("workbench", {})) - if set(source_node_keys) != set(values["nodes_map"].keys()): + def ensure_consistent_entries(self) -> Self: + source_node_keys = (NodeID(n) for n in self.source.get("workbench", {})) + if set(source_node_keys) != set(self.nodes_map.keys()): msg = "source project nodes do not fit with nodes_map entries" raise ValueError(msg) destination_node_keys = ( - NodeID(n) for n in values["destination"].get("workbench", {}) + NodeID(n) for n in self.destination.get("workbench", {}) ) - if set(destination_node_keys) != set(values["nodes_map"].values()): + if set(destination_node_keys) != set(self.nodes_map.values()): msg = "destination project nodes do not fit with nodes_map values" raise ValueError(msg) - return values + return self class SoftCopyBody(BaseModel): diff --git a/packages/models-library/src/models_library/utils/json_serialization.py b/packages/models-library/src/models_library/utils/json_serialization.py index 9bd0abd5ef6..ef11a2640cd 100644 --- a/packages/models-library/src/models_library/utils/json_serialization.py +++ b/packages/models-library/src/models_library/utils/json_serialization.py @@ -24,6 +24,7 @@ import orjson from pydantic import NameEmail, SecretBytes, SecretStr +from pydantic_core import Url from pydantic_extra_types.color import Color @@ -84,6 +85,7 @@ def decimal_encoder(dec_value: Decimal) -> int | float: SecretBytes: str, SecretStr: str, set: list, + Url: str, UUID: str, } diff --git a/packages/settings-library/src/settings_library/node_ports.py b/packages/settings-library/src/settings_library/node_ports.py index 522fcdd0991..8498c882e4d 100644 --- a/packages/settings-library/src/settings_library/node_ports.py +++ b/packages/settings-library/src/settings_library/node_ports.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import Final +from typing import Final, Self from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator @@ -22,14 +22,13 @@ def auth_required(self) -> bool: return self.STORAGE_USERNAME is not None and self.STORAGE_PASSWORD is not None @model_validator(mode="after") - @classmethod - def _validate_auth_fields(cls, values): - username = values.STORAGE_USERNAME - password = values.STORAGE_PASSWORD + def _validate_auth_fields(self) -> Self: + username = self.STORAGE_USERNAME + password = self.STORAGE_PASSWORD if (username is None) != (password is None): msg = f"Both {username=} and {password=} must be either set or unset!" raise ValueError(msg) - return values + return self class NodePortsSettings(BaseCustomSettings): diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index 0311ed28d7b..e3addbc6130 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -2,6 +2,7 @@ import logging import os from collections.abc import Callable +from enum import Enum from pprint import pformat from typing import Any @@ -64,7 +65,8 @@ def print_as_envfile( if verbose and field.description: typer.echo(f"# {field.description}") - + if isinstance(value, Enum): + value = value.value typer.echo(f"{name}={value}") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py index 7579c3eeb0c..b288a295db9 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, StorageFileID from models_library.users import UserID -from pydantic import ByteSize, TypeAdapter +from pydantic import TypeAdapter from servicelib.archiving_utils import unarchive_dir from servicelib.logging_utils import log_context from servicelib.progress_bar import ProgressBarData @@ -184,15 +184,6 @@ async def _delete_legacy_archive( ) -async def get_remote_size( - *, user_id: UserID, project_id: ProjectID, node_uuid: NodeID, source_path: Path -) -> ByteSize: - s3_object = __create_s3_object_key(project_id, node_uuid, source_path) - return await filemanager.get_path_size( - user_id=user_id, store_id=SIMCORE_LOCATION, s3_object=s3_object - ) - - async def push( user_id: UserID, project_id: ProjectID, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py index f3e2587fab7..b7180877037 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py @@ -15,7 +15,7 @@ from models_library.basic_types import IDStr, SHA256Str from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, TypeAdapter, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.file_utils import create_sha256_checksum from servicelib.progress_bar import ProgressBarData from settings_library.aws_s3_cli import AwsS3CliSettings @@ -189,7 +189,9 @@ async def download_path_from_s3( aws_s3_cli_settings, progress_bar, local_directory_path=local_path, - download_s3_link=TypeAdapter(AnyUrl).validate_python(f"{download_link}"), + download_s3_link=TypeAdapter(AnyUrl).validate_python( + f"{download_link}" + ), ) elif r_clone_settings: await r_clone.sync_s3_to_local( @@ -572,21 +574,6 @@ async def get_file_metadata( ) -async def get_path_size( - user_id: UserID, - store_id: LocationID, - s3_object: StorageFileID, - client_session: ClientSession | None = None, -) -> ByteSize: - file_metadata: FileMetaDataGet = await _get_file_meta_data( - user_id=user_id, - store_id=store_id, - s3_object=s3_object, - client_session=client_session, - ) - return ByteSize(file_metadata.file_size) - - async def delete_file( user_id: UserID, store_id: LocationID, diff --git a/scripts/release/monitor_release/settings.py b/scripts/release/monitor_release/settings.py index ba4f9c7e1d2..947732f2ceb 100644 --- a/scripts/release/monitor_release/settings.py +++ b/scripts/release/monitor_release/settings.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Final +from typing import Final, Self from dotenv import load_dotenv from pydantic import BaseModel, Field, HttpUrl, TypeAdapter, model_validator @@ -47,7 +47,7 @@ class ReleaseSettings(BaseSettings): model_config = SettingsConfigDict(extra="ignore") @model_validator(mode="after") - def deduce_portainer_url(self): + def deduce_portainer_url(self) -> Self: self.portainer_url = TypeAdapter(HttpUrl).validate_python( f"https://{self.PORTAINER_DOMAIN}" ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py index ee270aeb888..c505329af50 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py @@ -1,4 +1,5 @@ import re +from typing import Self from models_library.basic_regex import SIMPLE_VERSION_RE from models_library.services import ServiceMetaDataPublished @@ -49,7 +50,7 @@ class ContainerHostConfig(BaseModel): ) @model_validator(mode="after") - def ensure_memory_swap_is_not_unlimited(self) -> "ContainerHostConfig": + def ensure_memory_swap_is_not_unlimited(self) -> Self: if self.memory_swap is None: self.memory_swap = self.memory diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index a02f4ecd646..e5145f8f6bd 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -177,12 +177,12 @@ async def export_service_runs( ) # Create presigned S3 link - generated_url: str = await s3_client.create_single_presigned_download_link( + generated_url = await s3_client.create_single_presigned_download_link( bucket=s3_bucket_name, object_key=s3_object_key, expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC, ) - return generated_url + return f"{generated_url}" async def get_osparc_credits_aggregated_usages_page( diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index d5a191d6e58..c2766e7dc10 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -29,17 +29,30 @@ aiofiles==23.2.1 # aioboto3 aiohttp==3.9.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_aiohttp.in @@ -61,6 +74,8 @@ aiosignal==1.3.1 # via aiohttp alembic==1.13.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -97,17 +112,30 @@ botocore-stubs==1.34.69 # via types-aiobotocore certifi==2024.2.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -155,17 +183,30 @@ isodate==0.6.1 # via openapi-core jinja2==3.1.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiohttp-swagger @@ -195,17 +236,30 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator mako==1.3.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -302,17 +356,30 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in @@ -341,35 +408,80 @@ psycopg2-binary==2.9.9 # via # aiopg # sqlalchemy -pydantic==1.10.14 +pydantic==2.9.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.6.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -380,19 +492,34 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -401,17 +528,30 @@ pyyaml==6.0.1 # jsonschema-path redis==5.0.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -461,17 +601,30 @@ sniffio==1.3.1 # via anyio sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -519,6 +672,7 @@ typing-extensions==4.10.0 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 @@ -526,33 +680,59 @@ typing-extensions==4.10.0 # types-aiobotocore-ssm ujson==5.9.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiohttp-swagger urllib3==2.0.7 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index f0132fe4c7c..2915015ad89 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -11,6 +11,10 @@ aiosignal==1.3.1 # via # -c requirements/_base.txt # aiohttp +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto attrs==23.2.0 @@ -193,11 +197,15 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.4 # via moto pytest==8.3.3 @@ -238,7 +246,9 @@ python-dateutil==2.9.0.post0 # pandas # simcore-service-storage-sdk python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pytz==2024.2 # via pandas pyyaml==6.0.1 @@ -319,6 +329,7 @@ typing-extensions==4.10.0 # cfn-lint # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs tzdata==2024.2 # via pandas diff --git a/services/storage/src/simcore_service_storage/application.py b/services/storage/src/simcore_service_storage/application.py index 7ad35793bd7..77b207a513a 100644 --- a/services/storage/src/simcore_service_storage/application.py +++ b/services/storage/src/simcore_service_storage/application.py @@ -44,7 +44,7 @@ def create(settings: Settings) -> web.Application: _logger.debug( "Initializing app with settings:\n%s", - settings.json(indent=2, sort_keys=True), + settings.model_dump_json(indent=2), ) app = create_safe_application(None) diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py index 5e1f4cff09c..8fb9a162c52 100644 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py +++ b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py @@ -1,4 +1,3 @@ -import asyncio import logging from collections.abc import Callable from math import ceil @@ -9,7 +8,7 @@ from aiohttp.client import ClientSession from models_library.api_schemas_storage import DatCoreDatasetName from models_library.users import UserID -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from servicelib.aiohttp.application_keys import APP_CONFIG_KEY from servicelib.aiohttp.client_session import get_client_session from servicelib.utils import logged_gather @@ -73,7 +72,7 @@ async def _request( except aiohttp.ClientResponseError as exc: raise _DatcoreAdapterResponseError(status=exc.status, reason=f"{exc}") from exc - except asyncio.TimeoutError as exc: + except TimeoutError as exc: msg = f"datcore-adapter server timed-out: {exc}" raise DatcoreAdapterTimeoutError(msg) from exc @@ -122,7 +121,7 @@ async def check_service_health(app: web.Application) -> bool: session: ClientSession = get_client_session(app) try: await session.get(url, raise_for_status=True) - except (asyncio.TimeoutError, aiohttp.ClientError): + except (TimeoutError, aiohttp.ClientError): return False return True @@ -187,7 +186,7 @@ async def list_all_files_metadatas_in_dataset( ), ) return [ - FileMetaData.construct( + FileMetaData.model_construct( file_uuid=d["path"], location_id=DATCORE_ID, location=DATCORE_STR, @@ -229,7 +228,7 @@ async def get_file_download_presigned_link( dict[str, Any], await _request(app, api_key, api_secret, "GET", f"/files/{file_id}"), ) - url: AnyUrl = parse_obj_as(AnyUrl, file_download_data["link"]) + url: AnyUrl = TypeAdapter(AnyUrl).validate_python(file_download_data["link"]) return url diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py index c1322590eaa..71905e42381 100644 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py +++ b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py @@ -1,24 +1,24 @@ from functools import cached_property from models_library.basic_types import PortInt, VersionTag -from pydantic import AnyHttpUrl, Field +from pydantic import AnyHttpUrl, Field, TypeAdapter from settings_library.base import BaseCustomSettings class DatcoreAdapterSettings(BaseCustomSettings): DATCORE_ADAPTER_ENABLED: bool = True DATCORE_ADAPTER_HOST: str = "datcore-adapter" - DATCORE_ADAPTER_PORT: PortInt = PortInt(8000) + DATCORE_ADAPTER_PORT: PortInt = TypeAdapter(PortInt).validate_python(8000) DATCORE_ADAPTER_VTAG: VersionTag = Field( "v0", description="Datcore-adapter service API's version tag" ) @cached_property def endpoint(self) -> str: - endpoint: str = AnyHttpUrl.build( + endpoint = AnyHttpUrl.build( scheme="http", host=self.DATCORE_ADAPTER_HOST, - port=f"{self.DATCORE_ADAPTER_PORT}", + port=self.DATCORE_ADAPTER_PORT, path=f"/{self.DATCORE_ADAPTER_VTAG}", ) - return endpoint + return f"{endpoint}" diff --git a/services/storage/src/simcore_service_storage/db_file_meta_data.py b/services/storage/src/simcore_service_storage/db_file_meta_data.py index 21c519e5483..b742449ee00 100644 --- a/services/storage/src/simcore_service_storage/db_file_meta_data.py +++ b/services/storage/src/simcore_service_storage/db_file_meta_data.py @@ -32,7 +32,9 @@ async def upsert( ) -> FileMetaDataAtDB: # NOTE: upsert file_meta_data, if the file already exists, we update the whole row # so we get the correct time stamps - fmd_db = FileMetaDataAtDB.from_orm(fmd) if isinstance(fmd, FileMetaData) else fmd + fmd_db = ( + FileMetaDataAtDB.model_validate(fmd) if isinstance(fmd, FileMetaData) else fmd + ) insert_statement = pg_insert(file_meta_data).values(**jsonable_encoder(fmd_db)) on_update_statement = insert_statement.on_conflict_do_update( index_elements=[file_meta_data.c.file_id], set_=jsonable_encoder(fmd_db) @@ -40,11 +42,11 @@ async def upsert( result = await conn.execute(on_update_statement) row = await result.first() assert row # nosec - return FileMetaDataAtDB.from_orm(row) + return FileMetaDataAtDB.model_validate(row) async def insert(conn: SAConnection, fmd: FileMetaData) -> FileMetaDataAtDB: - fmd_db = FileMetaDataAtDB.from_orm(fmd) + fmd_db = FileMetaDataAtDB.model_validate(fmd) result = await conn.execute( file_meta_data.insert() .values(jsonable_encoder(fmd_db)) @@ -52,7 +54,7 @@ async def insert(conn: SAConnection, fmd: FileMetaData) -> FileMetaDataAtDB: ) row = await result.first() assert row # nosec - return FileMetaDataAtDB.from_orm(row) + return FileMetaDataAtDB.model_validate(row) async def get(conn: SAConnection, file_id: SimcoreS3FileID) -> FileMetaDataAtDB: @@ -60,7 +62,7 @@ async def get(conn: SAConnection, file_id: SimcoreS3FileID) -> FileMetaDataAtDB: query=sa.select(file_meta_data).where(file_meta_data.c.file_id == file_id) ) if row := await result.first(): - return FileMetaDataAtDB.from_orm(row) + return FileMetaDataAtDB.model_validate(row) raise FileMetaDataNotFoundError(file_id=file_id) @@ -83,9 +85,11 @@ def _list_filter_with_partial_file_id_stmt( conditions.append( sa.or_( file_meta_data.c.user_id == f"{user_id}", - file_meta_data.c.project_id.in_(f"{_}" for _ in project_ids) - if project_ids - else False, + ( + file_meta_data.c.project_id.in_(f"{_}" for _ in project_ids) + if project_ids + else False + ), ) ) @@ -130,7 +134,9 @@ async def list_filter_with_partial_file_id( offset=offset, ) - return [FileMetaDataAtDB.from_orm(row) async for row in await conn.execute(stmt)] + return [ + FileMetaDataAtDB.model_validate(row) async for row in await conn.execute(stmt) + ] async def list_fmds( @@ -158,7 +164,9 @@ async def list_fmds( ) ) - return [FileMetaDataAtDB.from_orm(row) async for row in await conn.execute(stmt)] + return [ + FileMetaDataAtDB.model_validate(row) async for row in await conn.execute(stmt) + ] async def total(conn: SAConnection) -> int: @@ -177,7 +185,7 @@ async def list_valid_uploads( file_meta_data.c.upload_expires_at == None # lgtm [py/test-equals-none] ) ): - fmd_at_db = FileMetaDataAtDB.from_orm(row) + fmd_at_db = FileMetaDataAtDB.model_validate(row) yield fmd_at_db diff --git a/services/storage/src/simcore_service_storage/db_projects.py b/services/storage/src/simcore_service_storage/db_projects.py index a8bf48db80e..dc680c491ee 100644 --- a/services/storage/src/simcore_service_storage/db_projects.py +++ b/services/storage/src/simcore_service_storage/db_projects.py @@ -22,7 +22,7 @@ async def list_valid_projects_in( ) ): with suppress(ValidationError): - yield ProjectAtDB.from_orm(row) + yield ProjectAtDB.model_validate(row) async def project_exists( diff --git a/services/storage/src/simcore_service_storage/exceptions.py b/services/storage/src/simcore_service_storage/exceptions.py index d41c6d16d75..937a3afdd06 100644 --- a/services/storage/src/simcore_service_storage/exceptions.py +++ b/services/storage/src/simcore_service_storage/exceptions.py @@ -1,37 +1,31 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class StorageRuntimeError(PydanticErrorMixin, RuntimeError): +class StorageRuntimeError(OsparcErrorMixin, RuntimeError): ... class DatabaseAccessError(StorageRuntimeError): - code = "database.access_error" msg_template: str = "Unexpected error while accessing database backend" class FileMetaDataNotFoundError(DatabaseAccessError): - code = "filemetadata.not_found_error" msg_template: str = "The file meta data for {file_id} was not found" class FileAccessRightError(DatabaseAccessError): - code = "file.access_right_error" msg_template: str = "Insufficient access rights to {access_right} data {file_id}" class ProjectAccessRightError(DatabaseAccessError): - code = "file.access_right_error" msg_template: str = ( "Insufficient access rights to {access_right} project {project_id}" ) class ProjectNotFoundError(DatabaseAccessError): - code = "project.not_found_error" msg_template: str = "Project {project_id} was not found" class LinkAlreadyExistsError(DatabaseAccessError): - code = "link.already_exists_error" msg_template: str = "The link {file_id} already exists" diff --git a/services/storage/src/simcore_service_storage/handlers_datasets.py b/services/storage/src/simcore_service_storage/handlers_datasets.py index 4041ba7e509..69b345247e2 100644 --- a/services/storage/src/simcore_service_storage/handlers_datasets.py +++ b/services/storage/src/simcore_service_storage/handlers_datasets.py @@ -69,6 +69,6 @@ async def get_files_metadata_dataset(request: web.Request) -> web.Response: expand_dirs=query_params.expand_dirs, ) return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet.from_orm(d)) for d in data]}, + {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, dumps=json_dumps, ) diff --git a/services/storage/src/simcore_service_storage/handlers_files.py b/services/storage/src/simcore_service_storage/handlers_files.py index 87cb60d5829..b7ad6ebb007 100644 --- a/services/storage/src/simcore_service_storage/handlers_files.py +++ b/services/storage/src/simcore_service_storage/handlers_files.py @@ -18,7 +18,7 @@ ) from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.utils.json_serialization import json_dumps -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -72,7 +72,7 @@ async def get_files_metadata(request: web.Request) -> web.Response: project_id=query_params.project_id, ) return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet.from_orm(d)) for d in data]}, + {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, dumps=json_dumps, ) @@ -87,7 +87,7 @@ async def get_file_metadata(request: web.Request) -> web.Response: ) path_params = parse_request_path_parameters_as(FilePathParams, request) log.debug( - "received call to get_files_metadata_dataset with %s", + "received call to get_file_metadata_dataset with %s", f"{path_params=}, {query_params=}", ) @@ -134,7 +134,10 @@ async def get_file_metadata(request: web.Request) -> web.Response: dumps=json_dumps, ) - return jsonable_encoder(FileMetaDataGet.from_orm(data)) # type: ignore[no-any-return] # middleware takes care of enveloping + return web.json_response( + {"data": jsonable_encoder(FileMetaDataGet(**data.model_dump()))}, + dumps=json_dumps, + ) @routes.get( @@ -177,7 +180,8 @@ async def upload_file(request: web.Request) -> web.Response: - client calls complete_upload handle which will reconstruct the file on S3 backend - client waits for completion to finish and then the file is accessible on S3 backend - Use-case v1: if query.file_size is not defined, returns a PresignedLink model (backward compatibility) + + Use-case v1: query.file_size is not defined, returns a PresignedLink model (backward compatibility) Use-case v1.1: if query.link_type=presigned or None, returns a presigned link (limited to a single 5GB file) Use-case v1.2: if query.link_type=s3, returns a s3 direct link (limited to a single 5TB file) @@ -205,10 +209,12 @@ async def upload_file(request: web.Request) -> web.Response: is_directory=query_params.is_directory, sha256_checksum=query_params.sha256_checksum, ) - if query_params.file_size is None and not query_params.is_directory: + if query_params.is_v1_upload: # return v1 response assert len(links.urls) == 1 # nosec - response = {"data": {"link": jsonable_encoder(links.urls[0], by_alias=True)}} + response = { + "data": {"link": jsonable_encoder(f"{links.urls[0]}", by_alias=True)} + } log.debug("Returning v1 response: %s", response) return web.json_response(response, dumps=json_dumps) @@ -233,11 +239,8 @@ async def upload_file(request: web.Request) -> web.Response: chunk_size=links.chunk_size, urls=links.urls, links=FileUploadLinks( - abort_upload=parse_obj_as(AnyUrl, f"{abort_url}"), - complete_upload=parse_obj_as( - AnyUrl, - f"{complete_url}", - ), + abort_upload=TypeAdapter(AnyUrl).validate_python(f"{abort_url}"), + complete_upload=TypeAdapter(AnyUrl).validate_python(f"{complete_url}"), ), ) log.debug("returning v2 response: %s", v2_response) @@ -306,7 +309,7 @@ async def complete_upload_file(request: web.Request) -> web.Response: complete_task_state_url = f"{request.url.scheme}://{ip_addr}:{port}{route}" response = FileUploadCompleteResponse( links=FileUploadCompleteLinks( - state=parse_obj_as(AnyUrl, complete_task_state_url) + state=TypeAdapter(AnyUrl).validate_python(complete_task_state_url) ) ) return web.json_response( @@ -408,4 +411,4 @@ async def copy_as_soft_link(request: web.Request): query_params.user_id, path_params.file_id, body.link_id ) - return jsonable_encoder(FileMetaDataGet.from_orm(file_link)) + return jsonable_encoder(FileMetaDataGet(**file_link.model_dump())) diff --git a/services/storage/src/simcore_service_storage/handlers_health.py b/services/storage/src/simcore_service_storage/handlers_health.py index eb94feb9bb4..5be7fda4e23 100644 --- a/services/storage/src/simcore_service_storage/handlers_health.py +++ b/services/storage/src/simcore_service_storage/handlers_health.py @@ -31,13 +31,12 @@ async def get_health(request: web.Request) -> web.Response: assert request # nosec return web.json_response( { - "data": HealthCheck.parse_obj( - { - "name": PROJECT_NAME, - "version": f"{VERSION}", - "api_version": API_VERSION, - } - ).dict(**RESPONSE_MODEL_POLICY) + "data": HealthCheck( + name=PROJECT_NAME, + version=f"{VERSION}", + api_version=API_VERSION, + status=None, + ).model_dump(**RESPONSE_MODEL_POLICY) }, dumps=json_dumps, ) @@ -69,7 +68,7 @@ async def get_status(request: web.Request) -> web.Response: "connected" if await is_pg_responsive(request.app) else "failed" ) - status = AppStatusCheck.parse_obj( + status = AppStatusCheck.model_validate( { "app_name": PROJECT_NAME, "version": f"{VERSION}", @@ -84,5 +83,5 @@ async def get_status(request: web.Request) -> web.Response: ) return web.json_response( - {"data": status.dict(exclude_unset=True)}, dumps=json_dumps + {"data": status.model_dump(exclude_unset=True)}, dumps=json_dumps ) diff --git a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py index 0f8e52fa7fc..47dd35bc6bc 100644 --- a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py @@ -53,7 +53,7 @@ async def get_or_create_temporary_s3_access(request: web.Request) -> web.Respons s3_settings: S3Settings = await sts.get_or_create_temporary_token_for_user( request.app, query_params.user_id ) - return web.json_response({"data": s3_settings.dict()}, dumps=json_dumps) + return web.json_response({"data": s3_settings.model_dump()}, dumps=json_dumps) async def _copy_folders_from_project( @@ -160,6 +160,6 @@ async def search_files(request: web.Request) -> web.Response: ) return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet.from_orm(d)) for d in data]}, + {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, dumps=json_dumps, ) diff --git a/services/storage/src/simcore_service_storage/models.py b/services/storage/src/simcore_service_storage/models.py index d05099edd06..2e21fc2483c 100644 --- a/services/storage/src/simcore_service_storage/models.py +++ b/services/storage/src/simcore_service_storage/models.py @@ -1,11 +1,14 @@ import datetime import urllib.parse from dataclasses import dataclass -from typing import Final, Literal, NamedTuple +from typing import Any, Literal, NamedTuple from uuid import UUID +import arrow from aws_library.s3 import UploadID from models_library.api_schemas_storage import ( + UNDEFINED_SIZE, + UNDEFINED_SIZE_TYPE, DatasetMetaDataGet, ETag, FileMetaDataGet, @@ -31,16 +34,14 @@ AnyUrl, BaseModel, ByteSize, - Extra, + ConfigDict, Field, - parse_obj_as, - root_validator, - validate_arguments, - validator, + TypeAdapter, + field_validator, + model_validator, + validate_call, ) -UNDEFINED_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, -1) - class DatasetMetaData(DatasetMetaDataGet): ... @@ -64,7 +65,7 @@ class FileMetaDataAtDB(BaseModel): user_id: UserID created_at: datetime.datetime file_id: SimcoreS3FileID - file_size: ByteSize + file_size: UNDEFINED_SIZE_TYPE | ByteSize last_modified: datetime.datetime entity_tag: ETag | None = None is_soft_link: bool @@ -73,9 +74,7 @@ class FileMetaDataAtDB(BaseModel): is_directory: bool sha256_checksum: SHA256Str | None = None - class Config: - orm_mode = True - extra = Extra.forbid + model_config = ConfigDict(from_attributes=True, extra="forbid") class FileMetaData(FileMetaDataGet): @@ -91,7 +90,7 @@ class FileMetaData(FileMetaDataGet): sha256_checksum: SHA256Str | None @classmethod - @validate_arguments + @validate_call def from_simcore_node( cls, user_id: UserID, @@ -103,7 +102,7 @@ def from_simcore_node( **file_meta_data_kwargs, ): parts = file_id.split("/") - now = datetime.datetime.utcnow() + now = arrow.utcnow().datetime fmd_kwargs = { "file_uuid": file_id, "location_id": location_id, @@ -113,9 +112,15 @@ def from_simcore_node( "file_name": parts[-1], "user_id": user_id, "project_id": ( - parse_obj_as(ProjectID, parts[0]) if is_uuid(parts[0]) else None + TypeAdapter(ProjectID).validate_python(parts[0]) + if is_uuid(parts[0]) + else None + ), + "node_id": ( + TypeAdapter(NodeID).validate_python(parts[1]) + if is_uuid(parts[1]) + else None ), - "node_id": parse_obj_as(NodeID, parts[1]) if is_uuid(parts[1]) else None, "file_id": file_id, "created_at": now, "last_modified": now, @@ -128,7 +133,7 @@ def from_simcore_node( "is_directory": False, } fmd_kwargs.update(**file_meta_data_kwargs) - return cls.parse_obj(fmd_kwargs) + return cls.model_validate(fmd_kwargs) @dataclass @@ -139,10 +144,7 @@ class UploadLinks: class StorageQueryParamsBase(BaseModel): user_id: UserID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict(populate_by_name=True, extra="forbid") class FilesMetadataDatasetQueryParams(StorageQueryParamsBase): @@ -163,9 +165,9 @@ class SyncMetadataQueryParams(BaseModel): class FileDownloadQueryParams(StorageQueryParamsBase): link_type: LinkType = LinkType.PRESIGNED - @validator("link_type", pre=True) + @field_validator("link_type", mode="before") @classmethod - def convert_from_lower_case(cls, v): + def convert_from_lower_case(cls, v: str) -> str: if v is not None: return f"{v}".upper() return v @@ -173,26 +175,39 @@ def convert_from_lower_case(cls, v): class FileUploadQueryParams(StorageQueryParamsBase): link_type: LinkType = LinkType.PRESIGNED - file_size: ByteSize | None + file_size: ByteSize | None = None # NOTE: in old legacy services this might happen is_directory: bool = False sha256_checksum: SHA256Str | None = None - @validator("link_type", pre=True) + @field_validator("link_type", mode="before") @classmethod - def convert_from_lower_case(cls, v): + def convert_from_lower_case(cls, v: str) -> str: if v is not None: return f"{v}".upper() return v - @root_validator() + @model_validator(mode="before") @classmethod - def when_directory_force_link_type_and_file_size(cls, values): - if values["is_directory"] is True: + def when_directory_force_link_type_and_file_size(cls, data: Any) -> Any: + assert isinstance(data, dict) + + if bool(data.get("is_directory", False)) is True: # sets directory size by default to undefined - values["file_size"] = UNDEFINED_SIZE + if int(data.get("file_size", -1)) < 0: + data["file_size"] = None # only 1 link will be returned manged by the uploader - values["link_type"] = LinkType.S3 - return values + data["link_type"] = LinkType.S3.value + return data + + @property + def is_v1_upload(self) -> bool: + """This returns True if the query params are missing the file_size query parameter, which was the case in the legacy services that have an old version of simcore-sdk + v1 rationale: + - client calls this handler, which returns a single link (either direct S3 or presigned) to the S3 backend + - client uploads the file + - storage relies on lazy update to find if the file is finished uploaded (when client calls get_file_meta_data, or if the dsm_cleaner goes over it after the upload time is expired) + """ + return self.file_size is None and self.is_directory is False class DeleteFolderQueryParams(StorageQueryParamsBase): @@ -211,17 +226,14 @@ class SearchFilesQueryParams(StorageQueryParamsBase): ) offset: int = Field(default=0, ge=0, description="Page offset") - _empty_is_none = validator("startswith", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("startswith", mode="before")( empty_str_to_none_pre_validator ) class LocationPathParams(BaseModel): location_id: LocationID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict(populate_by_name=True, extra="forbid") class FilesMetadataDatasetPathParams(LocationPathParams): @@ -231,9 +243,9 @@ class FilesMetadataDatasetPathParams(LocationPathParams): class FilePathParams(LocationPathParams): file_id: StorageFileID - @validator("file_id", pre=True) + @field_validator("file_id", mode="before") @classmethod - def unquote(cls, v): + def unquote(cls, v: str) -> str: if v is not None: return urllib.parse.unquote(f"{v}") return v @@ -250,9 +262,9 @@ class SimcoreS3FoldersParams(BaseModel): class CopyAsSoftLinkParams(BaseModel): file_id: StorageFileID - @validator("file_id", pre=True) + @field_validator("file_id", mode="before") @classmethod - def unquote(cls, v): + def unquote(cls, v: str) -> str: if v is not None: return urllib.parse.unquote(f"{v}") return v diff --git a/services/storage/src/simcore_service_storage/s3_utils.py b/services/storage/src/simcore_service_storage/s3_utils.py index 641bebf4e64..f40d33d531f 100644 --- a/services/storage/src/simcore_service_storage/s3_utils.py +++ b/services/storage/src/simcore_service_storage/s3_utils.py @@ -2,7 +2,7 @@ from collections import defaultdict from dataclasses import dataclass, field -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.aiohttp.long_running_tasks.server import ( ProgressMessage, ProgressPercent, @@ -55,7 +55,7 @@ def finalize_transfer(self) -> None: def copy_transfer_cb(self, total_bytes_copied: int, *, file_name: str) -> None: _logger.debug( "Copied %s of %s", - parse_obj_as(ByteSize, total_bytes_copied).human_readable(), + TypeAdapter(ByteSize).validate_python(total_bytes_copied).human_readable(), file_name, ) self._file_total_bytes_copied[file_name] = total_bytes_copied @@ -66,7 +66,7 @@ def copy_transfer_cb(self, total_bytes_copied: int, *, file_name: str) -> None: def upload_transfer_cb(self, bytes_transferred: int, *, file_name: str) -> None: _logger.debug( "Uploaded %s of %s", - parse_obj_as(ByteSize, bytes_transferred).human_readable(), + TypeAdapter(ByteSize).validate_python(bytes_transferred).human_readable(), file_name, ) self._file_total_bytes_copied[file_name] += bytes_transferred diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py index dc01d96f99e..f38bc170352 100644 --- a/services/storage/src/simcore_service_storage/settings.py +++ b/services/storage/src/simcore_service_storage/settings.py @@ -1,6 +1,13 @@ -from typing import Any - -from pydantic import Field, PositiveInt, root_validator, validator +from typing import Self + +from pydantic import ( + AliasChoices, + Field, + PositiveInt, + TypeAdapter, + field_validator, + model_validator, +) from settings_library.base import BaseCustomSettings from settings_library.basic_types import LogLevel, PortInt from settings_library.postgres import PostgresSettings @@ -14,10 +21,11 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): STORAGE_HOST: str = "0.0.0.0" # nosec - STORAGE_PORT: PortInt = PortInt(8080) + STORAGE_PORT: PortInt = TypeAdapter(PortInt).validate_python(8080) LOG_LEVEL: LogLevel = Field( - "INFO", env=["STORAGE_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + "INFO", + validation_alias=AliasChoices("STORAGE_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) STORAGE_MAX_WORKERS: PositiveInt = Field( @@ -35,15 +43,23 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): None, description="Pennsieve API secret ONLY for testing purposes" ) - STORAGE_POSTGRES: PostgresSettings = Field(auto_default_from_env=True) + STORAGE_POSTGRES: PostgresSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - STORAGE_REDIS: RedisSettings | None = Field(auto_default_from_env=True) + STORAGE_REDIS: RedisSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - STORAGE_S3: S3Settings = Field(auto_default_from_env=True) + STORAGE_S3: S3Settings = Field(json_schema_extra={"auto_default_from_env": True}) - STORAGE_TRACING: TracingSettings | None = Field(auto_default_from_env=True) + STORAGE_TRACING: TracingSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - DATCORE_ADAPTER: DatcoreAdapterSettings = Field(auto_default_from_env=True) + DATCORE_ADAPTER: DatcoreAdapterSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) STORAGE_SYNC_METADATA_TIMEOUT: PositiveInt = Field( 180, description="Timeout (seconds) for metadata sync task" @@ -64,22 +80,26 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): ) STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - False, - env=["STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + default=False, + validation_alias=AliasChoices( + "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - @validator("LOG_LEVEL") + @field_validator("LOG_LEVEL", mode="before") @classmethod - def _validate_loglevel(cls, value) -> str: + def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) return log_level - @root_validator() - @classmethod - def ensure_settings_consistency(cls, values: dict[str, Any]): - if values.get("STORAGE_CLEANER_INTERVAL_S") and not values.get("STORAGE_REDIS"): - raise ValueError( - "STORAGE_CLEANER_INTERVAL_S cleaner cannot be set without STORAGE_REDIS! Please correct settings." + @model_validator(mode="after") + def ensure_settings_consistency(self) -> Self: + if self.STORAGE_CLEANER_INTERVAL_S is not None and not self.STORAGE_REDIS: + msg = ( + "STORAGE_CLEANER_INTERVAL_S cleaner cannot be set without STORAGE_REDIS! " + "Please correct settings." ) - return values + raise ValueError(msg) + return self diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index db5a1ab288b..b6e7e57f1cc 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -20,7 +20,12 @@ S3MetaData, UploadedBytesTransferredCallback, ) -from models_library.api_schemas_storage import LinkType, S3BucketName, UploadedPart +from models_library.api_schemas_storage import ( + UNDEFINED_SIZE_TYPE, + LinkType, + S3BucketName, + UploadedPart, +) from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes_io import ( @@ -30,7 +35,7 @@ StorageFileID, ) from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, NonNegativeInt, parse_obj_as +from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter from servicelib.aiohttp.client_session import get_client_session from servicelib.aiohttp.long_running_tasks.server import TaskProgress from servicelib.logging_utils import log_context @@ -252,7 +257,7 @@ async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaDat raise FileAccessRightError(access_right="read", file_id=file_id) fmd = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if is_file_entry_valid(fmd): return convert_db_to_model(fmd) @@ -279,7 +284,7 @@ async def create_file_upload_links( # there was a multipart upload in progress beforehand, it MUST be # cancelled to prevent unwanted costs in AWS await self._clean_pending_upload( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if ( @@ -352,7 +357,7 @@ async def create_file_upload_links( # user wants just the s3 link s3_link = get_s3_client(self.app).compute_s3_url( bucket=self.simcore_bucket_name, - object_key=parse_obj_as(SimcoreS3FileID, file_id), + object_key=TypeAdapter(SimcoreS3FileID).validate_python(file_id), ) return UploadLinks( [s3_link], file_size_bytes or MAX_LINK_CHUNK_BYTE_SIZE[link_type] @@ -371,7 +376,7 @@ async def abort_file_upload( raise FileAccessRightError(access_right="write/delete", file_id=file_id) fmd: FileMetaDataAtDB = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if is_valid_managed_multipart_upload(fmd.upload_id): assert fmd.upload_id # nosec @@ -407,7 +412,7 @@ async def complete_file_upload( if not can.write: raise FileAccessRightError(access_right="write", file_id=file_id) fmd = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if is_valid_managed_multipart_upload(fmd.upload_id): @@ -455,12 +460,12 @@ async def create_file_download_link( ): raise S3KeyNotFoundError(key=file_id, bucket=self.simcore_bucket_name) return await self.__get_link( - parse_obj_as(SimcoreS3FileID, file_id), link_type + TypeAdapter(SimcoreS3FileID).validate_python(file_id), link_type ) # standard file link async with self.engine.acquire() as conn: fmd = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if not is_file_entry_valid(fmd): # try lazy update @@ -482,9 +487,8 @@ async def __ensure_read_access_rights( async def __get_link( self, s3_file_id: SimcoreS3FileID, link_type: LinkType ) -> AnyUrl: - link: AnyUrl = parse_obj_as( - AnyUrl, - f"s3://{self.simcore_bucket_name}/{urllib.parse.quote(s3_file_id)}", + link: AnyUrl = TypeAdapter(AnyUrl).validate_python( + f"s3://{self.simcore_bucket_name}/{urllib.parse.quote(s3_file_id)}" ) if link_type == LinkType.PRESIGNED: link = await get_s3_client(self.app).create_single_presigned_download_link( @@ -523,7 +527,7 @@ async def delete_file( # NOTE: deleting might be slow, so better ensure we release the connection async with self.engine.acquire() as conn: file: FileMetaDataAtDB = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) await get_s3_client(self.app).delete_objects_recursively( bucket=file.bucket_name, @@ -620,14 +624,16 @@ async def deep_copy_project_simcore_s3( f"{len(src_project_files)} files", log_duration=True, ): - sizes_and_num_files: list[tuple[ByteSize, int]] = await limited_gather( + sizes_and_num_files: list[ + tuple[ByteSize | UNDEFINED_SIZE_TYPE, int] + ] = await limited_gather( *[self._get_size_and_num_files(fmd) for fmd in src_project_files], limit=_MAX_PARALLEL_S3_CALLS, ) total_num_of_files = sum(n for _, n in sizes_and_num_files) - src_project_total_data_size: ByteSize = parse_obj_as( - ByteSize, sum(n for n, _ in sizes_and_num_files) - ) + src_project_total_data_size: ByteSize = TypeAdapter( + ByteSize + ).validate_python(sum(n for n, _ in sizes_and_num_files)) with log_context( _logger, logging.INFO, @@ -653,7 +659,7 @@ async def deep_copy_project_simcore_s3( self._copy_path_s3_s3( user_id, src_fmd=src_fmd, - dst_file_id=SimcoreS3FileID( + dst_file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{dst_project_uuid}/{new_node_id}/{src_fmd.object_name.split('/', maxsplit=2)[-1]}" ), bytes_transfered_cb=s3_transfered_data_cb.copy_transfer_cb, @@ -693,7 +699,7 @@ async def deep_copy_project_simcore_s3( async def _get_size_and_num_files( self, fmd: FileMetaDataAtDB - ) -> tuple[ByteSize, int]: + ) -> tuple[ByteSize | UNDEFINED_SIZE_TYPE, int]: if not fmd.is_directory: return fmd.file_size, 1 @@ -711,7 +717,7 @@ async def _get_size_and_num_files( total_size += sum(x.size for x in s3_objects) total_num_s3_objects += len(s3_objects) - return parse_obj_as(ByteSize, total_size), total_num_s3_objects + return TypeAdapter(ByteSize).validate_python(total_size), total_num_s3_objects async def search_owned_files( self, @@ -752,7 +758,7 @@ async def create_soft_link( ) -> FileMetaData: async with self.engine.acquire() as conn: if await db_file_meta_data.exists( - conn, parse_obj_as(SimcoreS3FileID, link_file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(link_file_id) ): raise LinkAlreadyExistsError(file_id=link_file_id) # validate target_uuid @@ -901,7 +907,7 @@ async def _update_fmd_from_other( s3_metadata = await get_s3_client(self.app).get_object_metadata( bucket=fmd.bucket_name, object_key=fmd.object_name ) - fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size) + fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.last_modified = s3_metadata.last_modified fmd.entity_tag = s3_metadata.e_tag else: @@ -938,12 +944,12 @@ async def _update_database_from_storage( s3_metadata = await self._get_s3_metadata(fmd) if not fmd.is_directory: assert isinstance(s3_metadata, S3MetaData) # nosec - fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size) + fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.last_modified = s3_metadata.last_modified fmd.entity_tag = s3_metadata.e_tag elif fmd.is_directory: assert isinstance(s3_metadata, S3DirectoryMetaData) # nosec - fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size) + fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.upload_expires_at = None fmd.upload_id = None async with self.engine.acquire() as conn: @@ -971,13 +977,15 @@ async def _copy_file_datcore_s3( ) assert dc_link.path # nosec filename = Path(dc_link.path).name - dst_file_id = SimcoreS3FileID(f"{dest_project_id}/{dest_node_id}/{filename}") + dst_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{dest_project_id}/{dest_node_id}/{filename}" + ) _logger.debug("copying %s to %s", f"{source_uuid=}", f"{dst_file_id=}") with tempfile.TemporaryDirectory() as tmpdir: local_file_path = Path(tmpdir) / filename # Downloads DATCore -> local - await download_to_file_or_raise(session, dc_link, local_file_path) + await download_to_file_or_raise(session, f"{dc_link}", local_file_path) # copying will happen using aioboto3, therefore multipart might happen async with self.engine.acquire() as conn: @@ -1068,7 +1076,7 @@ async def _create_fmd_for_upload( ) fmd = FileMetaData.from_simcore_node( user_id=user_id, - file_id=parse_obj_as(SimcoreS3FileID, file_id), + file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id), bucket=self.simcore_bucket_name, location_id=self.location_id, location_name=self.location_name, @@ -1085,7 +1093,9 @@ def create_simcore_s3_data_manager(app: web.Application) -> SimcoreS3DataManager assert cfg.STORAGE_S3 # nosec return SimcoreS3DataManager( engine=app[APP_AIOPG_ENGINE_KEY], - simcore_bucket_name=parse_obj_as(S3BucketName, cfg.STORAGE_S3.S3_BUCKET_NAME), + simcore_bucket_name=TypeAdapter(S3BucketName).validate_python( + cfg.STORAGE_S3.S3_BUCKET_NAME + ), app=app, settings=cfg, ) diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py index 3cb3cbfc399..e4a58549e31 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py @@ -1,6 +1,5 @@ from contextlib import suppress from pathlib import Path -from typing import cast from aiopg.sa.connection import SAConnection from aws_library.s3 import S3MetaData, SimcoreS3API @@ -10,7 +9,7 @@ SimcoreS3FileID, StorageFileID, ) -from pydantic import ByteSize, NonNegativeInt, parse_obj_as +from pydantic import ByteSize, NonNegativeInt, TypeAdapter from servicelib.utils import ensure_ends_with from . import db_file_meta_data @@ -56,7 +55,7 @@ async def expand_directory( location_id=fmd.location_id, location=fmd.location, bucket_name=fmd.bucket_name, - object_name=cast(SimcoreS3FileID, x.object_key), + object_name=x.object_key, user_id=fmd.user_id, # NOTE: to ensure users have a consistent experience the # `created_at` field is inherited from the last_modified @@ -64,8 +63,8 @@ async def expand_directory( # creation of the directory, the file's creation date # will not be 1 month in the passed. created_at=x.last_modified, - file_id=cast(SimcoreS3FileID, x.object_key), - file_size=parse_obj_as(ByteSize, x.size), + file_id=x.object_key, + file_size=TypeAdapter(ByteSize).validate_python(x.size), last_modified=x.last_modified, entity_tag=x.e_tag, is_soft_link=False, @@ -100,7 +99,7 @@ async def _get_fmd( ) -> FileMetaDataAtDB | None: with suppress(FileMetaDataNotFoundError): return await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, s3_file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(s3_file_id) ) return None @@ -114,7 +113,9 @@ async def _get_fmd( # could not extract a directory name from the provided path return None - directory_file_id = parse_obj_as(SimcoreS3FileID, directory_file_id_str) + directory_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + directory_file_id_str + ) directory_file_id_fmd = await _get_fmd(conn, directory_file_id) return directory_file_id if directory_file_id_fmd else None diff --git a/services/storage/src/simcore_service_storage/utils.py b/services/storage/src/simcore_service_storage/utils.py index 0baddfcfc9a..7abc18ed552 100644 --- a/services/storage/src/simcore_service_storage/utils.py +++ b/services/storage/src/simcore_service_storage/utils.py @@ -15,8 +15,8 @@ def convert_db_to_model(x: FileMetaDataAtDB) -> FileMetaData: - model: FileMetaData = FileMetaData.parse_obj( - x.dict() + model: FileMetaData = FileMetaData.model_validate( + x.model_dump() | { "file_uuid": x.file_id, "file_name": x.file_id.split("/")[-1], diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 1b3f634446c..ca6483c75ae 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -39,7 +39,7 @@ from models_library.projects_nodes_io import LocationID, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context @@ -174,15 +174,15 @@ def app_settings( monkeypatch.delenv("S3_ENDPOINT") setenvs_from_dict( monkeypatch, - s3_settings.dict(exclude={"S3_ENDPOINT"}), + s3_settings.model_dump(exclude={"S3_ENDPOINT"}), ) else: setenvs_from_dict( monkeypatch, - s3_settings.dict(), + s3_settings.model_dump(), ) test_app_settings = Settings.create_from_envs() - print(f"{test_app_settings.json(indent=2)=}") + print(f"{test_app_settings.model_dump_json(indent=2)=}") return test_app_settings @@ -259,7 +259,7 @@ async def _getter(file_id: SimcoreS3FileID) -> FileMetaDataGet: data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_fmd = parse_obj_as(FileMetaDataGet, data) + received_fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert received_fmd return received_fmd @@ -291,7 +291,7 @@ async def _link_creator( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_file_upload = parse_obj_as(FileUploadSchema, data) + received_file_upload = TypeAdapter(FileUploadSchema).validate_python(data) assert received_file_upload file_params.append((user_id, location_id, file_id)) return received_file_upload @@ -353,7 +353,7 @@ async def _uploader( file, file_upload_link ) # complete the upload - complete_url = URL(file_upload_link.links.complete_upload).relative() + complete_url = URL(f"{file_upload_link.links.complete_upload}").relative() with log_context(logging.INFO, f"completing upload of {file=}"): response = await client.post( f"{complete_url}", @@ -363,8 +363,10 @@ async def _uploader( data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate( + data + ) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() completion_etag = None async for attempt in AsyncRetrying( @@ -382,7 +384,7 @@ async def _uploader( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) if future.state == FileUploadCompleteState.NOK: msg = f"{data=}" raise ValueError(msg) @@ -430,7 +432,7 @@ def _creator( if file_base_path: s3_file_name = f"{file_base_path / file_name}" clean_path = Path(f"{project_id}/{node_id}/{s3_file_name}") - return SimcoreS3FileID(f"{clean_path}") + return TypeAdapter(SimcoreS3FileID).validate_python(f"{clean_path}") return _creator @@ -470,7 +472,7 @@ async def _directory_creator(dir_name: str): assert len(directory_file_upload.urls) == 1 # complete the upload - complete_url = URL(directory_file_upload.links.complete_upload).relative() + complete_url = URL(f"{directory_file_upload.links.complete_upload}").relative() response = await client.post( f"{complete_url}", json=jsonable_encoder(FileUploadCompletionBody(parts=[])), @@ -479,8 +481,8 @@ async def _directory_creator(dir_name: str): data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() # check that it finished updating assert client.app @@ -500,7 +502,7 @@ async def _directory_creator(dir_name: str): data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) assert future.state == FileUploadCompleteState.OK assert future.e_tag is None ctx.logger.info( @@ -535,7 +537,9 @@ async def _create_file(s: int, f: int): await storage_s3_client.upload_file( bucket=storage_s3_bucket, file=file, - object_key=SimcoreS3FileID(f"{clean_path}"), + object_key=TypeAdapter(SimcoreS3FileID).validate_python( + f"{clean_path}" + ), bytes_transfered_cb=None, ) diff --git a/services/storage/tests/fixtures/data_models.py b/services/storage/tests/fixtures/data_models.py index ae5816a427f..ab225928f62 100644 --- a/services/storage/tests/fixtures/data_models.py +++ b/services/storage/tests/fixtures/data_models.py @@ -18,7 +18,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.faker_factories import random_project, random_user from servicelib.utils import limited_gather from simcore_postgres_database.models.project_to_groups import project_to_groups @@ -45,7 +45,7 @@ async def _user_context(aiopg_engine: Engine, *, name: str) -> AsyncIterator[Use assert isinstance(row.id, int) try: - yield UserID(row.id) + yield TypeAdapter(UserID).validate_python(row.id) finally: async with aiopg_engine.acquire() as conn: await conn.execute(users.delete().where(users.c.id == row.id)) @@ -149,7 +149,7 @@ async def project_id( async def collaborator_id(aiopg_engine: Engine) -> AsyncIterator[UserID]: async with _user_context(aiopg_engine, name="collaborator") as new_user_id: - yield UserID(new_user_id) + yield TypeAdapter(UserID).validate_python(new_user_id) @pytest.fixture @@ -177,7 +177,7 @@ async def _() -> None: ) row = await result.fetchone() assert row - access_rights: dict[str, Any] = row[projects.c.access_rights] + access_rights: dict[str | int, Any] = row[projects.c.access_rights] access_rights[await _get_user_group(conn, user_id)] = { "read": True, @@ -279,22 +279,19 @@ async def random_project_with_files( async def _creator( num_nodes: int = 12, file_sizes: tuple[ByteSize, ...] = ( - parse_obj_as(ByteSize, "7Mib"), - parse_obj_as(ByteSize, "110Mib"), - parse_obj_as(ByteSize, "1Mib"), + TypeAdapter(ByteSize).validate_python("7Mib"), + TypeAdapter(ByteSize).validate_python("110Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), ), file_checksums: tuple[SHA256Str, ...] = ( - parse_obj_as( - SHA256Str, - "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd", + TypeAdapter(SHA256Str).validate_python( + "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd" ), - parse_obj_as( - SHA256Str, - "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e", + TypeAdapter(SHA256Str).validate_python( + "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e" ), - parse_obj_as( - SHA256Str, - "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3", + TypeAdapter(SHA256Str).validate_python( + "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3" ), ), ) -> tuple[ diff --git a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py index 39be9386497..5e94a17d3bc 100644 --- a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py +++ b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py @@ -56,7 +56,7 @@ def location_name() -> str: return SimcoreS3DataManager.get_location_name() -async def test_storage_client_used_in_simcore_sdk_0_3_2( +async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 client: TestClient, file_id: str, user_id: str, diff --git a/services/storage/tests/unit/test_cli.py b/services/storage/tests/unit/test_cli.py index cab69609fdd..7c86b85fcd7 100644 --- a/services/storage/tests/unit/test_cli.py +++ b/services/storage/tests/unit/test_cli.py @@ -29,7 +29,7 @@ def test_cli_settings_as_json( assert result.exit_code == os.EX_OK, result # reuse resulting json to build settings settings: dict = json.loads(result.stdout) - assert Settings.parse_obj(settings) + assert Settings.model_validate(settings) def test_cli_settings_env_file( @@ -41,9 +41,9 @@ def test_cli_settings_env_file( # reuse resulting env_file to build settings env_file = StringIO(result.stdout) - settings: dict = dotenv_values(stream=env_file) + settings = dotenv_values(stream=env_file) for key, value in settings.items(): with contextlib.suppress(json.decoder.JSONDecodeError): settings[key] = json.loads(str(value)) - assert Settings.parse_obj(settings) + assert Settings.model_validate(settings) diff --git a/services/storage/tests/unit/test_dsm.py b/services/storage/tests/unit/test_dsm.py index ae07ec94c9b..22c78955581 100644 --- a/services/storage/tests/unit/test_dsm.py +++ b/services/storage/tests/unit/test_dsm.py @@ -10,7 +10,7 @@ from faker import Faker from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.utils import limited_gather from simcore_service_storage.models import FileMetaData, S3BucketName from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -31,7 +31,7 @@ async def dsm_mockup_complete_db( cleanup_user_projects_file_metadata: None, faker: Faker, ) -> tuple[FileMetaData, FileMetaData]: - file_size = parse_obj_as(ByteSize, "10Mib") + file_size = TypeAdapter(ByteSize).validate_python("10Mib") uploaded_files = await limited_gather( *(upload_file(file_size, faker.file_name(), None) for _ in range(2)), limit=2, diff --git a/services/storage/tests/unit/test_dsm_dsmcleaner.py b/services/storage/tests/unit/test_dsm_dsmcleaner.py index 60f4f7f57a9..7862d4d7166 100644 --- a/services/storage/tests/unit/test_dsm_dsmcleaner.py +++ b/services/storage/tests/unit/test_dsm_dsmcleaner.py @@ -3,7 +3,6 @@ # pylint: disable=redefined-outer-name # pylint: disable=too-many-arguments # pylint: disable=too-many-branches -# pylint: disable=too-many-positional-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable @@ -23,7 +22,7 @@ from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import SimcoreS3DirectoryID, SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.parametrizations import byte_size_ids from simcore_postgres_database.storage_models import file_meta_data from simcore_service_storage import db_file_meta_data @@ -47,14 +46,18 @@ def disabled_dsm_cleaner_task(monkeypatch: pytest.MonkeyPatch): @pytest.fixture def simcore_directory_id(simcore_file_id: SimcoreS3FileID) -> SimcoreS3FileID: - return SimcoreS3FileID( - Path(SimcoreS3DirectoryID.from_simcore_s3_object(simcore_file_id)) + return TypeAdapter(SimcoreS3FileID).validate_python( + SimcoreS3DirectoryID.from_simcore_s3_object(simcore_file_id) ) @pytest.mark.parametrize( "file_size", - [ByteSize(0), parse_obj_as(ByteSize, "10Mib"), parse_obj_as(ByteSize, "100Mib")], + [ + TypeAdapter(ByteSize).validate_python("0"), + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), + ], ids=byte_size_ids, ) @pytest.mark.parametrize( @@ -120,7 +123,11 @@ async def test_regression_collaborator_creates_file_upload_links( @pytest.mark.parametrize( "file_size", - [ByteSize(0), parse_obj_as(ByteSize, "10Mib"), parse_obj_as(ByteSize, "100Mib")], + [ + ByteSize(0), + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), + ], ids=byte_size_ids, ) @pytest.mark.parametrize( @@ -207,7 +214,10 @@ async def test_clean_expired_uploads_deletes_expired_pending_uploads( @pytest.mark.parametrize( "file_size", - [parse_obj_as(ByteSize, "10Mib"), parse_obj_as(ByteSize, "100Mib")], + [ + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), + ], ids=byte_size_ids, ) @pytest.mark.parametrize("link_type", [LinkType.S3, LinkType.PRESIGNED]) @@ -287,7 +297,7 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi # check the entries were reverted async with aiopg_engine.acquire() as conn: reverted_fmd = await db_file_meta_data.get(conn, file_id) - assert original_fmd.dict(exclude={"created_at"}) == reverted_fmd.dict( + assert original_fmd.model_dump(exclude={"created_at"}) == reverted_fmd.model_dump( exclude={"created_at"} ) # check the S3 content is the old file @@ -303,7 +313,7 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi @pytest.mark.parametrize( "file_size", - [parse_obj_as(ByteSize, "100Mib")], + [TypeAdapter(ByteSize).validate_python("100Mib")], ids=byte_size_ids, ) @pytest.mark.parametrize("is_directory", [True, False]) @@ -353,7 +363,9 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation file_ids_to_upload: set[SimcoreS3FileID] = ( { - SimcoreS3FileID(f"{file_or_directory_id}/file{x}") + TypeAdapter(SimcoreS3FileID).validate_python( + f"{file_or_directory_id}/file{x}" + ) for x in range(FILES_IN_DIR) } if is_directory @@ -366,7 +378,7 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation object_key=file_id, file_size=file_size, expiration_secs=3600, - sha256_checksum=parse_obj_as(SHA256Str, _faker.sha256()), + sha256_checksum=TypeAdapter(SHA256Str).validate_python(_faker.sha256()), ) for file_id in file_ids_to_upload ] diff --git a/services/storage/tests/unit/test_dsm_soft_links.py b/services/storage/tests/unit/test_dsm_soft_links.py index ed0e01ea7f0..dd822ea2165 100644 --- a/services/storage/tests/unit/test_dsm_soft_links.py +++ b/services/storage/tests/unit/test_dsm_soft_links.py @@ -46,7 +46,7 @@ async def output_file( async with aiopg_engine.acquire() as conn: stmt = ( file_meta_data.insert() - .values(jsonable_encoder(FileMetaDataAtDB.from_orm(file))) + .values(jsonable_encoder(FileMetaDataAtDB.model_validate(file))) .returning(literal_column("*")) ) result = await conn.execute(stmt) diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 92408a33136..d207005c2d2 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -15,7 +15,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.parametrizations import ( @@ -73,7 +73,7 @@ async def test_get_files_metadata_dataset( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (n + 1) fmd = list_fmds[n] assert fmd.file_name == file.name @@ -100,7 +100,7 @@ async def test_get_datasets_metadata( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - list_datasets = parse_obj_as(list[DatasetMetaDataGet], data) + list_datasets = TypeAdapter(list[DatasetMetaDataGet]).validate_python(data) assert len(list_datasets) == 1 dataset = list_datasets[0] assert dataset.dataset_id == project_id diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index 5623c7e67c2..f9fc415d86a 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -11,12 +11,12 @@ import json import logging import urllib.parse -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import AbstractAsyncContextManager from dataclasses import dataclass from pathlib import Path from random import choice -from typing import Any, AsyncIterator, Literal +from typing import Any, Literal from uuid import uuid4 import pytest @@ -43,7 +43,7 @@ from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyHttpUrl, ByteSize, HttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, ByteSize, HttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context @@ -112,7 +112,7 @@ class SingleLinkParam: {}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="default_returns_single_presigned", ), @@ -121,13 +121,16 @@ class SingleLinkParam: {"link_type": "presigned"}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="presigned_returns_single_presigned", ), pytest.param( SingleLinkParam( - {"link_type": "s3"}, "s3", [], parse_obj_as(ByteSize, "5TiB") + {"link_type": "s3"}, + "s3", + [], + TypeAdapter(ByteSize).validate_python("5TiB"), ), id="s3_returns_single_s3_link", ), @@ -207,7 +210,7 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_file_upload_link = parse_obj_as(PresignedLink, data) + received_file_upload_link = TypeAdapter(PresignedLink).validate_python(data) assert received_file_upload_link file_params.append((user_id, location_id, file_id)) return received_file_upload_link @@ -238,7 +241,7 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi {}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="default_returns_single_presigned", ), @@ -247,13 +250,16 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi {"link_type": "presigned"}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="presigned_returns_single_presigned", ), pytest.param( SingleLinkParam( - {"link_type": "s3"}, "s3", [], parse_obj_as(ByteSize, "5TiB") + {"link_type": "s3"}, + "s3", + [], + TypeAdapter(ByteSize).validate_python("5TiB"), ), id="s3_returns_single_s3_link", ), @@ -314,50 +320,50 @@ class MultiPartParam: pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "10MiB"), + file_size=TypeAdapter(ByteSize).validate_python("10MiB"), expected_response=status.HTTP_200_OK, expected_num_links=1, - expected_chunk_size=parse_obj_as(ByteSize, "10MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("10MiB"), ), id="10MiB file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "100MiB"), + file_size=TypeAdapter(ByteSize).validate_python("100MiB"), expected_response=status.HTTP_200_OK, expected_num_links=10, - expected_chunk_size=parse_obj_as(ByteSize, "10MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("10MiB"), ), id="100MiB file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "5TiB"), + file_size=TypeAdapter(ByteSize).validate_python("5TiB"), expected_response=status.HTTP_200_OK, expected_num_links=8739, - expected_chunk_size=parse_obj_as(ByteSize, "600MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("600MiB"), ), id="5TiB file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "9431773844"), + file_size=TypeAdapter(ByteSize).validate_python("9431773844"), expected_response=status.HTTP_200_OK, expected_num_links=900, - expected_chunk_size=parse_obj_as(ByteSize, "10MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("10MiB"), ), id="9431773844B (8.8Gib) file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.S3, - file_size=parse_obj_as(ByteSize, "255GiB"), + file_size=TypeAdapter(ByteSize).validate_python("255GiB"), expected_response=status.HTTP_200_OK, expected_num_links=1, - expected_chunk_size=parse_obj_as(ByteSize, "255GiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("255GiB"), ), id="5TiB file,s3", ), @@ -409,8 +415,8 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin @pytest.mark.parametrize( "link_type, file_size", [ - (LinkType.PRESIGNED, parse_obj_as(ByteSize, "1000Mib")), - (LinkType.S3, parse_obj_as(ByteSize, "1000Mib")), + (LinkType.PRESIGNED, TypeAdapter(ByteSize).validate_python("1000Mib")), + (LinkType.S3, TypeAdapter(ByteSize).validate_python("1000Mib")), ], ids=byte_size_ids, ) @@ -449,7 +455,7 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( expected_upload_ids=([upload_id] if upload_id else None), ) # delete/abort file upload - abort_url = URL(upload_link.links.abort_upload).relative() + abort_url = URL(f"{upload_link.links.abort_upload}").relative() response = await client.post(f"{abort_url}") await assert_status(response, status.HTTP_204_NO_CONTENT) @@ -474,10 +480,10 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( @pytest.mark.parametrize( "link_type, file_size", [ - (LinkType.PRESIGNED, parse_obj_as(ByteSize, "10Mib")), - (LinkType.PRESIGNED, parse_obj_as(ByteSize, "1000Mib")), - (LinkType.S3, parse_obj_as(ByteSize, "10Mib")), - (LinkType.S3, parse_obj_as(ByteSize, "1000Mib")), + (LinkType.PRESIGNED, TypeAdapter(ByteSize).validate_python("10Mib")), + (LinkType.PRESIGNED, TypeAdapter(ByteSize).validate_python("1000Mib")), + (LinkType.S3, TypeAdapter(ByteSize).validate_python("10Mib")), + (LinkType.S3, TypeAdapter(ByteSize).validate_python("1000Mib")), ], ids=byte_size_ids, ) @@ -561,9 +567,11 @@ def complex_file_name(faker: Faker) -> str: @pytest.mark.parametrize( "file_size", [ - (parse_obj_as(ByteSize, "1Mib")), - (parse_obj_as(ByteSize, "500Mib")), - pytest.param(parse_obj_as(ByteSize, "7Gib"), marks=pytest.mark.heavy_load), + (TypeAdapter(ByteSize).validate_python("1Mib")), + (TypeAdapter(ByteSize).validate_python("500Mib")), + pytest.param( + TypeAdapter(ByteSize).validate_python("5Gib"), marks=pytest.mark.heavy_load + ), ], ids=byte_size_ids, ) @@ -578,8 +586,8 @@ async def test_upload_real_file( @pytest.mark.parametrize( "file_size", [ - (parse_obj_as(ByteSize, "1Mib")), - (parse_obj_as(ByteSize, "117Mib")), + (TypeAdapter(ByteSize).validate_python("1Mib")), + (TypeAdapter(ByteSize).validate_python("117Mib")), ], ids=byte_size_ids, ) @@ -614,7 +622,7 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w file, file_upload_link ) # complete the upload - complete_url = URL(file_upload_link.links.complete_upload).relative() + complete_url = URL(f"{file_upload_link.links.complete_upload}").relative() response = await client.post( f"{complete_url}", json=jsonable_encoder(FileUploadCompletionBody(parts=part_to_etag)), @@ -623,8 +631,8 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() # here we do not check now for the state completion. instead we simulate a restart where the tasks disappear client.app[UPLOAD_TASKS_KEY].clear() @@ -644,7 +652,7 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) assert future.state == FileUploadCompleteState.OK assert future.e_tag is not None completion_etag = future.e_tag @@ -686,7 +694,7 @@ async def test_upload_of_single_presigned_link_lazily_update_database_on_get( s3_client: S3Client, ): assert client.app - file_size = parse_obj_as(ByteSize, "500Mib") + file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() # create a file file = create_file_of_size(file_size, file_name) @@ -729,7 +737,7 @@ async def test_upload_real_file_with_s3_client( s3_client: S3Client, ): assert client.app - file_size = parse_obj_as(ByteSize, "500Mib") + file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() # create a file file = create_file_of_size(file_size, file_name) @@ -754,15 +762,15 @@ async def test_upload_real_file_with_s3_client( assert s3_metadata.e_tag == upload_e_tag # complete the upload - complete_url = URL(file_upload_link.links.complete_upload).relative() + complete_url = URL(f"{file_upload_link.links.complete_upload}").relative() with log_context(logging.INFO, f"completing upload of {file=}"): response = await client.post(f"{complete_url}", json={"parts": []}) response.raise_for_status() data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() completion_etag = None async for attempt in AsyncRetrying( reraise=True, @@ -779,7 +787,7 @@ async def test_upload_real_file_with_s3_client( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) if future.state != FileUploadCompleteState.OK: msg = f"{data=}" raise ValueError(msg) @@ -812,7 +820,10 @@ async def test_upload_real_file_with_s3_client( @pytest.mark.parametrize( "file_size", - [parse_obj_as(ByteSize, "160Mib"), parse_obj_as(ByteSize, "1Mib")], + [ + TypeAdapter(ByteSize).validate_python("160Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + ], ids=byte_size_ids, ) async def test_upload_twice_and_fail_second_time_shall_keep_first_version( @@ -865,7 +876,7 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( ) # 4. abort file upload - abort_url = URL(upload_link.links.abort_upload).relative() + abort_url = URL(f"{upload_link.links.abort_upload}").relative() response = await client.post(f"{abort_url}") await assert_status(response, status.HTTP_204_NO_CONTENT) @@ -888,7 +899,7 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( @pytest.fixture def file_size() -> ByteSize: - return parse_obj_as(ByteSize, "1Mib") + return TypeAdapter(ByteSize).validate_python("1Mib") async def _assert_file_downloaded( @@ -896,7 +907,7 @@ async def _assert_file_downloaded( ): dest_file = tmp_path / faker.file_name() async with ClientSession() as session: - response = await session.get(link) + response = await session.get(f"{link}") response.raise_for_status() with dest_file.open("wb") as fp: fp.write(await response.read()) @@ -916,7 +927,9 @@ async def test_download_file_no_file_was_uploaded( ): assert client.app - missing_file = parse_obj_as(SimcoreS3FileID, f"{project_id}/{node_id}/missing.file") + missing_file = TypeAdapter(SimcoreS3FileID).validate_python( + f"{project_id}/{node_id}/missing.file" + ) assert ( await storage_s3_client.object_exists( bucket=storage_s3_bucket, object_key=missing_file @@ -975,7 +988,7 @@ async def test_download_file_1_to_1_with_file_meta_data( assert not error assert data assert "link" in data - assert parse_obj_as(AnyHttpUrl, data["link"]) + assert TypeAdapter(AnyHttpUrl).validate_python(data["link"]) await _assert_file_downloaded( faker, tmp_path, link=data["link"], uploaded_file=uploaded_file ) @@ -1006,7 +1019,9 @@ async def test_download_file_from_inside_a_directory( file_name = "meta_data_entry_is_dir.file" file_to_upload_in_dir = create_file_of_size(file_size, file_name) - s3_file_id = parse_obj_as(SimcoreS3FileID, f"{dir_path_in_s3}/{file_name}") + s3_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{dir_path_in_s3}/{file_name}" + ) await storage_s3_client.upload_file( bucket=storage_s3_bucket, file=file_to_upload_in_dir, @@ -1034,7 +1049,7 @@ async def test_download_file_from_inside_a_directory( assert not error assert data assert "link" in data - assert parse_obj_as(AnyHttpUrl, data["link"]) + assert TypeAdapter(AnyHttpUrl).validate_python(data["link"]) await _assert_file_downloaded( faker, tmp_path, link=data["link"], uploaded_file=file_to_upload_in_dir ) @@ -1055,8 +1070,8 @@ async def test_download_file_the_file_is_missing_from_the_directory( assert directory_file_upload.urls[0].path dir_path_in_s3 = directory_file_upload.urls[0].path.strip("/") - missing_s3_file_id = parse_obj_as( - SimcoreS3FileID, f"{dir_path_in_s3}/missing_inside_dir.file" + missing_s3_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{dir_path_in_s3}/missing_inside_dir.file" ) download_url = ( client.app.router["download_file"] @@ -1083,8 +1098,8 @@ async def test_download_file_access_rights( assert client.app # project_id does not exist - missing_file = parse_obj_as( - SimcoreS3FileID, f"{faker.uuid4()}/{faker.uuid4()}/project_id_is_missing" + missing_file = TypeAdapter(SimcoreS3FileID).validate_python( + f"{faker.uuid4()}/{faker.uuid4()}/project_id_is_missing" ) assert ( await storage_s3_client.object_exists( @@ -1110,7 +1125,7 @@ async def test_download_file_access_rights( @pytest.mark.parametrize( "file_size", [ - pytest.param(parse_obj_as(ByteSize, "1Mib")), + pytest.param(TypeAdapter(ByteSize).validate_python("1Mib")), ], ids=byte_size_ids, ) @@ -1184,7 +1199,7 @@ async def test_copy_as_soft_link( # now let's try with whatever link id file, original_file_uuid = await upload_file( - parse_obj_as(ByteSize, "10Mib"), faker.file_name() + TypeAdapter(ByteSize).validate_python("10Mib"), faker.file_name() ) url = ( client.app.router["copy_as_soft_link"] @@ -1193,13 +1208,15 @@ async def test_copy_as_soft_link( ) .with_query(user_id=user_id) ) - link_id = SimcoreS3FileID(f"api/{node_id}/{faker.file_name()}") + link_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"api/{node_id}/{faker.file_name()}" + ) response = await client.post( f"{url}", json=jsonable_encoder(SoftCopyBody(link_id=link_id)) ) data, error = await assert_status(response, status.HTTP_200_OK) assert not error - fmd = parse_obj_as(FileMetaDataGet, data) + fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert fmd.file_id == link_id @@ -1223,7 +1240,7 @@ async def __list_files( response = await client.get(f"{get_url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - return parse_obj_as(list[FileMetaDataGet], data) + return TypeAdapter(list[FileMetaDataGet]).validate_python(data) async def _list_files_legacy( @@ -1257,9 +1274,9 @@ async def _list_files_and_directories( @pytest.mark.parametrize( "file_size", [ - parse_obj_as(ByteSize, "-1"), - parse_obj_as(ByteSize, "0"), - parse_obj_as(ByteSize, "1TB"), + ByteSize(-1), + TypeAdapter(ByteSize).validate_python("0"), + TypeAdapter(ByteSize).validate_python("1TB"), ], ) async def test_is_directory_link_forces_link_type_and_size( @@ -1329,7 +1346,7 @@ async def test_upload_file_is_directory_and_remove_content( location_id: LocationID, user_id: UserID, ): - FILE_SIZE_IN_DIR = parse_obj_as(ByteSize, "1Mib") + FILE_SIZE_IN_DIR = TypeAdapter(ByteSize).validate_python("1Mib") DIR_NAME = "some-dir" SUBDIR_COUNT = 4 FILE_COUNT = 5 @@ -1396,7 +1413,7 @@ async def test_listing_more_than_1000_objects_in_bucket( ): async with create_directory_with_files( dir_name="some-random", - file_size_in_dir=parse_obj_as(ByteSize, "1"), + file_size_in_dir=TypeAdapter(ByteSize).validate_python("1"), subdir_count=1, file_count=files_in_dir, ) as directory_file_upload: @@ -1427,19 +1444,19 @@ async def test_listing_with_project_id_filter( project, src_projects_list = await random_project_with_files( num_nodes=1, file_sizes=(ByteSize(1),), - file_checksums=(SHA256Str(faker.sha256()),), + file_checksums=(TypeAdapter(SHA256Str).validate_python(faker.sha256()),), ) _, _ = await random_project_with_files( num_nodes=1, file_sizes=(ByteSize(1),), - file_checksums=(SHA256Str(faker.sha256()),), + file_checksums=(TypeAdapter(SHA256Str).validate_python(faker.sha256()),), ) assert len(src_projects_list.keys()) > 0 - node_id = list(src_projects_list.keys())[0] + node_id = next(iter(src_projects_list.keys())) project_files_in_db = set(src_projects_list[node_id]) assert len(project_files_in_db) > 0 project_id = project["uuid"] - project_file_name = Path(choice(list(project_files_in_db))).name + project_file_name = Path(choice(list(project_files_in_db))).name # noqa: S311 assert client.app query = { @@ -1456,7 +1473,7 @@ async def test_listing_with_project_id_filter( response = await client.get(f"{url}") data, _ = await assert_status(response, status.HTTP_200_OK) - list_of_files = parse_obj_as(list[FileMetaDataGet], data) + list_of_files = TypeAdapter(list[FileMetaDataGet]).validate_python(data) if uuid_filter: assert len(list_of_files) == 1 diff --git a/services/storage/tests/unit/test_handlers_files_metadata.py b/services/storage/tests/unit/test_handlers_files_metadata.py index dd0c8138ebb..9abd834d21a 100644 --- a/services/storage/tests/unit/test_handlers_files_metadata.py +++ b/services/storage/tests/unit/test_handlers_files_metadata.py @@ -16,7 +16,7 @@ from models_library.api_schemas_storage import FileMetaDataGet, SimcoreS3FileID from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import status @@ -58,12 +58,12 @@ async def test_get_files_metadata( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds # now add some stuff there NUM_FILES = 10 - file_size = parse_obj_as(ByteSize, "15Mib") + file_size = TypeAdapter(ByteSize).validate_python("15Mib") files_owned_by_us = [ await upload_file(file_size, faker.file_name()) for _ in range(NUM_FILES) ] @@ -73,7 +73,7 @@ async def test_get_files_metadata( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == NUM_FILES # checks project_id filter! @@ -90,13 +90,13 @@ async def test_get_files_metadata( previous_data = deepcopy(data) data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (NUM_FILES) assert previous_data == data # create some more files but with a base common name NUM_FILES = 10 - file_size = parse_obj_as(ByteSize, "15Mib") + file_size = TypeAdapter(ByteSize).validate_python("15Mib") files_with_common_name = [ await upload_file(file_size, f"common_name-{faker.file_name()}") for _ in range(NUM_FILES) @@ -107,14 +107,14 @@ async def test_get_files_metadata( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (2 * NUM_FILES) # we can filter them now response = await client.get(f"{url.update_query(uuid_filter='common_name')}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (NUM_FILES) @@ -171,7 +171,7 @@ async def test_get_file_metadata( # now add some stuff there NUM_FILES = 10 - file_size = parse_obj_as(ByteSize, "15Mib") + file_size = TypeAdapter(ByteSize).validate_python("15Mib") files_owned_by_us = [] for _ in range(NUM_FILES): files_owned_by_us.append(await upload_file(file_size, faker.file_name())) @@ -188,6 +188,6 @@ async def test_get_file_metadata( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - fmd = parse_obj_as(FileMetaDataGet, data) + fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert fmd.file_id == selected_file_uuid assert fmd.file_size == selected_file.stat().st_size diff --git a/services/storage/tests/unit/test_handlers_health.py b/services/storage/tests/unit/test_handlers_health.py index d10b882b611..8705c4c8e36 100644 --- a/services/storage/tests/unit/test_handlers_health.py +++ b/services/storage/tests/unit/test_handlers_health.py @@ -26,7 +26,7 @@ async def test_health_check(client: TestClient): assert data assert not error - app_health = HealthCheck.parse_obj(data) + app_health = HealthCheck.model_validate(data) assert app_health.name == simcore_service_storage._meta.PROJECT_NAME # noqa: SLF001 assert app_health.version == str( simcore_service_storage._meta.VERSION @@ -41,7 +41,7 @@ async def test_health_status(client: TestClient): assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert ( app_status_check.app_name == simcore_service_storage._meta.PROJECT_NAME ) # noqa: SLF001 @@ -68,7 +68,7 @@ async def test_bad_health_status_if_bucket_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" # now delete the bucket await s3_client.delete_bucket(Bucket=storage_s3_bucket) @@ -77,7 +77,7 @@ async def test_bad_health_status_if_bucket_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "no access to S3 bucket" @@ -90,7 +90,7 @@ async def test_bad_health_status_if_s3_server_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" # now disable the s3 server mocked_aws_server.stop() @@ -99,7 +99,7 @@ async def test_bad_health_status_if_s3_server_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "failed" # start the server again mocked_aws_server.start() @@ -108,5 +108,5 @@ async def test_bad_health_status_if_s3_server_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" diff --git a/services/storage/tests/unit/test_handlers_simcore_s3.py b/services/storage/tests/unit/test_handlers_simcore_s3.py index e922f1f60f0..bcda9331f2b 100644 --- a/services/storage/tests/unit/test_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_handlers_simcore_s3.py @@ -26,7 +26,7 @@ from models_library.projects_nodes_io import NodeID, NodeIDStr, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -82,7 +82,7 @@ async def test_simcore_s3_access_returns_default(client: TestClient): data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_settings = S3Settings.parse_obj(data) + received_settings = S3Settings.model_validate(data) assert received_settings @@ -209,12 +209,12 @@ async def test_copy_folders_from_valid_project_with_one_large_file( ], ): # 1. create a src project with 1 large file - sha256_checksum: SHA256Str = parse_obj_as( - SHA256Str, "0b3216d95ec5a36c120ba16c88911dcf5ff655925d0fbdbc74cf95baf86de6fc" + sha256_checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python( + "0b3216d95ec5a36c120ba16c88911dcf5ff655925d0fbdbc74cf95baf86de6fc" ) src_project, src_projects_list = await random_project_with_files( 1, - (parse_obj_as(ByteSize, "210Mib"),), + (TypeAdapter(ByteSize).validate_python("210Mib"),), (sha256_checksum,), ) # 2. create a dst project without files @@ -233,7 +233,9 @@ async def test_copy_folders_from_valid_project_with_one_large_file( ) # check that file meta data was effectively copied for src_node_id in src_projects_list: - dst_node_id = nodes_map.get(NodeIDStr(f"{src_node_id}")) + dst_node_id = nodes_map.get( + TypeAdapter(NodeIDStr).validate_python(f"{src_node_id}") + ) assert dst_node_id for src_file_id, src_file in src_projects_list[src_node_id].items(): path: Any = src_file["path"] @@ -242,17 +244,18 @@ async def test_copy_folders_from_valid_project_with_one_large_file( assert isinstance(checksum, str) await assert_file_meta_data_in_db( aiopg_engine, - file_id=parse_obj_as( - SimcoreS3FileID, + file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{src_file_id}".replace( src_project["uuid"], dst_project["uuid"] - ).replace(f"{src_node_id}", f"{dst_node_id}"), + ).replace(f"{src_node_id}", f"{dst_node_id}") ), expected_entry_exists=True, expected_file_size=path.stat().st_size, expected_upload_id=None, expected_upload_expiration_date=None, - expected_sha256_checksum=SHA256Str(checksum), + expected_sha256_checksum=TypeAdapter(SHA256Str).validate_python( + checksum + ), ) @@ -292,7 +295,9 @@ async def test_copy_folders_from_valid_project( # check that file meta data was effectively copied for src_node_id in src_projects_list: - dst_node_id = nodes_map.get(NodeIDStr(f"{src_node_id}")) + dst_node_id = nodes_map.get( + TypeAdapter(NodeIDStr).validate_python(f"{src_node_id}") + ) assert dst_node_id for src_file_id, src_file in src_projects_list[src_node_id].items(): path: Any = src_file["path"] @@ -301,17 +306,18 @@ async def test_copy_folders_from_valid_project( assert isinstance(checksum, str) await assert_file_meta_data_in_db( aiopg_engine, - file_id=parse_obj_as( - SimcoreS3FileID, + file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{src_file_id}".replace( src_project["uuid"], dst_project["uuid"] - ).replace(f"{src_node_id}", f"{dst_node_id}"), + ).replace(f"{src_node_id}", f"{dst_node_id}") ), expected_entry_exists=True, expected_file_size=path.stat().st_size, expected_upload_id=None, expected_upload_expiration_date=None, - expected_sha256_checksum=SHA256Str(checksum), + expected_sha256_checksum=TypeAdapter(SHA256Str).validate_python( + checksum + ), ) @@ -394,9 +400,9 @@ async def with_random_project_with_files( ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],]: return await random_project_with_files( file_sizes=( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "2Mib"), - parse_obj_as(ByteSize, "5Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("2Mib"), + TypeAdapter(ByteSize).validate_python("5Mib"), ) ) @@ -472,7 +478,7 @@ async def uploaded_file_ids( for _ in range(expected_number_of_user_files): file_path, file_id = await upload_file( - file_size=parse_obj_as(ByteSize, "10Mib"), + file_size=TypeAdapter(ByteSize).validate_python("10Mib"), file_name=faker.file_name(), sha256_checksum=faker.sha256(), ) @@ -525,7 +531,7 @@ async def test_search_files_request( data, error = await assert_status(response, status.HTTP_200_OK) assert not error - found = parse_obj_as(list[FileMetaDataGet], data) + found = TypeAdapter(list[FileMetaDataGet]).validate_python(data) expected = uploaded_file_ids[ search_files_query_params.offset : search_files_query_params.offset @@ -548,7 +554,7 @@ async def test_search_files( ): assert client.app _file_name: str = faker.file_name() - _sha256_checksum: SHA256Str = parse_obj_as(SHA256Str, faker.sha256()) + _sha256_checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python(faker.sha256()) url = ( client.app.router["search_files"] @@ -571,12 +577,12 @@ async def test_search_files( data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds # let's upload some files now file, file_id = await upload_file( - file_size=parse_obj_as(ByteSize, "10Mib"), + file_size=TypeAdapter(ByteSize).validate_python("10Mib"), file_name=_file_name, sha256_checksum=_sha256_checksum, ) @@ -584,7 +590,7 @@ async def test_search_files( response = await client.post(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == 1 assert list_fmds[0].file_id == file_id assert list_fmds[0].file_size == file.stat().st_size @@ -600,7 +606,7 @@ async def test_search_files( response = await client.post(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == 1 assert list_fmds[0].file_id == file_id assert list_fmds[0].file_size == file.stat().st_size @@ -620,5 +626,5 @@ async def test_search_files( response = await client.post(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds diff --git a/services/storage/tests/unit/test_models.py b/services/storage/tests/unit/test_models.py index 0dbab6821d2..250b037e5cf 100644 --- a/services/storage/tests/unit/test_models.py +++ b/services/storage/tests/unit/test_models.py @@ -43,7 +43,7 @@ def test_file_id(file_id: str): assert parsed_file_id == file_id -def test_fmd_build(): +def test_fmd_build_api(): file_id = TypeAdapter(SimcoreS3FileID).validate_python(f"api/{uuid.uuid4()}/xx.dat") fmd = FileMetaData.from_simcore_node( user_id=12, @@ -64,6 +64,8 @@ def test_fmd_build(): assert fmd.location_id == SimcoreS3DataManager.get_location_id() assert fmd.bucket_name == "test-bucket" + +def test_fmd_build_webapi(): file_id = TypeAdapter(SimcoreS3FileID).validate_python( f"{uuid.uuid4()}/{uuid.uuid4()}/xx.dat" ) diff --git a/services/storage/tests/unit/test_simcore_s3_dsm.py b/services/storage/tests/unit/test_simcore_s3_dsm.py index 3f360b1505a..41c69355025 100644 --- a/services/storage/tests/unit/test_simcore_s3_dsm.py +++ b/services/storage/tests/unit/test_simcore_s3_dsm.py @@ -12,7 +12,7 @@ from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from simcore_service_storage import db_file_meta_data from simcore_service_storage.models import FileMetaData from simcore_service_storage.s3 import get_s3_client @@ -24,7 +24,7 @@ @pytest.fixture def file_size() -> ByteSize: - return parse_obj_as(ByteSize, "1") + return TypeAdapter(ByteSize).validate_python("1") @pytest.fixture @@ -47,7 +47,9 @@ async def test__copy_path_s3_s3( aiopg_engine: Engine, ): def _get_dest_file_id(src: SimcoreS3FileID) -> SimcoreS3FileID: - return parse_obj_as(SimcoreS3FileID, f"{Path(src).parent}/the-copy") + return TypeAdapter(SimcoreS3FileID).validate_python( + f"{Path(src).parent}/the-copy" + ) async def _copy_s3_path(s3_file_id_to_copy: SimcoreS3FileID) -> None: async with aiopg_engine.acquire() as conn: @@ -84,7 +86,7 @@ async def _count_files(s3_file_id: SimcoreS3FileID, expected_count: int) -> None assert directory_file_upload.urls[0].path s3_object = directory_file_upload.urls[0].path.lstrip("/") - s3_file_id_dir_src = parse_obj_as(SimcoreS3FileID, s3_object) + s3_file_id_dir_src = TypeAdapter(SimcoreS3FileID).validate_python(s3_object) s3_file_id_dir_dst = _get_dest_file_id(s3_file_id_dir_src) await _count_files(s3_file_id_dir_dst, expected_count=0) @@ -104,7 +106,7 @@ async def test_upload_and_search( user_id: UserID, faker: Faker, ): - checksum: SHA256Str = parse_obj_as(SHA256Str, faker.sha256()) + checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python(faker.sha256()) _, _ = await upload_file(file_size, "file1", sha256_checksum=checksum) _, _ = await upload_file(file_size, "file2", sha256_checksum=checksum) diff --git a/services/storage/tests/unit/test_utils.py b/services/storage/tests/unit/test_utils.py index 90dca22d42d..d0aef454df1 100644 --- a/services/storage/tests/unit/test_utils.py +++ b/services/storage/tests/unit/test_utils.py @@ -14,9 +14,10 @@ import pytest from aiohttp import ClientSession from faker import Faker +from models_library.api_schemas_storage import UNDEFINED_SIZE_TYPE from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from pydantic import ByteSize, HttpUrl, TypeAdapter, parse_obj_as +from pydantic import ByteSize, HttpUrl, TypeAdapter from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID from simcore_service_storage.models import ETag, FileMetaData, S3BucketName, UploadID from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -46,24 +47,30 @@ async def test_download_files(tmp_path: Path, httpbin_base_url: HttpUrl): [ (-1, None, None, None, False), (0, None, None, None, False), - (random.randint(1, 1000000), None, None, None, False), + (random.randint(1, 1000000), None, None, None, False), # noqa: S311 (-1, "some_valid_entity_tag", None, None, False), (0, "some_valid_entity_tag", None, None, False), ( - random.randint(1, 1000000), + random.randint(1, 1000000), # noqa: S311 "some_valid_entity_tag", "som_upload_id", None, False, ), ( - random.randint(1, 1000000), + random.randint(1, 1000000), # noqa: S311 "some_valid_entity_tag", None, - datetime.datetime.utcnow(), + datetime.datetime.now(datetime.UTC), False, ), - (random.randint(1, 1000000), "some_valid_entity_tag", None, None, True), + ( + random.randint(1, 1000000), # noqa: S311 + "some_valid_entity_tag", + None, + None, + True, + ), ], ) def test_file_entry_valid( @@ -84,7 +91,9 @@ def test_file_entry_valid( location_name=SimcoreS3DataManager.get_location_name(), sha256_checksum=None, ) - fmd.file_size = parse_obj_as(ByteSize, file_size) + fmd.file_size = TypeAdapter(UNDEFINED_SIZE_TYPE | ByteSize).validate_python( + file_size + ) fmd.entity_tag = entity_tag fmd.upload_id = upload_id fmd.upload_expires_at = upload_expires_at diff --git a/services/storage/tests/unit/test_utils_handlers.py b/services/storage/tests/unit/test_utils_handlers.py index a5f82a6b893..cc220ceb3e2 100644 --- a/services/storage/tests/unit/test_utils_handlers.py +++ b/services/storage/tests/unit/test_utils_handlers.py @@ -31,8 +31,7 @@ async def raising_handler( @pytest.fixture def mock_request(mocker: MockerFixture) -> web.Request: - mock = mocker.patch("aiohttp.web.Request", autospec=True) - return mock + return mocker.patch("aiohttp.web.Request", autospec=True) class FakeErrorModel(BaseModel): @@ -48,7 +47,10 @@ class FakeErrorModel(BaseModel): (ProjectNotFoundError(project_id="x"), web.HTTPNotFound), (FileAccessRightError(file_id="x", access_right="x"), web.HTTPForbidden), (ProjectAccessRightError(project_id="x", access_right="x"), web.HTTPForbidden), - (ValidationError(errors=[], model=FakeErrorModel), web.HTTPUnprocessableEntity), + ( + ValidationError.from_exception_data(title="test", line_errors=[]), + web.HTTPUnprocessableEntity, + ), (DBAPIError, web.HTTPServiceUnavailable), ], ) From 39aed6a3d9c49004568fc5b58b8649d9867b7e98 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 25 Oct 2024 13:17:08 +0200 Subject: [PATCH 21/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Upgrade=20API=20Serv?= =?UTF-8?q?er=20service=20(Pydantic=20v2)=20(#6580)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_schemas_webserver/projects.py | 12 +- .../api_schemas_webserver/wallets.py | 10 +- .../helpers/httpx_calls_capture_parameters.py | 26 +- services/api-server/Makefile | 8 +- services/api-server/openapi.json | 2079 ++++++++++------- services/api-server/requirements/_base.txt | 265 ++- .../api/routes/files.py | 10 +- .../api/routes/solvers.py | 4 +- .../api/routes/solvers_jobs.py | 2 +- .../api/routes/solvers_jobs_getters.py | 2 +- .../api/routes/studies.py | 2 +- .../core/application.py | 2 +- .../core/settings.py | 54 +- .../db/repositories/users.py | 8 +- .../exceptions/_base.py | 5 +- .../exceptions/handlers/_custom_errors.py | 3 +- .../handlers/_handlers_backend_errors.py | 6 +- .../exceptions/handlers/_http_exceptions.py | 6 +- .../handlers/_httpx_client_exceptions.py | 3 +- .../handlers/_log_streaming_errors.py | 5 +- .../exceptions/handlers/_validation_errors.py | 3 +- .../models/api_resources.py | 29 +- .../models/basic_types.py | 15 +- .../models/pagination.py | 30 +- .../models/schemas/errors.py | 9 +- .../models/schemas/files.py | 40 +- .../models/schemas/jobs.py | 70 +- .../models/schemas/meta.py | 10 +- .../models/schemas/profiles.py | 14 +- .../models/schemas/solvers.py | 36 +- .../models/schemas/studies.py | 5 +- .../services/catalog.py | 33 +- .../services/director_v2.py | 32 +- .../services/log_streaming.py | 17 +- .../services/solver_job_models_converters.py | 34 +- .../services/solver_job_outputs.py | 4 +- .../services/storage.py | 20 +- .../services/study_job_models_converters.py | 4 +- .../services/webserver.py | 66 +- services/api-server/tests/conftest.py | 4 +- .../tests/unit/_with_db/conftest.py | 4 +- .../tests/unit/_with_db/test_api_user.py | 4 +- .../_with_db/test_core_settings__with_db.py | 2 +- .../tests/unit/api_solvers/conftest.py | 4 +- .../api_solvers/test_api_routers_solvers.py | 2 +- .../test_api_routers_solvers_jobs.py | 26 +- .../test_api_routers_solvers_jobs_delete.py | 17 +- .../test_api_routers_solvers_jobs_logs.py | 14 +- .../test_api_routers_solvers_jobs_metadata.py | 18 +- .../test_api_routers_solvers_jobs_read.py | 11 +- .../test_api_routers_studies_jobs_metadata.py | 11 +- .../api_studies/test_api_routes_studies.py | 16 +- .../test_api_routes_studies_jobs.py | 13 +- .../unit/captures/test__mocks_captures.py | 8 +- services/api-server/tests/unit/conftest.py | 20 +- .../api-server/tests/unit/test__fastapi.py | 4 +- .../api-server/tests/unit/test_api_files.py | 15 +- .../api-server/tests/unit/test_api_health.py | 3 +- .../tests/unit/test_api_solver_jobs.py | 18 +- .../api-server/tests/unit/test_api_solvers.py | 3 +- .../api-server/tests/unit/test_api_wallets.py | 7 +- services/api-server/tests/unit/test_cli.py | 2 +- .../tests/unit/test_core_settings.py | 2 +- .../api-server/tests/unit/test_credits.py | 2 +- .../api-server/tests/unit/test_exceptions.py | 6 +- services/api-server/tests/unit/test_models.py | 2 +- .../tests/unit/test_models_schemas_files.py | 20 +- .../tests/unit/test_models_schemas_jobs.py | 6 +- .../tests/unit/test_models_schemas_solvers.py | 8 +- .../tests/unit/test_services_rabbitmq.py | 16 +- ...t_services_solver_job_models_converters.py | 28 +- .../unit/test_services_solver_job_outputs.py | 7 +- .../unit/test_utils_http_calls_capture.py | 14 +- 73 files changed, 2014 insertions(+), 1306 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index d02f37b0aac..b539de902a4 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -9,7 +9,7 @@ from models_library.folders import FolderID from models_library.workspaces import WorkspaceID -from pydantic import Field, HttpUrl, field_validator +from pydantic import ConfigDict, Field, HttpUrl, field_validator from ..api_schemas_long_running_tasks.tasks import TaskGet from ..basic_types import LongTruncatedStr, ShortTruncatedStr @@ -78,14 +78,18 @@ class ProjectGet(OutputSchema): ui: EmptyModel | StudyUI | None = None quality: dict[str, Any] = {} dev: dict | None = None - permalink: ProjectPermalink = FieldNotRequired() - workspace_id: WorkspaceID | None - folder_id: FolderID | None + permalink: ProjectPermalink | None = FieldNotRequired() + workspace_id: WorkspaceID | None = None + folder_id: FolderID | None = None _empty_description = field_validator("description", mode="before")( none_to_empty_str_pre_validator ) + model_config = ConfigDict( + frozen=False + ) + TaskProjectGet: TypeAlias = TaskGet diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index f9ebbd9fb2d..1a417e9671d 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -1,8 +1,8 @@ from datetime import datetime from decimal import Decimal -from typing import Literal, TypeAlias +from typing import Annotated, Literal, TypeAlias -from pydantic import ConfigDict, Field, HttpUrl, field_validator +from pydantic import ConfigDict, Field, HttpUrl, PlainSerializer, field_validator from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID @@ -21,9 +21,13 @@ class WalletGet(OutputSchema): created: datetime modified: datetime + model_config = ConfigDict( + frozen=False + ) + class WalletGetWithAvailableCredits(WalletGet): - available_credits: Decimal + available_credits: Annotated[Decimal, PlainSerializer(float)] class WalletGetPermissions(WalletGet): diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py index 25f2abc8cd0..5ebde2fb9d5 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py @@ -8,13 +8,13 @@ class CapturedParameterSchema(BaseModel): title: str | None = None type_: Literal["str", "int", "float", "bool"] | None = Field(None, alias="type") - pattern: str | None + pattern: str | None = None format_: Literal["uuid"] | None = Field(None, alias="format") - exclusiveMinimum: bool | None - minimum: int | None - anyOf: list["CapturedParameterSchema"] | None - allOf: list["CapturedParameterSchema"] | None - oneOf: list["CapturedParameterSchema"] | None + exclusiveMinimum: bool | None = None + minimum: int | float | None = None + anyOf: list["CapturedParameterSchema"] | None = None + allOf: list["CapturedParameterSchema"] | None = None + oneOf: list["CapturedParameterSchema"] | None = None class Config: validate_always = True @@ -34,15 +34,15 @@ def preprocess_type_(cls, val): @model_validator(mode="after") @classmethod def check_compatibility(cls, values): - type_ = values.get("type_") - pattern = values.get("pattern") - format_ = values.get("format_") - anyOf = values.get("anyOf") - allOf = values.get("allOf") - oneOf = values.get("oneOf") + type_ = values.type_ + pattern = values.pattern + format_ = values.format_ + anyOf = values.anyOf + allOf = values.allOf + oneOf = values.oneOf if not any([type_, oneOf, anyOf, allOf]): type_ = "str" # this default is introduced because we have started using json query params in the webserver - values["type_"] = type_ + values.type_ = type_ if type_ != "str" and any([pattern, format_]): msg = f"For {type_=} both {pattern=} and {format_=} must be None" raise ValueError(msg) diff --git a/services/api-server/Makefile b/services/api-server/Makefile index 14210bb0815..900c0e8e0f1 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -15,8 +15,8 @@ reqs: ## compiles pip requirements (.in -> .txt) # specification of the used openapi-generator-cli (see also https://github.com/ITISFoundation/openapi-generator) -OPENAPI_GENERATOR_NAME := itisfoundation/openapi-generator-cli-openapi-generator-v4.2.3 -OPENAPI_GENERATOR_TAG := v0 +OPENAPI_GENERATOR_NAME := openapitools/openapi-generator-cli +OPENAPI_GENERATOR_TAG := latest OPENAPI_GENERATOR_IMAGE := $(OPENAPI_GENERATOR_NAME):$(OPENAPI_GENERATOR_TAG) define _create_and_validate_openapi @@ -25,10 +25,6 @@ define _create_and_validate_openapi export API_SERVER_DEV_FEATURES_ENABLED=$1; \ python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ - # patching version until tools adapted - @sed -i 's/"openapi": "3.1.0",/"openapi": "3.0.2",/g' $@ - - # validates OAS file: $@ docker run --rm \ --volume "$(CURDIR):/local" \ diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 7965ae507f2..51ba054e5af 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -1,5 +1,5 @@ { - "openapi": "3.0.2", + "openapi": "3.1.0", "info": { "title": "osparc.io public API", "description": "osparc-simcore public API specifications", @@ -316,26 +316,38 @@ "summary": "Upload File", "description": "Uploads a single file to the system", "operationId": "upload_file", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "content-length", + "in": "header", "required": false, "schema": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Content-Length" - }, - "name": "content-length", - "in": "header" + } } ], "requestBody": { + "required": true, "content": { "multipart/form-data": { "schema": { "$ref": "#/components/schemas/Body_upload_file_v0_files_content_put" } } - }, - "required": true + } }, "responses": { "200": { @@ -418,12 +430,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } }, "post": { "tags": [ @@ -432,15 +439,20 @@ "summary": "Get Upload Links", "description": "Get upload links for uploading a file to storage", "operationId": "get_upload_links", + "security": [ + { + "HTTPBasic": [] + } + ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ClientFile" } } - }, - "required": true + } }, "responses": { "200": { @@ -523,12 +535,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/files/{file_id}": { @@ -539,16 +546,21 @@ "summary": "Get File", "description": "Gets metadata for a given file resource", "operationId": "get_file", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "file_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "File Id" - }, - "name": "file_id", - "in": "path" + } } ], "responses": { @@ -632,12 +644,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } }, "delete": { "tags": [ @@ -645,16 +652,21 @@ ], "summary": "Delete File", "operationId": "delete_file", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "file_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "File Id" - }, - "name": "file_id", - "in": "path" + } } ], "responses": { @@ -736,12 +748,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/files:search": { @@ -752,49 +759,68 @@ "summary": "Search Files Page", "description": "Search files", "operationId": "search_files_page", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "sha256_checksum", + "in": "query", "required": false, "schema": { - "type": "string", - "pattern": "^[a-fA-F0-9]{64}$", + "anyOf": [ + { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$" + }, + { + "type": "null" + } + ], "title": "Sha256 Checksum" - }, - "name": "sha256_checksum", - "in": "query" + } }, { + "name": "file_id", + "in": "query", "required": false, "schema": { - "type": "string", - "format": "uuid", + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], "title": "File Id" - }, - "name": "file_id", - "in": "query" + } }, { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" + "default": 50, + "title": "Limit" + } }, { + "name": "offset", + "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" + "default": 0, + "title": "Offset" + } } ], "responses": { @@ -878,12 +904,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/files/{file_id}:abort": { @@ -893,27 +914,32 @@ ], "summary": "Abort Multipart Upload", "operationId": "abort_multipart_upload", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "file_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "File Id" - }, - "name": "file_id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Body_abort_multipart_upload_v0_files__file_id__abort_post" } } - }, - "required": true + } }, "responses": { "200": { @@ -984,12 +1010,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/files/{file_id}:complete": { @@ -999,27 +1020,32 @@ ], "summary": "Complete Multipart Upload", "operationId": "complete_multipart_upload", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "file_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "File Id" - }, - "name": "file_id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Body_complete_multipart_upload_v0_files__file_id__complete_post" } } - }, - "required": true + } }, "responses": { "200": { @@ -1102,12 +1128,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/files/{file_id}/content": { @@ -1117,16 +1138,21 @@ ], "summary": "Download File", "operationId": "download_file", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "file_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "File Id" - }, - "name": "file_id", - "in": "path" + } } ], "responses": { @@ -1194,7 +1220,6 @@ } }, "200": { - "description": "Returns a arbitrary binary data", "content": { "application/octet-stream": { "schema": { @@ -1207,7 +1232,8 @@ "type": "string" } } - } + }, + "description": "Returns a arbitrary binary data" }, "422": { "description": "Validation Error", @@ -1219,12 +1245,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers": { @@ -1417,16 +1438,21 @@ "summary": "Get Latest Release of a Solver", "description": "Gets latest release of a solver", "operationId": "get_solver", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } } ], "responses": { @@ -1510,12 +1536,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases": { @@ -1526,16 +1547,21 @@ "summary": "List Solver Releases", "description": "Lists all releases of a given (one) solver\n\nSEE get_solver_releases_page for a paginated version of this function", "operationId": "list_solver_releases", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } } ], "responses": { @@ -1544,10 +1570,10 @@ "content": { "application/json": { "schema": { + "type": "array", "items": { "$ref": "#/components/schemas/Solver" }, - "type": "array", "title": "Response List Solver Releases V0 Solvers Solver Key Releases Get" } } @@ -1623,12 +1649,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}": { @@ -1639,26 +1660,31 @@ "summary": "Get Solver Release", "description": "Gets a specific release of a solver", "operationId": "get_solver_release", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } } ], "responses": { @@ -1742,12 +1768,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/ports": { @@ -1758,26 +1779,31 @@ "summary": "List Solver Ports", "description": "Lists inputs and outputs of a given solver\n\nNew in *version 0.5.0*", "operationId": "list_solver_ports", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } } ], "responses": { @@ -1861,12 +1887,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/pricing_plan": { @@ -1877,26 +1898,31 @@ "summary": "Get Solver Pricing Plan", "description": "Gets solver pricing plan\n\nNew in *version 0.7*", "operationId": "get_solver_pricing_plan", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } } ], "responses": { @@ -1980,55 +2006,105 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs": { - "get": { + "post": { "tags": [ "solvers" ], - "summary": "List Jobs", - "description": "List of jobs in a specific released solver (limited to 20 jobs)\n\n- DEPRECATION: This implementation and returned values are deprecated and the will be replaced by that of get_jobs_page\n- SEE `get_jobs_page` for paginated version of this function", - "operationId": "list_jobs", + "summary": "Create Job", + "description": "Creates a job in a specific release with given inputs.\n\nNOTE: This operation does **not** start the job", + "operationId": "create_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } + }, + { + "name": "hidden", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true, + "title": "Hidden" + } + }, + { + "name": "x-simcore-parent-project-uuid", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "X-Simcore-Parent-Project-Uuid" + } + }, + { + "name": "x-simcore-parent-node-id", + "in": "header", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "X-Simcore-Parent-Node-Id" + } } ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobInputs" + } + } + } + }, "responses": { - "200": { + "201": { "description": "Successful Response", "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/Job" - }, - "type": "array", - "title": "Response List Jobs V0 Solvers Solver Key Releases Version Jobs Get" + "$ref": "#/components/schemas/Job" } } } @@ -2113,89 +2189,53 @@ } } } - }, + } + }, + "get": { + "tags": [ + "solvers" + ], + "summary": "List Jobs", + "description": "List of jobs in a specific released solver (limited to 20 jobs)\n\n- DEPRECATION: This implementation and returned values are deprecated and the will be replaced by that of get_jobs_page\n- SEE `get_jobs_page` for paginated version of this function", + "operationId": "list_jobs", "security": [ { "HTTPBasic": [] } - ] - }, - "post": { - "tags": [ - "solvers" ], - "summary": "Create Job", - "description": "Creates a job in a specific release with given inputs.\n\nNOTE: This operation does **not** start the job", - "operationId": "create_job", "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" - }, - { - "required": false, - "schema": { - "type": "boolean", - "title": "Hidden", - "default": true - }, - "name": "hidden", - "in": "query" - }, - { - "required": false, - "schema": { - "type": "string", - "format": "uuid", - "title": "X-Simcore-Parent-Project-Uuid" - }, - "name": "x-simcore-parent-project-uuid", - "in": "header" - }, - { - "required": false, - "schema": { - "type": "string", - "format": "uuid", - "title": "X-Simcore-Parent-Node-Id" - }, - "name": "x-simcore-parent-node-id", - "in": "header" + } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JobInputs" - } - } - }, - "required": true - }, "responses": { - "201": { + "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Job" + "type": "array", + "items": { + "$ref": "#/components/schemas/Job" + }, + "title": "Response List Jobs V0 Solvers Solver Key Releases Version Jobs Get" } } } @@ -2280,64 +2320,57 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}": { - "get": { + "delete": { "tags": [ "solvers" ], - "summary": "Get Job", - "description": "Gets job of a given solver", - "operationId": "get_job", + "summary": "Delete Job", + "description": "Deletes an existing solver job\n\nNew in *version 0.7*", + "operationId": "delete_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Job" - } - } - } + "204": { + "description": "Successful Response" }, "402": { "description": "Payment required", @@ -2419,55 +2452,62 @@ } } } - }, + } + }, + "get": { + "tags": [ + "solvers" + ], + "summary": "Get Job", + "description": "Gets job of a given solver", + "operationId": "get_job", "security": [ { "HTTPBasic": [] } - ] - }, - "delete": { - "tags": [ - "solvers" ], - "summary": "Delete Job", - "description": "Deletes an existing solver job\n\nNew in *version 0.7*", - "operationId": "delete_job", "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { - "204": { - "description": "Successful Response" + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Job" + } + } + } }, "402": { "description": "Payment required", @@ -2549,12 +2589,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}:start": { @@ -2565,46 +2600,58 @@ "summary": "Start Job", "description": "Starts job job_id created with the solver solver_key:version\n\nAdded in *version 0.4.3*: query parameter `cluster_id`\nAdded in *version 0.6*: responds with a 202 when successfully starting a computation", "operationId": "start_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } }, { + "name": "cluster_id", + "in": "query", "required": false, "schema": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Cluster Id" - }, - "name": "cluster_id", - "in": "query" + } } ], "responses": { @@ -2718,12 +2765,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}:stop": { @@ -2733,36 +2775,41 @@ ], "summary": "Stop Job", "operationId": "stop_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -2856,12 +2903,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}:inspect": { @@ -2871,36 +2913,41 @@ ], "summary": "Inspect Job", "operationId": "inspect_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -2994,54 +3041,64 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}/metadata": { - "get": { + "patch": { "tags": [ "solvers" ], - "summary": "Get Job Custom Metadata", - "description": "Gets custom metadata from a job\n\nNew in *version 0.7*", - "operationId": "get_job_custom_metadata", + "summary": "Replace Job Custom Metadata", + "description": "Updates custom metadata from a job\n\nNew in *version 0.7*", + "operationId": "replace_job_custom_metadata", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobMetadataUpdate" + } + } + } + }, "responses": { "200": { "description": "Successful Response", @@ -3123,62 +3180,52 @@ } } } - }, + } + }, + "get": { + "tags": [ + "solvers" + ], + "summary": "Get Job Custom Metadata", + "description": "Gets custom metadata from a job\n\nNew in *version 0.7*", + "operationId": "get_job_custom_metadata", "security": [ { "HTTPBasic": [] } - ] - }, - "patch": { - "tags": [ - "solvers" ], - "summary": "Replace Job Custom Metadata", - "description": "Updates custom metadata from a job\n\nNew in *version 0.7*", - "operationId": "replace_job_custom_metadata", "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JobMetadataUpdate" - } - } - }, - "required": true - }, "responses": { "200": { "description": "Successful Response", @@ -3260,12 +3307,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/page": { @@ -3276,49 +3318,54 @@ "summary": "Get Jobs Page", "description": "List of jobs on a specific released solver (includes pagination)\n\nNew in *version 0.7*", "operationId": "get_jobs_page", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" + "default": 50, + "title": "Limit" + } }, { + "name": "offset", + "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" + "default": 0, + "title": "Offset" + } } ], "responses": { @@ -3412,12 +3459,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}/outputs": { @@ -3427,36 +3469,41 @@ ], "summary": "Get Job Outputs", "operationId": "get_job_outputs", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -3550,12 +3597,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}/outputs/logfile": { @@ -3566,36 +3608,41 @@ "summary": "Get Job Output Logfile", "description": "Special extra output with persistent logs file for the solver run.\n\n**NOTE**: this is not a log stream but a predefined output that is only\navailable after the job is done.\n\nNew in *version 0.4.0*", "operationId": "get_job_output_logfile", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -3603,7 +3650,6 @@ "description": "Successful Response" }, "200": { - "description": "Returns a log file", "content": { "application/octet-stream": { "schema": { @@ -3622,7 +3668,8 @@ "type": "string" } } - } + }, + "description": "Returns a log file" }, "404": { "description": "Log not found" @@ -3687,12 +3734,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}/wallet": { @@ -3703,36 +3745,41 @@ "summary": "Get Job Wallet", "description": "Get job wallet\n\nNew in *version 0.7*", "operationId": "get_job_wallet", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -3741,7 +3788,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WalletGetWithAvailableCredits" + "anyOf": [ + { + "$ref": "#/components/schemas/WalletGetWithAvailableCredits" + }, + { + "type": "null" + } + ], + "title": "Response Get Job Wallet V0 Solvers Solver Key Releases Version Jobs Job Id Wallet Get" } } } @@ -3826,12 +3881,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}/pricing_unit": { @@ -3842,36 +3892,41 @@ "summary": "Get Job Pricing Unit", "description": "Get job pricing unit\n\nNew in *version 0.7*", "operationId": "get_job_pricing_unit", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -3880,7 +3935,15 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PricingUnitGet" + "anyOf": [ + { + "$ref": "#/components/schemas/PricingUnitGet" + }, + { + "type": "null" + } + ], + "title": "Response Get Job Pricing Unit V0 Solvers Solver Key Releases Version Jobs Job Id Pricing Unit Get" } } } @@ -3955,12 +4018,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/solvers/{solver_key}/releases/{version}/jobs/{job_id}/logstream": { @@ -3970,36 +4028,41 @@ ], "summary": "Get Log Stream", "operationId": "get_log_stream", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "solver_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" - }, - "name": "solver_key", - "in": "path" + } }, { + "name": "version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" - }, - "name": "version", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4008,6 +4071,7 @@ "content": { "application/x-ndjson": { "schema": { + "type": "string", "anyOf": [ { "$ref": "#/components/schemas/JobLog" @@ -4016,7 +4080,6 @@ "$ref": "#/components/schemas/ErrorGet" } ], - "type": "string", "title": "Response 200 Get Log Stream V0 Solvers Solver Key Releases Version Jobs Job Id Logstream Get" } } @@ -4092,12 +4155,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies": { @@ -4108,29 +4166,34 @@ "summary": "List Studies", "description": "New in *version 0.5.0*", "operationId": "list_studies", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "minimum": 1, - "title": "Limit", - "default": 50 - }, - "name": "limit", - "in": "query" + "default": 50, + "title": "Limit" + } }, { + "name": "offset", + "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, - "title": "Offset", - "default": 0 - }, - "name": "offset", - "in": "query" + "default": 0, + "title": "Offset" + } } ], "responses": { @@ -4154,12 +4217,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}": { @@ -4170,16 +4228,21 @@ "summary": "Get Study", "description": "New in *version 0.5.0*", "operationId": "get_study", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } } ], "responses": { @@ -4213,12 +4276,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}:clone": { @@ -4228,36 +4286,55 @@ ], "summary": "Clone Study", "operationId": "clone_study", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "x-simcore-parent-project-uuid", + "in": "header", "required": false, "schema": { - "type": "string", - "format": "uuid", + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], "title": "X-Simcore-Parent-Project-Uuid" - }, - "name": "x-simcore-parent-project-uuid", - "in": "header" + } }, { + "name": "x-simcore-parent-node-id", + "in": "header", "required": false, "schema": { - "type": "string", - "format": "uuid", + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], "title": "X-Simcore-Parent-Node-Id" - }, - "name": "x-simcore-parent-node-id", - "in": "header" + } } ], "responses": { @@ -4291,12 +4368,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/ports": { @@ -4307,16 +4379,21 @@ "summary": "List Study Ports", "description": "Lists metadata on ports of a given study\n\nNew in *version 0.5.0*", "operationId": "list_study_ports", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } } ], "responses": { @@ -4350,12 +4427,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs": { @@ -4366,57 +4438,76 @@ "summary": "Create Study Job", "description": "hidden -- if True (default) hides project from UI", "operationId": "create_study_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "hidden", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Hidden", - "default": true - }, - "name": "hidden", - "in": "query" + "default": true, + "title": "Hidden" + } }, { + "name": "x-simcore-parent-project-uuid", + "in": "header", "required": false, "schema": { - "type": "string", - "format": "uuid", + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], "title": "X-Simcore-Parent-Project-Uuid" - }, - "name": "x-simcore-parent-project-uuid", - "in": "header" + } }, { + "name": "x-simcore-parent-node-id", + "in": "header", "required": false, "schema": { - "type": "string", - "format": "uuid", + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], "title": "X-Simcore-Parent-Node-Id" - }, - "name": "x-simcore-parent-node-id", - "in": "header" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobInputs" } } - }, - "required": true + } }, "responses": { "200": { @@ -4439,12 +4530,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}": { @@ -4455,26 +4541,31 @@ "summary": "Delete Study Job", "description": "Deletes an existing study job", "operationId": "delete_study_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4482,14 +4573,14 @@ "description": "Successful Response" }, "404": { - "description": "Not Found", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorGet" } } - } + }, + "description": "Not Found" }, "422": { "description": "Validation Error", @@ -4499,14 +4590,9 @@ "$ref": "#/components/schemas/HTTPValidationError" } } - } - } - }, - "security": [ - { - "HTTPBasic": [] + } } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}:start": { @@ -4517,36 +4603,48 @@ "summary": "Start Study Job", "description": "Changed in *version 0.6.0*: Now responds with a 202 when successfully starting a computation", "operationId": "start_study_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } }, { + "name": "cluster_id", + "in": "query", "required": false, "schema": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Cluster Id" - }, - "name": "cluster_id", - "in": "query" + } } ], "responses": { @@ -4660,12 +4758,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}:stop": { @@ -4675,26 +4768,31 @@ ], "summary": "Stop Study Job", "operationId": "stop_study_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4718,12 +4816,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}:inspect": { @@ -4733,26 +4826,31 @@ ], "summary": "Inspect Study Job", "operationId": "inspect_study_job", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4776,12 +4874,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}/outputs": { @@ -4791,26 +4884,31 @@ ], "summary": "Get Study Job Outputs", "operationId": "get_study_job_outputs", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4834,12 +4932,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}/outputs/log-links": { @@ -4849,26 +4942,31 @@ ], "summary": "Get download links for study job log files", "operationId": "get_study_job_output_logfile", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4892,12 +4990,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/studies/{study_id}/jobs/{job_id}/metadata": { @@ -4908,26 +5001,31 @@ "summary": "Get Study Job Custom Metadata", "description": "Get custom metadata from a study's job\n\nNew in *version 0.7*", "operationId": "get_study_job_custom_metadata", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "responses": { @@ -4951,12 +5049,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } }, "put": { "tags": [ @@ -4965,37 +5058,42 @@ "summary": "Replace Study Job Custom Metadata", "description": "Changes custom metadata of a study's job\n\nNew in *version 0.7*", "operationId": "replace_study_job_custom_metadata", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "study_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Study Id" - }, - "name": "study_id", - "in": "path" + } }, { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Job Id" - }, - "name": "job_id", - "in": "path" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/JobMetadataUpdate" } } - }, - "required": true + } }, "responses": { "200": { @@ -5018,12 +5116,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/wallets/default": { @@ -5131,15 +5224,20 @@ "summary": "Get Wallet", "description": "Get wallet\n\nNew in *version 0.7*", "operationId": "get_wallet", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { + "name": "wallet_id", + "in": "path", "required": true, "schema": { "type": "integer", "title": "Wallet Id" - }, - "name": "wallet_id", - "in": "path" + } } ], "responses": { @@ -5233,12 +5331,7 @@ } } } - }, - "security": [ - { - "HTTPBasic": [] - } - ] + } } }, "/v0/credits/price": { @@ -5321,7 +5414,16 @@ "description": "File name" }, "filesize": { - "type": "integer", + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*(\\d*\\.?\\d+)\\s*(\\w+)?" + }, + { + "type": "integer", + "minimum": 0 + } + ], "title": "Filesize", "description": "File size in bytes" }, @@ -5350,12 +5452,7 @@ "description": "The file resource id" }, "upload_schema": { - "allOf": [ - { - "$ref": "#/components/schemas/FileUploadData" - } - ], - "title": "Upload Schema", + "$ref": "#/components/schemas/FileUploadData", "description": "Schema for uploading file" } }, @@ -5400,18 +5497,39 @@ "description": "Name of the file with extension" }, "content_type": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Content Type", "description": "Guess of type content [EXPERIMENTAL]" }, "checksum": { - "type": "string", - "pattern": "^[a-fA-F0-9]{64}$", + "anyOf": [ + { + "type": "string", + "pattern": "^[a-fA-F0-9]{64}$" + }, + { + "type": "null" + } + ], "title": "Checksum", "description": "SHA256 hash of the file's content" }, "e_tag": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "E Tag", "description": "S3 entity tag" } @@ -5444,12 +5562,12 @@ "properties": { "chunk_size": { "type": "integer", + "minimum": 0, "title": "Chunk Size" }, "urls": { "items": { "type": "string", - "maxLength": 65536, "minLength": 1, "format": "uri" }, @@ -5475,14 +5593,27 @@ "title": "Productname" }, "usdPerCredit": { - "type": "number", - "minimum": 0.0, + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Usdpercredit", "description": "Price of a credit in USD. If None, then this product's price is UNDEFINED" }, "minPaymentAmountUsd": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Minpaymentamountusd", "description": "Minimum amount (included) in USD that can be paid for this productCan be None if this product's price is UNDEFINED" } @@ -5501,10 +5632,17 @@ "$ref": "#/components/schemas/UsersGroup" }, "organizations": { - "items": { - "$ref": "#/components/schemas/UsersGroup" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/UsersGroup" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Organizations", "default": [] }, @@ -5562,26 +5700,47 @@ "description": "Runner that executes job" }, "url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Url", "description": "Link to get this resource (self)" }, "runner_url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Runner Url", "description": "Link to the solver's job (parent collection)" }, "outputs_url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Outputs Url", "description": "Link to the job outputs (sub-collection)" } @@ -5599,14 +5758,14 @@ ], "title": "Job", "example": { + "created_at": "2021-01-22T23:59:52.322176", "id": "f622946d-fd29-35b9-a193-abdd1095167c", + "inputs_checksum": "12345", "name": "solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c", + "outputs_url": "https://api.osparc.io/v0/solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs", "runner_name": "solvers/isolve/releases/1.3.4", - "inputs_checksum": "12345", - "created_at": "2021-01-22T23:59:52.322176", - "url": "https://api.osparc.io/v0/solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c", "runner_url": "https://api.osparc.io/v0/solvers/isolve/releases/1.3.4", - "outputs_url": "https://api.osparc.io/v0/solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs" + "url": "https://api.osparc.io/v0/solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c" } }, "JobInputs": { @@ -5632,6 +5791,9 @@ { "items": {}, "type": "array" + }, + { + "type": "null" } ] }, @@ -5646,14 +5808,14 @@ "title": "JobInputs", "example": { "values": { - "x": 4.33, - "n": 55, - "title": "Temperature", "enabled": true, "input_file": { "filename": "input.txt", "id": "0a3b2c56-dbcd-4871-b93b-d454b7883f9f" - } + }, + "n": 55, + "title": "Temperature", + "x": 4.33 } } }, @@ -5665,8 +5827,15 @@ "title": "Job Id" }, "node_id": { - "type": "string", - "format": "uuid", + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], "title": "Node Id" }, "log_level": { @@ -5690,11 +5859,11 @@ "title": "JobLog", "example": { "job_id": "145beae4-a3a8-4fde-adbb-4e8257c2c083", - "node_id": "3742215e-6756-48d2-8b73-4d043065309f", "log_level": 10, "messages": [ "PROGRESS: 5/10" - ] + ], + "node_id": "3742215e-6756-48d2-8b73-4d043065309f" } }, "JobLogsMap": { @@ -5744,10 +5913,17 @@ "description": "Custom key-value map" }, "url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Url", "description": "Link to get this resource (self)" } @@ -5816,6 +5992,9 @@ { "items": {}, "type": "array" + }, + { + "type": "null" } ] }, @@ -5832,14 +6011,14 @@ "example": { "job_id": "99d9ac65-9f10-4e2f-a433-b5e412bb037b", "results": { + "enabled": false, "maxSAR": 4.33, "n": 55, - "title": "Specific Absorption Rate", - "enabled": false, "output_file": { "filename": "sar_matrix.txt", "id": "0a3b2c56-dbcd-4871-b93b-d454b7883f9f" - } + }, + "title": "Specific Absorption Rate" } } }, @@ -5867,14 +6046,28 @@ "description": "Last modification timestamp of the solver job" }, "started_at": { - "type": "string", - "format": "date-time", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], "title": "Started At", "description": "Timestamp that indicate the moment the solver starts execution or None if the event did not occur" }, "stopped_at": { - "type": "string", - "format": "date-time", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], "title": "Stopped At", "description": "Timestamp at which the solver finished or killed execution or None if the event did not occur" } @@ -5888,41 +6081,78 @@ "title": "JobStatus", "example": { "job_id": "145beae4-a3a8-4fde-adbb-4e8257c2c083", - "state": "STARTED", "progress": 3, - "submitted_at": "2021-04-01 07:15:54.631007", - "started_at": "2021-04-01 07:16:43.670610" + "started_at": "2021-04-01 07:16:43.670610", + "state": "STARTED", + "submitted_at": "2021-04-01 07:15:54.631007" } }, "Links": { "properties": { "first": { - "type": "string", - "title": "First", - "example": "/api/v1/users?limit=1&offset1" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "First" }, "last": { - "type": "string", - "title": "Last", - "example": "/api/v1/users?limit=1&offset1" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last" }, "self": { - "type": "string", - "title": "Self", - "example": "/api/v1/users?limit=1&offset1" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Self" }, "next": { - "type": "string", - "title": "Next", - "example": "/api/v1/users?limit=1&offset1" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next" }, "prev": { - "type": "string", - "title": "Prev", - "example": "/api/v1/users?limit=1&offset1" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prev" } }, "type": "object", + "required": [ + "first", + "last", + "self", + "next", + "prev" + ], "title": "Links" }, "LogLink": { @@ -5933,7 +6163,6 @@ }, "download_link": { "type": "string", - "maxLength": 65536, "minLength": 1, "format": "uri", "title": "Download Link" @@ -5958,24 +6187,29 @@ "title": "Version" }, "released": { - "additionalProperties": { - "type": "string", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Released", "description": "Maps every route's path tag with a released version" }, "docs_url": { "type": "string", - "maxLength": 65536, "minLength": 1, "format": "uri", "title": "Docs Url" }, "docs_dev_url": { "type": "string", - "maxLength": 65536, "minLength": 1, "format": "uri", "title": "Docs Dev Url" @@ -5990,14 +6224,14 @@ ], "title": "Meta", "example": { + "docs_dev_url": "https://api.osparc.io/dev/doc", + "docs_url": "https://api.osparc.io/dev/doc", "name": "simcore_service_foo", - "version": "2.4.45", "released": { "v1": "1.3.4", "v2": "2.4.45" }, - "docs_url": "https://api.osparc.io/dev/doc", - "docs_dev_url": "https://api.osparc.io/dev/doc" + "version": "2.4.45" } }, "OnePage_SolverPort_": { @@ -6019,8 +6253,7 @@ "required": [ "items" ], - "title": "OnePage[SolverPort]", - "description": "A single page is used to envelope a small sequence that does not require\npagination\n\nIf total > MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, we should consider extending this\nentrypoint to proper pagination" + "title": "OnePage[SolverPort]" }, "OnePage_StudyPort_": { "properties": { @@ -6041,8 +6274,7 @@ "required": [ "items" ], - "title": "OnePage[StudyPort]", - "description": "A single page is used to envelope a small sequence that does not require\npagination\n\nIf total > MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, we should consider extending this\nentrypoint to proper pagination" + "title": "OnePage[StudyPort]" }, "Page_File_": { "properties": { @@ -6054,18 +6286,39 @@ "title": "Items" }, "total": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Total" }, "limit": { - "type": "integer", - "minimum": 1, + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], "title": "Limit" }, "offset": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Offset" }, "links": { @@ -6075,6 +6328,9 @@ "type": "object", "required": [ "items", + "total", + "limit", + "offset", "links" ], "title": "Page[File]" @@ -6089,18 +6345,39 @@ "title": "Items" }, "total": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Total" }, "limit": { - "type": "integer", - "minimum": 1, + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], "title": "Limit" }, "offset": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Offset" }, "links": { @@ -6110,6 +6387,9 @@ "type": "object", "required": [ "items", + "total", + "limit", + "offset", "links" ], "title": "Page[Job]" @@ -6124,18 +6404,39 @@ "title": "Items" }, "total": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Total" }, "limit": { - "type": "integer", - "minimum": 1, + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], "title": "Limit" }, "offset": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Offset" }, "links": { @@ -6145,6 +6446,9 @@ "type": "object", "required": [ "items", + "total", + "limit", + "offset", "links" ], "title": "Page[Study]" @@ -6154,8 +6458,8 @@ "enum": [ "TIER" ], - "title": "PricingPlanClassification", - "description": "An enumeration." + "const": "TIER", + "title": "PricingPlanClassification" }, "PricingUnitGet": { "properties": { @@ -6174,7 +6478,7 @@ "title": "Unitextrainfo" }, "currentCostPerUnit": { - "type": "number", + "type": "string", "title": "Currentcostperunit" }, "default": { @@ -6195,16 +6499,28 @@ "Profile": { "properties": { "first_name": { - "type": "string", - "maxLength": 255, - "title": "First Name", - "example": "James" + "anyOf": [ + { + "type": "string", + "maxLength": 255 + }, + { + "type": "null" + } + ], + "title": "First Name" }, "last_name": { - "type": "string", - "maxLength": 255, - "title": "Last Name", - "example": "Maxwell" + "anyOf": [ + { + "type": "string", + "maxLength": 255 + }, + { + "type": "null" + } + ], + "title": "Last Name" }, "id": { "type": "integer", @@ -6221,11 +6537,25 @@ "$ref": "#/components/schemas/UserRoleEnum" }, "groups": { - "$ref": "#/components/schemas/Groups" + "anyOf": [ + { + "$ref": "#/components/schemas/Groups" + }, + { + "type": "null" + } + ] }, "gravatar_id": { - "type": "string", - "maxLength": 40, + "anyOf": [ + { + "type": "string", + "maxLength": 40 + }, + { + "type": "null" + } + ], "title": "Gravatar Id", "description": "md5 hash value of email to retrieve an avatar image from https://www.gravatar.com" } @@ -6238,40 +6568,52 @@ ], "title": "Profile", "example": { - "id": "20", "first_name": "James", - "last_name": "Maxwell", - "login": "james-maxwell@itis.swiss", - "role": "USER", + "gravatar_id": "9a8930a5b20d7048e37740bac5c1ca4f", "groups": { + "all": { + "description": "all users", + "gid": "1", + "label": "Everyone" + }, "me": { + "description": "primary group", "gid": "123", - "label": "maxy", - "description": "primary group" + "label": "maxy" }, - "organizations": [], - "all": { - "gid": "1", - "label": "Everyone", - "description": "all users" - } + "organizations": [] }, - "gravatar_id": "9a8930a5b20d7048e37740bac5c1ca4f" + "id": "20", + "last_name": "Maxwell", + "login": "james-maxwell@itis.swiss", + "role": "USER" } }, "ProfileUpdate": { "properties": { "first_name": { - "type": "string", - "maxLength": 255, - "title": "First Name", - "example": "James" + "anyOf": [ + { + "type": "string", + "maxLength": 255 + }, + { + "type": "null" + } + ], + "title": "First Name" }, "last_name": { - "type": "string", - "maxLength": 255, - "title": "Last Name", - "example": "Maxwell" + "anyOf": [ + { + "type": "string", + "maxLength": 255 + }, + { + "type": "null" + } + ], + "title": "Last Name" } }, "type": "object", @@ -6362,7 +6704,14 @@ "description": "Human readable name" }, "description": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Description" }, "maintainer": { @@ -6370,10 +6719,17 @@ "title": "Maintainer" }, "url": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" + }, + { + "type": "null" + } + ], "title": "Url", "description": "Link to get this resource" } @@ -6389,12 +6745,12 @@ "title": "Solver", "description": "A released solver with a specific version", "example": { - "id": "simcore/services/comp/isolve", - "version": "2.1.1", - "title": "iSolve", "description": "EM solver", + "id": "simcore/services/comp/isolve", "maintainer": "info@itis.swiss", - "url": "https://api.osparc.io/v0/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/2.1.1" + "title": "iSolve", + "url": "https://api.osparc.io/v0/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/2.1.1", + "version": "2.1.1" } }, "SolverPort": { @@ -6414,7 +6770,14 @@ "title": "Kind" }, "content_schema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Content Schema", "description": "jsonschema for the port's value. SEE https://json-schema.org" } @@ -6426,15 +6789,15 @@ ], "title": "SolverPort", "example": { - "key": "input_2", - "kind": "input", "content_schema": { + "maximum": 5, + "minimum": 0, "title": "Sleep interval", "type": "integer", - "x_unit": "second", - "minimum": 0, - "maximum": 5 - } + "x_unit": "second" + }, + "key": "input_2", + "kind": "input" } }, "Study": { @@ -6476,7 +6839,14 @@ "title": "Kind" }, "content_schema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Content Schema", "description": "jsonschema for the port's value. SEE https://json-schema.org" } @@ -6488,15 +6858,15 @@ ], "title": "StudyPort", "example": { - "key": "input_2", - "kind": "input", "content_schema": { + "maximum": 5, + "minimum": 0, "title": "Sleep interval", "type": "integer", - "x_unit": "second", - "minimum": 0, - "maximum": 5 - } + "x_unit": "second" + }, + "key": "input_2", + "kind": "input" } }, "UploadLinks": { @@ -6547,8 +6917,7 @@ "PRODUCT_OWNER", "ADMIN" ], - "title": "UserRoleEnum", - "description": "An enumeration." + "title": "UserRoleEnum" }, "UsersGroup": { "properties": { @@ -6561,7 +6930,14 @@ "title": "Label" }, "description": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Description" } }, @@ -6620,7 +6996,14 @@ "title": "Name" }, "description": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Description" }, "owner": { @@ -6630,7 +7013,14 @@ "minimum": 0 }, "thumbnail": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Thumbnail" }, "status": { @@ -6655,7 +7045,9 @@ "required": [ "walletId", "name", + "description", "owner", + "thumbnail", "status", "created", "modified", @@ -6669,8 +7061,7 @@ "ACTIVE", "INACTIVE" ], - "title": "WalletStatus", - "description": "An enumeration." + "title": "WalletStatus" } }, "securitySchemes": { diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 51e2544cf88..c3bfeb2b28d 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -23,17 +23,31 @@ aiofiles==23.2.1 # -r requirements/_base.in aiohttp==3.9.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -51,6 +65,8 @@ alembic==1.13.1 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -82,17 +98,31 @@ attrs==23.2.0 # jsonschema certifi==2024.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -109,17 +139,31 @@ click==8.1.7 # uvicorn cryptography==42.0.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -136,27 +180,15 @@ email-validator==2.1.1 # via # fastapi # pydantic -fast-depends==2.4.12 +fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.2 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator +fastapi-cli==0.0.5 + # via fastapi fastapi-pagination==0.12.31 # via -r requirements/_base.in faststream==0.5.28 @@ -189,17 +221,31 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -219,17 +265,31 @@ itsdangerous==2.1.2 # via fastapi jinja2==3.1.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -243,17 +303,31 @@ jsonschema==3.2.0 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in mako==1.3.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -331,17 +405,31 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -381,40 +469,89 @@ psycopg2-binary==2.9.9 # sqlalchemy pycparser==2.22 # via cffi -pydantic==1.10.14 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in # fast-depends # fastapi # fastapi-pagination + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # fastapi +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in + # fastapi pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -427,23 +564,37 @@ python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 # via - # pydantic + # pydantic-settings # uvicorn python-multipart==0.0.9 # via fastapi pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -454,17 +605,31 @@ pyyaml==6.0.1 # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -499,17 +664,31 @@ sniffio==1.3.1 # httpx sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -517,19 +696,33 @@ sqlalchemy==1.4.52 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # aiopg # alembic -starlette==0.27.0 +starlette==0.38.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -556,6 +749,8 @@ typer==0.12.3 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in + # fastapi-cli + # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 @@ -571,37 +766,66 @@ typing-extensions==4.10.0 # opentelemetry-sdk # pint # pydantic + # pydantic-core # typer ujson==5.9.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi urllib3==2.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -610,6 +834,7 @@ uvicorn==0.29.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/services/api-server/src/simcore_service_api_server/api/routes/files.py b/services/api-server/src/simcore_service_api_server/api/routes/files.py index 76110352782..c279f2ce303 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/files.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/files.py @@ -12,7 +12,7 @@ from fastapi_pagination.api import create_page from models_library.api_schemas_storage import ETag, FileUploadCompletionBody, LinkType from models_library.basic_types import SHA256Str -from pydantic import AnyUrl, ByteSize, PositiveInt, ValidationError, parse_obj_as +from pydantic import AnyUrl, ByteSize, PositiveInt, TypeAdapter, ValidationError from servicelib.fastapi.requests_decorators import cancel_on_disconnect from simcore_sdk.node_ports_common.constants import SIMCORE_LOCATION from simcore_sdk.node_ports_common.file_io_utils import UploadableFileObject @@ -361,7 +361,9 @@ async def abort_multipart_upload( abort_link: URL = await storage_client.create_abort_upload_link( file=file, query={"user_id": str(user_id)} ) - await abort_upload(abort_upload_link=parse_obj_as(AnyUrl, str(abort_link))) + await abort_upload( + abort_upload_link=TypeAdapter(AnyUrl).validate_python(str(abort_link)) + ) @router.post( @@ -393,7 +395,7 @@ async def complete_multipart_upload( e_tag: ETag = await complete_file_upload( uploaded_parts=uploaded_parts.parts, - upload_completion_link=parse_obj_as(AnyUrl, f"{complete_link}"), + upload_completion_link=TypeAdapter(AnyUrl).validate_python(f"{complete_link}"), ) file.e_tag = e_tag @@ -433,4 +435,4 @@ async def download_file( ) _logger.info("Downloading %s to %s ...", file_meta, presigned_download_link) - return RedirectResponse(presigned_download_link) + return RedirectResponse(f"{presigned_download_link}") diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 01e58dc2653..c172000bd9e 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -7,7 +7,6 @@ from httpx import HTTPStatusError from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet from pydantic import ValidationError -from pydantic.errors import PydanticValueError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import VersionStr @@ -230,7 +229,6 @@ async def get_solver_release( IndexError, ValidationError, HTTPStatusError, - PydanticValueError, ) as err: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -259,7 +257,7 @@ async def list_solver_ports( product_name=product_name, ) - return OnePage[SolverPort](items=ports) + return OnePage[SolverPort].model_validate(dict(items=ports)) @router.get( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 0cdbfdf7e0a..151a79c6871 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -50,7 +50,7 @@ def _compose_job_resource_name(solver_key, solver_version, job_id) -> str: """Creates a unique resource name for solver's jobs""" return Job.compose_resource_name( - parent_name=Solver.compose_resource_name(solver_key, solver_version), # type: ignore + parent_name=Solver.compose_resource_name(solver_key, solver_version), job_id=job_id, ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index fb98a858e47..4e36afe7ccd 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -334,7 +334,7 @@ async def get_job_output_logfile( f"{solver_key}/releases/{version}/jobs/{job_id}/outputs/logfile", presigned_download_link, ) - return RedirectResponse(presigned_download_link) + return RedirectResponse(f"{presigned_download_link}") # No log found ! raise HTTPException( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies.py b/services/api-server/src/simcore_service_api_server/api/routes/studies.py index c5e9c010368..392acd8c72a 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies.py @@ -29,7 +29,7 @@ def _create_study_from_project(project: ProjectGet) -> Study: assert isinstance(project, ProjectGet) # nosec - return Study.construct( + return Study.model_construct( uid=project.uuid, title=project.name, description=project.description, diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 39b05720a6d..a39617ce33d 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -54,7 +54,7 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: config_all_loggers( log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED ) - _logger.debug("App settings:\n%s", settings.json(indent=2)) + _logger.debug("App settings:\n%s", settings.model_dump_json(indent=2)) # Labeling title = "osparc.io public API" diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index 3c00b3489b9..05e9abe3234 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -1,8 +1,14 @@ from functools import cached_property from models_library.basic_types import BootModeEnum, LogLevel -from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr -from pydantic.class_validators import validator +from pydantic import ( + AliasChoices, + Field, + NonNegativeInt, + PositiveInt, + SecretStr, + field_validator, +) from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings from settings_library.director_v2 import DirectorV2Settings @@ -25,11 +31,13 @@ class WebServerSettings(WebServerBaseSettings, MixinSessionSettings): description="Secret key to encrypt cookies. " 'TIP: python3 -c "from cryptography.fernet import *; print(Fernet.generate_key())"', min_length=44, - env=["SESSION_SECRET_KEY", "WEBSERVER_SESSION_SECRET_KEY"], + validation_alias=AliasChoices( + "SESSION_SECRET_KEY", "WEBSERVER_SESSION_SECRET_KEY" + ), ) WEBSERVER_SESSION_NAME: str = DEFAULT_SESSION_COOKIE_NAME - @validator("WEBSERVER_SESSION_SECRET_KEY") + @field_validator("WEBSERVER_SESSION_SECRET_KEY") @classmethod def check_valid_fernet_key(cls, v): return cls.do_check_valid_fernet_key(v) @@ -42,21 +50,25 @@ class BasicSettings(BaseCustomSettings, MixinLoggingSettings): # DEVELOPMENT API_SERVER_DEV_FEATURES_ENABLED: bool = Field( default=False, - env=["API_SERVER_DEV_FEATURES_ENABLED", "FAKE_API_SERVER_ENABLED"], + validation_alias=AliasChoices( + "API_SERVER_DEV_FEATURES_ENABLED", "FAKE_API_SERVER_ENABLED" + ), ) # LOGGING LOG_LEVEL: LogLevel = Field( default=LogLevel.INFO.value, - env=["API_SERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices("API_SERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + validation_alias=AliasChoices( + "API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED" + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - @validator("LOG_LEVEL", pre=True) + @field_validator("LOG_LEVEL", mode="before") @classmethod def _validate_loglevel(cls, value) -> str: log_level: str = cls.validate_log_level(value) @@ -65,20 +77,29 @@ def _validate_loglevel(cls, value) -> str: class ApplicationSettings(BasicSettings): # DOCKER BOOT - SC_BOOT_MODE: BootModeEnum | None + SC_BOOT_MODE: BootModeEnum | None = None - API_SERVER_POSTGRES: PostgresSettings | None = Field(auto_default_from_env=True) + API_SERVER_POSTGRES: PostgresSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) API_SERVER_RABBITMQ: RabbitSettings | None = Field( - auto_default_from_env=True, description="settings for service/rabbitmq" + json_schema_extra={"auto_default_from_env": True}, + description="settings for service/rabbitmq", ) # SERVICES with http API - API_SERVER_WEBSERVER: WebServerSettings | None = Field(auto_default_from_env=True) - API_SERVER_CATALOG: CatalogSettings | None = Field(auto_default_from_env=True) - API_SERVER_STORAGE: StorageSettings | None = Field(auto_default_from_env=True) + API_SERVER_WEBSERVER: WebServerSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) + API_SERVER_CATALOG: CatalogSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) + API_SERVER_STORAGE: StorageSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) API_SERVER_DIRECTOR_V2: DirectorV2Settings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) API_SERVER_LOG_CHECK_TIMEOUT_SECONDS: NonNegativeInt = 3 * 60 API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True @@ -88,7 +109,8 @@ class ApplicationSettings(BasicSettings): API_SERVER_PROMETHEUS_INSTRUMENTATION_COLLECT_SECONDS: PositiveInt = 5 API_SERVER_PROFILING: bool = False API_SERVER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + description="settings for opentelemetry tracing", + json_schema_extra={"auto_default_from_env": True}, ) @cached_property diff --git a/services/api-server/src/simcore_service_api_server/db/repositories/users.py b/services/api-server/src/simcore_service_api_server/db/repositories/users.py index 15ea74f083b..3ae1ce00184 100644 --- a/services/api-server/src/simcore_service_api_server/db/repositories/users.py +++ b/services/api-server/src/simcore_service_api_server/db/repositories/users.py @@ -1,6 +1,6 @@ import sqlalchemy as sa from models_library.emails import LowerCaseEmailStr -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..tables import UserStatus, users from ._base import BaseRepository @@ -14,4 +14,8 @@ async def get_active_user_email(self, user_id: int) -> LowerCaseEmailStr | None: (users.c.id == user_id) & (users.c.status == UserStatus.ACTIVE) ) ) - return parse_obj_as(LowerCaseEmailStr, email) if email is not None else None + return ( + TypeAdapter(LowerCaseEmailStr).validate_python(email) + if email is not None + else None + ) diff --git a/services/api-server/src/simcore_service_api_server/exceptions/_base.py b/services/api-server/src/simcore_service_api_server/exceptions/_base.py index 5ea7664ec23..9101ae4164c 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/_base.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/_base.py @@ -1,8 +1,5 @@ -from typing import Any - from common_library.errors_classes import OsparcErrorMixin class ApiServerBaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + ... diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_custom_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_custom_errors.py index 48ab4aeab11..558b5191f59 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_custom_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_custom_errors.py @@ -8,8 +8,9 @@ from ._utils import create_error_json_response -async def custom_error_handler(request: Request, exc: CustomBaseError): +async def custom_error_handler(request: Request, exc: Exception): assert request # nosec + assert isinstance(exc, CustomBaseError) error_msg = f"{exc}" if isinstance(exc, InsufficientCreditsError): diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py index e1dc19c26ea..e46d0f8f977 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_handlers_backend_errors.py @@ -5,8 +5,8 @@ from ._utils import create_error_json_response -async def backend_error_handler( - request: Request, exc: BaseBackEndError -) -> JSONResponse: +async def backend_error_handler(request: Request, exc: Exception) -> JSONResponse: assert request # nosec + assert isinstance(exc, BaseBackEndError) + return create_error_json_response(f"{exc}", status_code=exc.status_code) diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py index f0a03e2605b..24be07387d7 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py @@ -1,10 +1,12 @@ -from fastapi import HTTPException +from starlette.exceptions import HTTPException from starlette.requests import Request from starlette.responses import JSONResponse from ._utils import create_error_json_response -async def http_exception_handler(request: Request, exc: HTTPException) -> JSONResponse: +async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse: assert request # nosec + assert isinstance(exc, HTTPException) + return create_error_json_response(exc.detail, status_code=exc.status_code) diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_httpx_client_exceptions.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_httpx_client_exceptions.py index 265e68c9115..99989de85e3 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_httpx_client_exceptions.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_httpx_client_exceptions.py @@ -14,13 +14,14 @@ _logger = logging.getLogger(__file__) -async def handle_httpx_client_exceptions(request: Request, exc: HTTPError): +async def handle_httpx_client_exceptions(request: Request, exc: Exception): """ Default httpx exception handler. See https://www.python-httpx.org/exceptions/ With this in place only HTTPStatusErrors need to be customized closer to the httpx client itself. """ assert request # nosec + assert isinstance(exc, HTTPError) status_code: Any detail: str diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_log_streaming_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_log_streaming_errors.py index 066ee5dd2d6..9340b9fcbd8 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_log_streaming_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_log_streaming_errors.py @@ -10,10 +10,9 @@ from ._utils import create_error_json_response -async def log_handling_error_handler( - request: Request, exc: LogStreamingBaseError -) -> JSONResponse: +async def log_handling_error_handler(request: Request, exc: Exception) -> JSONResponse: assert request # nosec + assert isinstance(exc, LogStreamingBaseError) msg = f"{exc}" status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_validation_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_validation_errors.py index de7fd25fecf..f7d5f0d7c93 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_validation_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_validation_errors.py @@ -10,9 +10,10 @@ async def http422_error_handler( request: Request, - exc: RequestValidationError | ValidationError, + exc: Exception, ) -> JSONResponse: assert request # nosec + assert isinstance(exc, RequestValidationError | ValidationError) return create_error_json_response( *exc.errors(), status_code=status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index 3f64fd323c0..e8bf8bf2bc1 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -1,9 +1,9 @@ import re import urllib.parse -from typing import Any +from typing import Annotated, Any, TypeAlias -from pydantic import BaseModel, Field -from pydantic.types import ConstrainedStr +from pydantic import BaseModel, Field, TypeAdapter +from pydantic.types import StringConstraints # RESOURCE NAMES https://cloud.google.com/apis/design/resource_names # @@ -30,18 +30,15 @@ _RELATIVE_RESOURCE_NAME_RE = r"^([^\s/]+/?){1,10}$" -class RelativeResourceName(ConstrainedStr): - regex = re.compile(_RELATIVE_RESOURCE_NAME_RE) - - class Config: - frozen = True - +RelativeResourceName: TypeAlias = Annotated[ + str, StringConstraints(pattern=_RELATIVE_RESOURCE_NAME_RE), Field(frozen=True) +] # NOTE: we quote parts in a single resource_name and unquote when split def parse_last_resource_id(resource_name: RelativeResourceName) -> str: - if match := RelativeResourceName.regex.match(resource_name): + if match := re.match(_RELATIVE_RESOURCE_NAME_RE, resource_name): last_quoted_part = match.group(1) return urllib.parse.unquote_plus(last_quoted_part) msg = f"Invalid '{resource_name=}' does not match RelativeResourceName" @@ -53,7 +50,7 @@ def compose_resource_name(*collection_or_resource_ids) -> RelativeResourceName: urllib.parse.quote_plus(f"{_id}".lstrip("/")) for _id in collection_or_resource_ids ] - return RelativeResourceName("/".join(quoted_parts)) + return TypeAdapter(RelativeResourceName).validate_python("/".join(quoted_parts)) def split_resource_name(resource_name: RelativeResourceName) -> list[str]: @@ -67,10 +64,12 @@ def split_resource_name(resource_name: RelativeResourceName) -> list[str]: # Resource IDs must be clearly documented whether they are assigned by the client, the server, or either # class BaseResource(BaseModel): - name: RelativeResourceName = Field(None, example="solvers/isolve/releases/1.2.3") - id: Any = Field(None, description="Resource ID", example="1.2.3") # noqa: A003 + name: RelativeResourceName = Field(None, examples=["solvers/isolve/releases/1.2.3"]) + id: Any = Field(None, description="Resource ID", examples=["1.2.3"]) # noqa: A003 class BaseCollection(BaseModel): - name: RelativeResourceName = Field(None, example="solvers/isolve/releases") - id: Any = Field(None, description="Collection ID", example="releases") # noqa: A003 + name: RelativeResourceName = Field(None, examples=["solvers/isolve/releases"]) + id: Any = Field( + None, description="Collection ID", examples=["releases"] + ) # noqa: A003 diff --git a/services/api-server/src/simcore_service_api_server/models/basic_types.py b/services/api-server/src/simcore_service_api_server/models/basic_types.py index 53ea6fe31ce..8e0c4c79af2 100644 --- a/services/api-server/src/simcore_service_api_server/models/basic_types.py +++ b/services/api-server/src/simcore_service_api_server/models/basic_types.py @@ -1,17 +1,14 @@ -import re +from typing import Annotated, TypeAlias from fastapi.responses import StreamingResponse from models_library.basic_regex import SIMPLE_VERSION_RE -from pydantic import ConstrainedStr +from pydantic import StringConstraints +VersionStr: TypeAlias = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=SIMPLE_VERSION_RE) +] -class VersionStr(ConstrainedStr): - strip_whitespace = True - regex = re.compile(SIMPLE_VERSION_RE) - - -class FileNameStr(ConstrainedStr): - strip_whitespace = True +FileNameStr: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True)] class LogStreamingResponse(StreamingResponse): diff --git a/services/api-server/src/simcore_service_api_server/models/pagination.py b/services/api-server/src/simcore_service_api_server/models/pagination.py index 6a3f5f96b17..1cabda966b5 100644 --- a/services/api-server/src/simcore_service_api_server/models/pagination.py +++ b/services/api-server/src/simcore_service_api_server/models/pagination.py @@ -7,7 +7,7 @@ """ from collections.abc import Sequence -from typing import Any, ClassVar, Generic, TypeAlias, TypeVar +from typing import Generic, TypeAlias, TypeVar from fastapi import Query from fastapi_pagination.customization import CustomizedPage, UseName, UseParamsFields @@ -18,8 +18,13 @@ MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, ) from models_library.utils.pydantic_tools_extension import FieldNotRequired -from pydantic import NonNegativeInt, validator -from pydantic.generics import GenericModel +from pydantic import ( + BaseModel, + ConfigDict, + NonNegativeInt, + ValidationInfo, + field_validator, +) T = TypeVar("T") @@ -41,7 +46,7 @@ PaginationParams: TypeAlias = _LimitOffsetParams -class OnePage(GenericModel, Generic[T]): +class OnePage(BaseModel, Generic[T]): """ A single page is used to envelope a small sequence that does not require pagination @@ -51,12 +56,12 @@ class OnePage(GenericModel, Generic[T]): """ items: Sequence[T] - total: NonNegativeInt = FieldNotRequired() + total: NonNegativeInt = FieldNotRequired(validate_default=True) - @validator("total", pre=True) + @field_validator("total", mode="before") @classmethod - def check_total(cls, v, values): - items = values["items"] + def _check_total(cls, v, info: ValidationInfo): + items = info.data.get("items", []) if v is None: return len(items) @@ -66,9 +71,9 @@ def check_total(cls, v, values): return v - class Config: - frozen = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + frozen=True, + json_schema_extra={ "examples": [ { "total": 1, @@ -78,7 +83,8 @@ class Config: "items": ["one"], }, ], - } + }, + ) __all__: tuple[str, ...] = ( diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/errors.py b/services/api-server/src/simcore_service_api_server/models/schemas/errors.py index 306ac959058..3243f5e44b9 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/errors.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/errors.py @@ -1,6 +1,6 @@ -from typing import Any, ClassVar +from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class ErrorGet(BaseModel): @@ -11,8 +11,8 @@ class ErrorGet(BaseModel): # - https://github.com/ITISFoundation/osparc-simcore/issues/2446 errors: list[Any] - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "errors": [ "some error message", @@ -20,3 +20,4 @@ class Config: ] } } + ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/files.py b/services/api-server/src/simcore_service_api_server/models/schemas/files.py index eece67dfa59..42686f0876e 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/files.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/files.py @@ -1,6 +1,6 @@ from mimetypes import guess_type from pathlib import Path -from typing import Any, ClassVar +from typing import Annotated from urllib.parse import quote as _quote from urllib.parse import unquote as _unquote from uuid import UUID, uuid3 @@ -14,18 +14,19 @@ AnyUrl, BaseModel, ByteSize, - ConstrainedStr, + ConfigDict, Field, - parse_obj_as, - validator, + StringConstraints, + TypeAdapter, + ValidationInfo, + field_validator, ) from servicelib.file_utils import create_sha256_checksum _NAMESPACE_FILEID_KEY = UUID("aa154444-d22d-4290-bb15-df37dba87865") -class FileName(ConstrainedStr): - strip_whitespace = True +FileName = Annotated[str, StringConstraints(strip_whitespace=True)] class ClientFile(BaseModel): @@ -46,7 +47,9 @@ class File(BaseModel): filename: str = Field(..., description="Name of the file with extension") content_type: str | None = Field( - default=None, description="Guess of type content [EXPERIMENTAL]" + default=None, + description="Guess of type content [EXPERIMENTAL]", + validate_default=True, ) sha256_checksum: SHA256Str | None = Field( default=None, @@ -55,9 +58,9 @@ class File(BaseModel): ) e_tag: ETag | None = Field(default=None, description="S3 entity tag") - class Config: - allow_population_by_field_name = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + populate_by_name=True, + json_schema_extra={ "examples": [ # complete { @@ -72,13 +75,14 @@ class Config: "filename": "whitepaper.pdf", }, ] - } + }, + ) - @validator("content_type", always=True, pre=True) + @field_validator("content_type", mode="before") @classmethod - def guess_content_type(cls, v, values): + def guess_content_type(cls, v, info: ValidationInfo): if v is None: - filename = values.get("filename") + filename = info.data.get("filename") if filename: mime_content_type, _ = guess_type(filename, strict=False) return mime_content_type @@ -133,8 +137,8 @@ async def create_from_client_file( @classmethod async def create_from_quoted_storage_id(cls, quoted_storage_id: str) -> "File": - storage_file_id: StorageFileID = parse_obj_as( - StorageFileID, _unquote(quoted_storage_id) # type: ignore[arg-type] + storage_file_id: StorageFileID = TypeAdapter(StorageFileID).validate_python( + _unquote(quoted_storage_id) ) _, fid, fname = Path(storage_file_id).parts return cls(id=UUID(fid), filename=fname, checksum=None) @@ -146,8 +150,8 @@ def create_id(cls, *keys) -> UUID: @property def storage_file_id(self) -> StorageFileID: """Get the StorageFileId associated with this file""" - return parse_obj_as( - StorageFileID, f"api/{self.id}/{self.filename}" # type: ignore[arg-type] + return TypeAdapter(StorageFileID).validate_python( + f"api/{self.id}/{self.filename}" ) @property diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index e6f5edf51b9..485d0c920c1 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -1,7 +1,7 @@ import datetime import hashlib import logging -from typing import Any, ClassVar, TypeAlias +from typing import Annotated, TypeAlias from uuid import UUID, uuid4 from models_library.projects import ProjectID @@ -9,24 +9,23 @@ from models_library.projects_state import RunningState from pydantic import ( BaseModel, - ConstrainedInt, - Extra, + ConfigDict, Field, HttpUrl, PositiveInt, StrictBool, StrictFloat, StrictInt, + TypeAdapter, ValidationError, - parse_obj_as, - validator, + ValidationInfo, + field_validator, ) from servicelib.logging_utils import LogLevelInt, LogMessageStr from starlette.datastructures import Headers from ...models.schemas.files import File from ...models.schemas.solvers import Solver -from .._utils_pydantic import BaseConfig from ..api_resources import ( RelativeResourceName, compose_resource_name, @@ -34,7 +33,6 @@ ) JobID: TypeAlias = UUID -assert JobID == ProjectID # ArgumentTypes are types used in the job inputs (see ResultsTypes) ArgumentTypes: TypeAlias = ( @@ -49,7 +47,7 @@ def _compute_keyword_arguments_checksum(kwargs: KeywordArguments): for key in sorted(kwargs.keys()): value = kwargs[key] if isinstance(value, File): - value = _compute_keyword_arguments_checksum(value.dict()) + value = _compute_keyword_arguments_checksum(value.model_dump()) else: value = str(value) _dump_str += f"{key}:{value}" @@ -70,10 +68,9 @@ class JobInputs(BaseModel): # TODO: gibt es platz fuer metadata? - class Config(BaseConfig): - frozen = True - allow_mutation = False - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + frozen=True, + json_schema_extra={ "example": { "values": { "x": 4.33, @@ -86,7 +83,8 @@ class Config(BaseConfig): }, } } - } + }, + ) def compute_checksum(self): return _compute_keyword_arguments_checksum(self.values) @@ -103,10 +101,9 @@ class JobOutputs(BaseModel): # TODO: an error might have occurred at the level of the job, i.e. affects all outputs, or only # on one specific output. - class Config(BaseConfig): - frozen = True - allow_mutation = False - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + frozen=True, + json_schema_extra={ "example": { "job_id": "99d9ac65-9f10-4e2f-a433-b5e412bb037b", "results": { @@ -120,7 +117,8 @@ class Config(BaseConfig): }, }, } - } + }, + ) def compute_results_checksum(self): return _compute_keyword_arguments_checksum(self.results) @@ -180,8 +178,8 @@ class Job(BaseModel): ..., description="Link to the job outputs (sub-collection)" ) - class Config(BaseConfig): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "id": "f622946d-fd29-35b9-a193-abdd1095167c", "name": "solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c", @@ -193,11 +191,12 @@ class Config(BaseConfig): "outputs_url": "https://api.osparc.io/v0/solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs", } } + ) - @validator("name", pre=True) + @field_validator("name", mode="before") @classmethod - def check_name(cls, v, values): - _id = str(values["id"]) + def check_name(cls, v, info: ValidationInfo): + _id = str(info.data["id"]) if not v.endswith(f"/{_id}"): msg = f"Resource name [{v}] and id [{_id}] do not match" raise ValueError(msg) @@ -225,7 +224,7 @@ def create_now( @classmethod def create_solver_job(cls, *, solver: Solver, inputs: JobInputs): return Job.create_now( - parent_name=solver.name, # type: ignore + parent_name=solver.name, inputs_checksum=inputs.compute_checksum(), ) @@ -248,9 +247,7 @@ def resource_name(self) -> str: return self.name -class PercentageInt(ConstrainedInt): - ge = 0 - le = 100 +PercentageInt: TypeAlias = Annotated[int, Field(ge=0, le=100)] class JobStatus(BaseModel): @@ -260,7 +257,7 @@ class JobStatus(BaseModel): job_id: JobID state: RunningState - progress: PercentageInt = Field(default=PercentageInt(0)) + progress: PercentageInt = Field(default=0) # Timestamps on states submitted_at: datetime.datetime = Field( @@ -275,8 +272,8 @@ class JobStatus(BaseModel): description="Timestamp at which the solver finished or killed execution or None if the event did not occur", ) - class Config(BaseConfig): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "job_id": "145beae4-a3a8-4fde-adbb-4e8257c2c083", "state": RunningState.STARTED, @@ -286,31 +283,31 @@ class Config(BaseConfig): "stopped_at": None, } } + ) class JobPricingSpecification(BaseModel): pricing_plan: PositiveInt = Field(..., alias="x-pricing-plan") pricing_unit: PositiveInt = Field(..., alias="x-pricing-unit") - class Config: - extra = Extra.ignore + model_config = ConfigDict(extra="ignore") @classmethod def create_from_headers(cls, headers: Headers) -> "JobPricingSpecification | None": try: - return parse_obj_as(JobPricingSpecification, headers) + return TypeAdapter(cls).validate_python(headers) except ValidationError: return None class JobLog(BaseModel): job_id: ProjectID - node_id: NodeID | None + node_id: NodeID | None = None log_level: LogLevelInt messages: list[LogMessageStr] - class Config(BaseConfig): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "job_id": "145beae4-a3a8-4fde-adbb-4e8257c2c083", "node_id": "3742215e-6756-48d2-8b73-4d043065309f", @@ -318,3 +315,4 @@ class Config(BaseConfig): "messages": ["PROGRESS: 5/10"], } } + ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/meta.py b/services/api-server/src/simcore_service_api_server/models/schemas/meta.py index ea358f1433f..64f80a45b9b 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/meta.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/meta.py @@ -1,15 +1,12 @@ -from typing import ClassVar - from models_library.api_schemas__common.meta import BaseMeta -from pydantic import AnyHttpUrl +from pydantic import AnyHttpUrl, ConfigDict class Meta(BaseMeta): docs_url: AnyHttpUrl docs_dev_url: AnyHttpUrl - - class Config: - schema_extra: ClassVar = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "simcore_service_foo", "version": "2.4.45", @@ -18,3 +15,4 @@ class Config: "docs_dev_url": "https://api.osparc.io/dev/doc", } } + ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py b/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py index 8f86f2e693f..76b283aa4a9 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/profiles.py @@ -1,17 +1,16 @@ from enum import auto -from typing import Any, ClassVar from models_library.emails import LowerCaseEmailStr from models_library.users import FirstNameStr, LastNameStr, UserID from models_library.utils.enums import StrAutoEnum -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from ..domain.groups import Groups class ProfileCommon(BaseModel): - first_name: FirstNameStr | None = Field(None, example="James") - last_name: LastNameStr | None = Field(None, example="Maxwell") + first_name: FirstNameStr | None = Field(None, examples=["James"]) + last_name: LastNameStr | None = Field(None, examples=["Maxwell"]) class ProfileUpdate(ProfileCommon): @@ -39,15 +38,15 @@ class Profile(ProfileCommon): max_length=40, ) - @validator("role", pre=True) + @field_validator("role", mode="before") @classmethod def enforce_role_upper(cls, v): if v: return v.upper() return v - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "id": "20", "first_name": "James", @@ -70,3 +69,4 @@ class Config: "gravatar_id": "9a8930a5b20d7048e37740bac5c1ca4f", } } + ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index a99017852a5..0e32f5343b5 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -1,12 +1,12 @@ import urllib.parse -from typing import Any, ClassVar, Literal +from typing import Annotated, Any, Literal import packaging.version from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE from models_library.services import ServiceMetaDataPublished from models_library.services_regex import COMPUTATIONAL_SERVICE_KEY_RE from packaging.version import Version -from pydantic import BaseModel, ConstrainedStr, Extra, Field, HttpUrl +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, StringConstraints from ..api_resources import compose_resource_name from ..basic_types import VersionStr @@ -30,9 +30,9 @@ SOLVER_RESOURCE_NAME_RE = r"^solvers/([^\s/]+)/releases/([\d\.]+)$" -class SolverKeyId(ConstrainedStr): - strip_whitespace = True - regex = COMPUTATIONAL_SERVICE_KEY_RE +SolverKeyId = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=COMPUTATIONAL_SERVICE_KEY_RE) +] class Solver(BaseModel): @@ -46,17 +46,16 @@ class Solver(BaseModel): # Human readables Identifiers title: str = Field(..., description="Human readable name") - description: str | None + description: str | None = None maintainer: str # TODO: consider released: Optional[datetime] required? # TODO: consider version_aliases: list[str] = [] # remaining tags # Get links to other resources url: HttpUrl | None = Field(..., description="Link to get this resource") - - class Config: - extra = Extra.ignore - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="ignore", + json_schema_extra={ "example": { "id": "simcore/services/comp/isolve", "version": "2.1.1", @@ -65,11 +64,12 @@ class Config: "maintainer": "info@itis.swiss", "url": "https://api.osparc.io/v0/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/2.1.1", } - } + }, + ) @classmethod def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> "Solver": - data = image_meta.dict( + data = image_meta.model_dump( include={"name", "key", "version", "description", "contact"}, ) @@ -114,7 +114,7 @@ class SolverPort(BaseModel): key: str = Field( ..., description="port identifier name", - regex=PUBLIC_VARIABLE_NAME_RE, + pattern=PUBLIC_VARIABLE_NAME_RE, title="Key name", ) kind: PortKindStr @@ -122,10 +122,9 @@ class SolverPort(BaseModel): None, description="jsonschema for the port's value. SEE https://json-schema.org", ) - - class Config: - extra = Extra.ignore - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="ignore", + json_schema_extra={ "example": { "key": "input_2", "kind": "input", @@ -137,4 +136,5 @@ class Config: "maximum": 5, }, } - } + }, + ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/studies.py b/services/api-server/src/simcore_service_api_server/models/schemas/studies.py index 96c63ee7910..a5f69a7ad08 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/studies.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/studies.py @@ -1,15 +1,16 @@ from typing import TypeAlias +from common_library.pydantic_networks_extension import AnyUrlLegacy from models_library import projects, projects_nodes_io from models_library.utils import pydantic_tools_extension -from pydantic import AnyUrl, BaseModel, Field +from pydantic import BaseModel, Field from .. import api_resources from . import solvers StudyID: TypeAlias = projects.ProjectID NodeName: TypeAlias = str -DownloadLink: TypeAlias = AnyUrl +DownloadLink: TypeAlias = AnyUrlLegacy class Study(BaseModel): diff --git a/services/api-server/src/simcore_service_api_server/services/catalog.py b/services/api-server/src/simcore_service_api_server/services/catalog.py index 56a7d648790..2bbc741b5f2 100644 --- a/services/api-server/src/simcore_service_api_server/services/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services/catalog.py @@ -5,11 +5,12 @@ from dataclasses import dataclass from functools import partial from operator import attrgetter +from typing import Final from fastapi import FastAPI, status from models_library.emails import LowerCaseEmailStr from models_library.services import ServiceMetaDataPublished, ServiceType -from pydantic import Extra, ValidationError, parse_obj_as, parse_raw_as +from pydantic import ConfigDict, TypeAdapter, ValidationError from settings_library.catalog import CatalogSettings from simcore_service_api_server.exceptions.backend_errors import ( ListSolversOrStudiesError, @@ -42,14 +43,12 @@ class TruncatedCatalogServiceOut(ServiceMetaDataPublished): that asks only what is needed. """ - owner: LowerCaseEmailStr | None - - class Config: - extra = Extra.ignore + owner: LowerCaseEmailStr | None = None + model_config = ConfigDict(extra="ignore") # Converters def to_solver(self) -> Solver: - data = self.dict( + data = self.model_dump( include={"name", "key", "version", "description", "contact", "owner"}, ) @@ -70,6 +69,17 @@ def to_solver(self) -> Solver: _exception_mapper = partial(service_exception_mapper, "Catalog") +TruncatedCatalogServiceOutAdapter: Final[ + TypeAdapter[TruncatedCatalogServiceOut] +] = TypeAdapter(TruncatedCatalogServiceOut) +TruncatedCatalogServiceOutListAdapter: Final[ + TypeAdapter[list[TruncatedCatalogServiceOut]] +] = TypeAdapter(list[TruncatedCatalogServiceOut]) + + +def _parse_response(type_adapter: TypeAdapter, response): + return type_adapter.validate_json(response.text) + @dataclass class CatalogApi(BaseServiceClientApi): @@ -99,7 +109,10 @@ async def list_solvers( services: list[ TruncatedCatalogServiceOut ] = await asyncio.get_event_loop().run_in_executor( - None, parse_raw_as, list[TruncatedCatalogServiceOut], response.text + None, + _parse_response, + TruncatedCatalogServiceOutListAdapter, + response, ) solvers = [] for service in services: @@ -115,7 +128,7 @@ async def list_solvers( # invalid items instead of returning error _logger.warning( "Skipping invalid service returned by catalog '%s': %s", - service.json(), + service.model_dump_json(), err, ) return solvers @@ -140,7 +153,7 @@ async def get_service( service: ( TruncatedCatalogServiceOut ) = await asyncio.get_event_loop().run_in_executor( - None, parse_raw_as, TruncatedCatalogServiceOut, response.text + None, _parse_response, TruncatedCatalogServiceOutAdapter, response ) assert ( # nosec service.service_type == ServiceType.COMPUTATIONAL @@ -167,7 +180,7 @@ async def get_service_ports( response.raise_for_status() - return parse_obj_as(list[SolverPort], response.json()) + return TypeAdapter(list[SolverPort]).validate_python(response.json()) async def list_latest_releases( self, *, user_id: int, product_name: str diff --git a/services/api-server/src/simcore_service_api_server/services/director_v2.py b/services/api-server/src/simcore_service_api_server/services/director_v2.py index ff31490b072..0134bcaf444 100644 --- a/services/api-server/src/simcore_service_api_server/services/director_v2.py +++ b/services/api-server/src/simcore_service_api_server/services/director_v2.py @@ -1,6 +1,5 @@ import logging from functools import partial -from typing import Any, ClassVar from uuid import UUID from fastapi import FastAPI @@ -8,7 +7,15 @@ from models_library.projects_nodes_io import NodeID from models_library.projects_pipeline import ComputationTask from models_library.projects_state import RunningState -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, PositiveInt, parse_raw_as +from pydantic import ( + AnyHttpUrl, + AnyUrl, + BaseModel, + ConfigDict, + Field, + PositiveInt, + TypeAdapter, +) from simcore_service_api_server.exceptions.backend_errors import ( JobNotFoundError, LogFileNotFoundError, @@ -40,18 +47,19 @@ class ComputationTaskGet(ComputationTask): def guess_progress(self) -> PercentageInt: # guess progress based on self.state if self.state in [RunningState.SUCCESS, RunningState.FAILED]: - return PercentageInt(100) - return PercentageInt(0) + return 100 + return 0 - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { - **ComputationTask.Config.schema_extra["examples"][0], + **ComputationTask.model_config["json_schema_extra"]["examples"][0], # type: ignore "url": "https://link-to-stop-computation", } ] } + ) class TaskLogFileGet(BaseModel): @@ -85,7 +93,7 @@ async def create_computation( }, ) response.raise_for_status() - task: ComputationTaskGet = ComputationTaskGet.parse_raw(response.text) + task: ComputationTaskGet = ComputationTaskGet.model_validate_json(response.text) return task @_exception_mapper({}) @@ -121,7 +129,7 @@ async def start_computation( }, ) response.raise_for_status() - task: ComputationTaskGet = ComputationTaskGet.parse_raw(response.text) + task: ComputationTaskGet = ComputationTaskGet.model_validate_json(response.text) return task @_exception_mapper({status.HTTP_404_NOT_FOUND: JobNotFoundError}) @@ -135,7 +143,7 @@ async def get_computation( }, ) response.raise_for_status() - task: ComputationTaskGet = ComputationTaskGet.parse_raw(response.text) + task: ComputationTaskGet = ComputationTaskGet.model_validate_json(response.text) return task @_exception_mapper({status.HTTP_404_NOT_FOUND: JobNotFoundError}) @@ -149,7 +157,7 @@ async def stop_computation( }, ) response.raise_for_status() - task: ComputationTaskGet = ComputationTaskGet.parse_raw(response.text) + task: ComputationTaskGet = ComputationTaskGet.model_validate_json(response.text) return task @_exception_mapper({status.HTTP_404_NOT_FOUND: JobNotFoundError}) @@ -179,7 +187,7 @@ async def get_computation_logs( response.raise_for_status() log_links: list[LogLink] = [] - for r in parse_raw_as(list[TaskLogFileGet], response.text or "[]"): + for r in TypeAdapter(list[TaskLogFileGet]).validate_json(response.text or "[]"): if r.download_link: log_links.append( LogLink(node_name=f"{r.task_id}", download_link=r.download_link) diff --git a/services/api-server/src/simcore_service_api_server/services/log_streaming.py b/services/api-server/src/simcore_service_api_server/services/log_streaming.py index dc031667fbb..6bfcc248414 100644 --- a/services/api-server/src/simcore_service_api_server/services/log_streaming.py +++ b/services/api-server/src/simcore_service_api_server/services/log_streaming.py @@ -53,7 +53,7 @@ async def __aexit__(self, exc_type, exc, tb): async def _distribute_logs(self, data: bytes): with log_catch(_logger, reraise=False): - got = LoggerRabbitMessage.parse_raw(data) + got = LoggerRabbitMessage.model_validate_json(data) item = JobLog( job_id=got.project_id, node_id=got.node_id, @@ -122,15 +122,13 @@ async def log_generator(self) -> AsyncIterable[str]: log: JobLog = await asyncio.wait_for( self._queue.get(), timeout=self._log_check_timeout ) - yield log.json() + _NEW_LINE - except TimeoutError: + yield log.model_dump_json() + _NEW_LINE + except asyncio.TimeoutError: done = await self._project_done() except BaseBackEndError as exc: _logger.info("%s", f"{exc}") - - yield ErrorGet(errors=[f"{exc}"]).json() + _NEW_LINE - + yield ErrorGet(errors=[f"{exc}"]).model_dump_json() + _NEW_LINE except Exception as exc: # pylint: disable=W0718 error_code = create_error_code(exc) user_error_msg = ( @@ -144,7 +142,10 @@ async def log_generator(self) -> AsyncIterable[str]: error_code=error_code, ) ) - yield ErrorGet(errors=[user_error_msg]).json() + _NEW_LINE - + yield ErrorGet( + errors=[ + MSG_INTERNAL_ERROR_USER_FRIENDLY_TEMPLATE + f" (OEC: {error_code})" + ] + ).model_dump_json() + _NEW_LINE finally: await self._log_distributor.deregister(self._job_id) diff --git a/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py b/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py index 137463e1263..a8988037f65 100644 --- a/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/services/solver_job_models_converters.py @@ -6,14 +6,14 @@ import urllib.parse import uuid from collections.abc import Callable -from datetime import datetime +from datetime import datetime, timezone from functools import lru_cache import arrow from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet from models_library.basic_types import KeyIDStr from models_library.projects_nodes import InputID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..models.basic_types import VersionStr from ..models.domain.projects import InputTypes, Node, SimCoreFileLink, StudyUI @@ -45,7 +45,7 @@ def format_datetime(snapshot: datetime) -> str: def now_str() -> str: # NOTE: backend MUST use UTC - return format_datetime(datetime.utcnow()) + return format_datetime(datetime.now(timezone.utc)) # CONVERTERS -------------- @@ -62,14 +62,14 @@ def create_node_inputs_from_job_inputs( node_inputs: dict[InputID, InputTypes] = {} for name, value in inputs.values.items(): - assert parse_obj_as(ArgumentTypes, value) == value # type: ignore # nosec - assert parse_obj_as(KeyIDStr, name) is not None # nosec + assert TypeAdapter(ArgumentTypes).validate_python(value) == value # type: ignore # nosec + assert TypeAdapter(KeyIDStr).validate_python(name) is not None # nosec if isinstance(value, File): # FIXME: ensure this aligns with storage policy node_inputs[KeyIDStr(name)] = SimCoreFileLink( store=0, - path=f"api/{value.id}/{value.filename}", # type: ignore[arg-type] + path=f"api/{value.id}/{value.filename}", label=value.filename, eTag=value.e_tag, ) @@ -88,10 +88,8 @@ def create_job_inputs_from_node_inputs(inputs: dict[InputID, InputTypes]) -> Job """ input_values: dict[str, ArgumentTypes] = {} for name, value in inputs.items(): - assert parse_obj_as(InputID, name) == name # nosec - assert ( # nosec - parse_obj_as(InputTypes, value) == value # type: ignore[arg-type] - ) + assert TypeAdapter(InputID).validate_python(name) == name # nosec + assert TypeAdapter(InputTypes).validate_python(value) == value # nosec if isinstance(value, SimCoreFileLink): # FIXME: ensure this aligns with storage policy @@ -141,15 +139,15 @@ def create_new_project_for_job( ) solver_service = Node( - key=solver.id, # type: ignore[arg-type] - version=solver.version, # type: ignore[arg-type] + key=solver.id, + version=solver.version, label=solver.title, inputs=solver_inputs, inputsUnits={}, ) # Ensembles project model so it can be used as input for create_project - job_info = job.json( + job_info = job.model_dump_json( include={"id", "name", "inputs_checksum", "created_at"}, indent=2 ) @@ -158,7 +156,7 @@ def create_new_project_for_job( name=job.name, # NOTE: this IS an identifier as well. MUST NOT be changed in the case of project APIs! description=f"Study associated to solver job:\n{job_info}", thumbnail="https://via.placeholder.com/170x120.png", # type: ignore[arg-type] - workbench={solver_id: solver_service}, # type: ignore[dict-item] + workbench={solver_id: solver_service}, ui=StudyUI( workbench={ f"{solver_id}": { # type: ignore[dict-item] @@ -208,10 +206,10 @@ def create_job_from_project( job = Job( id=job_id, - name=project.name, # type: ignore[arg-type] + name=project.name, inputs_checksum=job_inputs.compute_checksum(), created_at=project.creation_date, # type: ignore[arg-type] - runner_name=solver_name, # type: ignore + runner_name=solver_name, url=url_for( "get_job", solver_key=solver_key, @@ -231,7 +229,9 @@ def create_job_from_project( ), ) - assert all(getattr(job, f) for f in job.__fields__ if f.endswith("url")) # nosec + assert all( + getattr(job, f) for f in job.model_fields.keys() if f.endswith("url") + ) # nosec return job diff --git a/services/api-server/src/simcore_service_api_server/services/solver_job_outputs.py b/services/api-server/src/simcore_service_api_server/services/solver_job_outputs.py index 27ce26bd8f0..dac7610b5a3 100644 --- a/services/api-server/src/simcore_service_api_server/services/solver_job_outputs.py +++ b/services/api-server/src/simcore_service_api_server/services/solver_job_outputs.py @@ -4,7 +4,7 @@ import aiopg from models_library.projects import ProjectID, ProjectIDStr from models_library.projects_nodes_io import BaseFileLink, NodeID, NodeIDStr -from pydantic import StrictBool, StrictFloat, StrictInt, parse_obj_as +from pydantic import StrictBool, StrictFloat, StrictInt, TypeAdapter from simcore_sdk import node_ports_v2 from simcore_sdk.node_ports_v2 import DBManager, Nodeports from simcore_service_api_server.exceptions.backend_errors import ( @@ -44,7 +44,7 @@ async def get_solver_output_results( port.property_type, port.value, ) - assert parse_obj_as(ResultsTypes, port.value) == port.value # type: ignore # nosec + assert TypeAdapter(ResultsTypes).validate_python(port.value) == port.value # type: ignore # nosec solver_output_results[port.key] = port.value diff --git a/services/api-server/src/simcore_service_api_server/services/storage.py b/services/api-server/src/simcore_service_api_server/services/storage.py index 13920d8a931..442520c1582 100644 --- a/services/api-server/src/simcore_service_api_server/services/storage.py +++ b/services/api-server/src/simcore_service_api_server/services/storage.py @@ -72,9 +72,11 @@ async def list_files( ) response.raise_for_status() - files_metadata = Envelope[FileMetaDataArray].parse_raw(response.text).data + files_metadata = ( + Envelope[FileMetaDataArray].model_validate_json(response.text).data + ) files: list[StorageFileMetaData] = ( - [] if files_metadata is None else files_metadata.__root__ + [] if files_metadata is None else files_metadata.root ) return files @@ -107,9 +109,11 @@ async def search_owned_files( ) response.raise_for_status() - files_metadata = Envelope[FileMetaDataArray].parse_raw(response.text).data + files_metadata = ( + Envelope[FileMetaDataArray].model_validate_json(response.text).data + ) files: list[StorageFileMetaData] = ( - [] if files_metadata is None else files_metadata.__root__ + [] if files_metadata is None else files_metadata.root ) assert len(files) <= limit if limit else True # nosec return files @@ -127,7 +131,7 @@ async def get_download_link( response.raise_for_status() presigned_link: PresignedLink | None = ( - Envelope[PresignedLink].parse_raw(response.text).data + Envelope[PresignedLink].model_validate_json(response.text).data ) assert presigned_link is not None link: AnyUrl = presigned_link.link @@ -154,7 +158,7 @@ async def get_upload_links( ) response.raise_for_status() - enveloped_data = Envelope[FileUploadSchema].parse_raw(response.text) + enveloped_data = Envelope[FileUploadSchema].model_validate_json(response.text) assert enveloped_data.data # nosec return enveloped_data.data @@ -200,7 +204,9 @@ async def create_soft_link( ) response.raise_for_status() - stored_file_meta = Envelope[StorageFileMetaData].parse_raw(response.text).data + stored_file_meta = ( + Envelope[StorageFileMetaData].model_validate_json(response.text).data + ) assert stored_file_meta is not None file_meta: File = to_file_api_model(stored_file_meta) return file_meta diff --git a/services/api-server/src/simcore_service_api_server/services/study_job_models_converters.py b/services/api-server/src/simcore_service_api_server/services/study_job_models_converters.py index 8258e67129b..1ab18a85c0d 100644 --- a/services/api-server/src/simcore_service_api_server/services/study_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/services/study_job_models_converters.py @@ -13,7 +13,7 @@ from models_library.projects import DateTimeStr from models_library.projects_nodes import InputID from models_library.projects_nodes_io import LinkToFileTypes, NodeID, SimcoreS3FileID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..models.domain.projects import InputTypes, SimCoreFileLink from ..models.schemas.files import File @@ -107,7 +107,7 @@ async def create_job_outputs_from_project_outputs( and {"store", "path"}.issubset(value.keys()) ): assert ( # nosec - parse_obj_as(LinkToFileTypes, value) is not None # type: ignore[arg-type] + TypeAdapter(LinkToFileTypes).validate_python(value) is not None ) path = value["path"] diff --git a/services/api-server/src/simcore_service_api_server/services/webserver.py b/services/api-server/src/simcore_service_api_server/services/webserver.py index 0d265248dc2..5a2326258af 100644 --- a/services/api-server/src/simcore_service_api_server/services/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services/webserver.py @@ -102,7 +102,7 @@ def _get_lrt_urls(lrt_response: httpx.Response): # WARNING: this function is patched in patch_lrt_response_urls fixture - data = Envelope[TaskGet].parse_raw(lrt_response.text).data + data = Envelope[TaskGet].model_validate_json(lrt_response.text).data assert data is not None # nosec return data.status_href, data.result_href @@ -207,7 +207,7 @@ async def _page_projects( ) resp.raise_for_status() - return Page[ProjectGet].parse_raw(resp.text) + return Page[ProjectGet].model_validate_json(resp.text) async def _wait_for_long_running_task_results(self, lrt_response: httpx.Response): status_url, result_url = _get_lrt_urls(lrt_response) @@ -224,7 +224,9 @@ async def _wait_for_long_running_task_results(self, lrt_response: httpx.Response url=status_url, cookies=self.session_cookies ) get_response.raise_for_status() - task_status = Envelope[TaskStatus].parse_raw(get_response.text).data + task_status = ( + Envelope[TaskStatus].model_validate_json(get_response.text).data + ) assert task_status is not None # nosec if not task_status.done: msg = "Timed out creating project. TIP: Try again, or contact oSparc support if this is happening repeatedly" @@ -234,7 +236,7 @@ async def _wait_for_long_running_task_results(self, lrt_response: httpx.Response f"{result_url}", cookies=self.session_cookies ) result_response.raise_for_status() - return Envelope.parse_raw(result_response.text).data + return Envelope.model_validate_json(result_response.text).data # PROFILE -------------------------------------------------- @@ -242,7 +244,9 @@ async def _wait_for_long_running_task_results(self, lrt_response: httpx.Response async def get_me(self) -> Profile: response = await self.client.get("/me", cookies=self.session_cookies) response.raise_for_status() - profile: Profile | None = Envelope[Profile].parse_raw(response.text).data + profile: Profile | None = ( + Envelope[Profile].model_validate_json(response.text).data + ) assert profile is not None # nosec return profile @@ -250,7 +254,7 @@ async def get_me(self) -> Profile: async def update_me(self, *, profile_update: ProfileUpdate) -> Profile: response = await self.client.put( "/me", - json=profile_update.dict(exclude_none=True), + json=profile_update.model_dump(exclude_none=True), cookies=self.session_cookies, ) response.raise_for_status() @@ -282,7 +286,7 @@ async def create_project( ) response.raise_for_status() result = await self._wait_for_long_running_task_results(response) - return ProjectGet.parse_obj(result) + return ProjectGet.model_validate(result) @_exception_mapper(_JOB_STATUS_MAP) async def clone_project( @@ -307,7 +311,7 @@ async def clone_project( ) response.raise_for_status() result = await self._wait_for_long_running_task_results(response) - return ProjectGet.parse_obj(result) + return ProjectGet.model_validate(result) @_exception_mapper(_JOB_STATUS_MAP) async def get_project(self, *, project_id: UUID) -> ProjectGet: @@ -316,7 +320,7 @@ async def get_project(self, *, project_id: UUID) -> ProjectGet: cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[ProjectGet].parse_raw(response.text).data + data = Envelope[ProjectGet].model_validate_json(response.text).data assert data is not None # nosec return data @@ -360,7 +364,7 @@ async def get_project_metadata_ports( cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[list[StudyPort]].parse_raw(response.text).data + data = Envelope[list[StudyPort]].model_validate_json(response.text).data assert data is not None # nosec assert isinstance(data, list) # nosec return data @@ -374,7 +378,7 @@ async def get_project_metadata( cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[ProjectMetadataGet].parse_raw(response.text).data + data = Envelope[ProjectMetadataGet].model_validate_json(response.text).data assert data is not None # nosec return data @@ -397,7 +401,7 @@ async def update_project_metadata( json=jsonable_encoder(ProjectMetadataUpdate(custom=metadata)), ) response.raise_for_status() - data = Envelope[ProjectMetadataGet].parse_raw(response.text).data + data = Envelope[ProjectMetadataGet].model_validate_json(response.text).data assert data is not None # nosec return data @@ -411,7 +415,7 @@ async def get_project_node_pricing_unit( ) response.raise_for_status() - data = Envelope[PricingUnitGet].parse_raw(response.text).data + data = Envelope[PricingUnitGet].model_validate_json(response.text).data assert data is not None # nosec return data @@ -466,7 +470,9 @@ async def update_project_inputs( ) response.raise_for_status() data: dict[NodeID, ProjectInputGet] | None = ( - Envelope[dict[NodeID, ProjectInputGet]].parse_raw(response.text).data + Envelope[dict[NodeID, ProjectInputGet]] + .model_validate_json(response.text) + .data ) assert data is not None # nosec return data @@ -483,7 +489,9 @@ async def get_project_inputs( response.raise_for_status() data: dict[NodeID, ProjectInputGet] | None = ( - Envelope[dict[NodeID, ProjectInputGet]].parse_raw(response.text).data + Envelope[dict[NodeID, ProjectInputGet]] + .model_validate_json(response.text) + .data ) assert data is not None # nosec return data @@ -500,7 +508,9 @@ async def get_project_outputs( response.raise_for_status() data: dict[NodeID, dict[str, Any]] | None = ( - Envelope[dict[NodeID, dict[str, Any]]].parse_raw(response.text).data + Envelope[dict[NodeID, dict[str, Any]]] + .model_validate_json(response.text) + .data ) assert data is not None # nosec return data @@ -525,7 +535,11 @@ async def get_default_wallet(self) -> WalletGetWithAvailableCredits: cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[WalletGetWithAvailableCredits].parse_raw(response.text).data + data = ( + Envelope[WalletGetWithAvailableCredits] + .model_validate_json(response.text) + .data + ) assert data is not None # nosec return data @@ -536,7 +550,11 @@ async def get_wallet(self, *, wallet_id: int) -> WalletGetWithAvailableCredits: cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[WalletGetWithAvailableCredits].parse_raw(response.text).data + data = ( + Envelope[WalletGetWithAvailableCredits] + .model_validate_json(response.text) + .data + ) assert data is not None # nosec return data @@ -547,7 +565,7 @@ async def get_project_wallet(self, *, project_id: ProjectID) -> WalletGet | None cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[WalletGet].parse_raw(response.text).data + data = Envelope[WalletGet].model_validate_json(response.text).data assert data is not None # nosec return data @@ -560,7 +578,7 @@ async def get_product_price(self) -> GetCreditPrice: cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[GetCreditPrice].parse_raw(response.text).data + data = Envelope[GetCreditPrice].model_validate_json(response.text).data assert data is not None # nosec return data @@ -577,10 +595,12 @@ async def get_service_pricing_plan( cookies=self.session_cookies, ) response.raise_for_status() - pricing_plan_get = Envelope[PricingPlanGet].parse_raw(response.text).data + pricing_plan_get = ( + Envelope[PricingPlanGet].model_validate_json(response.text).data + ) if pricing_plan_get: - return ServicePricingPlanGet.construct( - **pricing_plan_get.dict(exclude={"is_active"}) + return ServicePricingPlanGet.model_construct( + **pricing_plan_get.model_dump(exclude={"is_active"}) ) return None diff --git a/services/api-server/tests/conftest.py b/services/api-server/tests/conftest.py index e2b885bffa9..57533c02438 100644 --- a/services/api-server/tests/conftest.py +++ b/services/api-server/tests/conftest.py @@ -5,11 +5,11 @@ import sys from pathlib import Path +from pydantic import TypeAdapter import pytest import simcore_service_api_server from dotenv import dotenv_values from models_library.projects import ProjectID -from pydantic import parse_obj_as from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_api_server.models.schemas.jobs import JobID @@ -106,4 +106,4 @@ def tests_utils_dir(project_tests_dir: Path) -> Path: @pytest.fixture def job_id(project_id: ProjectID) -> JobID: - return parse_obj_as(JobID, project_id) + return TypeAdapter(JobID).validate_python(project_id) diff --git a/services/api-server/tests/unit/_with_db/conftest.py b/services/api-server/tests/unit/_with_db/conftest.py index 57450561ce4..22b2f6b4c84 100644 --- a/services/api-server/tests/unit/_with_db/conftest.py +++ b/services/api-server/tests/unit/_with_db/conftest.py @@ -161,7 +161,7 @@ def app_environment( assert "API_SERVER_POSTGRES" not in envs # Should be sufficient to create settings - print(PostgresSettings.create_from_envs().json(indent=1)) + print(PostgresSettings.create_from_envs().model_dump_json(indent=1)) return envs @@ -260,7 +260,7 @@ async def _generate_fake_api_key(n: PositiveInt): row = await result.fetchone() assert row _generate_fake_api_key.row_ids.append(row.id) - yield ApiKeyInDB.from_orm(row) + yield ApiKeyInDB.model_validate(row) _generate_fake_api_key.row_ids = [] yield _generate_fake_api_key diff --git a/services/api-server/tests/unit/_with_db/test_api_user.py b/services/api-server/tests/unit/_with_db/test_api_user.py index 87d2de26c64..0a42177867b 100644 --- a/services/api-server/tests/unit/_with_db/test_api_user.py +++ b/services/api-server/tests/unit/_with_db/test_api_user.py @@ -32,7 +32,7 @@ def mocked_webserver_service_api(app: FastAPI): ) as respx_mock: # NOTE: webserver-api uses the same schema as api-server! # in-memory fake data - me = deepcopy(Profile.Config.schema_extra["example"]) + me = deepcopy(Profile.model_config["json_schema_extra"]["example"]) def _get_me(request): return httpx.Response(status.HTTP_200_OK, json={"data": me}) @@ -86,6 +86,6 @@ async def test_update_profile( ) assert resp.status_code == status.HTTP_200_OK, resp.text - profile = Profile.parse_obj(resp.json()) + profile = Profile.model_validate(resp.json()) assert profile.first_name == "Oliver" assert profile.last_name == "Heaviside" diff --git a/services/api-server/tests/unit/_with_db/test_core_settings__with_db.py b/services/api-server/tests/unit/_with_db/test_core_settings__with_db.py index f61c1ae4153..78b9ae20b7b 100644 --- a/services/api-server/tests/unit/_with_db/test_core_settings__with_db.py +++ b/services/api-server/tests/unit/_with_db/test_core_settings__with_db.py @@ -12,7 +12,7 @@ def test_unit_with_db_app_environment(app_environment: EnvVarsDict): settings = ApplicationSettings.create_from_envs() - print("captured settings: \n", settings.json(indent=2)) + print("captured settings: \n", settings.model_dump_json(indent=2)) assert settings.SC_BOOT_MODE == BootModeEnum.PRODUCTION assert settings.log_level == logging.DEBUG diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index a7b813776da..ec8bf7d5630 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -96,8 +96,8 @@ async def mocked_directorv2_service( stop_time: Final[datetime] = datetime.now() + timedelta(seconds=5) def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: - task = ComputationTaskGet.parse_obj( - ComputationTaskGet.Config.schema_extra["examples"][0] + task = ComputationTaskGet.model_validate( + ComputationTaskGet.model_config["json_schema_extra"]["examples"][0] ) if datetime.now() > stop_time: task.state = RunningState.SUCCESS diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index ebdcfc59950..d4a6cf80a76 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -39,7 +39,7 @@ async def test_list_solvers( for item in data: solver = Solver(**item) - print(solver.json(indent=1, exclude_unset=True)) + print(solver.model_dump_json(indent=1, exclude_unset=True)) # use link to get the same solver assert solver.url diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index ed3ae76cfbd..7846dbe1e84 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -18,7 +18,7 @@ from fastapi import FastAPI from models_library.services import ServiceMetaDataPublished from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyUrl, HttpUrl, parse_obj_as +from pydantic import AnyUrl, HttpUrl, TypeAdapter from respx import MockRouter from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.core.settings import ApplicationSettings @@ -52,10 +52,10 @@ def presigned_download_link( node_id: str, bucket_name: str, mocked_s3_server_url: HttpUrl, -) -> Iterator[AnyUrl]: +) -> AnyUrl: s3_client = boto3.client( "s3", - endpoint_url=mocked_s3_server_url, + endpoint_url=f"{mocked_s3_server_url}", # Some fake auth, otherwise botocore.exceptions.NoCredentialsError: Unable to locate credentials aws_secret_access_key="xxx", # noqa: S106 aws_access_key_id="xxx", @@ -79,7 +79,7 @@ def presigned_download_link( print("generated link", presigned_url) # SEE also https://gist.github.com/amarjandu/77a7d8e33623bae1e4e5ba40dc043cb9 - return parse_obj_as(AnyUrl, presigned_url) + return TypeAdapter(AnyUrl).validate_python(presigned_url) @pytest.fixture @@ -119,7 +119,7 @@ def mocked_directorv2_service_api( json=[ { "task_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", - "download_link": presigned_download_link, + "download_link": f"{presigned_download_link}", } ], ) @@ -131,7 +131,7 @@ def test_download_presigned_link( presigned_download_link: AnyUrl, tmp_path: Path, project_id: str, node_id: str ): """Cheks that the generation of presigned_download_link works as expected""" - r = httpx.get(presigned_download_link) + r = httpx.get(f"{presigned_download_link}") ## pprint(dict(r.headers)) # r.headers looks like: # { @@ -191,9 +191,9 @@ async def test_solver_logs( # was a re-direction resp0 = resp.history[0] assert resp0.status_code == status.HTTP_307_TEMPORARY_REDIRECT - assert resp0.headers["location"] == presigned_download_link + assert resp0.headers["location"] == f"{presigned_download_link}" - assert resp.url == presigned_download_link + assert f"{resp.url}" == f"{presigned_download_link}" pprint(dict(resp.headers)) # noqa: T203 @@ -247,7 +247,7 @@ async def test_run_solver_job( ).respond( status.HTTP_201_CREATED, json=jsonable_encoder( - ComputationTaskGet.parse_obj( + ComputationTaskGet.model_validate( { "id": project_id, "state": "UNKNOWN", @@ -314,7 +314,7 @@ async def test_run_solver_job( example = next( e - for e in ServiceMetaDataPublished.Config.schema_extra["examples"] + for e in ServiceMetaDataPublished.model_config["json_schema_extra"]["examples"] if "boot-options" in e ) @@ -350,7 +350,7 @@ async def test_run_solver_job( # Tests https://github.com/ITISFoundation/osparc-issues/issues/948 "a_list": [1, 2, 3], } - ).dict(), + ).model_dump(), ) assert resp.status_code == status.HTTP_201_CREATED @@ -358,7 +358,7 @@ async def test_run_solver_job( assert mocked_webserver_service_api["get_task_status"].called assert mocked_webserver_service_api["get_task_result"].called - job = Job.parse_obj(resp.json()) + job = Job.model_validate(resp.json()) # Start Job resp = await client.post( @@ -369,5 +369,5 @@ async def test_run_solver_job( assert resp.status_code == status.HTTP_202_ACCEPTED assert mocked_directorv2_service_api["inspect_computation"].called - job_status = JobStatus.parse_obj(resp.json()) + job_status = JobStatus.model_validate(resp.json()) assert job_status.progress == 0.0 diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index 000a586836a..bdb3886ebec 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -11,7 +11,7 @@ import pytest from faker import Faker from models_library.basic_regex import UUID_RE_BASE -from pydantic import parse_file_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from servicelib.common_headers import ( @@ -42,7 +42,7 @@ def mocked_backend_services_apis_for_delete_non_existing_project( template = environment.get_template(mock_name) def _response(request: httpx.Request, project_id: str): - capture = HttpApiCallCaptureModel.parse_raw( + capture = HttpApiCallCaptureModel.model_validate_json( template.render(project_id=project_id) ) return httpx.Response( @@ -91,9 +91,8 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( mock_name = "on_create_job.json" # fixture - captures = parse_file_as( - list[HttpApiCallCaptureModel], - project_tests_dir / "mocks" / mock_name, + captures = TypeAdapter(list[HttpApiCallCaptureModel]).validate_json( + Path(project_tests_dir / "mocks" / mock_name).read_text() ) capture = captures[0] @@ -137,10 +136,10 @@ async def test_create_and_delete_solver_job( "x": 3.14, "n": 42, } - ).dict(), + ).model_dump(), ) assert resp.status_code == status.HTTP_201_CREATED - job = Job.parse_obj(resp.json()) + job = Job.model_validate(resp.json()) # Delete Job after creation resp = await client.delete( @@ -224,7 +223,7 @@ def create_project_side_effect(request: httpx.Request): "x": 3.14, "n": 42, } - ).dict(), + ).model_dump(), ) assert resp.status_code == status.HTTP_201_CREATED - job = Job.parse_obj(resp.json()) + job = Job.model_validate(resp.json()) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py index a30404606d7..eb821e46d01 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py @@ -76,7 +76,7 @@ def fake_project_for_streaming( assert isinstance(response_body := GET_PROJECT.response_body, dict) assert (data := response_body.get("data")) is not None - fake_project = ProjectGet.parse_obj(data) + fake_project = ProjectGet.model_validate(data) fake_project.workbench = {faker.uuid4(): faker.uuid4()} mocker.patch( "simcore_service_api_server.api.dependencies.webserver.AuthSession.get_project", @@ -113,8 +113,8 @@ async def test_log_streaming( response.raise_for_status() if not disconnect: async for line in response.aiter_lines(): - job_log = JobLog.parse_raw(line) - pprint(job_log.json()) + job_log = JobLog.model_validate_json(line) + pprint(job_log.model_dump()) collected_messages += job_log.messages assert fake_log_distributor.deregister_is_called @@ -160,12 +160,12 @@ async def test_logstreaming_job_not_found_exception( response.raise_for_status() async for line in response.aiter_lines(): try: - job_log = JobLog.parse_raw(line) - pprint(job_log.json()) + job_log = JobLog.model_validate_json(line) + pprint(job_log.model_dump()) except ValidationError: - error = ErrorGet.parse_raw(line) + error = ErrorGet.model_validate_json(line) _received_error = True - print(error.json()) + print(error.model_dump()) assert fake_log_distributor.deregister_is_called assert _received_error diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py index ccf9b40b565..6b62c89b6b8 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py @@ -10,7 +10,7 @@ import pytest from faker import Faker from models_library.basic_regex import UUID_RE_BASE -from pydantic import parse_file_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG @@ -45,8 +45,8 @@ def mocked_backend( captures = { c.name: c - for c in parse_file_as( - list[HttpApiCallCaptureModel], project_tests_dir / "mocks" / mock_name + for c in TypeAdapter(list[HttpApiCallCaptureModel]).validate_json( + Path(project_tests_dir / "mocks" / mock_name).read_text() ) } @@ -112,10 +112,10 @@ async def test_get_and_update_job_metadata( "title": "Temperature", "enabled": True, } - ).dict(), + ).model_dump(), ) assert resp.status_code == status.HTTP_201_CREATED - job = Job.parse_obj(resp.json()) + job = Job.model_validate(resp.json()) # Get metadata resp = await client.get( @@ -123,7 +123,7 @@ async def test_get_and_update_job_metadata( auth=auth, ) assert resp.status_code == status.HTTP_200_OK - job_meta = JobMetadata.parse_obj(resp.json()) + job_meta = JobMetadata.model_validate(resp.json()) assert job_meta.metadata == {} @@ -132,11 +132,11 @@ async def test_get_and_update_job_metadata( resp = await client.patch( f"/{API_VTAG}/solvers/{solver_key}/releases/{solver_version}/jobs/{job.id}/metadata", auth=auth, - json=JobMetadataUpdate(metadata=my_metadata).dict(), + json=JobMetadataUpdate(metadata=my_metadata).model_dump(), ) assert resp.status_code == status.HTTP_200_OK - job_meta = JobMetadata.parse_obj(resp.json()) + job_meta = JobMetadata.model_validate(resp.json()) assert job_meta.metadata == my_metadata # Get metadata after update @@ -145,7 +145,7 @@ async def test_get_and_update_job_metadata( auth=auth, ) assert resp.status_code == status.HTTP_200_OK - job_meta = JobMetadata.parse_obj(resp.json()) + job_meta = JobMetadata.model_validate(resp.json()) assert job_meta.metadata == my_metadata diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index 1dbf8b3fa0f..b51c580eb82 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -7,7 +7,7 @@ import httpx import pytest -from pydantic import parse_file_as, parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG @@ -28,9 +28,8 @@ def mocked_backend( project_tests_dir: Path, ) -> MockBackendRouters: mock_name = "on_list_jobs.json" - captures = parse_file_as( - list[HttpApiCallCaptureModel], - project_tests_dir / "mocks" / mock_name, + captures = TypeAdapter(list[HttpApiCallCaptureModel]).validate_json( + Path(project_tests_dir / "mocks" / mock_name).read_text() ) capture = captures[0] @@ -78,7 +77,7 @@ async def test_list_solver_jobs( f"/{API_VTAG}/solvers/{solver_key}/releases/{solver_version}/jobs", auth=auth ) assert resp.status_code == status.HTTP_200_OK - jobs = parse_obj_as(list[Job], resp.json()) + jobs = TypeAdapter(list[Job]).validate_python(resp.json()) # list jobs (w/ pagination) resp = await client.get( @@ -88,7 +87,7 @@ async def test_list_solver_jobs( ) assert resp.status_code == status.HTTP_200_OK - jobs_page = parse_obj_as(Page[Job], resp.json()) + jobs_page = TypeAdapter(Page[Job]).validate_python(resp.json()) assert jobs_page.items == jobs diff --git a/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py b/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py index 098718c3738..d1fae307589 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py +++ b/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py @@ -12,7 +12,7 @@ import httpx import pytest from fastapi.encoders import jsonable_encoder -from pydantic import parse_file_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from pytest_simcore.helpers.httpx_calls_capture_parameters import PathDescription from respx import MockRouter @@ -38,9 +38,12 @@ def mocked_backend( # load captures = { c.name: c - for c in parse_file_as( - list[HttpApiCallCaptureModel], - project_tests_dir / "mocks" / "test_get_and_update_study_job_metadata.json", + for c in TypeAdapter(list[HttpApiCallCaptureModel]).validate_json( + Path( + project_tests_dir + / "mocks" + / "test_get_and_update_study_job_metadata.json" + ).read_text(), ) } diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py index 1893e6e068e..d5369bb0314 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py @@ -12,7 +12,7 @@ import pytest from faker import Faker from fastapi import status -from pydantic import parse_file_as, parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from servicelib.common_headers import ( @@ -40,8 +40,8 @@ def mocked_backend( captures = { c.name: c - for c in parse_file_as( - list[HttpApiCallCaptureModel], project_tests_dir / "mocks" / mock_name + for c in TypeAdapter(list[HttpApiCallCaptureModel]).validate_json( + Path(project_tests_dir / "mocks" / mock_name).read_text() ) } @@ -84,7 +84,7 @@ async def test_studies_read_workflow( resp = await client.get(f"/{API_VTAG}/studies", auth=auth) assert resp.status_code == status.HTTP_200_OK - studies = parse_obj_as(list[Study], resp.json()["items"]) + studies = TypeAdapter(list[Study]).validate_python(resp.json()["items"]) assert len(studies) == 1 assert studies[0].uid == study_id @@ -96,28 +96,28 @@ async def test_studies_read_workflow( resp = await client.get(f"/{API_VTAG}/studies/{study_id}", auth=auth) assert resp.status_code == status.HTTP_200_OK - study = parse_obj_as(Study, resp.json()) + study = TypeAdapter(Study).validate_python(resp.json()) assert study.uid == study_id # get ports resp = await client.get(f"/{API_VTAG}/studies/{study_id}/ports", auth=auth) assert resp.status_code == status.HTTP_200_OK - ports = parse_obj_as(list[StudyPort], resp.json()["items"]) + ports = TypeAdapter(list[StudyPort]).validate_python(resp.json()["items"]) assert len(ports) == (resp.json()["total"]) # get_study with non-existing uuid inexistent_study_id = StudyID("15531b1a-2565-11ee-ab43-02420a000031") resp = await client.get(f"/{API_VTAG}/studies/{inexistent_study_id}", auth=auth) assert resp.status_code == status.HTTP_404_NOT_FOUND - error = parse_obj_as(ErrorGet, resp.json()) + error = TypeAdapter(ErrorGet).validate_python(resp.json()) assert f"{inexistent_study_id}" in error.errors[0] resp = await client.get( f"/{API_VTAG}/studies/{inexistent_study_id}/ports", auth=auth ) assert resp.status_code == status.HTTP_404_NOT_FOUND - error = parse_obj_as(ErrorGet, resp.json()) + error = TypeAdapter(ErrorGet).validate_python(resp.json()) assert f"{inexistent_study_id}" in error.errors[0] diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py index 61d91fa9d94..811818a8939 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py @@ -14,7 +14,6 @@ import respx from faker import Faker from fastapi import status -from pydantic import parse_obj_as from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, @@ -45,7 +44,7 @@ async def test_studies_jobs_workflow( resp = await client.get("/v0/studies/{study_id}", auth=auth) assert resp.status_code == status.HTTP_200_OK - study = parse_obj_as(Study, resp.json()) + study = Study.model_validate(resp.json()) assert study.uid == study_id # Lists study jobs @@ -56,7 +55,7 @@ async def test_studies_jobs_workflow( resp = await client.post("/v0/studies/{study_id}/jobs", auth=auth) assert resp.status_code == status.HTTP_201_CREATED - job = parse_obj_as(Job, resp.json()) + job = Job.model_validate(resp.json()) job_id = job.id # Get Study Job @@ -358,7 +357,7 @@ async def test_get_job_logs( f"{API_VTAG}/studies/{_study_id}/jobs/{_job_id}/outputs/log-links", auth=auth ) assert response.status_code == status.HTTP_200_OK - _ = JobLogsMap.parse_obj(response.json()) + _ = JobLogsMap.model_validate(response.json()) async def test_get_study_outputs( @@ -394,17 +393,17 @@ async def test_get_study_outputs( }, ) assert response.status_code == status.HTTP_200_OK - _job = Job.parse_obj(response.json()) + _job = Job.model_validate(response.json()) _job_id = _job.id response = await client.post( f"/{API_VTAG}/studies/{_study_id}/jobs/{_job_id}:start", auth=auth ) assert response.status_code == status.HTTP_202_ACCEPTED - _ = JobStatus.parse_obj(response.json()) + _ = JobStatus.model_validate(response.json()) response = await client.post( f"/{API_VTAG}/studies/{_study_id}/jobs/{_job_id}/outputs", auth=auth ) assert response.status_code == status.HTTP_200_OK - _ = JobOutputs.parse_obj(response.json()) + _ = JobOutputs.model_validate(response.json()) diff --git a/services/api-server/tests/unit/captures/test__mocks_captures.py b/services/api-server/tests/unit/captures/test__mocks_captures.py index 4c04cca224d..81297e1bbe5 100644 --- a/services/api-server/tests/unit/captures/test__mocks_captures.py +++ b/services/api-server/tests/unit/captures/test__mocks_captures.py @@ -14,7 +14,7 @@ import jsonref import pytest import respx -from pydantic import parse_file_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from pytest_simcore.helpers.httpx_calls_capture_openapi import _determine_path from pytest_simcore.helpers.httpx_calls_capture_parameters import ( @@ -86,9 +86,9 @@ def test_openapion_capture_mock( assert mock_capture_path.exists() assert mock_capture_path.name.endswith(".json") - captures = parse_file_as( - list[HttpApiCallCaptureModel] | HttpApiCallCaptureModel, mock_capture_path - ) + captures = TypeAdapter( + list[HttpApiCallCaptureModel] | HttpApiCallCaptureModel + ).validate_json(mock_capture_path.read_text()) if not isinstance(captures, list): captures = [ diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index e8324bcc0b7..939f7bf9170 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -36,7 +36,7 @@ from models_library.users import UserID from moto.server import ThreadedMotoServer from packaging.version import Version -from pydantic import EmailStr, HttpUrl, parse_obj_as +from pydantic import EmailStr, HttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict @@ -72,7 +72,7 @@ def app_environment( ) # should be sufficient to create settings - print(ApplicationSettings.create_from_envs().json(indent=1)) + print(ApplicationSettings.create_from_envs().model_dump_json(indent=1)) return env_vars @@ -202,7 +202,9 @@ def mocked_s3_server_url() -> Iterator[HttpUrl]: ) # pylint: disable=protected-access - endpoint_url = parse_obj_as(HttpUrl, f"http://{server._ip_address}:{server._port}") + endpoint_url = TypeAdapter(HttpUrl).validate_python( + f"http://{server._ip_address}:{server._port}" + ) print(f"--> started mock S3 server on {endpoint_url}") server.start() @@ -363,7 +365,7 @@ def mocked_storage_service_api_base( "api_version": "1.0.0", "version": "1.0.0", }, - ).dict(), + ).model_dump(), ) assert openapi["paths"]["/v0/status"]["get"]["operationId"] == "get_status" @@ -376,7 +378,7 @@ def mocked_storage_service_api_base( "url": faker.url(), "diagnostics_url": faker.url(), } - ).dict(), + ).model_dump(mode="json"), ) # SEE https://github.com/pcrespov/sandbox-python/blob/f650aad57aced304aac9d0ad56c00723d2274ad0/respx-lib/test_disable_mock.py @@ -448,7 +450,7 @@ def patch_lrt_response_urls(mocker: MockerFixture): def _() -> MagicMock: def _get_lrt_urls(lrt_response: httpx.Response): # NOTE: this function is needed to mock - data = Envelope[TaskGet].parse_raw(lrt_response.text).data + data = Envelope[TaskGet].model_validate_json(lrt_response.text).data assert data is not None # nosec def _patch(href): @@ -497,7 +499,7 @@ def _set_result_and_get_reponse(self, result: Any): status_href=f"{settings.API_SERVER_WEBSERVER.api_base_url}/tasks/{task_id}", result_href=f"{settings.API_SERVER_WEBSERVER.api_base_url}/tasks/{task_id}/result", abort_href=f"{settings.API_SERVER_WEBSERVER.api_base_url}/tasks/{task_id}", - ).dict() + ).model_dump() }, ) @@ -511,7 +513,7 @@ def create_project_task(self, request: httpx.Request): if from_study := query.get("from_study"): return self.clone_project_task(request=request, project_id=from_study) project_create = json.loads(request.content) - project_get = ProjectGet.parse_obj( + project_get = ProjectGet.model_validate( { "creationDate": "2018-07-01T11:13:43Z", "lastChangeDate": "2018-07-01T11:13:43Z", @@ -525,7 +527,7 @@ def create_project_task(self, request: httpx.Request): def clone_project_task(self, request: httpx.Request, *, project_id: str): assert GET_PROJECT.response_body - project_get = ProjectGet.parse_obj( + project_get = ProjectGet.model_validate( { "creationDate": "2018-07-01T11:13:43Z", "lastChangeDate": "2018-07-01T11:13:43Z", diff --git a/services/api-server/tests/unit/test__fastapi.py b/services/api-server/tests/unit/test__fastapi.py index 6cf2e6f13c9..4eaddee4437 100644 --- a/services/api-server/tests/unit/test__fastapi.py +++ b/services/api-server/tests/unit/test__fastapi.py @@ -122,8 +122,8 @@ def test_fastapi_route_name_parsing(client: TestClient, app: FastAPI, faker: Fak # Ensures ':' is allowed in routes # SEE https://github.com/encode/starlette/pull/1657 - solver_key = Solver.Config.schema_extra["example"]["id"] - version = Solver.Config.schema_extra["example"]["version"] + solver_key = Solver.model_config["json_schema_extra"]["example"]["id"] + version = Solver.model_config["json_schema_extra"]["example"]["version"] job_id = faker.uuid4() # Checks whether parse correctly ":action" suffix diff --git a/services/api-server/tests/unit/test_api_files.py b/services/api-server/tests/unit/test_api_files.py index 4d45e0e5528..bbbef802188 100644 --- a/services/api-server/tests/unit/test_api_files.py +++ b/services/api-server/tests/unit/test_api_files.py @@ -24,7 +24,7 @@ UploadedPart, ) from models_library.basic_types import SHA256Str -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, @@ -67,8 +67,7 @@ def file(cls) -> File: @classmethod def client_file(cls) -> ClientFile: - return parse_obj_as( - ClientFile, + return TypeAdapter(ClientFile).validate_python( { "filename": cls._file_name, "filesize": cls._file_size, @@ -103,7 +102,7 @@ async def test_list_files_legacy( assert response.status_code == status.HTTP_200_OK - parse_obj_as(File, response.json()) + TypeAdapter(File).validate_python(response.json()) assert response.json() == [ { @@ -251,7 +250,9 @@ async def test_get_upload_links( payload: dict[str, str] = response.json() assert response.status_code == status.HTTP_200_OK - client_upload_schema: ClientFileUploadData = ClientFileUploadData.parse_obj(payload) + client_upload_schema: ClientFileUploadData = ClientFileUploadData.model_validate( + payload + ) if follow_up_request == "complete": body = { @@ -267,7 +268,7 @@ async def test_get_upload_links( payload: dict[str, str] = response.json() assert response.status_code == status.HTTP_200_OK - file: File = parse_obj_as(File, payload) + file: File = File.model_validate(payload) assert file.sha256_checksum == DummyFileData.checksum() elif follow_up_request == "abort": body = { @@ -331,7 +332,7 @@ def side_effect_callback( response = await client.get(f"{API_VTAG}/files:search", auth=auth, params=query) assert response.status_code == status.HTTP_200_OK - page: Page[File] = parse_obj_as(Page[File], response.json()) + page: Page[File] = TypeAdapter(Page[File]).validate_python(response.json()) assert len(page.items) == page.total file = page.items[0] if "sha256_checksum" in query: diff --git a/services/api-server/tests/unit/test_api_health.py b/services/api-server/tests/unit/test_api_health.py index d548c525406..d421703b0f2 100644 --- a/services/api-server/tests/unit/test_api_health.py +++ b/services/api-server/tests/unit/test_api_health.py @@ -10,7 +10,6 @@ from fastapi import FastAPI, status from httpx import AsyncClient from models_library.app_diagnostics import AppStatusCheck -from pydantic import parse_obj_as from respx import MockRouter from simcore_service_api_server._meta import API_VTAG @@ -54,4 +53,4 @@ async def test_get_service_state( "url": "http://api.testserver.io/state", } - assert parse_obj_as(AppStatusCheck, response.json()) + assert AppStatusCheck.model_validate(response.json()) diff --git a/services/api-server/tests/unit/test_api_solver_jobs.py b/services/api-server/tests/unit/test_api_solver_jobs.py index 524adc7300c..3f1e642a6ad 100644 --- a/services/api-server/tests/unit/test_api_solver_jobs.py +++ b/services/api-server/tests/unit/test_api_solver_jobs.py @@ -17,7 +17,7 @@ from models_library.api_schemas_webserver.resource_usage import PricingUnitGet from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from models_library.generics import Envelope -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, @@ -182,7 +182,7 @@ def _get_pricing_unit_side_effect( ) if capture_file == "get_job_pricing_unit_success.json": assert response.status_code == status.HTTP_200_OK - _ = parse_obj_as(PricingUnitGet, response.json()) + _ = TypeAdapter(PricingUnitGet).validate_python(response.json()) elif capture_file == "get_job_pricing_unit_invalid_job.json": assert response.status_code == status.HTTP_404_NOT_FOUND elif capture_file == "get_job_pricing_unit_invalid_solver.json": @@ -342,7 +342,7 @@ async def test_start_solver_job_conflict( ) assert response.status_code == status.HTTP_200_OK - job_status = JobStatus.parse_obj(response.json()) + job_status = JobStatus.model_validate(response.json()) assert f"{job_status.job_id}" == _job_id @@ -364,7 +364,7 @@ def _stop_job_side_effect( path_params: dict[str, Any], capture: HttpApiCallCaptureModel, ) -> Any: - task = ComputationTaskGet.parse_obj(capture.response_body) + task = ComputationTaskGet.model_validate(capture.response_body) task.id = UUID(_job_id) return jsonable_encoder(task) @@ -384,7 +384,7 @@ def _stop_job_side_effect( ) assert response.status_code == status.HTTP_200_OK - status_ = JobStatus.parse_obj(response.json()) + status_ = JobStatus.model_validate(response.json()) assert status_.job_id == UUID(_job_id) @@ -416,9 +416,11 @@ def _wallet_side_effect( path_params: dict[str, Any], capture: HttpApiCallCaptureModel, ): - wallet = parse_obj_as( - Envelope[WalletGetWithAvailableCredits], capture.response_body - ).data + wallet = ( + TypeAdapter(Envelope[WalletGetWithAvailableCredits]) + .validate_python(capture.response_body) + .data + ) assert wallet is not None wallet.available_credits = ( Decimal(10.0) if sufficient_credits else Decimal(-10.0) diff --git a/services/api-server/tests/unit/test_api_solvers.py b/services/api-server/tests/unit/test_api_solvers.py index dada48a762c..31e8ccb7f59 100644 --- a/services/api-server/tests/unit/test_api_solvers.py +++ b/services/api-server/tests/unit/test_api_solvers.py @@ -10,7 +10,6 @@ from fastapi import status from httpx import AsyncClient from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet -from pydantic import parse_obj_as from pytest_simcore.helpers.httpx_calls_capture_models import CreateRespxMockCallback from simcore_service_api_server._meta import API_VTAG @@ -50,4 +49,4 @@ async def test_get_solver_pricing_plan( ) assert expected_status_code == response.status_code if response.status_code == status.HTTP_200_OK: - _ = parse_obj_as(ServicePricingPlanGet, response.json()) + _ = ServicePricingPlanGet.model_validate(response.json()) diff --git a/services/api-server/tests/unit/test_api_wallets.py b/services/api-server/tests/unit/test_api_wallets.py index 0edf09e7f90..cad3bf5e285 100644 --- a/services/api-server/tests/unit/test_api_wallets.py +++ b/services/api-server/tests/unit/test_api_wallets.py @@ -11,7 +11,6 @@ from fastapi import status from httpx import AsyncClient from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits -from pydantic import parse_obj_as from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, @@ -53,8 +52,8 @@ def _get_wallet_side_effect( response = await client.get(f"{API_VTAG}/wallets/{wallet_id}", auth=auth) if "success" in capture: assert response.status_code == 200 - wallet: WalletGetWithAvailableCredits = parse_obj_as( - WalletGetWithAvailableCredits, response.json() + wallet: WalletGetWithAvailableCredits = ( + WalletGetWithAvailableCredits.model_validate(response.json()) ) assert wallet.wallet_id == wallet_id elif "failure" in capture: @@ -78,4 +77,4 @@ async def test_get_default_wallet( response = await client.get(f"{API_VTAG}/wallets/default", auth=auth) assert response.status_code == status.HTTP_200_OK - _ = parse_obj_as(WalletGetWithAvailableCredits, response.json()) + _ = WalletGetWithAvailableCredits.model_validate(response.json()) diff --git a/services/api-server/tests/unit/test_cli.py b/services/api-server/tests/unit/test_cli.py index 5ae5ca4c10f..febeca14b1f 100644 --- a/services/api-server/tests/unit/test_cli.py +++ b/services/api-server/tests/unit/test_cli.py @@ -29,7 +29,7 @@ def test_cli_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, result.output print(result.output) - settings = ApplicationSettings.parse_raw(result.output) + settings = ApplicationSettings.model_validate_json(result.output) assert settings == ApplicationSettings.create_from_envs() diff --git a/services/api-server/tests/unit/test_core_settings.py b/services/api-server/tests/unit/test_core_settings.py index fbb9f875b65..feb5052ab0f 100644 --- a/services/api-server/tests/unit/test_core_settings.py +++ b/services/api-server/tests/unit/test_core_settings.py @@ -37,4 +37,4 @@ def app_environment( def test_unit_app_environment(app_environment: EnvVarsDict): assert app_environment settings = ApplicationSettings.create_from_envs() - print("captured settings: \n", settings.json(indent=2)) + print("captured settings: \n", settings.model_dump_json(indent=2)) diff --git a/services/api-server/tests/unit/test_credits.py b/services/api-server/tests/unit/test_credits.py index 8c2dfd7dd74..3630e218754 100644 --- a/services/api-server/tests/unit/test_credits.py +++ b/services/api-server/tests/unit/test_credits.py @@ -23,4 +23,4 @@ async def test_get_credits_price( response = await client.get(f"{API_VTAG}/credits/price", auth=auth) assert response.status_code == status.HTTP_200_OK - _ = GetCreditPrice.parse_obj(response.json()) + _ = GetCreditPrice.model_validate(response.json()) diff --git a/services/api-server/tests/unit/test_exceptions.py b/services/api-server/tests/unit/test_exceptions.py index 48d75a38928..6949e72d088 100644 --- a/services/api-server/tests/unit/test_exceptions.py +++ b/services/api-server/tests/unit/test_exceptions.py @@ -67,7 +67,7 @@ async def test_raised_http_exception(client: httpx.AsyncClient): assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE - got = ErrorGet.parse_raw(response.text) + got = ErrorGet.model_validate_json(response.text) assert got.errors == ["fail message"] @@ -78,7 +78,7 @@ async def test_fastapi_http_exception_respond_with_error_model( assert response.status_code == status.HTTP_404_NOT_FOUND - got = ErrorGet.parse_raw(response.text) + got = ErrorGet.model_validate_json(response.text) assert got.errors == [HTTPStatus(response.status_code).phrase] @@ -87,7 +87,7 @@ async def test_custom_error_handlers(client: httpx.AsyncClient): assert response.status_code == status.HTTP_424_FAILED_DEPENDENCY - got = ErrorGet.parse_raw(response.text) + got = ErrorGet.model_validate_json(response.text) assert got.errors == [f"{MissingWalletError(job_id=123)}"] diff --git a/services/api-server/tests/unit/test_models.py b/services/api-server/tests/unit/test_models.py index 06ee47d86c4..b3e1f48a57a 100644 --- a/services/api-server/tests/unit/test_models.py +++ b/services/api-server/tests/unit/test_models.py @@ -21,7 +21,7 @@ def test_api_server_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): - assert model_cls.parse_obj( + assert model_cls.model_validate( example_data ), f"Failed {example_name} : {json.dumps(example_data)}" diff --git a/services/api-server/tests/unit/test_models_schemas_files.py b/services/api-server/tests/unit/test_models_schemas_files.py index ded5c379b58..2ae9c4e5144 100644 --- a/services/api-server/tests/unit/test_models_schemas_files.py +++ b/services/api-server/tests/unit/test_models_schemas_files.py @@ -14,7 +14,7 @@ from models_library.api_schemas_storage import FileMetaDataGet as StorageFileMetaData from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import StorageFileID -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError from simcore_service_api_server.models.schemas.files import File from simcore_service_api_server.services.storage import to_file_api_model @@ -34,8 +34,8 @@ def expected_sha256sum() -> SHA256Str: # $ echo -n "This is a test" | md5sum - # ce114e4501d2f4e2dcea3e17b546f339 - # - _sha256sum: SHA256Str = parse_obj_as( - SHA256Str, "c7be1ed902fb8dd4d48997c6452f5d7e509fbcdbe2808b16bcf4edce4c07d14e" + _sha256sum: SHA256Str = TypeAdapter(SHA256Str).validate_python( + "c7be1ed902fb8dd4d48997c6452f5d7e509fbcdbe2808b16bcf4edce4c07d14e" ) assert hashlib.sha256(FILE_CONTENT.encode()).hexdigest() == _sha256sum return _sha256sum @@ -81,10 +81,10 @@ async def test_create_filemetadata_from_starlette_uploadfile( def test_convert_between_file_models(): storage_file_meta = StorageFileMetaData( - **StorageFileMetaData.Config.schema_extra["examples"][1] + **StorageFileMetaData.model_config["json_schema_extra"]["examples"][1] ) - storage_file_meta.file_id = parse_obj_as( - StorageFileID, f"api/{uuid4()}/extensionless" + storage_file_meta.file_id = TypeAdapter(StorageFileID).validate_python( + f"api/{uuid4()}/extensionless" ) apiserver_file_meta = to_file_api_model(storage_file_meta) @@ -94,11 +94,13 @@ def test_convert_between_file_models(): assert apiserver_file_meta.e_tag == storage_file_meta.entity_tag with pytest.raises(ValueError): - storage_file_meta.file_id = parse_obj_as( - StorageFileID, f"{uuid4()}/{uuid4()}/foo.txt" + storage_file_meta.file_id = TypeAdapter(StorageFileID).validate_python( + f"{uuid4()}/{uuid4()}/foo.txt" ) to_file_api_model(storage_file_meta) with pytest.raises(ValidationError): - storage_file_meta.file_id = parse_obj_as(StorageFileID, "api/NOTUUID/foo.txt") + storage_file_meta.file_id = TypeAdapter(StorageFileID).validate_python( + "api/NOTUUID/foo.txt" + ) to_file_api_model(storage_file_meta) diff --git a/services/api-server/tests/unit/test_models_schemas_jobs.py b/services/api-server/tests/unit/test_models_schemas_jobs.py index c1e3a905fbc..73486cf0f55 100644 --- a/services/api-server/tests/unit/test_models_schemas_jobs.py +++ b/services/api-server/tests/unit/test_models_schemas_jobs.py @@ -46,8 +46,8 @@ def _deepcopy_and_shuffle(src): return deepcopy(src) shuffled_raw = _deepcopy_and_shuffle(raw) - inputs1 = JobInputs.parse_obj(raw) - inputs2 = JobInputs.parse_obj(shuffled_raw) + inputs1 = JobInputs.model_validate(raw) + inputs2 = JobInputs.model_validate(shuffled_raw) print(inputs1) print(inputs2) @@ -87,7 +87,7 @@ def test_job_resouce_names_has_associated_url(app: FastAPI): def test_parsing_job_custom_metadata(job_id: JobID, faker: Faker): job_name = faker.name() - got = Envelope[ProjectMetadataGet].parse_raw( + got = Envelope[ProjectMetadataGet].model_validate_json( textwrap.dedent( f""" {{ diff --git a/services/api-server/tests/unit/test_models_schemas_solvers.py b/services/api-server/tests/unit/test_models_schemas_solvers.py index a8dd693622a..0e81e60aa86 100644 --- a/services/api-server/tests/unit/test_models_schemas_solvers.py +++ b/services/api-server/tests/unit/test_models_schemas_solvers.py @@ -12,7 +12,7 @@ def test_solvers_sorting_by_name_and_version(faker: Faker): # SEE https://packaging.pypa.io/en/latest/version.html # have a solver - one_solver = Solver(**Solver.Config.schema_extra["example"]) + one_solver = Solver(**Solver.model_config["json_schema_extra"]["example"]) assert isinstance(one_solver.pep404_version, Version) major, minor, micro = one_solver.pep404_version.release @@ -20,14 +20,16 @@ def test_solvers_sorting_by_name_and_version(faker: Faker): # and a different version of the same # NOTE: that id=None so that it can be re-coputed - earlier_release = one_solver.copy( + earlier_release = one_solver.model_copy( update={"version": f"{one_solver.version}beta"}, deep=True ) assert earlier_release.pep404_version.is_prerelease assert earlier_release.pep404_version < one_solver.pep404_version # and yet a completely different solver - another_solver = one_solver.copy(update={"id": "simcore/services/comp/zSolve"}) + another_solver = one_solver.model_copy( + update={"id": "simcore/services/comp/zSolve"} + ) assert one_solver.id != another_solver.id assert one_solver.pep404_version == another_solver.pep404_version diff --git a/services/api-server/tests/unit/test_services_rabbitmq.py b/services/api-server/tests/unit/test_services_rabbitmq.py index ee68615c8f1..77116e711eb 100644 --- a/services/api-server/tests/unit/test_services_rabbitmq.py +++ b/services/api-server/tests/unit/test_services_rabbitmq.py @@ -332,8 +332,8 @@ async def log_streamer_with_distributor( log_distributor: LogDistributor, ) -> AsyncIterable[LogStreamer]: def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: - task = ComputationTaskGet.parse_obj( - ComputationTaskGet.Config.schema_extra["examples"][0] + task = ComputationTaskGet.model_validate( + ComputationTaskGet.model_config["json_schema_extra"]["examples"][0] ) if computation_done(): task.state = RunningState.SUCCESS @@ -378,7 +378,7 @@ async def _log_publisher(): collected_messages: list[str] = [] async for log in log_streamer_with_distributor.log_generator(): - job_log: JobLog = JobLog.parse_raw(log) + job_log: JobLog = JobLog.model_validate_json(log) assert len(job_log.messages) == 1 assert job_log.job_id == project_id collected_messages.append(job_log.messages[0]) @@ -420,13 +420,13 @@ def routing_key(self) -> str: log_level=logging.INFO, ) with pytest.raises(ValidationError): - LoggerRabbitMessage.parse_obj(log_rabbit_message.dict()) + LoggerRabbitMessage.model_validate(log_rabbit_message.model_dump()) await produce_logs("expected", log_message=log_rabbit_message) ii: int = 0 async for log in log_streamer_with_distributor.log_generator(): - _ = JobLog.parse_raw(log) + _ = JobLog.model_validate_json(log) ii += 1 assert ii == 0 @@ -448,7 +448,9 @@ async def test_log_generator(mocker: MockFixture, faker: Faker): published_logs: list[str] = [] for _ in range(10): - job_log = JobLog.parse_obj(JobLog.Config.schema_extra["example"]) + job_log = JobLog.model_validate( + JobLog.model_config["json_schema_extra"]["example"] + ) msg = faker.text() published_logs.append(msg) job_log.messages = [msg] @@ -456,7 +458,7 @@ async def test_log_generator(mocker: MockFixture, faker: Faker): collected_logs: list[str] = [] async for log in log_streamer.log_generator(): - job_log = JobLog.parse_raw(log) + job_log = JobLog.model_validate_json(log) assert len(job_log.messages) == 1 collected_logs.append(job_log.messages[0]) diff --git a/services/api-server/tests/unit/test_services_solver_job_models_converters.py b/services/api-server/tests/unit/test_services_solver_job_models_converters.py index 28f8be422f9..1016096dce5 100644 --- a/services/api-server/tests/unit/test_services_solver_job_models_converters.py +++ b/services/api-server/tests/unit/test_services_solver_job_models_converters.py @@ -6,7 +6,7 @@ from faker import Faker from models_library.projects import Project from models_library.projects_nodes import InputsDict, InputTypes, SimCoreFileLink -from pydantic import create_model, parse_obj_as +from pydantic import RootModel, TypeAdapter, create_model from simcore_service_api_server.models.schemas.files import File from simcore_service_api_server.models.schemas.jobs import ArgumentTypes, Job, JobInputs from simcore_service_api_server.models.schemas.solvers import Solver @@ -20,7 +20,7 @@ def test_create_project_model_for_job(faker: Faker): - solver = Solver.parse_obj( + solver = Solver.model_validate( { "id": "simcore/services/comp/itis/sleeper", "version": "2.0.2", @@ -31,7 +31,7 @@ def test_create_project_model_for_job(faker: Faker): } ) - inputs = JobInputs.parse_obj( + inputs = JobInputs.model_validate( { "values": { "input_3": False, # Fail after sleep ? @@ -46,7 +46,7 @@ def test_create_project_model_for_job(faker: Faker): } ) - print(inputs.json(indent=2)) + print(inputs.model_dump_json(indent=2)) job = Job.create_solver_job(solver=solver, inputs=inputs) @@ -77,7 +77,7 @@ def test_job_to_node_inputs_conversion(): } ) for value in job_inputs.values.values(): - assert parse_obj_as(ArgumentTypes, value) == value + assert TypeAdapter(ArgumentTypes).validate_python(value) == value node_inputs: InputsDict = { "x": 4.33, @@ -94,22 +94,22 @@ def test_job_to_node_inputs_conversion(): } for value in node_inputs.values(): - assert parse_obj_as(InputTypes, value) == value + assert TypeAdapter(InputTypes).validate_python(value) == value # test transformations in both directions got_node_inputs = create_node_inputs_from_job_inputs(inputs=job_inputs) got_job_inputs = create_job_inputs_from_node_inputs(inputs=node_inputs) - NodeInputs = create_model("NodeInputs", __root__=(dict[str, InputTypes], ...)) - print(NodeInputs.parse_obj(got_node_inputs).json(indent=2)) - print(got_job_inputs.json(indent=2)) + NodeInputs = create_model("NodeInputs", __base__=RootModel[dict[str, InputTypes]]) + print(NodeInputs.model_validate(got_node_inputs).model_dump_json(indent=2)) + print(got_job_inputs.model_dump_json(indent=2)) assert got_job_inputs == job_inputs assert got_node_inputs == node_inputs def test_create_job_from_project(faker: Faker): - project = Project.parse_obj( + project = Project.model_validate( { "uuid": "f925e30f-19de-42dc-acab-3ce93ea0a0a7", "name": "simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/2.0.2/jobs/f925e30f-19de-42dc-acab-3ce93ea0a0a7", @@ -181,7 +181,7 @@ def test_create_job_from_project(faker: Faker): }, ) - expected_job = Job.parse_obj( + expected_job = Job.model_validate( { "id": "f925e30f-19de-42dc-acab-3ce93ea0a0a7", "name": "simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/2.0.2/jobs/f925e30f-19de-42dc-acab-3ce93ea0a0a7", @@ -207,12 +207,12 @@ def fake_url_for(*args, **kwargs): assert job.id == project.uuid assert job.name == project.name - url_field_names = {name for name in job.__fields__ if name.endswith("url")} + url_field_names = {name for name in job.model_fields if name.endswith("url")} assert all(getattr(job, _) for _ in url_field_names) # this tends to be a problem assert job.inputs_checksum == expected_job.inputs_checksum - assert job.dict(exclude=url_field_names) == expected_job.dict( + assert job.model_dump(exclude=url_field_names) == expected_job.model_dump( exclude=url_field_names ) @@ -222,7 +222,7 @@ def test_create_jobstatus_from_task(): from simcore_service_api_server.models.schemas.jobs import JobStatus from simcore_service_api_server.services.director_v2 import ComputationTaskGet - task = ComputationTaskGet.parse_obj({}) # TODO: + task = ComputationTaskGet.model_validate({}) # TODO: job_status: JobStatus = create_jobstatus_from_task(task) assert job_status.job_id == task.id diff --git a/services/api-server/tests/unit/test_services_solver_job_outputs.py b/services/api-server/tests/unit/test_services_solver_job_outputs.py index 04765e420f7..b02022e5daa 100644 --- a/services/api-server/tests/unit/test_services_solver_job_outputs.py +++ b/services/api-server/tests/unit/test_services_solver_job_outputs.py @@ -3,7 +3,7 @@ # pylint: disable=unused-variable import types -from typing import get_args, get_origin +from typing import Union, get_args, get_origin from simcore_service_api_server.models.schemas.jobs import ArgumentTypes, File from simcore_service_api_server.services.solver_job_outputs import ( @@ -16,10 +16,11 @@ def test_resultstypes_and_argument_type_sync(): # I/O types returned by node-ports must be one-to-one mapped # with those returned as output results - assert get_origin(ArgumentTypes) == types.UnionType + # Python 3.10 and later treats unions with | as types.UnionType + assert get_origin(ArgumentTypes) in (types.UnionType, Union) argument_types_args = set(get_args(ArgumentTypes)) - assert get_origin(ResultsTypes) == types.UnionType + assert get_origin(ResultsTypes) in (types.UnionType, Union) results_types_args = set(get_args(ResultsTypes)) # files are in the inputs as File (or Raises KeyError if not) diff --git a/services/api-server/tests/unit/test_utils_http_calls_capture.py b/services/api-server/tests/unit/test_utils_http_calls_capture.py index 536c42f36e5..9ef5e034529 100644 --- a/services/api-server/tests/unit/test_utils_http_calls_capture.py +++ b/services/api-server/tests/unit/test_utils_http_calls_capture.py @@ -22,7 +22,7 @@ async def test_capture_http_call( ): # CAPTURE async with httpx.AsyncClient() as client: - response: httpx.Response = await client.get(f"{httpbin_base_url}/json") + response: httpx.Response = await client.get(f"{httpbin_base_url}json") print(response) _request: httpx.Request = response.request @@ -32,7 +32,7 @@ async def test_capture_http_call( response, name="get_json", enhance_from_openapi_specs=False ) - print(captured.json(indent=1)) + print(captured.model_dump_json(indent=1)) # MOCK with respx.mock( @@ -64,7 +64,7 @@ async def test_capture_http_dynamic_call( sample_uid = faker.uuid4() # used during test sampling response: httpx.Response = await client.post( - f"{httpbin_base_url}/anything/{sample_uid}", + f"{httpbin_base_url}anything/{sample_uid}", params={"n": 42}, json={ "resource_id": sample_uid, @@ -89,7 +89,7 @@ async def test_capture_http_dynamic_call( assert found.groupdict() == {"resouce_uid": sample_uid} # subs_json = re.sub(f"{resource_uid}", pattern, captured.json()) - # new_capture = HttpApiCallCaptureModel.parse_raw(subs_json) + # new_capture = HttpApiCallCaptureModel.model_validate_json(subs_json) # MOCK with respx.mock( @@ -110,7 +110,7 @@ async def test_capture_http_dynamic_call( other_uid = faker.uuid4() - response: httpx.Response = await client.post( + response = await client.post( f"http://test.it/anything/{other_uid}", params={"n": 42}, json={ @@ -140,6 +140,6 @@ def test_template_capture(project_tests_dir: Path, faker: Faker): # loads parametrized capture # replace in response and solve - capture = HttpApiCallCaptureModel.parse_raw(template.render(context)) - print(capture.json(indent=1)) + capture = HttpApiCallCaptureModel.model_validate_json(template.render(context)) + print(capture.model_dump_json(indent=1)) assert capture.path == url_path From e38cf4e7bd4bf5f56cb2eadf611ee302d000182b Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Mon, 28 Oct 2024 08:06:12 +0100 Subject: [PATCH 22/53] =?UTF-8?q?=E2=99=BB=EF=B8=8FFix=20settings=20librar?= =?UTF-8?q?y=20tests=20(#6605)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/common-library/requirements/_base.in | 2 + .../common-library/requirements/_base.txt | 7 ++ .../src/common_library}/json_serialization.py | 0 .../tests/test_json_serialization.py | 97 +++++++++++++++++++ .../models_library/service_settings_labels.py | 2 +- .../utils/_original_fastapi_encoders.py | 2 +- .../utils/labels_annotations.py | 2 +- .../utils/specs_substitution.py | 3 +- .../tests/test_utils_json_serialization.py | 70 +------------ .../src/pytest_simcore/helpers/aws_ec2.py | 2 +- .../tests/test__usecase_jupytermath.py | 2 +- .../src/servicelib/aiohttp/client_session.py | 2 +- .../long_running_tasks/_error_handlers.py | 2 +- .../aiohttp/long_running_tasks/_routes.py | 2 +- .../aiohttp/long_running_tasks/_server.py | 2 +- .../servicelib/aiohttp/requests_validation.py | 2 +- .../servicelib/aiohttp/rest_middlewares.py | 2 +- .../src/servicelib/aiohttp/rest_responses.py | 3 +- .../src/servicelib/aiohttp/rest_utils.py | 2 +- .../servicelib/utils_profiling_middleware.py | 2 +- .../tests/aiohttp/test_client_session.py | 2 +- .../tests/aiohttp/test_requests_validation.py | 2 +- .../tests/aiohttp/test_rest_middlewares.py | 2 +- .../deferred_tasks/test_deferred_tasks.py | 2 +- .../settings-library/requirements/_base.txt | 8 ++ .../src/settings_library/utils_cli.py | 2 +- .../tests/test__pydantic_settings.py | 5 +- .../node_ports_v2/serialization_v2.py | 2 +- .../api/dependencies/webserver.py | 2 +- .../models/_utils_pydantic.py | 2 +- .../utils/buffer_machines_pool_core.py | 2 +- .../unit/test_modules_buffer_machine_core.py | 2 +- .../services/director.py | 2 +- .../tests/unit/test_utils_clusters.py | 2 +- .../api/routes/dynamic_services.py | 2 +- .../modules/db/events.py | 2 +- .../dynamic_sidecar/docker_api/_core.py | 2 +- .../dynamic_sidecar/docker_compose_specs.py | 2 +- .../docker_service_specs/sidecar.py | 2 +- .../scheduler/_core/_event_create_sidecars.py | 2 +- .../simcore_service_director_v2/utils/dask.py | 2 +- ...dules_dynamic_sidecar_client_api_public.py | 2 +- ...es_dynamic_sidecar_docker_service_specs.py | 2 +- .../services/director_v2/_thin_client.py | 2 +- services/dynamic-sidecar/tests/conftest.py | 2 +- ...test_modules_system_monitor__disk_usage.py | 2 +- .../src/simcore_service_payments/cli.py | 2 +- .../handlers_datasets.py | 2 +- .../simcore_service_storage/handlers_files.py | 2 +- .../handlers_health.py | 2 +- .../handlers_locations.py | 2 +- .../handlers_simcore_s3.py | 2 +- .../storage/src/simcore_service_storage/s3.py | 2 +- .../simcore_service_webserver/db/_aiopg.py | 2 +- .../director_v2/_handlers.py | 2 +- .../exporter/_formatter/template_json.py | 2 +- .../groups/_handlers.py | 2 +- .../login/_registration_api.py | 2 +- .../simcore_service_webserver/login/utils.py | 2 +- .../meta_modeling/_iterations.py | 2 +- .../projects/_crud_api_create.py | 2 +- .../projects/_crud_handlers.py | 2 +- .../projects/_nodes_handlers.py | 2 +- .../projects/_ports_handlers.py | 2 +- .../projects/projects_api.py | 2 +- .../publications/_handlers.py | 2 +- .../socketio/server.py | 2 +- .../statics/_events.py | 2 +- .../users/schemas.py | 2 +- .../utils_aiohttp.py | 2 +- .../utils_rate_limiting.py | 2 +- .../version_control/_handlers_base.py | 2 +- .../version_control/db.py | 2 +- services/web/server/tests/conftest.py | 2 +- .../tests/integration/02/test_computation.py | 2 +- .../isolated/test_application_settings.py | 2 +- .../unit/isolated/test_products_model.py | 2 +- .../test_studies_dispatcher_handlers.py | 2 +- .../test_meta_modeling_iterations.py | 2 +- 79 files changed, 192 insertions(+), 143 deletions(-) rename packages/{models-library/src/models_library/utils => common-library/src/common_library}/json_serialization.py (100%) create mode 100644 packages/common-library/tests/test_json_serialization.py diff --git a/packages/common-library/requirements/_base.in b/packages/common-library/requirements/_base.in index 73cc02bceb7..2277d690e64 100644 --- a/packages/common-library/requirements/_base.in +++ b/packages/common-library/requirements/_base.in @@ -3,4 +3,6 @@ # --constraint ../../../requirements/constraints.txt +orjson pydantic +pydantic-extra-types diff --git a/packages/common-library/requirements/_base.txt b/packages/common-library/requirements/_base.txt index 32a8575abea..a0162daa052 100644 --- a/packages/common-library/requirements/_base.txt +++ b/packages/common-library/requirements/_base.txt @@ -1,11 +1,18 @@ annotated-types==0.7.0 # via pydantic +orjson==3.10.10 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in + # pydantic-extra-types pydantic-core==2.23.4 # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/_base.in typing-extensions==4.12.2 # via # pydantic diff --git a/packages/models-library/src/models_library/utils/json_serialization.py b/packages/common-library/src/common_library/json_serialization.py similarity index 100% rename from packages/models-library/src/models_library/utils/json_serialization.py rename to packages/common-library/src/common_library/json_serialization.py diff --git a/packages/common-library/tests/test_json_serialization.py b/packages/common-library/tests/test_json_serialization.py new file mode 100644 index 00000000000..7684497e493 --- /dev/null +++ b/packages/common-library/tests/test_json_serialization.py @@ -0,0 +1,97 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import json +from copy import deepcopy +from typing import Annotated, Any, TypeAlias +from uuid import uuid4 + +import pytest +from common_library.json_serialization import ( + JsonNamespace, + SeparatorTuple, + json_dumps, + json_loads, +) +from faker import Faker +from pydantic import Field, TypeAdapter +from pydantic.json import pydantic_encoder + + +@pytest.fixture +def fake_data_dict(faker: Faker) -> dict[str, Any]: + data = { + "uuid_as_UUID": faker.uuid4(cast_to=None), + "uuid_as_str": faker.uuid4(), + "int": faker.pyint(), + "float": faker.pyfloat(), + "str": faker.pystr(), + "dict": faker.pydict(), + "list": faker.pylist(), + } + data["object"] = deepcopy(data) + return data + + +def test_json_dump_variants(): + + uuid_obj = uuid4() + + with pytest.raises(TypeError) as exc_info: + json.dumps(uuid_obj) + + assert str(exc_info.value) == "Object of type UUID is not JSON serializable" + + assert json_dumps(uuid_obj) == json.dumps(str(uuid_obj)) + + +def test_serialized_non_str_dict_keys(): + # tests orjson.OPT_NON_STR_KEYS option + + # if a dict has a key of a type other than str it will NOT raise + json_dumps({1: "foo"}) + + +ConstrainedFloat: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] + + +def test_serialized_constraint_floats(): + # test extension of ENCODERS_BY_TYPE used in pydantic_encoder + + json_dumps({"value": 1.0}) + + # TypeError: Type is not JSON serializable: ProgressPercent + json_dumps({"value": TypeAdapter(ConstrainedFloat).validate_python(1.0)}) + + +def _expected_json_dumps(obj: Any, default=pydantic_encoder, **json_dumps_kwargs): + if "indent" not in json_dumps_kwargs: + json_dumps_kwargs.setdefault( + "separators", + SeparatorTuple(item_separator=",", key_separator=":"), # compact separators + ) + return json.dumps(obj, default=default, **json_dumps_kwargs) + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({}, id="no-kw"), + pytest.param({"sort_keys": True}, id="sort_keys-kw"), + pytest.param( + {"separators": (",", ":")}, id="default_separators-kw" + ), # NOTE: e.g. engineio.packet has `self.json.dumps(self.data, separators=(',', ':'))` + pytest.param( + {"indent": 2}, id="indent-kw" + ), # NOTE: only one-to-one with indent=2 + ], +) +def test_compatiblity_with_json_interface( + fake_data_dict: dict[str, Any], kwargs: dict[str, Any] +): + orjson_dump = JsonNamespace.dumps(fake_data_dict, **kwargs) + json_dump = _expected_json_dumps(fake_data_dict, **kwargs) + + # NOTE: cannot compare dumps directly because orjson compacts it more + assert json_loads(orjson_dump) == json_loads(json_dump) diff --git a/packages/models-library/src/models_library/service_settings_labels.py b/packages/models-library/src/models_library/service_settings_labels.py index 851b1880cc3..0a632a7b70d 100644 --- a/packages/models-library/src/models_library/service_settings_labels.py +++ b/packages/models-library/src/models_library/service_settings_labels.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, Literal, TypeAlias +from common_library.json_serialization import json_dumps from pydantic import ( BaseModel, ByteSize, @@ -23,7 +24,6 @@ from .generics import ListModel from .service_settings_nat_rule import NATRule from .services_resources import DEFAULT_SINGLE_SERVICE_NAME -from .utils.json_serialization import json_dumps _BaseConfig = ConfigDict( extra="forbid", arbitrary_types_allowed=True, ignored_types=(cached_property,) diff --git a/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py b/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py index 4f09ae6a379..0b48328d91b 100644 --- a/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py +++ b/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py @@ -10,7 +10,7 @@ from types import GeneratorType from typing import Any, Callable, Union -from models_library.utils.json_serialization import ENCODERS_BY_TYPE +from common_library.json_serialization import ENCODERS_BY_TYPE from pydantic import BaseModel from pydantic_core import PydanticUndefined, PydanticUndefinedType from typing_extensions import Annotated, Doc diff --git a/packages/models-library/src/models_library/utils/labels_annotations.py b/packages/models-library/src/models_library/utils/labels_annotations.py index ec8d1e7fd6c..26c7d7d73f5 100644 --- a/packages/models-library/src/models_library/utils/labels_annotations.py +++ b/packages/models-library/src/models_library/utils/labels_annotations.py @@ -8,7 +8,7 @@ from json.decoder import JSONDecodeError from typing import Any, TypeAlias -from .json_serialization import json_dumps +from common_library.json_serialization import json_dumps LabelsAnnotationsDict: TypeAlias = dict[str, str] diff --git a/packages/models-library/src/models_library/utils/specs_substitution.py b/packages/models-library/src/models_library/utils/specs_substitution.py index d8a7e9cf161..d1278d69912 100644 --- a/packages/models-library/src/models_library/utils/specs_substitution.py +++ b/packages/models-library/src/models_library/utils/specs_substitution.py @@ -1,10 +1,9 @@ from typing import Any, NamedTuple, TypeAlias, cast from common_library.errors_classes import OsparcErrorMixin - +from common_library.json_serialization import json_dumps, json_loads from pydantic import StrictBool, StrictFloat, StrictInt -from .json_serialization import json_dumps, json_loads from .string_substitution import ( SubstitutionsDict, TextTemplate, diff --git a/packages/models-library/tests/test_utils_json_serialization.py b/packages/models-library/tests/test_utils_json_serialization.py index 4e3839fc6e9..a229c16b75d 100644 --- a/packages/models-library/tests/test_utils_json_serialization.py +++ b/packages/models-library/tests/test_utils_json_serialization.py @@ -3,43 +3,14 @@ # pylint: disable=unused-variable -import json from copy import deepcopy from typing import Any from uuid import uuid4 import pytest +from common_library.json_serialization import json_dumps, json_loads from faker import Faker -from models_library.api_schemas_long_running_tasks.base import ProgressPercent from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import ( - JsonNamespace, - SeparatorTuple, - json_dumps, - json_loads, -) -from pydantic.json import pydantic_encoder - - -def _expected_json_dumps(obj: Any, default=pydantic_encoder, **json_dumps_kwargs): - if "indent" not in json_dumps_kwargs: - json_dumps_kwargs.setdefault( - "separators", - SeparatorTuple(item_separator=",", key_separator=":"), # compact separators - ) - return json.dumps(obj, default=default, **json_dumps_kwargs) - - -def test_json_dump_variants(): - - uuid_obj = uuid4() - - with pytest.raises(TypeError) as exc_info: - json.dumps(uuid_obj) - - assert str(exc_info.value) == "Object of type UUID is not JSON serializable" - - assert json_dumps(uuid_obj) == json.dumps(str(uuid_obj)) @pytest.fixture @@ -75,42 +46,3 @@ def test_serialization_of_nested_dicts(fake_data_dict: dict[str, Any]): dump = json_dumps(obj) assert json_loads(dump) == jsonable_encoder(obj) - - -@pytest.mark.parametrize( - "kwargs", - [ - pytest.param({}, id="no-kw"), - pytest.param({"sort_keys": True}, id="sort_keys-kw"), - pytest.param( - {"separators": (",", ":")}, id="default_separators-kw" - ), # NOTE: e.g. engineio.packet has `self.json.dumps(self.data, separators=(',', ':'))` - pytest.param( - {"indent": 2}, id="indent-kw" - ), # NOTE: only one-to-one with indent=2 - ], -) -def test_compatiblity_with_json_interface( - fake_data_dict: dict[str, Any], kwargs: dict[str, Any] -): - orjson_dump = JsonNamespace.dumps(fake_data_dict, **kwargs) - json_dump = _expected_json_dumps(fake_data_dict, **kwargs) - - # NOTE: cannot compare dumps directly because orjson compacts it more - assert json_loads(orjson_dump) == json_loads(json_dump) - - -def test_serialized_non_str_dict_keys(): - # tests orjson.OPT_NON_STR_KEYS option - - # if a dict has a key of a type other than str it will NOT raise - json_dumps({1: "foo"}) - - -def test_serialized_constraint_floats(): - # test extension of ENCODERS_BY_TYPE used in pydantic_encoder - - json_dumps({"value": 1.0}) - - # TypeError: Type is not JSON serializable: ProgressPercent - json_dumps({"value": ProgressPercent(1.0)}) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/aws_ec2.py b/packages/pytest-simcore/src/pytest_simcore/helpers/aws_ec2.py index cf54561f843..1e992f4ee45 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/aws_ec2.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/aws_ec2.py @@ -1,8 +1,8 @@ import base64 from collections.abc import Sequence +from common_library.json_serialization import json_dumps from models_library.docker import DockerGenericTag -from models_library.utils.json_serialization import json_dumps from types_aiobotocore_ec2 import EC2Client from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType from types_aiobotocore_ec2.type_defs import FilterTypeDef, InstanceTypeDef, TagTypeDef diff --git a/packages/service-integration/tests/test__usecase_jupytermath.py b/packages/service-integration/tests/test__usecase_jupytermath.py index e49f9b0512a..4b816a18198 100644 --- a/packages/service-integration/tests/test__usecase_jupytermath.py +++ b/packages/service-integration/tests/test__usecase_jupytermath.py @@ -13,7 +13,7 @@ import pytest import yaml -from models_library.utils.json_serialization import json_loads +from common_library.json_serialization import json_loads from service_integration import cli from typer.testing import CliRunner, Result diff --git a/packages/service-library/src/servicelib/aiohttp/client_session.py b/packages/service-library/src/servicelib/aiohttp/client_session.py index 26e4e7d579c..40e49c76a4b 100644 --- a/packages/service-library/src/servicelib/aiohttp/client_session.py +++ b/packages/service-library/src/servicelib/aiohttp/client_session.py @@ -2,7 +2,7 @@ from typing import cast from aiohttp import ClientSession, ClientTimeout, web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from ..utils import ( get_http_client_request_aiohttp_connect_timeout, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py index ef1e91a0197..9e67b2b7f4d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_error_handlers.py @@ -1,7 +1,7 @@ import logging from aiohttp import web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from ...long_running_tasks._errors import ( TaskCancelledError, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index d2e2dda98bf..1906c0bc93f 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -2,7 +2,7 @@ from typing import Any from aiohttp import web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from pydantic import BaseModel from servicelib.aiohttp import status diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 47640c491f2..1d97f9320c8 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -5,8 +5,8 @@ from typing import Any from aiohttp import web +from common_library.json_serialization import json_dumps from common_library.pydantic_networks_extension import AnyHttpUrlLegacy -from models_library.utils.json_serialization import json_dumps from pydantic import PositiveFloat, TypeAdapter from ...aiohttp import status diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index ce4a97d54a1..5a588041682 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -13,7 +13,7 @@ from typing import TypeAlias, TypeVar, Union from aiohttp import web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from ..mimetype_constants import MIMETYPE_APPLICATION_JSON diff --git a/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py b/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py index d12ea2147d8..d5f5a04283b 100644 --- a/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py +++ b/packages/service-library/src/servicelib/aiohttp/rest_middlewares.py @@ -13,7 +13,7 @@ from aiohttp.web_request import Request from aiohttp.web_response import StreamResponse from common_library.error_codes import create_error_code -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from ..logging_errors import create_troubleshotting_log_kwargs from ..mimetype_constants import MIMETYPE_APPLICATION_JSON diff --git a/packages/service-library/src/servicelib/aiohttp/rest_responses.py b/packages/service-library/src/servicelib/aiohttp/rest_responses.py index 52aa87497d8..1cce04ae015 100644 --- a/packages/service-library/src/servicelib/aiohttp/rest_responses.py +++ b/packages/service-library/src/servicelib/aiohttp/rest_responses.py @@ -1,6 +1,7 @@ """ Utils to check, convert and compose server responses for the RESTApi """ + import inspect import json from collections.abc import Mapping @@ -9,7 +10,7 @@ from aiohttp import web, web_exceptions from aiohttp.web_exceptions import HTTPError, HTTPException -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.aiohttp.status import HTTP_200_OK from ..mimetype_constants import MIMETYPE_APPLICATION_JSON diff --git a/packages/service-library/src/servicelib/aiohttp/rest_utils.py b/packages/service-library/src/servicelib/aiohttp/rest_utils.py index b25a72301d6..e026e211977 100644 --- a/packages/service-library/src/servicelib/aiohttp/rest_utils.py +++ b/packages/service-library/src/servicelib/aiohttp/rest_utils.py @@ -2,7 +2,7 @@ from aiohttp import web from aiohttp.web import RouteDef, RouteTableDef -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps class EnvelopeFactory: diff --git a/packages/service-library/src/servicelib/utils_profiling_middleware.py b/packages/service-library/src/servicelib/utils_profiling_middleware.py index 22bc7c7d29e..5b5038e335e 100644 --- a/packages/service-library/src/servicelib/utils_profiling_middleware.py +++ b/packages/service-library/src/servicelib/utils_profiling_middleware.py @@ -4,7 +4,7 @@ from contextlib import contextmanager from typing import Final -from models_library.utils.json_serialization import json_dumps, json_loads +from common_library.json_serialization import json_dumps, json_loads from pyinstrument import Profiler from .mimetype_constants import MIMETYPE_APPLICATION_JSON, MIMETYPE_APPLICATION_ND_JSON diff --git a/packages/service-library/tests/aiohttp/test_client_session.py b/packages/service-library/tests/aiohttp/test_client_session.py index ccc9356f07e..74b91655c31 100644 --- a/packages/service-library/tests/aiohttp/test_client_session.py +++ b/packages/service-library/tests/aiohttp/test_client_session.py @@ -10,7 +10,7 @@ from aiohttp import web from aiohttp.client import ClientSession from aiohttp.test_utils import TestServer -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.aiohttp.application_keys import APP_CLIENT_SESSION_KEY from servicelib.aiohttp.client_session import ( get_client_session, diff --git a/packages/service-library/tests/aiohttp/test_requests_validation.py b/packages/service-library/tests/aiohttp/test_requests_validation.py index 4e1b4f4e2e7..2094349932f 100644 --- a/packages/service-library/tests/aiohttp/test_requests_validation.py +++ b/packages/service-library/tests/aiohttp/test_requests_validation.py @@ -9,8 +9,8 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient +from common_library.json_serialization import json_dumps from faker import Faker -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( diff --git a/packages/service-library/tests/aiohttp/test_rest_middlewares.py b/packages/service-library/tests/aiohttp/test_rest_middlewares.py index 554982e53cd..00f371544b7 100644 --- a/packages/service-library/tests/aiohttp/test_rest_middlewares.py +++ b/packages/service-library/tests/aiohttp/test_rest_middlewares.py @@ -14,7 +14,7 @@ from aiohttp import web from aiohttp.test_utils import TestClient from common_library.error_codes import parse_error_code -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.aiohttp import status from servicelib.aiohttp.rest_middlewares import ( _FMSG_INTERNAL_ERROR_USER_FRIENDLY_WITH_OEC, diff --git a/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py b/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py index a03b87c4151..6dbc5d3d764 100644 --- a/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py +++ b/packages/service-library/tests/deferred_tasks/test_deferred_tasks.py @@ -16,8 +16,8 @@ import psutil import pytest from aiohttp.test_utils import unused_port +from common_library.json_serialization import json_dumps from common_library.serialization import model_dump_with_secrets -from models_library.utils.json_serialization import json_dumps from pydantic import NonNegativeFloat, NonNegativeInt from pytest_mock import MockerFixture from servicelib import redis as servicelib_redis diff --git a/packages/settings-library/requirements/_base.txt b/packages/settings-library/requirements/_base.txt index 64117c65d55..3d57d1af6c2 100644 --- a/packages/settings-library/requirements/_base.txt +++ b/packages/settings-library/requirements/_base.txt @@ -6,15 +6,23 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py +orjson==3.10.10 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in pydantic==2.9.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in + # pydantic-extra-types # pydantic-settings pydantic-core==2.23.4 # via pydantic +pydantic-extra-types==2.9.0 + # via -r requirements/../../../packages/common-library/requirements/_base.in pydantic-settings==2.5.2 # via -r requirements/_base.in pygments==2.18.0 diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index e3addbc6130..8c28dfe9cb5 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -8,8 +8,8 @@ import rich import typer +from common_library.json_serialization import json_dumps from common_library.serialization import model_dump_with_secrets -from models_library.utils.json_serialization import json_dumps from pydantic import ValidationError from pydantic_core import to_jsonable_python from pydantic_settings import BaseSettings diff --git a/packages/settings-library/tests/test__pydantic_settings.py b/packages/settings-library/tests/test__pydantic_settings.py index 2d52c1096e7..976395b1b10 100644 --- a/packages/settings-library/tests/test__pydantic_settings.py +++ b/packages/settings-library/tests/test__pydantic_settings.py @@ -132,8 +132,9 @@ def test_construct(monkeypatch): # from __init__ settings_from_init = Settings( VALUE=1, - VALUE_ALSO_REQUIRED=10, VALUE_NULLABLE_REQUIRED=None, + VALUE_NULLABLE_REQUIRED_AS_WELL=None, + VALUE_REQUIRED_AS_WELL=32, ) print(settings_from_init.model_dump_json(exclude_unset=True, indent=1)) @@ -145,6 +146,8 @@ def test_construct(monkeypatch): "VALUE": "1", "VALUE_ALSO_REQUIRED": "10", "VALUE_NULLABLE_REQUIRED": "null", + "VALUE_NULLABLE_REQUIRED_AS_WELL": "null", + "VALUE_REQUIRED_AS_WELL": "32", }, ) # WARNING: set this env to None would not work w/o ``parse_none`` validator! bug??? diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index f4d74711e18..510f7b14fbb 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -5,8 +5,8 @@ from typing import Any import pydantic +from common_library.json_serialization import json_dumps from models_library.projects_nodes_io import NodeID -from models_library.utils.json_serialization import json_dumps from models_library.utils.nodes import compute_node_hash from packaging import version from settings_library.aws_s3_cli import AwsS3CliSettings diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/webserver.py b/services/api-server/src/simcore_service_api_server/api/dependencies/webserver.py index e79b7e83057..94cfed68b55 100644 --- a/services/api-server/src/simcore_service_api_server/api/dependencies/webserver.py +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/webserver.py @@ -1,10 +1,10 @@ import time from typing import Annotated +from common_library.json_serialization import json_dumps from cryptography.fernet import Fernet from fastapi import Depends, FastAPI, HTTPException, status from fastapi.requests import Request -from models_library.utils.json_serialization import json_dumps from servicelib.rest_constants import X_PRODUCT_NAME_HEADER from ..._constants import MSG_BACKEND_SERVICE_UNAVAILABLE diff --git a/services/api-server/src/simcore_service_api_server/models/_utils_pydantic.py b/services/api-server/src/simcore_service_api_server/models/_utils_pydantic.py index ad519ca7d24..150cac87b97 100644 --- a/services/api-server/src/simcore_service_api_server/models/_utils_pydantic.py +++ b/services/api-server/src/simcore_service_api_server/models/_utils_pydantic.py @@ -1,4 +1,4 @@ -from models_library.utils.json_serialization import json_dumps, json_loads +from common_library.json_serialization import json_dumps, json_loads class BaseConfig: diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py index dcbba97015d..66ff7972306 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py @@ -3,9 +3,9 @@ from typing import Final from aws_library.ec2 import AWS_TAG_VALUE_MAX_LENGTH, AWSTagKey, AWSTagValue, EC2Tags +from common_library.json_serialization import json_dumps from fastapi import FastAPI from models_library.docker import DockerGenericTag -from models_library.utils.json_serialization import json_dumps from pydantic import TypeAdapter from ..constants import ( diff --git a/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py b/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py index 8135ac9d2c4..c81369a89bb 100644 --- a/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py +++ b/services/autoscaling/tests/unit/test_modules_buffer_machine_core.py @@ -17,11 +17,11 @@ import pytest import tenacity from aws_library.ec2 import AWSTagKey, EC2InstanceBootSpecific +from common_library.json_serialization import json_dumps from faker import Faker from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from models_library.docker import DockerGenericTag -from models_library.utils.json_serialization import json_dumps from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.aws_ec2 import ( diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/services/director.py index 7c6925902f4..ee1327b475c 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/services/director.py @@ -7,10 +7,10 @@ from typing import Any import httpx +from common_library.json_serialization import json_dumps from fastapi import FastAPI, HTTPException from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion -from models_library.utils.json_serialization import json_dumps from servicelib.logging_utils import log_context from starlette import status from tenacity.asyncio import AsyncRetrying diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index 20cde00c047..55190cb46a1 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -15,6 +15,7 @@ EC2InstanceBootSpecific, EC2InstanceData, ) +from common_library.json_serialization import json_dumps from faker import Faker from models_library.api_schemas_clusters_keeper.clusters import ClusterState from models_library.clusters import ( @@ -22,7 +23,6 @@ NoAuthentication, TLSAuthentication, ) -from models_library.utils.json_serialization import json_dumps from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from simcore_service_clusters_keeper.core.settings import ApplicationSettings diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py index 750f634bb3b..743d7ae2a66 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py @@ -3,6 +3,7 @@ from typing import Annotated, Final import httpx +from common_library.json_serialization import json_dumps from fastapi import APIRouter, Depends, Header, HTTPException, Request from fastapi.responses import RedirectResponse from models_library.api_schemas_directorv2.dynamic_services import ( @@ -18,7 +19,6 @@ from models_library.service_settings_labels import SimcoreServiceLabels from models_library.services import ServiceKeyVersion from models_library.users import UserID -from models_library.utils.json_serialization import json_dumps from pydantic import NonNegativeFloat, NonNegativeInt from servicelib.fastapi.requests_decorators import cancel_on_disconnect from servicelib.logging_utils import log_decorator diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/events.py b/services/director-v2/src/simcore_service_director_v2/modules/db/events.py index 63a74aec2df..5570664bd22 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/events.py @@ -2,8 +2,8 @@ from aiopg.sa import Engine, create_engine from aiopg.sa.engine import get_dialect +from common_library.json_serialization import json_dumps from fastapi import FastAPI -from models_library.utils.json_serialization import json_dumps from servicelib.retry_policies import PostgresRetryPolicyUponInitialization from settings_library.postgres import PostgresSettings from simcore_postgres_database.utils_aiopg import ( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index b36f17af9e5..dd80c484279 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -5,6 +5,7 @@ import aiodocker from aiodocker.utils import clean_filters, clean_map +from common_library.json_serialization import json_dumps from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec from models_library.docker import to_simcore_runtime_docker_label_key @@ -12,7 +13,6 @@ from models_library.projects_networks import DockerNetworkName from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState -from models_library.utils.json_serialization import json_dumps from servicelib.utils import logged_gather from starlette import status from tenacity import TryAgain, retry diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py index c3e0a06fe64..b9d38ca6502 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py @@ -2,6 +2,7 @@ from copy import deepcopy from typing import Any, Final, TypeAlias, TypedDict +from common_library.json_serialization import json_dumps from fastapi.applications import FastAPI from models_library.docker import DockerGenericTag, StandardSimcoreDockerLabels from models_library.products import ProductName @@ -21,7 +22,6 @@ ) from models_library.users import UserID from models_library.utils.docker_compose import replace_env_vars_in_compose_spec -from models_library.utils.json_serialization import json_dumps from pydantic import ByteSize from servicelib.resources import CPU_RESOURCE_LIMIT_KEY, MEM_RESOURCE_LIMIT_KEY from settings_library.docker_registry import RegistrySettings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index b788e455cf3..001e549bf57 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -2,6 +2,7 @@ from copy import deepcopy from typing import Any, NamedTuple +from common_library.json_serialization import json_dumps from models_library.aiodocker_api import AioDockerServiceSpec from models_library.basic_types import BootModeEnum, PortInt from models_library.callbacks_mapping import CallbacksMapping @@ -14,7 +15,6 @@ ) from models_library.resource_tracker import HardwareInfo from models_library.service_settings_labels import SimcoreServiceSettingsLabel -from models_library.utils.json_serialization import json_dumps from pydantic import ByteSize, parse_obj_as from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.efs_guardian import efs_manager diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py index 0d7f2f8288f..6475e324950 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py @@ -3,6 +3,7 @@ import logging from typing import Any, Final +from common_library.json_serialization import json_dumps from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec @@ -17,7 +18,6 @@ ) from models_library.service_settings_labels import SimcoreServiceSettingsLabel from models_library.services import RunID -from models_library.utils.json_serialization import json_dumps from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient from simcore_postgres_database.models.comp_tasks import NodeClass diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index c6229bacdc8..452f1ba50a9 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -8,6 +8,7 @@ import dask_gateway # type: ignore[import-untyped] import distributed from aiopg.sa.engine import Engine +from common_library.json_serialization import json_dumps from dask_task_models_library.container_tasks.io import ( FileUrl, PortValue, @@ -29,7 +30,6 @@ from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID -from models_library.utils.json_serialization import json_dumps from pydantic import AnyUrl, ByteSize, ValidationError, parse_obj_as from servicelib.logging_utils import log_catch, log_context from simcore_sdk import node_ports_v2 diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py index 69c1b9bc11b..a38a5a06197 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py @@ -6,11 +6,11 @@ from unittest.mock import AsyncMock import pytest +from common_library.json_serialization import json_dumps from faker import Faker from fastapi import FastAPI, status from httpx import HTTPError, Response from models_library.sidecar_volumes import VolumeCategory, VolumeStatus -from models_library.utils.json_serialization import json_dumps from pydantic import AnyHttpUrl, parse_obj_as from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index a05e4cd84da..edca0dfe03c 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -10,6 +10,7 @@ import pytest import respx +from common_library.json_serialization import json_dumps from faker import Faker from fastapi import FastAPI from fastapi.encoders import jsonable_encoder @@ -25,7 +26,6 @@ SimcoreServiceSettingsLabel, ) from models_library.services import RunID, ServiceKeyVersion -from models_library.utils.json_serialization import json_dumps from models_library.wallets import WalletInfo from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py index e823216576b..9b7fd4be9d4 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py @@ -1,6 +1,7 @@ import datetime from typing import cast +from common_library.json_serialization import json_dumps from fastapi import FastAPI, status from httpx import Response, Timeout from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( @@ -8,7 +9,6 @@ ) from models_library.projects_nodes_io import NodeID from models_library.services_resources import ServiceResourcesDictHelpers -from models_library.utils.json_serialization import json_dumps from servicelib.common_headers import ( X_DYNAMIC_SIDECAR_REQUEST_DNS, X_DYNAMIC_SIDECAR_REQUEST_SCHEME, diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 0dc55477b47..04e76ecd7e2 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -14,13 +14,13 @@ import pytest import simcore_service_dynamic_sidecar +from common_library.json_serialization import json_dumps from faker import Faker from models_library.projects import ProjectID from models_library.projects_nodes import NodeID from models_library.services import RunID from models_library.services_creation import CreateServiceMetricsAdditionalParams from models_library.users import UserID -from models_library.utils.json_serialization import json_dumps from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import ( diff --git a/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py b/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py index 97a65d4553b..06270e171ca 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_system_monitor__disk_usage.py @@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, Mock import pytest +from common_library.json_serialization import json_dumps from faker import Faker from fastapi import FastAPI from models_library.api_schemas_dynamic_sidecar.telemetry import ( @@ -17,7 +18,6 @@ from models_library.projects_nodes_io import NodeID from models_library.services_types import RunID from models_library.users import UserID -from models_library.utils.json_serialization import json_dumps from psutil._common import sdiskusage from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture diff --git a/services/payments/src/simcore_service_payments/cli.py b/services/payments/src/simcore_service_payments/cli.py index db884e5f99f..d493ba884bd 100644 --- a/services/payments/src/simcore_service_payments/cli.py +++ b/services/payments/src/simcore_service_payments/cli.py @@ -3,7 +3,7 @@ import os import typer -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.utils_secrets import generate_password, generate_token_secret_key from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings diff --git a/services/storage/src/simcore_service_storage/handlers_datasets.py b/services/storage/src/simcore_service_storage/handlers_datasets.py index 69b345247e2..2cd510bcf79 100644 --- a/services/storage/src/simcore_service_storage/handlers_datasets.py +++ b/services/storage/src/simcore_service_storage/handlers_datasets.py @@ -2,9 +2,9 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.json_serialization import json_dumps from models_library.api_schemas_storage import FileMetaDataGet from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from servicelib.aiohttp.requests_validation import ( parse_request_path_parameters_as, parse_request_query_parameters_as, diff --git a/services/storage/src/simcore_service_storage/handlers_files.py b/services/storage/src/simcore_service_storage/handlers_files.py index b7ad6ebb007..0f78fdaeea5 100644 --- a/services/storage/src/simcore_service_storage/handlers_files.py +++ b/services/storage/src/simcore_service_storage/handlers_files.py @@ -5,6 +5,7 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.json_serialization import json_dumps from models_library.api_schemas_storage import ( FileMetaDataGet, FileUploadCompleteFutureResponse, @@ -17,7 +18,6 @@ SoftCopyBody, ) from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( diff --git a/services/storage/src/simcore_service_storage/handlers_health.py b/services/storage/src/simcore_service_storage/handlers_health.py index 5be7fda4e23..eb97bafe1ba 100644 --- a/services/storage/src/simcore_service_storage/handlers_health.py +++ b/services/storage/src/simcore_service_storage/handlers_health.py @@ -8,9 +8,9 @@ from aiohttp import web from aws_library.s3 import S3AccessError +from common_library.json_serialization import json_dumps from models_library.api_schemas_storage import HealthCheck, S3BucketName from models_library.app_diagnostics import AppStatusCheck -from models_library.utils.json_serialization import json_dumps from pydantic import TypeAdapter from servicelib.rest_constants import RESPONSE_MODEL_POLICY diff --git a/services/storage/src/simcore_service_storage/handlers_locations.py b/services/storage/src/simcore_service_storage/handlers_locations.py index 0144d1c0910..494c1821f15 100644 --- a/services/storage/src/simcore_service_storage/handlers_locations.py +++ b/services/storage/src/simcore_service_storage/handlers_locations.py @@ -4,9 +4,9 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.json_serialization import json_dumps from models_library.api_schemas_storage import FileLocation from models_library.projects_nodes_io import StorageFileID -from models_library.utils.json_serialization import json_dumps from servicelib.aiohttp.application_keys import ( APP_CONFIG_KEY, APP_FIRE_AND_FORGET_TASKS_KEY, diff --git a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py index 47dd35bc6bc..248c415b23f 100644 --- a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py @@ -3,10 +3,10 @@ from aiohttp import web from aiohttp.web import RouteTableDef +from common_library.json_serialization import json_dumps from models_library.api_schemas_storage import FileMetaDataGet, FoldersBody from models_library.projects import ProjectID from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from servicelib.aiohttp import status from servicelib.aiohttp.long_running_tasks.server import ( TaskProgress, diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py index 8814db3757a..a3fa2c3e10b 100644 --- a/services/storage/src/simcore_service_storage/s3.py +++ b/services/storage/src/simcore_service_storage/s3.py @@ -8,7 +8,7 @@ from aiohttp import web from aws_library.s3 import SimcoreS3API -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.logging_utils import log_context from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log diff --git a/services/web/server/src/simcore_service_webserver/db/_aiopg.py b/services/web/server/src/simcore_service_webserver/db/_aiopg.py index f6944e5ef67..4a45a0a00fb 100644 --- a/services/web/server/src/simcore_service_webserver/db/_aiopg.py +++ b/services/web/server/src/simcore_service_webserver/db/_aiopg.py @@ -10,7 +10,7 @@ from aiohttp import web from aiopg.sa import Engine, create_engine -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.aiohttp.aiopg_utils import is_pg_responsive from servicelib.aiohttp.application_keys import APP_AIOPG_ENGINE_KEY from servicelib.logging_utils import log_context diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py b/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py index f794fa6f148..daffd3be369 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py @@ -3,11 +3,11 @@ from typing import Any from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.api_schemas_webserver.computations import ComputationStart from models_library.clusters import ClusterID from models_library.projects import ProjectID from models_library.users import UserID -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Field, ValidationError, parse_obj_as from pydantic.types import NonNegativeInt from servicelib.aiohttp import status diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/template_json.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/template_json.py index a8ddfd32f2a..e543510155c 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/template_json.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/template_json.py @@ -1,7 +1,7 @@ from pathlib import Path import aiofiles -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from ...projects.models import ProjectDict diff --git a/services/web/server/src/simcore_service_webserver/groups/_handlers.py b/services/web/server/src/simcore_service_webserver/groups/_handlers.py index b6284bed8bf..ba366aaef46 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_handlers.py @@ -4,6 +4,7 @@ from typing import Literal from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.api_schemas_webserver.groups import ( AllUsersGroups, GroupUserGet, @@ -11,7 +12,6 @@ ) from models_library.emails import LowerCaseEmailStr from models_library.users import GroupID, UserID -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Extra, Field, parse_obj_as from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( diff --git a/services/web/server/src/simcore_service_webserver/login/_registration_api.py b/services/web/server/src/simcore_service_webserver/login/_registration_api.py index 1dfd1a5a500..b11dca73662 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration_api.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration_api.py @@ -5,9 +5,9 @@ from aiohttp import web from captcha.image import ImageCaptcha +from common_library.json_serialization import json_dumps from models_library.emails import LowerCaseEmailStr from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from PIL.Image import Image from pydantic import EmailStr, PositiveInt, ValidationError, parse_obj_as from servicelib.utils_secrets import generate_passcode diff --git a/services/web/server/src/simcore_service_webserver/login/utils.py b/services/web/server/src/simcore_service_webserver/login/utils.py index 0e5c4ef47f2..07cc9a4154c 100644 --- a/services/web/server/src/simcore_service_webserver/login/utils.py +++ b/services/web/server/src/simcore_service_webserver/login/utils.py @@ -2,9 +2,9 @@ from typing import Any from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.products import ProductName from models_library.users import UserID -from models_library.utils.json_serialization import json_dumps from pydantic import PositiveInt from servicelib.aiohttp import observer from servicelib.aiohttp.rest_models import LogMessageType diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py index c4b16f12caf..4cfd2bfca24 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py @@ -10,13 +10,13 @@ from typing import Any, Literal, Optional from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.basic_types import KeyIDStr, SHA1Str from models_library.function_services_catalog import is_iterator_service from models_library.projects import ProjectID from models_library.projects_nodes import Node, OutputID, OutputTypes from models_library.projects_nodes_io import NodeID from models_library.services import ServiceMetaDataPublished -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, ValidationError from pydantic.fields import Field from pydantic.types import PositiveInt diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 37416912e15..4886b3b6242 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -5,6 +5,7 @@ from typing import Any, TypeAlias from aiohttp import web +from common_library.json_serialization import json_dumps from jsonschema import ValidationError as JsonSchemaValidationError from models_library.api_schemas_long_running_tasks.base import ProgressPercent from models_library.api_schemas_webserver.projects import ProjectGet @@ -13,7 +14,6 @@ from models_library.projects_state import ProjectStatus from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from models_library.workspaces import UserWorkspaceAccessRightsDB from pydantic import parse_obj_as from servicelib.aiohttp.long_running_tasks.server import TaskProgress diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 11d1f701b32..8041e83a07d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -9,6 +9,7 @@ import logging from aiohttp import web +from common_library.json_serialization import json_dumps from jsonschema import ValidationError as JsonSchemaValidationError from models_library.api_schemas_webserver.projects import ( EmptyModel, @@ -24,7 +25,6 @@ from models_library.rest_pagination import Page from models_library.rest_pagination_utils import paginate_data from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from pydantic import parse_obj_as from servicelib.aiohttp import status from servicelib.aiohttp.long_running_tasks.server import start_long_running_task diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index fd7a21eaad6..daeb1e454e0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -7,6 +7,7 @@ import logging from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.api_schemas_catalog.service_access_rights import ( ServiceAccessRightsGet, ) @@ -31,7 +32,6 @@ from models_library.services_resources import ServiceResourcesDict from models_library.users import GroupID from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Field, parse_obj_as from servicelib.aiohttp import status from servicelib.aiohttp.long_running_tasks.server import ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index 0d2fb6f3eca..40e33893428 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -9,6 +9,7 @@ from typing import Any, Literal from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.api_schemas_webserver.projects_ports import ( ProjectInputGet, ProjectInputUpdate, @@ -20,7 +21,6 @@ from models_library.projects_nodes_io import NodeID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from models_library.utils.services_io import JsonSchemaDict from pydantic import BaseModel, Field, parse_obj_as from servicelib.aiohttp.requests_validation import ( diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 8a5e69c39ce..38cd5368044 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -22,6 +22,7 @@ from uuid import UUID, uuid4 from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.dynamic_services import ( GetProjectInactivityResponse, @@ -60,7 +61,6 @@ from models_library.socketio import SocketMessageDict from models_library.users import GroupID, UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.utils.json_serialization import json_dumps from models_library.wallets import ZERO_CREDITS, WalletID, WalletInfo from models_library.workspaces import UserWorkspaceAccessRightsDB from pydantic import ByteSize, parse_obj_as diff --git a/services/web/server/src/simcore_service_webserver/publications/_handlers.py b/services/web/server/src/simcore_service_webserver/publications/_handlers.py index 2d7feef016f..2653bba1390 100644 --- a/services/web/server/src/simcore_service_webserver/publications/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/publications/_handlers.py @@ -1,8 +1,8 @@ import logging from aiohttp import MultipartReader, hdrs, web +from common_library.json_serialization import json_dumps from json2html import json2html # type: ignore[import-untyped] -from models_library.utils.json_serialization import json_dumps from servicelib.aiohttp import status from servicelib.mimetype_constants import ( MIMETYPE_APPLICATION_JSON, diff --git a/services/web/server/src/simcore_service_webserver/socketio/server.py b/services/web/server/src/simcore_service_webserver/socketio/server.py index fed57e4d57e..617af26072f 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/server.py +++ b/services/web/server/src/simcore_service_webserver/socketio/server.py @@ -3,7 +3,7 @@ from collections.abc import AsyncIterator from aiohttp import web -from models_library.utils.json_serialization import JsonNamespace +from common_library.json_serialization import JsonNamespace from servicelib.socketio_utils import cleanup_socketio_async_pubsub_manager from socketio import AsyncAioPikaManager, AsyncServer # type: ignore[import-untyped] diff --git a/services/web/server/src/simcore_service_webserver/statics/_events.py b/services/web/server/src/simcore_service_webserver/statics/_events.py index 49cef6deb64..4348fa5ef3d 100644 --- a/services/web/server/src/simcore_service_webserver/statics/_events.py +++ b/services/web/server/src/simcore_service_webserver/statics/_events.py @@ -6,7 +6,7 @@ from aiohttp import web from aiohttp.client import ClientSession from aiohttp.client_exceptions import ClientConnectionError, ClientError -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from servicelib.aiohttp.client_session import get_client_session from tenacity.asyncio import AsyncRetrying from tenacity.before import before_log diff --git a/services/web/server/src/simcore_service_webserver/users/schemas.py b/services/web/server/src/simcore_service_webserver/users/schemas.py index 53c9ee9b756..cae3f5b7df5 100644 --- a/services/web/server/src/simcore_service_webserver/users/schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/schemas.py @@ -2,12 +2,12 @@ from typing import Any, ClassVar, Literal from uuid import UUID +from common_library.json_serialization import json_dumps from models_library.api_schemas_webserver._base import OutputSchema from models_library.api_schemas_webserver.groups import AllUsersGroups from models_library.api_schemas_webserver.users_preferences import AggregatedPreferences from models_library.emails import LowerCaseEmailStr from models_library.users import FirstNameStr, LastNameStr, UserID -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Field, root_validator, validator from simcore_postgres_database.models.users import UserRole diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index ae35a58ee6f..4adaa5ef468 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -6,8 +6,8 @@ from aiohttp import web from aiohttp.web_exceptions import HTTPError, HTTPException from aiohttp.web_routedef import RouteDef, RouteTableDef +from common_library.json_serialization import json_dumps from models_library.generics import Envelope -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Field from pydantic.generics import GenericModel from servicelib.common_headers import X_FORWARDED_PROTO diff --git a/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py b/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py index 3c16f2b3340..8c1117300ca 100644 --- a/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py +++ b/services/web/server/src/simcore_service_webserver/utils_rate_limiting.py @@ -5,7 +5,7 @@ from typing import Callable, NamedTuple from aiohttp.web_exceptions import HTTPTooManyRequests -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps class RateLimitSetup(NamedTuple): diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py index 171cf937704..c40b27946f2 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py +++ b/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py @@ -3,7 +3,7 @@ from typing import Any from aiohttp import web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from pydantic.error_wrappers import ValidationError from servicelib.aiohttp.typing_extension import Handler diff --git a/services/web/server/src/simcore_service_webserver/version_control/db.py b/services/web/server/src/simcore_service_webserver/version_control/db.py index 9ae6162c09e..ee884df6e9c 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/db.py +++ b/services/web/server/src/simcore_service_webserver/version_control/db.py @@ -7,9 +7,9 @@ import sqlalchemy as sa from aiopg.sa import SAConnection from aiopg.sa.result import RowProxy +from common_library.json_serialization import json_dumps from models_library.basic_types import SHA1Str from models_library.projects import ProjectIDStr -from models_library.utils.json_serialization import json_dumps from pydantic.types import NonNegativeInt, PositiveInt from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.projects_version_control import ( diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index a57dfd3c852..76fc55eba28 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -16,11 +16,11 @@ import pytest import simcore_service_webserver from aiohttp.test_utils import TestClient +from common_library.json_serialization import json_dumps from faker import Faker from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.projects_state import ProjectState -from models_library.utils.json_serialization import json_dumps from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.dict_tools import ConfigDict from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict diff --git a/services/web/server/tests/integration/02/test_computation.py b/services/web/server/tests/integration/02/test_computation.py index 0a73402e68d..4dcb55e19d9 100644 --- a/services/web/server/tests/integration/02/test_computation.py +++ b/services/web/server/tests/integration/02/test_computation.py @@ -15,8 +15,8 @@ import pytest import sqlalchemy as sa from aiohttp.test_utils import TestClient +from common_library.json_serialization import json_dumps from models_library.projects_state import RunningState -from models_library.utils.json_serialization import json_dumps from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import status from servicelib.aiohttp.application import create_safe_application diff --git a/services/web/server/tests/unit/isolated/test_application_settings.py b/services/web/server/tests/unit/isolated/test_application_settings.py index 65fe54ff483..17e3502bfa5 100644 --- a/services/web/server/tests/unit/isolated/test_application_settings.py +++ b/services/web/server/tests/unit/isolated/test_application_settings.py @@ -6,7 +6,7 @@ import pytest from aiohttp import web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from pydantic import HttpUrl, parse_obj_as from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_webserver.application_settings import ( diff --git a/services/web/server/tests/unit/isolated/test_products_model.py b/services/web/server/tests/unit/isolated/test_products_model.py index 45ecbd0f4c3..31c4ac60699 100644 --- a/services/web/server/tests/unit/isolated/test_products_model.py +++ b/services/web/server/tests/unit/isolated/test_products_model.py @@ -6,7 +6,7 @@ from typing import Any import pytest -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from pydantic import BaseModel from simcore_service_webserver.products._db import Product diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py index 7933270984a..420adf921ac 100644 --- a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -16,8 +16,8 @@ from aiohttp import ClientResponse, ClientSession from aiohttp.test_utils import TestClient, TestServer from aioresponses import aioresponses +from common_library.json_serialization import json_dumps from models_library.projects_state import ProjectLocked, ProjectStatus -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, ByteSize, parse_obj_as from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py index 20cb885bdfa..ec22f84f441 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py @@ -7,11 +7,11 @@ import pytest from aiohttp import ClientResponse from aiohttp.test_utils import TestClient +from common_library.json_serialization import json_dumps from faker import Faker from models_library.projects import Project from models_library.projects_nodes import Node from models_library.services_resources import ServiceResourcesDict -from models_library.utils.json_serialization import json_dumps from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict From afa0b3cc1abde648546de7f998d5fe4c79b35be8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:50:11 +0100 Subject: [PATCH 23/53] =?UTF-8?q?=20=E2=AC=86=EF=B8=8F=20upgrades=20`iter?= =?UTF-8?q?=5Fmodel=5Fexamples=5Fin=5Fmodule`=20to=20enable=20all=20exampl?= =?UTF-8?q?e=20tests=20(#6632)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pytest_simcore/pydantic_models.py | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py index 04d285a601e..05e85361eba 100644 --- a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py @@ -82,21 +82,20 @@ def _is_model_cls(obj) -> bool: for model_name, model_cls in inspect.getmembers(module, _is_model_cls): assert model_name # nosec if ( - (config_cls := model_cls.model_config) - and inspect.isclass(config_cls) - and is_strict_inner(model_cls, config_cls) - and (schema_extra := getattr(config_cls, "schema_extra", {})) - and isinstance(schema_extra, dict) + (model_config := model_cls.model_config) + and isinstance(model_config, dict) + and (json_schema_extra := model_config.get("json_schema_extra", {})) + and isinstance(json_schema_extra, dict) ): - if "example" in schema_extra: + if "example" in json_schema_extra: yield ModelExample( model_cls=model_cls, example_name="example", - example_data=schema_extra["example"], + example_data=json_schema_extra["example"], ) - elif "examples" in schema_extra: - for index, example in enumerate(schema_extra["examples"]): + elif "examples" in json_schema_extra: + for index, example in enumerate(json_schema_extra["examples"]): yield ModelExample( model_cls=model_cls, example_name=f"examples_{index}", @@ -120,10 +119,10 @@ def model_cls_examples(model_cls: type[BaseModel]) -> dict[str, dict[str, Any]]: "SEE https://pydantic-docs.helpmanual.io/usage/schema/#schema-customization" ) + json_schema_extra: dict = model_cls.model_config.get("json_schema_extra", {}) + # checks exampleS setup in schema_extra - examples_list = copy.deepcopy( - model_cls.model_config["json_schema_extra"].get("examples", []) - ) + examples_list = copy.deepcopy(json_schema_extra.get("examples", [])) assert isinstance(examples_list, list), ( "OpenAPI and json-schema differ regarding the format for exampleS." "The former is a dict and the latter an array. " @@ -132,15 +131,12 @@ def model_cls_examples(model_cls: type[BaseModel]) -> dict[str, dict[str, Any]]: "SEE https://swagger.io/docs/specification/adding-examples/" ) - # check example in schema_extra - example = copy.deepcopy(model_cls.model_config["json_schema_extra"].get("example")) - # collect all examples and creates fixture -> {example-name: example, ...} examples = { - f"{model_cls.__name__}.example[{index}]": example - for index, example in enumerate(examples_list) + f"{model_cls.__name__}.example[{index}]": example_ + for index, example_ in enumerate(examples_list) } - if example: + if example := copy.deepcopy(json_schema_extra.get("example")): examples[f"{model_cls.__name__}.example"] = example return examples From 9e76d4ff4888407becfa4a1bcd4c7bdf87e13890 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:54:35 +0100 Subject: [PATCH 24/53] =?UTF-8?q?=E2=AC=86=EF=B8=8FPydantic=20V2:=20Divers?= =?UTF-8?q?e=20fixes=20after=20merges=20from=20master=20(#6627)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env-devel | 4 +- .../api_schemas__common/errors.py | 3 +- .../api_schemas_webserver/projects.py | 43 +++++++++++-------- .../api_schemas_webserver/projects_nodes.py | 25 ++++++----- .../api_schemas_webserver/wallets.py | 13 +++--- .../src/models_library/folders.py | 5 ++- .../src/models_library/projects.py | 8 +--- .../src/models_library/rest_filters.py | 3 +- .../models_library/utils/common_validators.py | 2 +- .../utils/pydantic_tools_extension.py | 19 +------- .../test_utils_pydantic_tools_extension.py | 11 ++--- .../utils_projects.py | 8 ++-- .../models/pagination.py | 4 +- .../models/schemas/studies.py | 10 ++--- .../services/director_v2.py | 18 ++++---- services/api-server/tests/unit/conftest.py | 2 + .../core/settings.py | 3 -- services/dynamic-scheduler/tests/conftest.py | 6 +-- .../src/simcore_service_efs_guardian/_meta.py | 12 ++++-- .../core/settings.py | 11 ++++- .../services/efs_manager.py | 26 ++++++----- .../unit/test_efs_removal_policy_task.py | 2 +- .../src/simcore_service_payments/models/db.py | 15 ++++--- .../diagnostics/_handlers.py | 5 +-- .../rest/_handlers.py | 8 ++-- 25 files changed, 126 insertions(+), 140 deletions(-) diff --git a/.env-devel b/.env-devel index 7da5d779a2e..b857ad27651 100644 --- a/.env-devel +++ b/.env-devel @@ -104,11 +104,11 @@ DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKE DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[] DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={} -DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600 +DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00 # DIRECTOR_V2 ---- DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG DYNAMIC_SCHEDULER_PROFILING=1 -DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600 +DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' diff --git a/packages/models-library/src/models_library/api_schemas__common/errors.py b/packages/models-library/src/models_library/api_schemas__common/errors.py index 92ed48088d0..d1f7d6aa34d 100644 --- a/packages/models-library/src/models_library/api_schemas__common/errors.py +++ b/packages/models-library/src/models_library/api_schemas__common/errors.py @@ -4,7 +4,6 @@ from pydantic import BaseModel, Field from ..basic_types import IDStr -from ..utils.pydantic_tools_extension import NOT_REQUIRED class DefaultApiError(BaseModel): @@ -13,7 +12,7 @@ class DefaultApiError(BaseModel): description="Error identifier as a code or a name. " "Mainly for machine-machine communication purposes.", ) - detail: Any | None = Field(NOT_REQUIRED, description="Human readable error message") + detail: Any | None = Field(default=None, description="Human readable error message") @classmethod def from_status_code( diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index a892df7fe92..976ee74a712 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -8,13 +8,12 @@ from datetime import datetime from typing import Any, Literal, TypeAlias -from models_library.folders import FolderID -from models_library.workspaces import WorkspaceID from pydantic import ConfigDict, Field, HttpUrl, field_validator from ..api_schemas_long_running_tasks.tasks import TaskGet from ..basic_types import LongTruncatedStr, ShortTruncatedStr from ..emails import LowerCaseEmailStr +from ..folders import FolderID from ..projects import ClassifierID, DateTimeStr, NodesDict, ProjectID from ..projects_access import AccessRights, GroupIDStr from ..projects_state import ProjectState @@ -24,7 +23,7 @@ none_to_empty_str_pre_validator, null_or_none_str_to_none_validator, ) -from ..utils.pydantic_tools_extension import FieldNotRequired +from ..workspaces import WorkspaceID from ._base import EmptyModel, InputSchema, OutputSchema from .permalinks import ProjectPermalink @@ -74,12 +73,16 @@ class ProjectGet(OutputSchema): prj_owner: LowerCaseEmailStr access_rights: dict[GroupIDStr, AccessRights] tags: list[int] - classifiers: list[ClassifierID] = [] + classifiers: list[ClassifierID] = Field( + default_factory=list, json_schema_extra={"default": []} + ) state: ProjectState | None = None ui: EmptyModel | StudyUI | None = None - quality: dict[str, Any] = {} + quality: dict[str, Any] = Field( + default_factory=dict, json_schema_extra={"default": {}} + ) dev: dict | None - permalink: ProjectPermalink = FieldNotRequired() + permalink: ProjectPermalink | None = None workspace_id: WorkspaceID | None folder_id: FolderID | None trashed_at: datetime | None @@ -107,13 +110,15 @@ class ProjectReplace(InputSchema): last_change_date: DateTimeStr workbench: NodesDict access_rights: dict[GroupIDStr, AccessRights] - tags: list[int] | None = [] + tags: list[int] | None = Field( + default_factory=list, json_schema_extra={"default": []} + ) classifiers: list[ClassifierID] | None = Field( - default_factory=list, + default_factory=list, json_schema_extra={"default": []} ) ui: StudyUI | None = None quality: dict[str, Any] = Field( - default_factory=dict, + default_factory=dict, json_schema_extra={"default": {}} ) _empty_is_none = field_validator("thumbnail", mode="before")( @@ -122,16 +127,16 @@ class ProjectReplace(InputSchema): class ProjectPatch(InputSchema): - name: ShortTruncatedStr = FieldNotRequired() - description: LongTruncatedStr = FieldNotRequired() - thumbnail: HttpUrl = FieldNotRequired() - access_rights: dict[GroupIDStr, AccessRights] = FieldNotRequired() - classifiers: list[ClassifierID] = FieldNotRequired() - dev: dict | None = FieldNotRequired() - ui: StudyUI | None = FieldNotRequired() - quality: dict[str, Any] = FieldNotRequired() - - _empty_is_none = field_validator("thumbnail", allow_reuse=True, pre=True)( + name: ShortTruncatedStr | None = Field(default=None) + description: LongTruncatedStr | None = Field(default=None) + thumbnail: HttpUrl | None = Field(default=None) + access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None) + classifiers: list[ClassifierID] | None = Field(default=None) + dev: dict | None = Field(default=None) + ui: StudyUI | None = Field(default=None) + quality: dict[str, Any] | None = Field(default=None) + + _empty_is_none = field_validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 0fddd5267b3..7b65b84cb5f 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -10,7 +10,6 @@ from ..services import ServiceKey, ServicePortKey, ServiceVersion from ..services_enums import ServiceState from ..services_resources import ServiceResourcesDict -from ..utils.pydantic_tools_extension import FieldNotRequired from ._base import InputSchemaWithoutCamelCase, OutputSchema assert ServiceResourcesDict # nosec @@ -27,19 +26,19 @@ class NodeCreate(InputSchemaWithoutCamelCase): class NodePatch(InputSchemaWithoutCamelCase): - service_key: ServiceKey = FieldNotRequired(alias="key") - service_version: ServiceVersion = FieldNotRequired(alias="version") - label: str = FieldNotRequired() - inputs: InputsDict = FieldNotRequired() - inputs_required: list[InputID] = FieldNotRequired(alias="inputsRequired") - input_nodes: list[NodeID] = FieldNotRequired(alias="inputNodes") - progress: float | None = FieldNotRequired( - ge=0, le=100 + service_key: ServiceKey | None = Field(default=None, alias="key") + service_version: ServiceVersion | None = Field(default=None, alias="version") + label: str | None = Field(default=None) + inputs: InputsDict = Field(default=None) + inputs_required: list[InputID] | None = Field(default=None, alias="inputsRequired") + input_nodes: list[NodeID] | None = Field(default=None, alias="inputNodes") + progress: float | None = Field( + default=None, ge=0, le=100 ) # NOTE: it is used by frontend for File Picker progress - boot_options: BootOptions = FieldNotRequired(alias="bootOptions") - outputs: dict[ - str, Any - ] = FieldNotRequired() # NOTE: it is used by frontend for File Picker + boot_options: BootOptions | None = Field(default=None, alias="bootOptions") + outputs: dict[str, Any] | None = Field( + default=None + ) # NOTE: it is used by frontend for File Picker class NodeCreated(OutputSchema): diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index 1a417e9671d..3ba88301f88 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -6,7 +6,6 @@ from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID -from ..utils.pydantic_tools_extension import FieldNotRequired from ..wallets import WalletID, WalletStatus from ._base import InputSchema, OutputSchema @@ -21,9 +20,7 @@ class WalletGet(OutputSchema): created: datetime modified: datetime - model_config = ConfigDict( - frozen=False - ) + model_config = ConfigDict(frozen=False) class WalletGetWithAvailableCredits(WalletGet): @@ -60,7 +57,7 @@ class PutWalletBodyParams(OutputSchema): class CreateWalletPayment(InputSchema): price_dollars: AmountDecimal - comment: str = FieldNotRequired(max_length=100) + comment: str | None = Field(default=None, max_length=100) class WalletPaymentInitiated(OutputSchema): @@ -77,15 +74,15 @@ class PaymentTransaction(OutputSchema): price_dollars: Decimal wallet_id: WalletID osparc_credits: Decimal - comment: str = FieldNotRequired() + comment: str | None = Field(default=None) created_at: datetime completed_at: datetime | None # SEE PaymentTransactionState enum state: Literal["PENDING", "SUCCESS", "FAILED", "CANCELED"] = Field( ..., alias="completedStatus" ) - state_message: str = FieldNotRequired() - invoice_url: HttpUrl = FieldNotRequired() + state_message: str | None = Field(default=None) + invoice_url: HttpUrl | None = Field(default=None) class PaymentMethodInitiated(OutputSchema): diff --git a/packages/models-library/src/models_library/folders.py b/packages/models-library/src/models_library/folders.py index 829e3865c51..f514439d732 100644 --- a/packages/models-library/src/models_library/folders.py +++ b/packages/models-library/src/models_library/folders.py @@ -1,10 +1,11 @@ from datetime import datetime from typing import TypeAlias -from models_library.users import GroupID, UserID -from models_library.workspaces import WorkspaceID from pydantic import BaseModel, ConfigDict, Field, PositiveInt +from .users import GroupID, UserID +from .workspaces import WorkspaceID + FolderID: TypeAlias = PositiveInt diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 2873833cbea..634b2004998 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -99,7 +99,7 @@ class ProjectAtDB(BaseProjectModel): prj_owner: int | None = Field(..., description="The project owner id") published: bool | None = Field( - False, description="Defines if a study is available publicly" + default=False, description="Defines if a study is available publicly" ) @field_validator("project_type", mode="before") @@ -183,8 +183,4 @@ class Project(BaseProjectModel): alias="trashedAt", ) - model_config = ConfigDict( - description="Document that stores metadata, pipeline and UI setup of a study", - title="osparc-simcore project", - extra="forbid", - ) + model_config = ConfigDict(title="osparc-simcore project", extra="forbid") diff --git a/packages/models-library/src/models_library/rest_filters.py b/packages/models-library/src/models_library/rest_filters.py index 70a1aeb777d..1527a2e6170 100644 --- a/packages/models-library/src/models_library/rest_filters.py +++ b/packages/models-library/src/models_library/rest_filters.py @@ -1,7 +1,6 @@ from typing import Generic, TypeVar from pydantic import BaseModel, Field, Json -from pydantic.generics import GenericModel class Filters(BaseModel): @@ -15,7 +14,7 @@ class Filters(BaseModel): FilterT = TypeVar("FilterT", bound=Filters) -class FiltersQueryParameters(GenericModel, Generic[FilterT]): +class FiltersQueryParameters(BaseModel, Generic[FilterT]): filters: Json[FilterT] | None = Field( # pylint: disable=unsubscriptable-object default=None, description="Custom filter query parameter encoded as JSON", diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index c60586dfa92..4e6ab8a4d69 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -8,7 +8,7 @@ class MyModel(BaseModel): thumbnail: str | None - _empty_is_none = validator("thumbnail", allow_reuse=True, pre=True)( + _empty_is_none = validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) diff --git a/packages/models-library/src/models_library/utils/pydantic_tools_extension.py b/packages/models-library/src/models_library/utils/pydantic_tools_extension.py index 1078120a0a6..81112418c56 100644 --- a/packages/models-library/src/models_library/utils/pydantic_tools_extension.py +++ b/packages/models-library/src/models_library/utils/pydantic_tools_extension.py @@ -1,7 +1,6 @@ -import functools -from typing import Final, TypeVar +from typing import TypeVar -from pydantic import Field, TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError T = TypeVar("T") @@ -11,17 +10,3 @@ def parse_obj_or_none(type_: type[T], obj) -> T | None: return TypeAdapter(type_).validate_python(obj) except ValidationError: return None - - -# -# NOTE: Helper to define non-nullable optional fields -# SEE details in test/test_utils_pydantic_tools_extension.py -# -# Two usage styles: -# -# class Model(BaseModel): -# value: FieldNotRequired(description="some optional field") -# other: Field(NOT_REQUIRED, description="alternative") -# -NOT_REQUIRED: Final = None -FieldNotRequired = functools.partial(Field, default=NOT_REQUIRED) diff --git a/packages/models-library/tests/test_utils_pydantic_tools_extension.py b/packages/models-library/tests/test_utils_pydantic_tools_extension.py index 0bf8abdbb41..174233c7adb 100644 --- a/packages/models-library/tests/test_utils_pydantic_tools_extension.py +++ b/packages/models-library/tests/test_utils_pydantic_tools_extension.py @@ -1,7 +1,4 @@ -from models_library.utils.pydantic_tools_extension import ( - FieldNotRequired, - parse_obj_or_none, -) +from models_library.utils.pydantic_tools_extension import parse_obj_or_none from pydantic import BaseModel, Field, StrictInt @@ -10,7 +7,7 @@ class MyModel(BaseModel): b: int | None = Field(...) c: int = 42 d: int | None = None - e: int = FieldNotRequired(description="optional non-nullable") + e: int = Field(default=324, description="optional non-nullable") def test_schema(): @@ -27,7 +24,7 @@ def test_schema(): "title": "D", }, "e": { - "default": None, + "default": 324, "title": "E", "type": "integer", "description": "optional non-nullable", @@ -39,7 +36,7 @@ def test_schema(): def test_only_required(): model = MyModel(a=1, b=2) - assert model.model_dump() == {"a": 1, "b": 2, "c": 42, "d": None, "e": None} + assert model.model_dump() == {"a": 1, "b": 2, "c": 42, "d": None, "e": 324} assert model.model_dump(exclude_unset=True) == {"a": 1, "b": 2} diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py index 1e44e400b5d..8dfd97b759b 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py @@ -1,8 +1,8 @@ import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime import sqlalchemy as sa -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.errors import PydanticErrorMixin from sqlalchemy.ext.asyncio import AsyncConnection @@ -37,5 +37,5 @@ async def get_project_last_change_date( row = result.first() if row is None: raise DBProjectNotFoundError(project_uuid=project_uuid) - date = parse_obj_as(datetime, row[0]) - return date.replace(tzinfo=timezone.utc) + date = TypeAdapter(datetime).validate_python(row[0]) + return date.replace(tzinfo=UTC) diff --git a/services/api-server/src/simcore_service_api_server/models/pagination.py b/services/api-server/src/simcore_service_api_server/models/pagination.py index 1cabda966b5..e2578437653 100644 --- a/services/api-server/src/simcore_service_api_server/models/pagination.py +++ b/services/api-server/src/simcore_service_api_server/models/pagination.py @@ -17,10 +17,10 @@ DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, ) -from models_library.utils.pydantic_tools_extension import FieldNotRequired from pydantic import ( BaseModel, ConfigDict, + Field, NonNegativeInt, ValidationInfo, field_validator, @@ -56,7 +56,7 @@ class OnePage(BaseModel, Generic[T]): """ items: Sequence[T] - total: NonNegativeInt = FieldNotRequired(validate_default=True) + total: NonNegativeInt | None = Field(default=None, validate_default=True) @field_validator("total", mode="before") @classmethod diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/studies.py b/services/api-server/src/simcore_service_api_server/models/schemas/studies.py index a5f69a7ad08..eb93027ebe6 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/studies.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/studies.py @@ -1,22 +1,20 @@ from typing import TypeAlias -from common_library.pydantic_networks_extension import AnyUrlLegacy from models_library import projects, projects_nodes_io -from models_library.utils import pydantic_tools_extension -from pydantic import BaseModel, Field +from pydantic import AnyUrl, BaseModel, Field from .. import api_resources from . import solvers StudyID: TypeAlias = projects.ProjectID NodeName: TypeAlias = str -DownloadLink: TypeAlias = AnyUrlLegacy +DownloadLink: TypeAlias = AnyUrl class Study(BaseModel): uid: StudyID - title: str = pydantic_tools_extension.FieldNotRequired() - description: str = pydantic_tools_extension.FieldNotRequired() + title: str | None = None + description: str | None = None @classmethod def compose_resource_name(cls, study_key) -> api_resources.RelativeResourceName: diff --git a/services/api-server/src/simcore_service_api_server/services/director_v2.py b/services/api-server/src/simcore_service_api_server/services/director_v2.py index 0134bcaf444..5ddbc4d2bca 100644 --- a/services/api-server/src/simcore_service_api_server/services/director_v2.py +++ b/services/api-server/src/simcore_service_api_server/services/director_v2.py @@ -16,14 +16,11 @@ PositiveInt, TypeAdapter, ) -from simcore_service_api_server.exceptions.backend_errors import ( - JobNotFoundError, - LogFileNotFoundError, -) from starlette import status from ..core.settings import DirectorV2Settings from ..db.repositories.groups_extra_properties import GroupsExtraPropertiesRepository +from ..exceptions.backend_errors import JobNotFoundError, LogFileNotFoundError from ..exceptions.service_errors_utils import service_exception_mapper from ..models.schemas.jobs import PercentageInt from ..models.schemas.studies import JobLogsMap, LogLink @@ -186,12 +183,13 @@ async def get_computation_logs( # probably not found response.raise_for_status() - log_links: list[LogLink] = [] - for r in TypeAdapter(list[TaskLogFileGet]).validate_json(response.text or "[]"): - if r.download_link: - log_links.append( - LogLink(node_name=f"{r.task_id}", download_link=r.download_link) - ) + log_links: list[LogLink] = [ + LogLink(node_name=f"{r.task_id}", download_link=r.download_link) + for r in TypeAdapter(list[TaskLogFileGet]).validate_json( + response.text or "[]" + ) + if r.download_link + ] return JobLogsMap(log_links=log_links) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 939f7bf9170..f6b932f9281 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -518,6 +518,8 @@ def create_project_task(self, request: httpx.Request): "creationDate": "2018-07-01T11:13:43Z", "lastChangeDate": "2018-07-01T11:13:43Z", "prjOwner": "owner@email.com", + "dev": None, + "trashed_at": None, **project_create, } ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 0c85785a596..f0a82bebcf0 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,7 +1,6 @@ import datetime from pydantic import AliasChoices, Field, TypeAdapter, field_validator -from pydantic_settings import SettingsConfigDict from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.director_v2 import DirectorV2Settings @@ -51,8 +50,6 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def _validate_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - model_config = SettingsConfigDict(extra="allow") - class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables diff --git a/services/dynamic-scheduler/tests/conftest.py b/services/dynamic-scheduler/tests/conftest.py index 2cb14086b2a..8d5b52096c1 100644 --- a/services/dynamic-scheduler/tests/conftest.py +++ b/services/dynamic-scheduler/tests/conftest.py @@ -60,7 +60,7 @@ def docker_compose_service_dynamic_scheduler_env_vars( for name, value in environment.items(): try: envs[name] = string.Template(value).substitute(env_devel_dict) - except (KeyError, ValueError) as err: # noqa: PERF203 + except (KeyError, ValueError) as err: pytest.fail( f"{err}: {value} is not defined in .env-devel but used as RHS in docker-compose services['dynamic-schdlr'].environment[{name}]" ) @@ -112,8 +112,8 @@ def disable_status_monitor_setup(mocker: MockerFixture) -> None: mocker.patch(f"{_PATH_APPLICATION}.setup_status_monitor") -MAX_TIME_FOR_APP_TO_STARTUP = 10 -MAX_TIME_FOR_APP_TO_SHUTDOWN = 10 +MAX_TIME_FOR_APP_TO_STARTUP: Final[float] = 10 +MAX_TIME_FOR_APP_TO_SHUTDOWN: Final[float] = 10 @pytest.fixture diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py b/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py index 9d489cc179e..1d0965003fb 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/_meta.py @@ -9,17 +9,21 @@ from models_library.basic_types import VersionStr, VersionTag from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter _current_distribution = distribution("simcore-service-efs-guardian") __version__: str = version("simcore-service-efs-guardian") APP_NAME: Final[str] = _current_distribution.metadata["Name"] -API_VERSION: Final[VersionStr] = parse_obj_as(VersionStr, __version__) +API_VERSION: Final[VersionStr] = TypeAdapter(VersionStr).validate_python(__version__) VERSION: Final[Version] = Version(__version__) -API_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") -RPC_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") +API_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + f"v{VERSION.major}" +) +RPC_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + f"v{VERSION.major}" +) def get_summary() -> str: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index c383a3f8d14..135678db336 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -9,7 +9,14 @@ LogLevel, VersionTag, ) -from pydantic import AliasChoices, ByteSize, Field, PositiveInt, field_validator +from pydantic import ( + AliasChoices, + ByteSize, + Field, + PositiveInt, + TypeAdapter, + field_validator, +) from settings_library.base import BaseCustomSettings from settings_library.efs import AwsEfsSettings from settings_library.postgres import PostgresSettings @@ -61,7 +68,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Linux group name that the EFS and Simcore linux users are part of" ) EFS_DEFAULT_USER_SERVICE_SIZE_BYTES: ByteSize = Field( - default=parse_obj_as(ByteSize, "500GiB") + default=TypeAdapter(ByteSize).validate_python("500GiB") ) EFS_REMOVAL_POLICY_TASK_AGE_LIMIT_TIMEDELTA: datetime.timedelta = Field( default=datetime.timedelta(days=10), diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py index 0c7e7267954..a09e4ce16cd 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py @@ -7,7 +7,7 @@ from fastapi import FastAPI from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter, ValidationError from ..core.settings import ApplicationSettings, get_application_settings from . import efs_manager_utils @@ -103,9 +103,9 @@ async def list_projects_across_whole_efs(self) -> list[ProjectID]: for child in _dir_path.iterdir(): if child.is_dir(): try: - _project_id = parse_obj_as(ProjectID, child.name) + _project_id = TypeAdapter(ProjectID).validate_python(child.name) project_uuids.append(_project_id) - except ValueError: + except ValidationError: _logger.error( "This is not a project ID. This should not happen! %s", _dir_path / child.name, @@ -130,17 +130,15 @@ async def remove_project_efs_data(self, project_id: ProjectID) -> None: try: shutil.rmtree(_dir_path) _logger.info("%s has been deleted.", _dir_path) - except FileNotFoundError as e: - _logger.error("Directory %s does not exist. Error: %s", _dir_path, e) - except PermissionError as e: - _logger.error( - "Permission denied when trying to delete %s. Error: %s", - _dir_path, - e, + except FileNotFoundError: + _logger.exception("Directory %s does not exist.", _dir_path) + except PermissionError: + _logger.exception( + "Permission denied when trying to delete %s.", _dir_path ) - except NotADirectoryError as e: - _logger.error("%s is not a directory. Error: %s", _dir_path, e) - except OSError as e: - _logger.error("Issue with path: %s Error: %s", _dir_path, e) + except NotADirectoryError: + _logger.exception("%s is not a directory.", _dir_path) + except OSError: + _logger.exception("Issue with path: %s", _dir_path) else: _logger.error("%s does not exist.", _dir_path) diff --git a/services/efs-guardian/tests/unit/test_efs_removal_policy_task.py b/services/efs-guardian/tests/unit/test_efs_removal_policy_task.py index ddc58bb5b8f..cd57d865002 100644 --- a/services/efs-guardian/tests/unit/test_efs_removal_policy_task.py +++ b/services/efs-guardian/tests/unit/test_efs_removal_policy_task.py @@ -56,7 +56,7 @@ def app_environment( **app_environment, **postgres_env_vars_dict, "POSTGRES_CLIENT_NAME": "efs-guardian-service-pg-client", - "EFS_REMOVAL_POLICY_TASK_AGE_LIMIT_TIMEDELTA": "3600", + "EFS_REMOVAL_POLICY_TASK_AGE_LIMIT_TIMEDELTA": "01:00:00", }, ) diff --git a/services/payments/src/simcore_service_payments/models/db.py b/services/payments/src/simcore_service_payments/models/db.py index 8811dbcc139..c858bbc2435 100644 --- a/services/payments/src/simcore_service_payments/models/db.py +++ b/services/payments/src/simcore_service_payments/models/db.py @@ -1,5 +1,6 @@ import datetime from decimal import Decimal +from typing import Any from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID from models_library.emails import LowerCaseEmailStr @@ -13,7 +14,7 @@ PaymentTransactionState, ) -_EXAMPLE_AFTER_INIT = { +_EXAMPLE_AFTER_INIT: dict[str, Any] = { "payment_id": "12345", "price_dollars": 10.99, "osparc_credits": 5.0, @@ -27,6 +28,8 @@ "invoice_pdf_url": None, "initiated_at": "2023-09-27T10:00:00", "state": PaymentTransactionState.PENDING, + "completed_at": None, + "state_message": None, } @@ -50,10 +53,10 @@ class PaymentsTransactionsDB(BaseModel): from_attributes=True, json_schema_extra={ "examples": [ - _EXAMPLE_AFTER_INIT, # type:ignore[list-item] + _EXAMPLE_AFTER_INIT, # successful completion { - **_EXAMPLE_AFTER_INIT, # type:ignore[dict-item] + **_EXAMPLE_AFTER_INIT, "invoice_url": "https://my-fake-pdf-link.com", "stripe_invoice_id": "12345", "invoice_pdf_url": "https://my-fake-pdf-link.com", @@ -73,6 +76,8 @@ class PaymentsTransactionsDB(BaseModel): "wallet_id": _EXAMPLE_AFTER_INIT["wallet_id"], "initiated_at": _EXAMPLE_AFTER_INIT["initiated_at"], "state": InitPromptAckFlowState.PENDING, + "completed_at": None, + "state_message": None, } @@ -89,10 +94,10 @@ class PaymentsMethodsDB(BaseModel): from_attributes=True, json_schema_extra={ "examples": [ - _EXAMPLE_AFTER_INIT_PAYMENT_METHOD, # type:ignore[list-item] + _EXAMPLE_AFTER_INIT_PAYMENT_METHOD, # successful completion { - **_EXAMPLE_AFTER_INIT_PAYMENT_METHOD, # type:ignore[dict-item] + **_EXAMPLE_AFTER_INIT_PAYMENT_METHOD, "completed_at": "2023-09-27T10:00:15", "state": "SUCCESS", "state_message": "Payment method completed successfully", diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py index 13154bf5723..3171d0a80fb 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py @@ -9,8 +9,7 @@ from aiohttp import ClientError, ClientSession, web from models_library.app_diagnostics import AppStatusCheck -from models_library.utils.pydantic_tools_extension import FieldNotRequired -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, Field, parse_obj_as from servicelib.aiohttp.client_session import get_client_session from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.utils import logged_gather @@ -32,7 +31,7 @@ class StatusDiagnosticsQueryParam(BaseModel): - top_tracemalloc: int = FieldNotRequired() + top_tracemalloc: int | None = Field(default=None) class StatusDiagnosticsGet(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/rest/_handlers.py b/services/web/server/src/simcore_service_webserver/rest/_handlers.py index 0cf11ad4810..085b69cb0f6 100644 --- a/services/web/server/src/simcore_service_webserver/rest/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/rest/_handlers.py @@ -2,12 +2,12 @@ """ + import datetime import logging from typing import Any from aiohttp import web -from models_library.utils.pydantic_tools_extension import FieldNotRequired from pydantic import BaseModel, parse_obj_as from servicelib.aiohttp import status @@ -85,9 +85,9 @@ async def get_config(request: web.Request): class _ScheduledMaintenanceGet(BaseModel): - start: datetime.datetime = FieldNotRequired() - end: datetime.datetime = FieldNotRequired() - reason: str = FieldNotRequired() + start: datetime.datetime | None = None + end: datetime.datetime | None = None + reason: str | None = None @routes.get(f"/{API_VTAG}/scheduled_maintenance", name="get_scheduled_maintenance") From 34c9f3e0401457ddfa197e21be062608d0fa3675 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 31 Oct 2024 17:18:50 +0100 Subject: [PATCH 25/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20pydantic=20migration?= =?UTF-8?q?=20catalog=20(#6629)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_schemas_catalog/services.py | 3 +- .../src/models_library/app_diagnostics.py | 7 +- .../services_metadata_editable.py | 5 +- .../src/models_library/services_resources.py | 7 +- .../tests/test_services_resources.py | 21 + .../src/settings_library/application.py | 3 +- scripts/common-service.Makefile | 4 +- services/api-server/Makefile | 5 - .../exceptions/handlers/_http_exceptions.py | 2 +- services/catalog/VERSION | 2 +- services/catalog/openapi.json | 2457 ++++++++++++----- services/catalog/requirements/_base.txt | 137 +- services/catalog/requirements/prod.txt | 1 + services/catalog/setup.cfg | 6 +- .../src/simcore_service_catalog/_meta.py | 3 - .../api/rest/_health.py | 2 +- .../simcore_service_catalog/api/rest/_meta.py | 6 +- .../api/rest/_services.py | 14 +- .../api/rest/_services_resources.py | 26 +- .../api/rest/_services_specifications.py | 2 +- .../src/simcore_service_catalog/cli.py | 9 +- .../core/application.py | 2 +- .../core/background_tasks.py | 2 +- .../simcore_service_catalog/core/settings.py | 60 +- .../db/repositories/groups.py | 8 +- .../db/repositories/services.py | 20 +- .../exceptions/errors.py | 5 +- .../exceptions/handlers/_http_error.py | 13 +- .../exceptions/handlers/_validation_error.py | 8 +- .../src/simcore_service_catalog/main.py | 4 +- .../models/services_db.py | 36 +- .../models/services_specifications.py | 4 +- .../services/access_rights.py | 5 +- .../services/compatibility.py | 9 +- .../services/director.py | 2 +- .../services/function_services.py | 2 +- .../services/manifest.py | 2 +- .../services/services_api.py | 21 +- .../utils/service_resources.py | 39 +- services/catalog/tests/unit/conftest.py | 8 +- .../tests/unit/test__model_examples.py | 28 + services/catalog/tests/unit/test_cli.py | 34 + .../catalog/tests/unit/test_core_settings.py | 22 + .../tests/unit/test_services_compatibility.py | 7 +- .../unit/test_utils_service_resources.py | 4 +- .../catalog/tests/unit/with_dbs/conftest.py | 15 +- .../with_dbs/test_api_rest_services__get.py | 2 +- .../with_dbs/test_api_rest_services__list.py | 18 +- .../test_api_rest_services_access_rights.py | 6 +- .../test_api_rest_services_resources.py | 55 +- .../test_api_rest_services_specifications.py | 126 +- .../tests/unit/with_dbs/test_api_rpc.py | 4 +- .../unit/with_dbs/test_db_repositories.py | 22 +- .../with_dbs/test_services_access_rights.py | 38 +- 54 files changed, 2383 insertions(+), 970 deletions(-) create mode 100644 packages/models-library/tests/test_services_resources.py create mode 100644 services/catalog/tests/unit/test__model_examples.py create mode 100644 services/catalog/tests/unit/test_cli.py create mode 100644 services/catalog/tests/unit/test_core_settings.py diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index db386a8714f..bce573a2494 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -244,9 +244,10 @@ class ServiceGetV2(BaseModel): quality: dict[str, Any] = {} history: list[ServiceRelease] = Field( - default=[], + default_factory=list, description="history of releases for this service at this point in time, starting from the newest to the oldest." " It includes current release.", + json_schema_extra={"default": []}, ) model_config = ConfigDict( diff --git a/packages/models-library/src/models_library/app_diagnostics.py b/packages/models-library/src/models_library/app_diagnostics.py index ce8c9331eae..a8652e84db2 100644 --- a/packages/models-library/src/models_library/app_diagnostics.py +++ b/packages/models-library/src/models_library/app_diagnostics.py @@ -7,12 +7,15 @@ class AppStatusCheck(BaseModel): app_name: str = Field(..., description="Application name") version: str = Field(..., description="Application's version") services: dict[str, Any] = Field( - default={}, description="Other backend services connected from this service" + default_factory=dict, + description="Other backend services connected from this service", + json_schema_extra={"default": {}}, ) sessions: dict[str, Any] | None = Field( - default={}, + default_factory=dict, description="Client sessions info. If single session per app, then is denoted as main", + json_schema_extra={"default": {}}, ) url: AnyUrl | None = Field( diff --git a/packages/models-library/src/models_library/services_metadata_editable.py b/packages/models-library/src/models_library/services_metadata_editable.py index 4ad106225c0..c4436583503 100644 --- a/packages/models-library/src/models_library/services_metadata_editable.py +++ b/packages/models-library/src/models_library/services_metadata_editable.py @@ -33,7 +33,9 @@ class ServiceMetaDataEditable(ServiceBaseDisplay): "If now>=deprecated, the service is retired", ) classifiers: list[str] | None - quality: dict[str, Any] = {} + quality: dict[str, Any] = Field( + default_factory=dict, json_schema_extra={"default": {}} + ) model_config = ConfigDict( json_schema_extra={ @@ -60,6 +62,7 @@ class ServiceMetaDataEditable(ServiceBaseDisplay): for n in range(1, 11) }, }, + "classifiers": [], } } ) diff --git a/packages/models-library/src/models_library/services_resources.py b/packages/models-library/src/models_library/services_resources.py index 7a2b65456a4..175c56f968a 100644 --- a/packages/models-library/src/models_library/services_resources.py +++ b/packages/models-library/src/models_library/services_resources.py @@ -33,13 +33,14 @@ CPU_100_PERCENT: Final[int] = int(1 * GIGA) -class ResourceValue(BaseModel): +class ResourceValue(BaseModel, validate_assignment=True): limit: StrictInt | StrictFloat | str reservation: StrictInt | StrictFloat | str @model_validator(mode="before") @classmethod def _ensure_limits_are_equal_or_above_reservations(cls, values): + # WARNING: this does not validate ON-ASSIGNMENT! if isinstance(values["reservation"], str): # in case of string, the limit is the same as the reservation values["limit"] = values["reservation"] @@ -56,10 +57,8 @@ def set_reservation_same_as_limit(self) -> None: def set_value(self, value: StrictInt | StrictFloat | str) -> None: self.limit = self.reservation = value - model_config = ConfigDict(validate_assignment=True) - -ResourcesDict = dict[ResourceName, ResourceValue] +ResourcesDict: TypeAlias = dict[ResourceName, ResourceValue] class BootMode(StrAutoEnum): diff --git a/packages/models-library/tests/test_services_resources.py b/packages/models-library/tests/test_services_resources.py new file mode 100644 index 00000000000..3bc4c83c0ec --- /dev/null +++ b/packages/models-library/tests/test_services_resources.py @@ -0,0 +1,21 @@ +import pytest +from models_library.services_resources import ResourceValue + + +@pytest.mark.xfail() +def test_reservation_is_cap_by_limit_on_assigment_pydantic_2_bug(): + + res = ResourceValue(limit=10, reservation=30) + assert res.limit == 10 + assert res.reservation == 10 + + # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.validate_assignment + # before-validators DO NOT work on Assignment!!! + # SEE https://github.com/pydantic/pydantic/issues/7105 + res.reservation = 30 + assert res.reservation == 10 + + # update here is not validated neither + # + # res.model_copy(update={"reservation": 30}) + # diff --git a/packages/settings-library/src/settings_library/application.py b/packages/settings-library/src/settings_library/application.py index e7d588c7023..43740b69fd5 100644 --- a/packages/settings-library/src/settings_library/application.py +++ b/packages/settings-library/src/settings_library/application.py @@ -1,7 +1,7 @@ from pydantic import Field, PositiveInt from .base import BaseCustomSettings -from .basic_types import BuildTargetEnum +from .basic_types import BootMode, BuildTargetEnum class BaseApplicationSettings(BaseCustomSettings): @@ -16,6 +16,7 @@ class BaseApplicationSettings(BaseCustomSettings): SC_VCS_URL: str | None = None # @Dockerfile + SC_BOOT_MODE: BootMode | None = None SC_BOOT_TARGET: BuildTargetEnum | None = None SC_HEALTHCHECK_TIMEOUT: PositiveInt | None = Field( default=None, diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index e999b5b9e75..138b9ae27fc 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -192,8 +192,8 @@ _assert_target_defined: # specification of the used openapi-generator-cli (see also https://github.com/ITISFoundation/openapi-generator) -OPENAPI_GENERATOR_NAME := itisfoundation/openapi-generator-cli-openapi-generator-v4.2.3 -OPENAPI_GENERATOR_TAG := v0 +OPENAPI_GENERATOR_NAME := openapitools/openapi-generator-cli +OPENAPI_GENERATOR_TAG := latest OPENAPI_GENERATOR_IMAGE := $(OPENAPI_GENERATOR_NAME):$(OPENAPI_GENERATOR_TAG) define validate_openapi_specs diff --git a/services/api-server/Makefile b/services/api-server/Makefile index 900c0e8e0f1..5b675b22ce4 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -14,11 +14,6 @@ reqs: ## compiles pip requirements (.in -> .txt) cp .env-devel $@ -# specification of the used openapi-generator-cli (see also https://github.com/ITISFoundation/openapi-generator) -OPENAPI_GENERATOR_NAME := openapitools/openapi-generator-cli -OPENAPI_GENERATOR_TAG := latest -OPENAPI_GENERATOR_IMAGE := $(OPENAPI_GENERATOR_NAME):$(OPENAPI_GENERATOR_TAG) - define _create_and_validate_openapi # generating openapi specs file under $< (NOTE: Skips DEV FEATURES since this OAS is the 'offically released'!) @source .env; \ diff --git a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py index 24be07387d7..bdff166096b 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/handlers/_http_exceptions.py @@ -7,6 +7,6 @@ async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse: assert request # nosec - assert isinstance(exc, HTTPException) + assert isinstance(exc, HTTPException) # nosec return create_error_json_response(exc.detail, status_code=exc.status_code) diff --git a/services/catalog/VERSION b/services/catalog/VERSION index 79a2734bbf3..09a3acfa138 100644 --- a/services/catalog/VERSION +++ b/services/catalog/VERSION @@ -1 +1 @@ -0.5.0 \ No newline at end of file +0.6.0 \ No newline at end of file diff --git a/services/catalog/openapi.json b/services/catalog/openapi.json index ebacaf11616..c5663631059 100644 --- a/services/catalog/openapi.json +++ b/services/catalog/openapi.json @@ -71,37 +71,44 @@ "operationId": "get_service_resources_v0_services__service_key___service_version__resources_get", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { - "description": "if passed, and that user has custom resources, they will be merged with default resources and returned.", + "name": "user_id", + "in": "query", "required": false, "schema": { - "type": "integer", - "exclusiveMinimum": true, - "title": "User Id", + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": true, + "minimum": 0 + }, + { + "type": "null" + } + ], "description": "if passed, and that user has custom resources, they will be merged with default resources and returned.", - "minimum": 0 + "title": "User Id" }, - "name": "user_id", - "in": "query" + "description": "if passed, and that user has custom resources, they will be merged with default resources and returned." } ], "responses": { @@ -110,9 +117,6 @@ "content": { "application/json": { "schema": { - "additionalProperties": { - "$ref": "#/components/schemas/ImageResources" - }, "type": "object", "title": "Response Get Service Resources V0 Services Service Key Service Version Resources Get" } @@ -141,47 +145,47 @@ "operationId": "get_service_specifications_v0_services__service_key___service_version__specifications_get", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "user_id", + "in": "query", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "User Id", "minimum": 0 - }, - "name": "user_id", - "in": "query" + } }, { - "description": "if True only the version specs will be retrieved, if False the latest version will be used instead", + "name": "strict", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Strict", "description": "if True only the version specs will be retrieved, if False the latest version will be used instead", - "default": false + "default": false, + "title": "Strict" }, - "name": "strict", - "in": "query" + "description": "if True only the version specs will be retrieved, if False the latest version will be used instead" } ], "responses": { @@ -218,42 +222,42 @@ "operationId": "list_service_ports_v0_services__service_key___service_version__ports_get", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "user_id", + "in": "query", "required": true, "schema": { "type": "integer", "title": "User Id" - }, - "name": "user_id", - "in": "query" + } }, { + "name": "x-simcore-products-name", + "in": "header", "required": false, "schema": { "type": "string", "title": "X-Simcore-Products-Name" - }, - "name": "x-simcore-products-name", - "in": "header" + } } ], "responses": { @@ -262,10 +266,10 @@ "content": { "application/json": { "schema": { + "type": "array", "items": { "$ref": "#/components/schemas/ServicePortGet" }, - "type": "array", "title": "Response List Service Ports V0 Services Service Key Service Version Ports Get" } } @@ -294,42 +298,42 @@ "operationId": "get_service_access_rights_v0_services__service_key___service_version__accessRights_get", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "user_id", + "in": "query", "required": true, "schema": { "type": "integer", "title": "User Id" - }, - "name": "user_id", - "in": "query" + } }, { + "name": "x-simcore-products-name", + "in": "header", "required": true, "schema": { "type": "string", "title": "X-Simcore-Products-Name" - }, - "name": "x-simcore-products-name", - "in": "header" + } } ], "responses": { @@ -365,34 +369,34 @@ "operationId": "list_services_v0_services_get", "parameters": [ { + "name": "user_id", + "in": "query", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "User Id", "minimum": 0 - }, - "name": "user_id", - "in": "query" + } }, { + "name": "details", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Details", - "default": true - }, - "name": "details", - "in": "query" + "default": true, + "title": "Details" + } }, { + "name": "x-simcore-products-name", + "in": "header", "required": true, "schema": { "type": "string", "title": "X-Simcore-Products-Name" - }, - "name": "x-simcore-products-name", - "in": "header" + } } ], "responses": { @@ -401,10 +405,10 @@ "content": { "application/json": { "schema": { + "type": "array", "items": { "$ref": "#/components/schemas/ServiceGet" }, - "type": "array", "title": "Response List Services V0 Services Get" } } @@ -432,42 +436,42 @@ "operationId": "get_service_v0_services__service_key___service_version__get", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "user_id", + "in": "query", "required": true, "schema": { "type": "integer", "title": "User Id" - }, - "name": "user_id", - "in": "query" + } }, { + "name": "x-simcore-products-name", + "in": "header", "required": false, "schema": { "type": "string", "title": "X-Simcore-Products-Name" - }, - "name": "x-simcore-products-name", - "in": "header" + } } ], "responses": { @@ -501,53 +505,60 @@ "operationId": "update_service_v0_services__service_key___service_version__patch", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "user_id", + "in": "query", "required": true, "schema": { "type": "integer", "title": "User Id" - }, - "name": "user_id", - "in": "query" + } }, { + "name": "x-simcore-products-name", + "in": "header", "required": false, "schema": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "X-Simcore-Products-Name" - }, - "name": "x-simcore-products-name", - "in": "header" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ServiceUpdate" } } - }, - "required": true + } }, "responses": { "200": { @@ -590,7 +601,14 @@ "description": "Email address" }, "affiliation": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Affiliation" } }, @@ -633,8 +651,8 @@ ], "title": "Badge", "example": { - "name": "osparc.io", "image": "https://img.shields.io/website-up-down-green-red/https/itisfoundation.github.io.svg?label=documentation", + "name": "osparc.io", "url": "https://itisfoundation.github.io/" } }, @@ -650,11 +668,18 @@ "title": "Version" }, "released": { - "additionalProperties": { - "type": "string", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Released", "description": "Maps every route's path tag with a released version" } @@ -667,25 +692,35 @@ "title": "BaseMeta", "example": { "name": "simcore_service_foo", - "version": "2.4.45", "released": { "v1": "1.3.4", "v2": "2.4.45" - } + }, + "version": "2.4.45" } }, "BindOptions": { "properties": { "Propagation": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Propagation" + }, + { + "type": "null" } ], "description": "A propagation mode with the value `[r]private`, `[r]shared`, or `[r]slave`." }, "NonRecursive": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Nonrecursive", "description": "Disable recursive bind mount.", "default": false @@ -720,8 +755,7 @@ "GPU", "MPI" ], - "title": "BootMode", - "description": "An enumeration." + "title": "BootMode" }, "BootOption": { "properties": { @@ -767,26 +801,49 @@ "Config1": { "properties": { "File": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/File1" + }, + { + "type": "null" } ], - "title": "File", "description": "File represents a specific target that is backed by a file.\n\n


\n\n> **Note**: `Configs.File` and `Configs.Runtime` are mutually exclusive\n" }, "Runtime": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Runtime", "description": "Runtime represents a target that is not mounted into the\ncontainer but is used by the task\n\n


\n\n> **Note**: `Configs.File` and `Configs.Runtime` are mutually\n> exclusive\n" }, "ConfigID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Configid", "description": "ConfigID represents the ID of the specific config that we're\nreferencing.\n" }, "ConfigName": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Configname", "description": "ConfigName is the name of the config that this references,\nbut this is just provided for lookup/display purposes. The\nconfig in the reference will be identified by its ID.\n" } @@ -797,218 +854,403 @@ "ContainerSpec": { "properties": { "Image": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Image", "description": "The image name to use for the container" }, "Labels": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Labels", "description": "User-defined key/value data." }, "Command": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Command", "description": "The command to be run in the image." }, "Args": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Args", "description": "Arguments to the command." }, "Hostname": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Hostname", "description": "The hostname to use for the container, as a valid\n[RFC 1123](https://tools.ietf.org/html/rfc1123) hostname.\n" }, "Env": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Env", "description": "A list of environment variables in the form `VAR=value`.\n" }, "Dir": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Dir", "description": "The working directory for commands to run in." }, "User": { - "type": "string", - "title": "User", - "description": "The user inside the container." + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "User", + "description": "The user inside the container." }, "Groups": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Groups", "description": "A list of additional groups that the container process will run as.\n" }, "Privileges": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Privileges" + }, + { + "type": "null" } ], - "title": "Privileges", "description": "Security options for the container" }, "TTY": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Tty", "description": "Whether a pseudo-TTY should be allocated." }, "OpenStdin": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Openstdin", "description": "Open `stdin`" }, "ReadOnly": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Readonly", "description": "Mount the container's root filesystem as read only." }, "Mounts": { - "items": { - "$ref": "#/components/schemas/Mount" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Mount" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Mounts", "description": "Specification for mounts to be added to containers created as part\nof the service.\n" }, "StopSignal": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Stopsignal", "description": "Signal to stop the container." }, "StopGracePeriod": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Stopgraceperiod", "description": "Amount of time to wait for the container to terminate before\nforcefully killing it.\n" }, "HealthCheck": { - "$ref": "#/components/schemas/HealthConfig" + "anyOf": [ + { + "$ref": "#/components/schemas/HealthConfig" + }, + { + "type": "null" + } + ] }, "Hosts": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Hosts", "description": "A list of hostname/IP mappings to add to the container's `hosts`\nfile. The format of extra hosts is specified in the\n[hosts(5)](http://man7.org/linux/man-pages/man5/hosts.5.html)\nman page:\n\n IP_address canonical_hostname [aliases...]\n" }, "DNSConfig": { - "allOf": [ + "anyOf": [ { - "$ref": "#/components/schemas/DNSConfig" + "$ref": "#/components/schemas/DnsConfig" + }, + { + "type": "null" } ], - "title": "Dnsconfig", "description": "Specification for DNS related configurations in resolver configuration\nfile (`resolv.conf`).\n" }, "Secrets": { - "items": { - "$ref": "#/components/schemas/Secret" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Secret" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Secrets", "description": "Secrets contains references to zero or more secrets that will be\nexposed to the service.\n" }, "Configs": { - "items": { - "$ref": "#/components/schemas/Config1" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Config1" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Configs", "description": "Configs contains references to zero or more configs that will be\nexposed to the service.\n" }, "Isolation": { - "allOf": [ + "anyOf": [ + { + "$ref": "#/components/schemas/Isolation1" + }, { - "$ref": "#/components/schemas/Isolation" + "type": "null" } ], "description": "Isolation technology of the containers running the service.\n(Windows only)\n" }, "Init": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Init", "description": "Run an init inside the container that forwards signals and reaps\nprocesses. This field is omitted if empty, and the default (as\nconfigured on the daemon) is used.\n" }, "Sysctls": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Sysctls", "description": "Set kernel namedspaced parameters (sysctls) in the container.\nThe Sysctls option on services accepts the same sysctls as the\nare supported on containers. Note that while the same sysctls are\nsupported, no guarantees or checks are made about their\nsuitability for a clustered environment, and it's up to the user\nto determine whether a given sysctl will work properly in a\nService.\n" }, "CapabilityAdd": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Capabilityadd", - "description": "A list of kernel capabilities to add to the default set\nfor the container.\n", - "example": [ - "CAP_NET_RAW", - "CAP_SYS_ADMIN", - "CAP_SYS_CHROOT", - "CAP_SYSLOG" - ] + "description": "A list of kernel capabilities to add to the default set\nfor the container.\n" }, "CapabilityDrop": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Capabilitydrop", - "description": "A list of kernel capabilities to drop from the default set\nfor the container.\n", - "example": [ - "CAP_NET_RAW" - ] + "description": "A list of kernel capabilities to drop from the default set\nfor the container.\n" }, "Ulimits": { - "items": { - "$ref": "#/components/schemas/Ulimit1" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Ulimit" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Ulimits", "description": "A list of resource limits to set in the container. For example: `{\"Name\": \"nofile\", \"Soft\": 1024, \"Hard\": 2048}`\"\n" } }, "type": "object", "title": "ContainerSpec", - "description": " Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." + "description": "Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." }, "CredentialSpec": { "properties": { "Config": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Config", - "description": "Load credential spec from a Swarm Config with the given ID.\nThe specified config must also be present in the Configs\nfield with the Runtime property set.\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", - "example": "0bt9dmxjvjiqermk6xrop3ekq" + "description": "Load credential spec from a Swarm Config with the given ID.\nThe specified config must also be present in the Configs\nfield with the Runtime property set.\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n" }, "File": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "File", - "description": "Load credential spec from this file. The file is read by\nthe daemon, and must be present in the `CredentialSpecs`\nsubdirectory in the docker data directory, which defaults\nto `C:\\ProgramData\\Docker\\` on Windows.\n\nFor example, specifying `spec.json` loads\n`C:\\ProgramData\\Docker\\CredentialSpecs\\spec.json`.\n\n


\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", - "example": "spec.json" + "description": "Load credential spec from this file. The file is read by\nthe daemon, and must be present in the `CredentialSpecs`\nsubdirectory in the docker data directory, which defaults\nto `C:\\ProgramData\\Docker\\` on Windows.\n\nFor example, specifying `spec.json` loads\n`C:\\ProgramData\\Docker\\CredentialSpecs\\spec.json`.\n\n


\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n" }, "Registry": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Registry", "description": "Load credential spec from this value in the Windows\nregistry. The specified registry value must be located in:\n\n`HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Virtualization\\Containers\\CredentialSpecs`\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n" } @@ -1017,63 +1259,112 @@ "title": "CredentialSpec", "description": "CredentialSpec for managed service account (Windows only)" }, - "DNSConfig": { + "DiscreteResourceSpec": { + "properties": { + "Kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Kind" + }, + "Value": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Value" + } + }, + "type": "object", + "title": "DiscreteResourceSpec" + }, + "DnsConfig": { "properties": { "Nameservers": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Nameservers", "description": "The IP addresses of the name servers." }, "Search": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Search", "description": "A search list for host-name lookup." }, "Options": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Options", "description": "A list of internal resolver variables to be modified (e.g.,\n`debug`, `ndots:3`, etc.).\n" } }, "type": "object", - "title": "DNSConfig", - "description": " Specification for DNS related configurations in resolver configuration\nfile (`resolv.conf`)." - }, - "DiscreteResourceSpec": { - "properties": { - "Kind": { - "type": "string", - "title": "Kind" - }, - "Value": { - "type": "integer", - "title": "Value" - } - }, - "type": "object", - "title": "DiscreteResourceSpec" + "title": "DnsConfig", + "description": "Specification for DNS related configurations in resolver configuration\nfile (`resolv.conf`)." }, "DriverConfig": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name", "description": "Name of the driver to use to create the volume." }, "Options": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Options", "description": "key/value map of driver specific options." } @@ -1085,31 +1376,61 @@ "EndpointPortConfig": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name" }, "Protocol": { - "$ref": "#/components/schemas/Type" + "anyOf": [ + { + "$ref": "#/components/schemas/Type" + }, + { + "type": "null" + } + ] }, "TargetPort": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Targetport", "description": "The port inside the container." }, "PublishedPort": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Publishedport", "description": "The port on the swarm hosts." }, "PublishMode": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/PublishMode" + }, + { + "type": "null" } ], "description": "The mode in which port is published.\n\n


\n\n- \"ingress\" makes the target port accessible on every node,\n regardless of whether there is a task for the service running on\n that node or not.\n- \"host\" bypasses the routing mesh and publish the port directly on\n the swarm node where that service is running.\n", - "default": "ingress", - "example": "ingress" + "default": "ingress" } }, "type": "object", @@ -1118,19 +1439,29 @@ "EndpointSpec": { "properties": { "Mode": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Mode1" + }, + { + "type": "null" } ], "description": "The mode of resolution to use for internal load balancing between tasks.\n", "default": "vip" }, "Ports": { - "items": { - "$ref": "#/components/schemas/EndpointPortConfig" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/EndpointPortConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Ports", "description": "List of exposed ports that this service is accessible on from the\noutside. Ports can only be provided if `vip` resolution mode is used.\n" } @@ -1147,7 +1478,7 @@ "rollback" ], "title": "FailureAction", - "description": " Action to take if an updated task fails to run, or stops running\nduring the update." + "description": "Action to take if an updated task fails to run, or stops running\nduring the update." }, "FailureAction1": { "type": "string", @@ -1156,27 +1487,55 @@ "pause" ], "title": "FailureAction1", - "description": " Action to take if an rolled back task fails to run, or stops\nrunning during the rollback." + "description": "Action to take if an rolled back task fails to run, or stops\nrunning during the rollback." }, "File": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name", "description": "Name represents the final filename in the filesystem.\n" }, "UID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Uid", "description": "UID represents the file UID." }, "GID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Gid", "description": "GID represents the file GID." }, "Mode": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Mode", "description": "Mode represents the FileMode of the file." } @@ -1188,37 +1547,79 @@ "File1": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name", "description": "Name represents the final filename in the filesystem.\n" }, "UID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Uid", "description": "UID represents the file UID." }, "GID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Gid", "description": "GID represents the file GID." }, "Mode": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Mode", "description": "Mode represents the FileMode of the file." } }, "type": "object", "title": "File1", - "description": " File represents a specific target that is backed by a file.\n\n


\n\n> **Note**: `Configs.File` and `Configs.Runtime` are mutually exclusive" + "description": "File represents a specific target that is backed by a file.\n\n


\n\n> **Note**: `Configs.File` and `Configs.Runtime` are mutually exclusive" }, "GenericResource": { "properties": { "NamedResourceSpec": { - "$ref": "#/components/schemas/NamedResourceSpec" + "anyOf": [ + { + "$ref": "#/components/schemas/NamedResourceSpec" + }, + { + "type": "null" + } + ] }, "DiscreteResourceSpec": { - "$ref": "#/components/schemas/DiscreteResourceSpec" + "anyOf": [ + { + "$ref": "#/components/schemas/DiscreteResourceSpec" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -1230,27 +1631,7 @@ }, "type": "array", "title": "GenericResources", - "description": "User-defined resources can be either Integer resources (e.g, `SSD=3`) or\nString resources (e.g, `GPU=UUID1`).\n", - "example": [ - { - "DiscreteResourceSpec": { - "Kind": "SSD", - "Value": 3 - } - }, - { - "NamedResourceSpec": { - "Kind": "GPU", - "Value": "UUID1" - } - }, - { - "NamedResourceSpec": { - "Kind": "GPU", - "Value": "UUID2" - } - } - ] + "description": "User-defined resources can be either Integer resources (e.g, `SSD=3`) or\nString resources (e.g, `GPU=UUID1`)." }, "HTTPValidationError": { "properties": { @@ -1268,30 +1649,65 @@ "HealthConfig": { "properties": { "Test": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Test", "description": "The test to perform. Possible values are:\n\n- `[]` inherit healthcheck from image or parent image\n- `[\"NONE\"]` disable healthcheck\n- `[\"CMD\", args...]` exec arguments directly\n- `[\"CMD-SHELL\", command]` run command with system's default shell\n" }, "Interval": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Interval", "description": "The time to wait between checks in nanoseconds. It should be 0 or at\nleast 1000000 (1 ms). 0 means inherit.\n" }, "Timeout": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Timeout", "description": "The time to wait before considering the check to have hung. It should\nbe 0 or at least 1000000 (1 ms). 0 means inherit.\n" }, "Retries": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Retries", "description": "The number of consecutive failures needed to consider a container as\nunhealthy. 0 means inherit.\n" }, "StartPeriod": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Startperiod", "description": "Start period for the container to initialize before starting\nhealth-retries countdown in nanoseconds. It should be 0 or at least\n1000000 (1 ms). 0 means inherit.\n" } @@ -1320,6 +1736,7 @@ "$ref": "#/components/schemas/BootMode" }, "type": "array", + "title": "Boot Modes", "description": "describe how a service shall be booted, using CPU, MPI, openMP or GPU", "default": [ "CPU" @@ -1335,6 +1752,14 @@ "example": { "image": "simcore/service/dynamic/pretty-intense:1.0.0", "resources": { + "AIRAM": { + "limit": 1, + "reservation": 1 + }, + "ANY_resource": { + "limit": "some_value", + "reservation": "some_value" + }, "CPU": { "limit": 4, "reservation": 0.1 @@ -1346,46 +1771,56 @@ "VRAM": { "limit": 1, "reservation": 1 - }, - "AIRAM": { - "limit": 1, - "reservation": 1 - }, - "ANY_resource": { - "limit": "some_value", - "reservation": "some_value" } } } }, - "Isolation": { + "Isolation1": { "type": "string", "enum": [ "default", "process", "hyperv" ], - "title": "Isolation", - "description": "Isolation technology of the container. (Windows only)" + "title": "Isolation1", + "description": "Isolation technology of the containers running the service.\n(Windows only)" }, "Limit": { "properties": { "NanoCPUs": { - "type": "integer", - "title": "Nanocpus", - "example": 4000000000 + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Nanocpus" }, "MemoryBytes": { - "type": "integer", - "title": "Memorybytes", - "example": 8272408576 + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Memorybytes" }, "Pids": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Pids", "description": "Limits the maximum number of PIDs in the container. Set `0` for unlimited.\n", - "default": 0, - "example": 100 + "default": 0 } }, "type": "object", @@ -1395,41 +1830,78 @@ "LogDriver1": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name" }, "Options": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Options" } }, "type": "object", "title": "LogDriver1", - "description": " Specifies the log driver to use for tasks created from this spec. If\nnot present, the default one for the swarm will be used, finally\nfalling back to the engine default if not specified." + "description": "Specifies the log driver to use for tasks created from this spec. If\nnot present, the default one for the swarm will be used, finally\nfalling back to the engine default if not specified." }, "Mode": { "properties": { "Replicated": { - "$ref": "#/components/schemas/Replicated" + "anyOf": [ + { + "$ref": "#/components/schemas/Replicated" + }, + { + "type": "null" + } + ] }, "Global": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Global" }, "ReplicatedJob": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/ReplicatedJob" + }, + { + "type": "null" } ], - "title": "Replicatedjob", "description": "The mode used for services with a finite number of tasks that run\nto a completed state.\n" }, "GlobalJob": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Globaljob", "description": "The mode used for services which run a task to the completed state\non each valid node.\n" } @@ -1450,58 +1922,95 @@ "Mount": { "properties": { "Target": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Target", "description": "Container path." }, "Source": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Source", "description": "Mount source (e.g. a volume name, a host path)." }, "Type": { - "allOf": [ + "anyOf": [ { - "$ref": "#/components/schemas/Type1" + "$ref": "#/components/schemas/Type2" + }, + { + "type": "null" } ], "description": "The mount type. Available types:\n\n- `bind` Mounts a file or directory from the host into the container. Must exist prior to creating the container.\n- `volume` Creates a volume with the given name and options (or uses a pre-existing volume with the same name and options). These are **not** removed when the container is removed.\n- `tmpfs` Create a tmpfs with the given options. The mount source cannot be specified for tmpfs.\n- `npipe` Mounts a named pipe from the host into the container. Must exist prior to creating the container.\n" }, "ReadOnly": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Readonly", "description": "Whether the mount should be read-only." }, "Consistency": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Consistency", "description": "The consistency requirement for the mount: `default`, `consistent`, `cached`, or `delegated`." }, "BindOptions": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/BindOptions" + }, + { + "type": "null" } ], - "title": "Bindoptions", "description": "Optional configuration for the `bind` type." }, "VolumeOptions": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/VolumeOptions" + }, + { + "type": "null" } ], - "title": "Volumeoptions", "description": "Optional configuration for the `volume` type." }, "TmpfsOptions": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/TmpfsOptions" + }, + { + "type": "null" } ], - "title": "Tmpfsoptions", "description": "Optional configuration for the `tmpfs` type." } }, @@ -1511,11 +2020,25 @@ "NamedResourceSpec": { "properties": { "Kind": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Kind" }, "Value": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Value" } }, @@ -1525,23 +2048,44 @@ "NetworkAttachmentConfig": { "properties": { "Target": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Target", "description": "The target network for attachment. Must be a network name or ID.\n" }, "Aliases": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Aliases", "description": "Discoverable alternate names for the service on this network.\n" }, "DriverOpts": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Driveropts", "description": "Driver attachment options for the network target.\n" } @@ -1553,14 +2097,21 @@ "NetworkAttachmentSpec": { "properties": { "ContainerID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Containerid", "description": "ID of the container represented by this task" } }, "type": "object", "title": "NetworkAttachmentSpec", - "description": " Read-only spec type for non-swarm containers attached to swarm overlay\nnetworks.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." + "description": "Read-only spec type for non-swarm containers attached to swarm overlay\nnetworks.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." }, "Order": { "type": "string", @@ -1569,56 +2120,74 @@ "start-first" ], "title": "Order", - "description": " The order of operations when rolling out an updated task. Either\nthe old task is shut down before the new task is started, or the\nnew task is started before the old task is shut down." + "description": "The order of operations when rolling out an updated task. Either\nthe old task is shut down before the new task is started, or the\nnew task is started before the old task is shut down." + }, + "Order1": { + "type": "string", + "enum": [ + "stop-first", + "start-first" + ], + "title": "Order1", + "description": "The order of operations when rolling back a task. Either the old\ntask is shut down before the new task is started, or the new task\nis started before the old task is shut down." }, "Placement": { "properties": { "Constraints": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Constraints", - "description": "An array of constraint expressions to limit the set of nodes where\na task can be scheduled. Constraint expressions can either use a\n_match_ (`==`) or _exclude_ (`!=`) rule. Multiple constraints find\nnodes that satisfy every expression (AND match). Constraints can\nmatch node or Docker Engine labels as follows:\n\nnode attribute | matches | example\n---------------------|--------------------------------|-----------------------------------------------\n`node.id` | Node ID | `node.id==2ivku8v2gvtg4`\n`node.hostname` | Node hostname | `node.hostname!=node-2`\n`node.role` | Node role (`manager`/`worker`) | `node.role==manager`\n`node.platform.os` | Node operating system | `node.platform.os==windows`\n`node.platform.arch` | Node architecture | `node.platform.arch==x86_64`\n`node.labels` | User-defined node labels | `node.labels.security==high`\n`engine.labels` | Docker Engine's labels | `engine.labels.operatingsystem==ubuntu-14.04`\n\n`engine.labels` apply to Docker Engine labels like operating system,\ndrivers, etc. Swarm administrators add `node.labels` for operational\npurposes by using the [`node update endpoint`](#operation/NodeUpdate).\n", - "example": [ - "node.hostname!=node3.corp.example.com", - "node.role!=manager", - "node.labels.type==production", - "node.platform.os==linux", - "node.platform.arch==x86_64" - ] + "description": "An array of constraint expressions to limit the set of nodes where\na task can be scheduled. Constraint expressions can either use a\n_match_ (`==`) or _exclude_ (`!=`) rule. Multiple constraints find\nnodes that satisfy every expression (AND match). Constraints can\nmatch node or Docker Engine labels as follows:\n\nnode attribute | matches | example\n---------------------|--------------------------------|-----------------------------------------------\n`node.id` | Node ID | `node.id==2ivku8v2gvtg4`\n`node.hostname` | Node hostname | `node.hostname!=node-2`\n`node.role` | Node role (`manager`/`worker`) | `node.role==manager`\n`node.platform.os` | Node operating system | `node.platform.os==windows`\n`node.platform.arch` | Node architecture | `node.platform.arch==x86_64`\n`node.labels` | User-defined node labels | `node.labels.security==high`\n`engine.labels` | Docker Engine's labels | `engine.labels.operatingsystem==ubuntu-14.04`\n\n`engine.labels` apply to Docker Engine labels like operating system,\ndrivers, etc. Swarm administrators add `node.labels` for operational\npurposes by using the [`node update endpoint`](#operation/NodeUpdate).\n" }, "Preferences": { - "items": { - "$ref": "#/components/schemas/Preference" - }, - "type": "array", - "title": "Preferences", - "description": "Preferences provide a way to make the scheduler aware of factors\nsuch as topology. They are provided in order from highest to\nlowest precedence.\n", - "example": [ + "anyOf": [ { - "Spread": { - "SpreadDescriptor": "node.labels.datacenter" - } + "items": { + "$ref": "#/components/schemas/Preference" + }, + "type": "array" }, { - "Spread": { - "SpreadDescriptor": "node.labels.rack" - } + "type": "null" } - ] + ], + "title": "Preferences", + "description": "Preferences provide a way to make the scheduler aware of factors\nsuch as topology. They are provided in order from highest to\nlowest precedence.\n" }, "MaxReplicas": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Maxreplicas", "description": "Maximum number of replicas for per node (default value is 0, which\nis unlimited)\n", "default": 0 }, "Platforms": { - "items": { - "$ref": "#/components/schemas/Platform" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Platform" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Platforms", "description": "Platforms stores all the platforms that the service's image can\nrun on. This field is used in the platform filter for scheduling.\nIf empty, then the platform filter is off, meaning there are no\nscheduling restrictions.\n" } @@ -1629,16 +2198,28 @@ "Platform": { "properties": { "Architecture": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Architecture", - "description": "Architecture represents the hardware architecture (for example,\n`x86_64`).\n", - "example": "x86_64" + "description": "Architecture represents the hardware architecture (for example,\n`x86_64`).\n" }, "OS": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Os", - "description": "OS represents the Operating System (for example, `linux` or `windows`).\n", - "example": "linux" + "description": "OS represents the Operating System (for example, `linux` or `windows`).\n" } }, "type": "object", @@ -1648,62 +2229,114 @@ "PluginPrivilege": { "properties": { "Name": { - "type": "string", - "title": "Name", - "example": "network" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" }, "Description": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Description" }, "Value": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Value", - "example": [ - "host" - ] + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Value" } }, "type": "object", "title": "PluginPrivilege", - "description": " Describes a permission the user has to accept upon installing\nthe plugin." + "description": "Describes a permission the user has to accept upon installing\nthe plugin." }, "PluginSpec": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name", "description": "The name or 'alias' to use for the plugin." }, "Remote": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Remote", "description": "The plugin image reference to use." }, "Disabled": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Disabled", "description": "Disable the plugin once scheduled." }, "PluginPrivilege": { - "items": { - "$ref": "#/components/schemas/PluginPrivilege" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PluginPrivilege" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Pluginprivilege" } }, "type": "object", "title": "PluginSpec", - "description": " Plugin spec for the service. *(Experimental release only.)*\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." + "description": "Plugin spec for the service. *(Experimental release only.)*\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." }, "Preference": { "properties": { "Spread": { - "$ref": "#/components/schemas/Spread" + "anyOf": [ + { + "$ref": "#/components/schemas/Spread" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -1712,21 +2345,25 @@ "Privileges": { "properties": { "CredentialSpec": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/CredentialSpec" + }, + { + "type": "null" } ], - "title": "Credentialspec", "description": "CredentialSpec for managed service account (Windows only)" }, "SELinuxContext": { - "allOf": [ + "anyOf": [ + { + "$ref": "#/components/schemas/SeLinuxContext" + }, { - "$ref": "#/components/schemas/SELinuxContext" + "type": "null" } ], - "title": "Selinuxcontext", "description": "SELinux labels of the container" } }, @@ -1754,12 +2391,19 @@ "host" ], "title": "PublishMode", - "description": " The mode in which port is published.\n\n


\n\n- \"ingress\" makes the target port accessible on every node,\n regardless of whether there is a task for the service running on\n that node or not.\n- \"host\" bypasses the routing mesh and publish the port directly on\n the swarm node where that service is running." + "description": "The mode in which port is published.\n\n


\n\n- \"ingress\" makes the target port accessible on every node,\n regardless of whether there is a task for the service running on\n that node or not.\n- \"host\" bypasses the routing mesh and publish the port directly on\n the swarm node where that service is running." }, "Replicated": { "properties": { "Replicas": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Replicas" } }, @@ -1769,40 +2413,73 @@ "ReplicatedJob": { "properties": { "MaxConcurrent": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Maxconcurrent", "description": "The maximum number of replicas to run simultaneously.\n", "default": 1 }, "TotalCompletions": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Totalcompletions", "description": "The total number of replicas desired to reach the Completed\nstate. If unset, will default to the value of `MaxConcurrent`\n" } }, "type": "object", "title": "ReplicatedJob", - "description": " The mode used for services with a finite number of tasks that run\nto a completed state." + "description": "The mode used for services with a finite number of tasks that run\nto a completed state." }, "ResourceObject": { "properties": { "NanoCPUs": { - "type": "integer", - "title": "Nanocpus", - "example": 4000000000 + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Nanocpus" }, "MemoryBytes": { - "type": "integer", - "title": "Memorybytes", - "example": 8272408576 + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Memorybytes" }, "GenericResources": { - "$ref": "#/components/schemas/GenericResources" + "anyOf": [ + { + "$ref": "#/components/schemas/GenericResources" + }, + { + "type": "null" + } + ] } }, "type": "object", "title": "ResourceObject", - "description": " An object describing the resources which can be advertised by a node and\nrequested by a task." + "description": "An object describing the resources which can be advertised by a node and\nrequested by a task." }, "ResourceValue": { "properties": { @@ -1845,51 +2522,79 @@ "Resources1": { "properties": { "Limits": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Limit" + }, + { + "type": "null" } ], - "title": "Limits", "description": "Define resources limits." }, "Reservations": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/ResourceObject" + }, + { + "type": "null" } ], - "title": "Reservations", "description": "Define resources reservation." } }, "type": "object", "title": "Resources1", - "description": " Resource requirements which apply to each individual container created\nas part of the service." + "description": "Resource requirements which apply to each individual container created\nas part of the service." }, "RestartPolicy1": { "properties": { "Condition": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Condition" + }, + { + "type": "null" } ], "description": "Condition for restart." }, "Delay": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Delay", "description": "Delay between restart attempts." }, "MaxAttempts": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Maxattempts", "description": "Maximum attempts to restart a given container before giving up\n(default value is 0, which is ignored).\n", "default": 0 }, "Window": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Window", "description": "Windows is the time window used to evaluate the restart policy\n(default value is 0, which is unbounded).\n", "default": 0 @@ -1897,43 +2602,77 @@ }, "type": "object", "title": "RestartPolicy1", - "description": " Specification for the restart policy which applies to containers\ncreated as part of this service." + "description": "Specification for the restart policy which applies to containers\ncreated as part of this service." }, "RollbackConfig": { "properties": { "Parallelism": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Parallelism", "description": "Maximum number of tasks to be rolled back in one iteration (0 means\nunlimited parallelism).\n" }, "Delay": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Delay", "description": "Amount of time between rollback iterations, in nanoseconds.\n" }, "FailureAction": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/FailureAction1" + }, + { + "type": "null" } ], "description": "Action to take if an rolled back task fails to run, or stops\nrunning during the rollback.\n" }, "Monitor": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Monitor", "description": "Amount of time to monitor each rolled back task for failures, in\nnanoseconds.\n" }, "MaxFailureRatio": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], "title": "Maxfailureratio", "description": "The fraction of tasks that may fail during a rollback before the\nfailure action is invoked, specified as a floating point number\nbetween 0 and 1.\n", "default": 0 }, "Order": { - "allOf": [ + "anyOf": [ { - "$ref": "#/components/schemas/Order" + "$ref": "#/components/schemas/Order1" + }, + { + "type": "null" } ], "description": "The order of operations when rolling back a task. Either the old\ntask is shut down before the new task is started, or the new task\nis started before the old task is shut down.\n" @@ -1943,56 +2682,107 @@ "title": "RollbackConfig", "description": "Specification for the rollback strategy of the service." }, - "SELinuxContext": { + "SeLinuxContext": { "properties": { "Disable": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Disable", "description": "Disable SELinux" }, "User": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "User", "description": "SELinux user label" }, "Role": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Role", "description": "SELinux role label" }, "Type": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Type", "description": "SELinux type label" }, "Level": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Level", "description": "SELinux level label" } }, "type": "object", - "title": "SELinuxContext", + "title": "SeLinuxContext", "description": "SELinux labels of the container" }, "Secret": { "properties": { "File": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/File" + }, + { + "type": "null" } ], - "title": "File", "description": "File represents a specific target that is backed by a file.\n" }, "SecretID": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Secretid", "description": "SecretID represents the ID of the specific secret that we're\nreferencing.\n" }, "SecretName": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Secretname", "description": "SecretName is the name of the secret that this references,\nbut this is just provided for lookup/display purposes. The\nsecret in the reference will be identified by its ID.\n" } @@ -2054,14 +2844,17 @@ "name": { "type": "string", "title": "Name", - "description": "Display name: short, human readable name for the node", - "example": "Fast Counter" + "description": "Display name: short, human readable name for the node" }, "thumbnail": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Thumbnail", "description": "url to the thumbnail" }, @@ -2077,21 +2870,42 @@ "default": false }, "version_display": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Version Display", "description": "A user-friendly or marketing name for the release. This can be used to reference the release in a more readable and recognizable format, such as 'Matterhorn Release,' 'Spring Update,' or 'Holiday Edition.' This name is not used for version comparison but is useful for communication and documentation purposes." }, "deprecated": { - "type": "string", - "format": "date-time", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], "title": "Deprecated", "description": "Owner can set the date to retire the service. Three possibilities:If None, the service is marked as `published`;If now=deprecated, the service is retired" }, "classifiers": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Classifiers" }, "quality": { @@ -2100,10 +2914,17 @@ "default": {} }, "accessRights": { - "additionalProperties": { - "$ref": "#/components/schemas/ServiceGroupAccessRights" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/components/schemas/ServiceGroupAccessRights" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Accessrights", "description": "service access rights per group id" }, @@ -2120,30 +2941,47 @@ "description": "service version number" }, "release_date": { - "type": "string", - "format": "date-time", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], "title": "Release Date", "description": "A timestamp when the specific version of the service was released. This field helps in tracking the timeline of releases and understanding the sequence of updates. A timestamp string should be formatted as YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [\u00b1]HH[:]MM]" }, "integration-version": { - "type": "string", - "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "anyOf": [ + { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" + }, + { + "type": "null" + } + ], "title": "Integration-Version", "description": "This version is used to maintain backward compatibility when there are changes in the way a service is integrated into the framework" }, "type": { - "allOf": [ - { - "$ref": "#/components/schemas/ServiceType" - } - ], + "$ref": "#/components/schemas/ServiceType", "description": "service type" }, "badges": { - "items": { - "$ref": "#/components/schemas/Badge" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Badge" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Badges", "deprecated": true }, @@ -2162,48 +3000,88 @@ "description": "email to correspond to the authors about the node" }, "inputs": { - "additionalProperties": { - "$ref": "#/components/schemas/ServiceInput" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Inputs", "description": "definition of the inputs of this node" }, "outputs": { - "additionalProperties": { - "$ref": "#/components/schemas/ServiceOutput" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Outputs", "description": "definition of the outputs of this node" }, "boot-options": { - "additionalProperties": { - "$ref": "#/components/schemas/BootOption" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Boot-Options", "description": "Service defined boot options. These get injected in the service as env variables." }, "min-visible-inputs": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Min-Visible-Inputs", "description": "The number of 'data type inputs' displayed by default in the UI. When None all 'data type inputs' are displayed." }, "progress_regexp": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Progress Regexp", "description": "regexp pattern for detecting computational service's progress" }, "image_digest": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Image Digest", "description": "Image manifest digest. Note that this is NOT injected as an image label" }, "owner": { - "type": "string", - "format": "email", + "anyOf": [ + { + "type": "string", + "format": "email" + }, + { + "type": "null" + } + ], "title": "Owner" } }, @@ -2211,16 +3089,17 @@ "required": [ "name", "description", + "classifiers", "key", "version", "type", "authors", "contact", "inputs", - "outputs" + "outputs", + "owner" ], - "title": "ServiceGet", - "description": "Service metadata at publication time\n\n- read-only (can only be changed overwriting the image labels in the registry)\n- base metaddata\n- injected in the image labels\n\nNOTE: This model is serialized in .osparc/metadata.yml and in the labels of the docker image" + "title": "ServiceGet" }, "ServiceGroupAccessRights": { "properties": { @@ -2243,7 +3122,14 @@ "ServiceInput": { "properties": { "displayOrder": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], "title": "Displayorder", "description": "DEPRECATED: new display order is taken from the item position. This will be removed.", "deprecated": true @@ -2251,14 +3137,12 @@ "label": { "type": "string", "title": "Label", - "description": "short name for the property", - "example": "Age" + "description": "short name for the property" }, "description": { "type": "string", "title": "Description", - "description": "description of the property", - "example": "Age in seconds since 1970" + "description": "description of the property" }, "type": { "type": "string", @@ -2267,21 +3151,38 @@ "description": "data type expected on this input glob matching for data type is allowed" }, "contentSchema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Contentschema", "description": "jsonschema of this input/output. Required when type='ref_contentSchema'" }, "fileToKeyMap": { - "additionalProperties": { - "type": "string", - "pattern": "^[-_a-zA-Z0-9]+$" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Filetokeymap", "description": "Place the data associated with the named keys in files" }, "unit": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Unit", "description": "Units, when it refers to a physical quantity", "deprecated": true @@ -2299,18 +3200,23 @@ }, { "type": "string" + }, + { + "type": "null" } ], "title": "Defaultvalue", "deprecated": true }, "widget": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Widget" + }, + { + "type": "null" } ], - "title": "Widget", "description": "custom widget to use instead of the default one determined from the data-type" } }, @@ -2327,7 +3233,14 @@ "ServiceOutput": { "properties": { "displayOrder": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], "title": "Displayorder", "description": "DEPRECATED: new display order is taken from the item position. This will be removed.", "deprecated": true @@ -2335,14 +3248,12 @@ "label": { "type": "string", "title": "Label", - "description": "short name for the property", - "example": "Age" + "description": "short name for the property" }, "description": { "type": "string", "title": "Description", - "description": "description of the property", - "example": "Age in seconds since 1970" + "description": "description of the property" }, "type": { "type": "string", @@ -2351,32 +3262,51 @@ "description": "data type expected on this input glob matching for data type is allowed" }, "contentSchema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Contentschema", "description": "jsonschema of this input/output. Required when type='ref_contentSchema'" }, "fileToKeyMap": { - "additionalProperties": { - "type": "string", - "pattern": "^[-_a-zA-Z0-9]+$" - }, - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Filetokeymap", "description": "Place the data associated with the named keys in files" }, "unit": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Unit", "description": "Units, when it refers to a physical quantity", "deprecated": true }, "widget": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Widget" + }, + { + "type": "null" } ], - "title": "Widget", "description": "custom widget to use instead of the default one determined from the data-type", "deprecated": true } @@ -2388,8 +3318,7 @@ "description", "type" ], - "title": "ServiceOutput", - "description": "Base class for service input/outputs" + "title": "ServiceOutput" }, "ServicePortGet": { "properties": { @@ -2408,11 +3337,25 @@ "title": "Kind" }, "content_media_type": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Content Media Type" }, "content_schema": { - "type": "object", + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], "title": "Content Schema", "description": "jsonschema for the port's value. SEE https://json-schema.org/understanding-json-schema/" } @@ -2424,72 +3367,113 @@ ], "title": "ServicePortGet", "example": { - "key": "input_1", - "kind": "input", "content_schema": { + "maximum": 5, + "minimum": 0, "title": "Sleep interval", "type": "integer", - "x_unit": "second", - "minimum": 0, - "maximum": 5 - } + "x_unit": "second" + }, + "key": "input_1", + "kind": "input" } }, "ServiceSpec": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name", "description": "Name of the service." }, "Labels": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Labels", "description": "User-defined key/value metadata." }, "TaskTemplate": { - "$ref": "#/components/schemas/TaskSpec" + "anyOf": [ + { + "$ref": "#/components/schemas/TaskSpec" + }, + { + "type": "null" + } + ] }, "Mode": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Mode" + }, + { + "type": "null" } ], - "title": "Mode", "description": "Scheduling mode for the service." }, "UpdateConfig": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/UpdateConfig" + }, + { + "type": "null" } ], - "title": "Updateconfig", "description": "Specification for the update strategy of the service." }, "RollbackConfig": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/RollbackConfig" + }, + { + "type": "null" } ], - "title": "Rollbackconfig", "description": "Specification for the rollback strategy of the service." }, "Networks": { - "items": { - "$ref": "#/components/schemas/NetworkAttachmentConfig" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/NetworkAttachmentConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Networks", "description": "Specifies which networks the service should attach to." }, "EndpointSpec": { - "$ref": "#/components/schemas/EndpointSpec" + "anyOf": [ + { + "$ref": "#/components/schemas/EndpointSpec" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -2499,21 +3483,25 @@ "ServiceSpecificationsGet": { "properties": { "sidecar": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/ServiceSpec" + }, + { + "type": "null" } ], - "title": "Sidecar", "description": "schedule-time specifications for the service sidecar (follows Docker Service creation API, see https://docs.docker.com/engine/api/v1.25/#operation/ServiceCreate)" }, "service": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/ServiceSpec" + }, + { + "type": "null" } ], - "title": "Service", "description": "schedule-time specifications specifications for the service (follows Docker Service creation API (specifically only the Resources part), see https://docs.docker.com/engine/api/v1.41/#tag/Service/operation/ServiceCreate" } }, @@ -2528,32 +3516,56 @@ "frontend", "backend" ], - "title": "ServiceType", - "description": "An enumeration." + "title": "ServiceType" }, "ServiceUpdate": { "properties": { "accessRights": { - "additionalProperties": { - "$ref": "#/components/schemas/ServiceGroupAccessRights" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/components/schemas/ServiceGroupAccessRights" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Accessrights", "description": "service access rights per group id" }, "name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name" }, "thumbnail": { - "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Thumbnail" }, "description": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Description" }, "description_ui": { @@ -2562,20 +3574,41 @@ "default": false }, "version_display": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Version Display" }, "deprecated": { - "type": "string", - "format": "date-time", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], "title": "Deprecated", "description": "Owner can set the date to retire the service. Three possibilities:If None, the service is marked as `published`;If now=deprecated, the service is retired" }, "classifiers": { - "items": { - "type": "string" - }, - "type": "array", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Classifiers" }, "quality": { @@ -2585,6 +3618,12 @@ } }, "type": "object", + "required": [ + "name", + "thumbnail", + "description", + "classifiers" + ], "title": "ServiceUpdate", "example": { "accessRights": { @@ -2601,13 +3640,23 @@ "write_access": false } }, - "name": "My Human Readable Service Name", - "description": "An interesting service that does something", "classifiers": [ "RRID:SCR_018997", "RRID:SCR_019001" ], + "description": "An interesting service that does something", + "name": "My Human Readable Service Name", "quality": { + "annotations": { + "certificationLink": "", + "certificationStatus": "Uncertified", + "documentation": "", + "limitations": "", + "purpose": "", + "standards": "", + "vandv": "" + }, + "enabled": true, "tsr": { "r01": { "level": 3, @@ -2649,16 +3698,6 @@ "level": 0, "references": "" } - }, - "enabled": true, - "annotations": { - "vandv": "", - "purpose": "", - "standards": "", - "limitations": "", - "documentation": "", - "certificationLink": "", - "certificationStatus": "Uncertified" } } } @@ -2666,7 +3705,14 @@ "Spread": { "properties": { "SpreadDescriptor": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Spreaddescriptor", "description": "label descriptor, such as `engine.labels.az`.\n" } @@ -2706,78 +3752,118 @@ "TaskSpec": { "properties": { "PluginSpec": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/PluginSpec" + }, + { + "type": "null" } ], - "title": "Pluginspec", "description": "Plugin spec for the service. *(Experimental release only.)*\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n" }, "ContainerSpec": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/ContainerSpec" + }, + { + "type": "null" } ], - "title": "Containerspec", "description": "Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n" }, "NetworkAttachmentSpec": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/NetworkAttachmentSpec" + }, + { + "type": "null" } ], - "title": "Networkattachmentspec", "description": "Read-only spec type for non-swarm containers attached to swarm overlay\nnetworks.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`.\n" }, "Resources": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Resources1" + }, + { + "type": "null" } ], - "title": "Resources", "description": "Resource requirements which apply to each individual container created\nas part of the service.\n" }, "RestartPolicy": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/RestartPolicy1" + }, + { + "type": "null" } ], - "title": "Restartpolicy", "description": "Specification for the restart policy which applies to containers\ncreated as part of this service.\n" }, "Placement": { - "$ref": "#/components/schemas/Placement" + "anyOf": [ + { + "$ref": "#/components/schemas/Placement" + }, + { + "type": "null" + } + ] }, "ForceUpdate": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Forceupdate", "description": "A counter that triggers an update even if no relevant parameters have\nbeen changed.\n" }, "Runtime": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Runtime", "description": "Runtime is the type of runtime specified for the task executor.\n" }, "Networks": { - "items": { - "$ref": "#/components/schemas/NetworkAttachmentConfig" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/NetworkAttachmentConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Networks", "description": "Specifies which networks the service should attach to." }, "LogDriver": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/LogDriver1" + }, + { + "type": "null" } ], - "title": "Logdriver", "description": "Specifies the log driver to use for tasks created from this spec. If\nnot present, the default one for the swarm will be used, finally\nfalling back to the engine default if not specified.\n" } }, @@ -2805,12 +3891,26 @@ "TmpfsOptions": { "properties": { "SizeBytes": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Sizebytes", "description": "The size for the tmpfs mount in bytes." }, "Mode": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Mode", "description": "The permission mode for the tmpfs mount in an integer." } @@ -2826,10 +3926,9 @@ "udp", "sctp" ], - "title": "Type", - "description": "An enumeration." + "title": "Type" }, - "Type1": { + "Type2": { "type": "string", "enum": [ "bind", @@ -2837,65 +3936,120 @@ "tmpfs", "npipe" ], - "title": "Type1", - "description": " The mount type:\n\n- `bind` a mount of a file or directory from the host into the container.\n- `volume` a docker volume with the given `Name`.\n- `tmpfs` a `tmpfs`.\n- `npipe` a named pipe from the host into the container." + "title": "Type2", + "description": "The mount type. Available types:\n\n- `bind` Mounts a file or directory from the host into the container. Must exist prior to creating the container.\n- `volume` Creates a volume with the given name and options (or uses a pre-existing volume with the same name and options). These are **not** removed when the container is removed.\n- `tmpfs` Create a tmpfs with the given options. The mount source cannot be specified for tmpfs.\n- `npipe` Mounts a named pipe from the host into the container. Must exist prior to creating the container." }, - "Ulimit1": { + "Ulimit": { "properties": { "Name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Name", "description": "Name of ulimit" }, "Soft": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Soft", "description": "Soft limit" }, "Hard": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Hard", "description": "Hard limit" } }, "type": "object", - "title": "Ulimit1" + "title": "Ulimit" }, "UpdateConfig": { "properties": { "Parallelism": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Parallelism", "description": "Maximum number of tasks to be updated in one iteration (0 means\nunlimited parallelism).\n" }, "Delay": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Delay", "description": "Amount of time between updates, in nanoseconds." }, "FailureAction": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/FailureAction" + }, + { + "type": "null" } ], "description": "Action to take if an updated task fails to run, or stops running\nduring the update.\n" }, "Monitor": { - "type": "integer", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], "title": "Monitor", "description": "Amount of time to monitor each updated task for failures, in\nnanoseconds.\n" }, "MaxFailureRatio": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], "title": "Maxfailureratio", "description": "The fraction of tasks that may fail during an update before the\nfailure action is invoked, specified as a floating point number\nbetween 0 and 1.\n", "default": 0 }, "Order": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/Order" + }, + { + "type": "null" } ], "description": "The order of operations when rolling out an updated task. Either\nthe old task is shut down before the new task is started, or the\nnew task is started before the old task is shut down.\n" @@ -2941,26 +4095,42 @@ "VolumeOptions": { "properties": { "NoCopy": { - "type": "boolean", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "title": "Nocopy", "description": "Populate volume with data from the target.", "default": false }, "Labels": { - "additionalProperties": { - "type": "string" - }, - "type": "object", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "title": "Labels", "description": "User-defined key/value metadata." }, "DriverConfig": { - "allOf": [ + "anyOf": [ { "$ref": "#/components/schemas/DriverConfig" + }, + { + "type": "null" } ], - "title": "Driverconfig", "description": "Map of driver specific options" } }, @@ -2971,11 +4141,7 @@ "Widget": { "properties": { "type": { - "allOf": [ - { - "$ref": "#/components/schemas/WidgetType" - } - ], + "$ref": "#/components/schemas/WidgetType", "description": "type of the property" }, "details": { @@ -3004,8 +4170,7 @@ "TextArea", "SelectBox" ], - "title": "WidgetType", - "description": "An enumeration." + "title": "WidgetType" } } } diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index d78334d994c..e5aeb9fc368 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -12,11 +12,18 @@ aiofiles==23.2.1 # via -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.9.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -26,6 +33,8 @@ aiosignal==1.3.1 # via aiohttp alembic==1.13.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -53,11 +62,18 @@ attrs==23.2.0 # referencing certifi==2024.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -83,18 +99,13 @@ email-validator==2.1.1 # pydantic fast-depends==2.4.12 # via faststream -fastapi==0.99.1 +fastapi==0.115.3 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator +fastapi-cli==0.0.5 + # via fastapi faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 @@ -119,11 +130,18 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -142,11 +160,18 @@ itsdangerous==2.1.2 # via fastapi jinja2==3.1.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -158,11 +183,18 @@ jsonschema-specifications==2023.7.1 # via jsonschema mako==1.3.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -235,11 +267,18 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in @@ -263,25 +302,54 @@ psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==1.10.14 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # fastapi +pydantic-settings==2.6.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # fastapi pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -290,17 +358,24 @@ python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 # via - # pydantic + # pydantic-settings # uvicorn python-multipart==0.0.9 # via fastapi pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -309,11 +384,18 @@ pyyaml==6.0.1 # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -348,22 +430,36 @@ sniffio==1.3.1 # httpx sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.27.0 +starlette==0.41.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -379,6 +475,7 @@ typer==0.12.3 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # fastapi-cli types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 @@ -390,24 +487,39 @@ typing-extensions==4.10.0 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer ujson==5.9.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi urllib3==2.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -415,6 +527,7 @@ uvicorn==0.29.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/services/catalog/requirements/prod.txt b/services/catalog/requirements/prod.txt index a830c6815bd..c72846d7091 100644 --- a/services/catalog/requirements/prod.txt +++ b/services/catalog/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/catalog/setup.cfg b/services/catalog/setup.cfg index 144dbc1a4b9..ba46449cb68 100644 --- a/services/catalog/setup.cfg +++ b/services/catalog/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.6.0 commit = True message = services/catalog version: {current_version} → {new_version} tag = False @@ -9,10 +9,10 @@ commit_args = --no-verify [tool:pytest] asyncio_mode = auto -markers = +markers = testit: "marks test to run during development" [mypy] -plugins = +plugins = pydantic.mypy sqlalchemy.ext.mypy.plugin diff --git a/services/catalog/src/simcore_service_catalog/_meta.py b/services/catalog/src/simcore_service_catalog/_meta.py index 6aab1be93b2..770d24a4e28 100644 --- a/services/catalog/src/simcore_service_catalog/_meta.py +++ b/services/catalog/src/simcore_service_catalog/_meta.py @@ -1,6 +1,3 @@ -""" Application's metadata - -""" from typing import Final from models_library.basic_types import VersionStr diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_health.py b/services/catalog/src/simcore_service_catalog/api/rest/_health.py index aa59a59181a..a4360dff292 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_health.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_health.py @@ -7,4 +7,4 @@ @router.get("/") async def check_service_health(): - return f"{__name__}@{datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}" + return f"{__name__}@{datetime.datetime.now(tz=datetime.UTC).isoformat()}" diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_meta.py b/services/catalog/src/simcore_service_catalog/api/rest/_meta.py index 24369f696e1..e78a06ddb61 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_meta.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_meta.py @@ -1,7 +1,5 @@ from fastapi import APIRouter from models_library.api_schemas__common.meta import BaseMeta -from models_library.basic_types import VersionStr -from pydantic import parse_obj_as from ..._meta import API_VERSION, API_VTAG @@ -12,6 +10,6 @@ async def get_service_metadata(): return BaseMeta( name=__name__.split(".")[0], - version=parse_obj_as(VersionStr, API_VERSION), - released={API_VTAG: parse_obj_as(VersionStr, API_VERSION)}, + version=API_VERSION, + released={API_VTAG: API_VERSION}, ) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index 68a9f6490ba..f8d4bddabb0 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -43,7 +43,7 @@ def _compose_service_details( # compose service from registry and DB service = service_in_registry service.update( - service_in_db.dict(exclude_unset=True, exclude={"owner"}), + service_in_db.model_dump(exclude_unset=True, exclude={"owner"}), access_rights={rights.gid: rights for rights in service_access_rights_in_db}, owner=service_owner if service_owner else None, ) @@ -121,7 +121,7 @@ async def list_services( # NOTE: here validation is not necessary since key,version were already validated # in terms of time, this takes the most return [ - ServiceGet.construct( + ServiceGet.model_construct( key=key, version=version, name="nodetails", @@ -132,6 +132,8 @@ async def list_services( inputs={}, outputs={}, deprecated=services_in_db[(key, version)].deprecated, + classifiers=[], + owner=None, ) for key, version in services_in_db ] @@ -254,10 +256,10 @@ async def get_service( ) # access is allowed, override some of the values with what is in the db - service_in_manifest = service_in_manifest.copy( - update=service_in_db.dict(exclude_unset=True, exclude={"owner"}) + service_data.update( + service_in_manifest.model_dump(exclude_unset=True, by_alias=True) + | service_in_db.model_dump(exclude_unset=True, exclude={"owner"}) ) - service_data.update(service_in_manifest.dict(exclude_unset=True, by_alias=True)) return service_data @@ -322,7 +324,7 @@ async def update_service( ServiceMetaDataAtDB( key=service_key, version=service_version, - **updated_service.dict(exclude_unset=True), + **updated_service.model_dump(exclude_unset=True), ) ) # let's modify the service access rights (they can be added/removed/modified) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 701c4b41f3d..e8d552c7059 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -20,7 +20,7 @@ ServiceResourcesDictHelpers, ) from models_library.utils.docker_compose import replace_env_vars_in_compose_spec -from pydantic import parse_obj_as, parse_raw_as +from pydantic import TypeAdapter from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS from ...db.repositories.services import ServicesRepository @@ -61,7 +61,7 @@ def _compute_service_available_boot_modes( if not isinstance(entry.value, dict): _logger.warning( "resource %s for %s got invalid type", - f"{entry.dict()!r}", + f"{entry.model_dump()!r}", f"{service_key}:{service_version}", ) continue @@ -99,7 +99,7 @@ def _resources_from_settings( if not isinstance(entry.value, dict): _logger.warning( "resource %s for %s got invalid type", - f"{entry.dict()!r}", + f"{entry.model_dump()!r}", f"{service_key}:{service_version}", ) continue @@ -156,8 +156,7 @@ async def _get_service_labels( def _get_service_settings( labels: dict[str, Any] ) -> list[SimcoreServiceSettingLabelEntry]: - service_settings = parse_raw_as( - list[SimcoreServiceSettingLabelEntry], + service_settings = TypeAdapter(list[SimcoreServiceSettingLabelEntry]).validate_json( labels.get(SIMCORE_SERVICE_SETTINGS_LABELS, "[]"), ) _logger.debug("received %s", f"{service_settings=}") @@ -181,7 +180,9 @@ async def get_service_resources( ], user_groups: Annotated[list[GroupAtDB], Depends(list_user_groups)], ) -> ServiceResourcesDict: - image_version = parse_obj_as(DockerGenericTag, f"{service_key}:{service_version}") + image_version = TypeAdapter(DockerGenericTag).validate_python( + f"{service_key}:{service_version}" + ) if is_function_service(service_key): return ServiceResourcesDictHelpers.create_from_single_service( image_version, default_service_resources @@ -196,10 +197,9 @@ async def get_service_resources( image_version, default_service_resources ) - service_spec: ComposeSpecLabelDict | None = parse_raw_as( - ComposeSpecLabelDict | None, # type: ignore[arg-type] - service_labels.get(SIMCORE_SERVICE_COMPOSE_SPEC_LABEL, "null"), - ) + service_spec: ComposeSpecLabelDict | None = TypeAdapter( + ComposeSpecLabelDict | None + ).validate_json(service_labels.get(SIMCORE_SERVICE_COMPOSE_SPEC_LABEL, "null")) _logger.debug("received %s", f"{service_spec=}") if service_spec is None: @@ -235,7 +235,9 @@ async def get_service_resources( ) full_service_spec: ComposeSpecLabelDict = yaml.safe_load(stringified_service_spec) - service_to_resources: ServiceResourcesDict = parse_obj_as(ServiceResourcesDict, {}) + service_to_resources: ServiceResourcesDict = TypeAdapter( + ServiceResourcesDict + ).validate_python({}) for spec_key, spec_data in full_service_spec["services"].items(): # image can be: @@ -277,7 +279,7 @@ async def get_service_resources( spec_service_resources, user_specific_service_specs.service ) - service_to_resources[spec_key] = ImageResources.parse_obj( + service_to_resources[spec_key] = ImageResources.model_validate( { "image": image, "resources": spec_service_resources, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index d5481b3cefd..49751a196e8 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -70,7 +70,7 @@ async def get_service_specifications( if not service_specs: # nothing found, let's return the default then - service_specs = default_service_specifications.copy() + service_specs = default_service_specifications.model_copy() _logger.debug("returning %s", f"{service_specs=}") return service_specs diff --git a/services/catalog/src/simcore_service_catalog/cli.py b/services/catalog/src/simcore_service_catalog/cli.py index 5218b369ecc..0d4fbf5107b 100644 --- a/services/catalog/src/simcore_service_catalog/cli.py +++ b/services/catalog/src/simcore_service_catalog/cli.py @@ -5,9 +5,13 @@ from settings_library.http_client_request import ClientRequestSettings from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings -from settings_library.utils_cli import create_settings_command, print_as_envfile +from settings_library.utils_cli import ( + create_settings_command, + create_version_callback, + print_as_envfile, +) -from ._meta import PROJECT_NAME +from ._meta import PROJECT_NAME, __version__ from .core.settings import ApplicationSettings, DirectorSettings _logger = logging.getLogger(__name__) @@ -18,6 +22,7 @@ main.command()( create_settings_command(settings_cls=ApplicationSettings, logger=_logger) ) +main.callback()(create_version_callback(__version__)) @main.command() diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index a28dc8c5a32..a753f206a3e 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -29,7 +29,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: settings = ApplicationSettings.create_from_envs() assert settings # nosec - _logger.debug(settings.json(indent=2)) + _logger.debug(settings.model_dump_json(indent=2)) app = FastAPI( debug=settings.SC_BOOT_MODE diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index e6eb7c59fcb..5e513246732 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -89,7 +89,7 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version: # set the service in the DB await services_repo.create_or_update_service( - ServiceMetaDataAtDB(**service_metadata.dict(), owner=owner_gid), + ServiceMetaDataAtDB(**service_metadata.model_dump(), owner=owner_gid), service_access_rights, ) diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index 6235dcfd37f..38f9e22c5c2 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -5,9 +5,10 @@ from models_library.api_schemas_catalog.services_specifications import ( ServiceSpecifications, ) -from models_library.basic_types import BootModeEnum, BuildTargetEnum, LogLevel -from models_library.services_resources import ResourcesDict -from pydantic import ByteSize, Field, PositiveInt, parse_obj_as +from models_library.basic_types import LogLevel +from models_library.services_resources import ResourcesDict, ResourceValue +from pydantic import AliasChoices, ByteSize, Field, PositiveInt, TypeAdapter +from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.http_client_request import ClientRequestSettings from settings_library.postgres import PostgresSettings @@ -28,37 +29,31 @@ def base_url(self) -> str: return f"http://{self.DIRECTOR_HOST}:{self.DIRECTOR_PORT}/{self.DIRECTOR_VTAG}" -_DEFAULT_RESOURCES: Final[ResourcesDict] = parse_obj_as( - ResourcesDict, - { - "CPU": { - "limit": 0.1, - "reservation": 0.1, - }, - "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), - }, - }, +_in_bytes = TypeAdapter(ByteSize).validate_python + +_DEFAULT_RESOURCES: Final[ResourcesDict] = ResourcesDict( + CPU=ResourceValue(limit=0.1, reservation=0.1), + RAM=ResourceValue(limit=_in_bytes("2Gib"), reservation=_in_bytes("2Gib")), ) + _DEFAULT_SERVICE_SPECIFICATIONS: Final[ ServiceSpecifications -] = ServiceSpecifications.parse_obj({}) - +] = ServiceSpecifications.model_validate({}) -class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): - # docker environs - SC_BOOT_MODE: BootModeEnum | None - SC_BOOT_TARGET: BuildTargetEnum | None - CATALOG_LOG_LEVEL: LogLevel = Field( +class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): + LOG_LEVEL: LogLevel = Field( LogLevel.INFO.value, - env=["CATALOG_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "CATALOG_LOG_LEVEL", "CATALOG_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=["CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + validation_alias=AliasChoices( + "CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED" + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) CATALOG_DEV_FEATURES_ENABLED: bool = Field( @@ -66,15 +61,21 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables development features. WARNING: make sure it is disabled in production .env file!", ) - CATALOG_POSTGRES: PostgresSettings | None = Field(auto_default_from_env=True) + CATALOG_POSTGRES: PostgresSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - CATALOG_RABBITMQ: RabbitSettings = Field(auto_default_from_env=True) + CATALOG_RABBITMQ: RabbitSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) CATALOG_CLIENT_REQUEST: ClientRequestSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) - CATALOG_DIRECTOR: DirectorSettings | None = Field(auto_default_from_env=True) + CATALOG_DIRECTOR: DirectorSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True @@ -89,5 +90,6 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): _DEFAULT_SERVICE_SPECIFICATIONS ) CATALOG_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py b/services/catalog/src/simcore_service_catalog/db/repositories/groups.py index 4f339846301..8a1540b3f1a 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/groups.py @@ -3,7 +3,7 @@ import sqlalchemy as sa from models_library.emails import LowerCaseEmailStr from models_library.groups import GroupAtDB -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.types import PositiveInt from ...exceptions.errors import UninitializedGroupError @@ -15,7 +15,7 @@ class GroupsRepository(BaseRepository): async def list_user_groups(self, user_id: int) -> list[GroupAtDB]: async with self.db_engine.connect() as conn: return [ - GroupAtDB.from_orm(row) + GroupAtDB.model_validate(row) async for row in await conn.stream( sa.select(groups) .select_from( @@ -66,7 +66,7 @@ async def get_user_email_from_gid( email = await conn.scalar( sa.select(users.c.email).where(users.c.primary_gid == gid) ) - return cast(LowerCaseEmailStr, f"{email}") if email else None + return email or None async def list_user_emails_from_gids( self, gids: set[PositiveInt] @@ -79,7 +79,7 @@ async def list_user_emails_from_gids( ) ): service_owners[row[users.c.primary_gid]] = ( - parse_obj_as(LowerCaseEmailStr, row[users.c.email]) + TypeAdapter(LowerCaseEmailStr).validate_python(row[users.c.email]) if row[users.c.email] else None ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index 0f611c932b8..bae22e11597 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -15,7 +15,7 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import GroupID, UserID from psycopg2.errors import ForeignKeyViolation -from pydantic import PositiveInt, ValidationError, parse_obj_as +from pydantic import PositiveInt, TypeAdapter, ValidationError from simcore_postgres_database.utils_services import create_select_latest_services_query from sqlalchemy import literal_column from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -62,7 +62,7 @@ def _merge_specs( merged_spec = {} for spec in itertools.chain([everyone_spec], team_specs.values(), [user_spec]): if spec is not None: - merged_spec.update(spec.dict(include={"sidecar", "service"})) + merged_spec.update(spec.model_dump(include={"sidecar", "service"})) return merged_spec @@ -229,7 +229,7 @@ async def create_or_update_service( result = await conn.execute( # pylint: disable=no-value-for-parameter services_meta_data.insert() - .values(**new_service.dict(by_alias=True)) + .values(**new_service.model_dump(by_alias=True)) .returning(literal_column("*")) ) row = result.first() @@ -252,7 +252,7 @@ async def update_service(self, patched_service: ServiceMetaDataAtDB) -> None: & (services_meta_data.c.version == patched_service.version) ) .values( - **patched_service.dict( + **patched_service.model_dump( by_alias=True, exclude_unset=True, exclude={"key", "version"}, @@ -441,7 +441,11 @@ async def get_service_history( result = await conn.execute(stmt_history) row = result.one_or_none() - return parse_obj_as(list[ReleaseFromDB], row.history) if row else None + return ( + TypeAdapter(list[ReleaseFromDB]).validate_python(row.history) + if row + else None + ) # Service Access Rights ---- @@ -500,7 +504,7 @@ async def upsert_service_access_rights( # update the services_access_rights table (some might be added/removed/modified) for rights in new_access_rights: insert_stmt = pg_insert(services_access_rights).values( - **rights.dict(by_alias=True) + **rights.model_dump(by_alias=True) ) on_update_stmt = insert_stmt.on_conflict_do_update( index_elements=[ @@ -509,7 +513,7 @@ async def upsert_service_access_rights( services_access_rights.c.gid, services_access_rights.c.product_name, ], - set_=rights.dict( + set_=rights.model_dump( by_alias=True, exclude_unset=True, exclude={"key", "version", "gid", "product_name"}, @@ -617,5 +621,5 @@ async def get_service_specifications( if merged_specifications := _merge_specs( everyone_specs, teams_specs, primary_specs ): - return ServiceSpecifications.parse_obj(merged_specifications) + return ServiceSpecifications.model_validate(merged_specifications) return None # mypy diff --git a/services/catalog/src/simcore_service_catalog/exceptions/errors.py b/services/catalog/src/simcore_service_catalog/exceptions/errors.py index 0384088d37c..84010d9a700 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/errors.py +++ b/services/catalog/src/simcore_service_catalog/exceptions/errors.py @@ -1,11 +1,8 @@ -from typing import Any - from common_library.errors_classes import OsparcErrorMixin class CatalogBaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + ... class RepositoryError(CatalogBaseError): diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py index a28839dad42..7af4b5d93bd 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py +++ b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py @@ -1,13 +1,14 @@ -from collections.abc import Callable -from typing import Awaitable - from fastapi import HTTPException from fastapi.encoders import jsonable_encoder from starlette.requests import Request from starlette.responses import JSONResponse +from starlette.types import HTTPExceptionHandler + +async def http_error_handler(request: Request, exc: Exception) -> JSONResponse: + assert request # nosec + assert isinstance(exc, HTTPException) # nosec -async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse: return JSONResponse( content=jsonable_encoder({"errors": [exc.detail]}), status_code=exc.status_code ) @@ -15,7 +16,7 @@ async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse: def make_http_error_handler_for_exception( status_code: int, exception_cls: type[BaseException] -) -> Callable[[Request, type[BaseException]], Awaitable[JSONResponse]]: +) -> HTTPExceptionHandler: """ Produces a handler for BaseException-type exceptions which converts them into an error JSON response with a given status code @@ -29,4 +30,4 @@ async def _http_error_handler(_: Request, exc: type[BaseException]) -> JSONRespo content=jsonable_encoder({"errors": [str(exc)]}), status_code=status_code ) - return _http_error_handler + return _http_error_handler # type: ignore[return-value] diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py index 23aaa1d0f4e..8e3ad77f15d 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py +++ b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py @@ -1,17 +1,13 @@ from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import RequestValidationError from fastapi.openapi.constants import REF_PREFIX from fastapi.openapi.utils import validation_error_response_definition -from pydantic import ValidationError from starlette.requests import Request from starlette.responses import JSONResponse from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY -async def http422_error_handler( - _: Request, - exc: RequestValidationError | ValidationError, -) -> JSONResponse: +async def http422_error_handler(_: Request, exc: Exception) -> JSONResponse: + assert hasattr(exc, "errors") # nosec return JSONResponse( content=jsonable_encoder({"errors": exc.errors()}), status_code=HTTP_422_UNPROCESSABLE_ENTITY, diff --git a/services/catalog/src/simcore_service_catalog/main.py b/services/catalog/src/simcore_service_catalog/main.py index f5c9f4ee97c..cd8f0405858 100644 --- a/services/catalog/src/simcore_service_catalog/main.py +++ b/services/catalog/src/simcore_service_catalog/main.py @@ -11,8 +11,8 @@ _the_settings = ApplicationSettings.create_from_envs() # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 -logging.basicConfig(level=_the_settings.CATALOG_LOG_LEVEL.value) # NOSONAR -logging.root.setLevel(_the_settings.CATALOG_LOG_LEVEL.value) +logging.basicConfig(level=_the_settings.log_level) # NOSONAR +logging.root.setLevel(_the_settings.log_level) config_all_loggers( log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED ) diff --git a/services/catalog/src/simcore_service_catalog/models/services_db.py b/services/catalog/src/simcore_service_catalog/models/services_db.py index 2fd92f479ac..0412ba6878a 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_db.py +++ b/services/catalog/src/simcore_service_catalog/models/services_db.py @@ -1,24 +1,27 @@ from datetime import datetime -from typing import Any, ClassVar +from typing import Annotated, Any from models_library.products import ProductName from models_library.services_access import ServiceGroupAccessRights from models_library.services_base import ServiceKeyVersion from models_library.services_metadata_editable import ServiceMetaDataEditable from models_library.services_types import ServiceKey, ServiceVersion -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, HttpUrl from pydantic.types import PositiveInt from simcore_postgres_database.models.services_compatibility import CompatiblePolicyDict class ServiceMetaDataAtDB(ServiceKeyVersion, ServiceMetaDataEditable): - # for a partial update all members must be Optional - classifiers: list[str] | None = Field(default_factory=list) - owner: PositiveInt | None + # for a partial update all Editable members must be Optional + name: str | None = None + thumbnail: Annotated[str, HttpUrl] | None = None + description: str | None = None - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + classifiers: list[str] | None = Field(default_factory=list) + owner: PositiveInt | None = None + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "example": { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", @@ -49,7 +52,8 @@ class Config: }, }, } - } + }, + ) class ReleaseFromDB(BaseModel): @@ -83,19 +87,18 @@ class ServiceWithHistoryFromDB(BaseModel): assert ( # nosec - set(ReleaseFromDB.__fields__) + set(ReleaseFromDB.model_fields) .difference({"compatibility_policy"}) - .issubset(set(ServiceWithHistoryFromDB.__fields__)) + .issubset(set(ServiceWithHistoryFromDB.model_fields)) ) class ServiceAccessRightsAtDB(ServiceKeyVersion, ServiceGroupAccessRights): gid: PositiveInt product_name: ProductName - - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "example": { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", @@ -106,4 +109,5 @@ class Config: "created": "2021-01-18 12:46:57.7315", "modified": "2021-01-19 12:45:00", } - } + }, + ) diff --git a/services/catalog/src/simcore_service_catalog/models/services_specifications.py b/services/catalog/src/simcore_service_catalog/models/services_specifications.py index ce40b492f07..d53e56a8c56 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/models/services_specifications.py @@ -3,6 +3,7 @@ ) from models_library.services import ServiceKey, ServiceVersion from models_library.users import GroupID +from pydantic import ConfigDict class ServiceSpecificationsAtDB(ServiceSpecifications): @@ -10,5 +11,4 @@ class ServiceSpecificationsAtDB(ServiceSpecifications): service_version: ServiceVersion gid: GroupID - class Config(ServiceSpecifications.Config): - orm_mode: bool = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/catalog/src/simcore_service_catalog/services/access_rights.py b/services/catalog/src/simcore_service_catalog/services/access_rights.py index d35e7d4e5c0..0c83baf2a01 100644 --- a/services/catalog/src/simcore_service_catalog/services/access_rights.py +++ b/services/catalog/src/simcore_service_catalog/services/access_rights.py @@ -143,8 +143,7 @@ async def evaluate_auto_upgrade_policy( ) service_access_rights = [ - access.copy( - exclude={"created", "modified"}, + access.model_copy( update={"version": service_metadata.version}, deep=True, ) @@ -170,7 +169,7 @@ def _get_target(access: ServiceAccessRightsAtDB) -> tuple[str | int, ...]: def _get_flags(access: ServiceAccessRightsAtDB) -> dict[str, bool]: """Extracts only""" - flags = access.dict(include={"execute_access", "write_access"}) + flags = access.model_dump(include={"execute_access", "write_access"}) return cast(dict[str, bool], flags) access_flags_map: dict[tuple[str | int, ...], dict[str, bool]] = {} diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/services/compatibility.py index 1e9ea2b9a48..9c21e8b7ea7 100644 --- a/services/catalog/src/simcore_service_catalog/services/compatibility.py +++ b/services/catalog/src/simcore_service_catalog/services/compatibility.py @@ -8,7 +8,6 @@ from models_library.users import UserID from packaging.specifiers import SpecifierSet from packaging.version import Version -from pydantic import parse_obj_as from simcore_service_catalog.utils.versioning import as_version from ..db.repositories.services import ServicesRepository @@ -79,12 +78,12 @@ async def _evaluate_custom_compatibility( return Compatibility( can_update_to=CompatibleService( key=other_service_key, - version=parse_obj_as(ServiceVersion, f"{latest_version}"), + version=f"{latest_version}", ) ) return Compatibility( can_update_to=CompatibleService( - version=parse_obj_as(ServiceVersion, f"{latest_version}"), + version=f"{latest_version}", ) ) @@ -116,9 +115,7 @@ async def evaluate_service_compatibility_map( released_versions, ): compatibility = Compatibility( - can_update_to=CompatibleService( - version=parse_obj_as(ServiceVersion, f"{latest_version}") - ) + can_update_to=CompatibleService(version=f"{latest_version}") ) result[release.version] = compatibility diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/services/director.py index ee1327b475c..a762892dd77 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/services/director.py @@ -148,7 +148,7 @@ async def get_service( # NOTE: the fact that it returns a list of one element is a defect of the director API assert isinstance(data, list) # nosec assert len(data) == 1 # nosec - return ServiceMetaDataPublished.parse_obj(data[0]) + return ServiceMetaDataPublished.model_validate(data[0]) async def setup_director(app: FastAPI) -> None: diff --git a/services/catalog/src/simcore_service_catalog/services/function_services.py b/services/catalog/src/simcore_service_catalog/services/function_services.py index 006da49d413..93abd9466f8 100644 --- a/services/catalog/src/simcore_service_catalog/services/function_services.py +++ b/services/catalog/src/simcore_service_catalog/services/function_services.py @@ -14,7 +14,7 @@ def _as_dict(model_instance: ServiceMetaDataPublished) -> dict[str, Any]: - return model_instance.dict(by_alias=True, exclude_unset=True) + return model_instance.model_dump(by_alias=True, exclude_unset=True) def get_function_service(key, version) -> ServiceMetaDataPublished: diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index aa6caf52618..bf7c26a6b63 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -64,7 +64,7 @@ async def get_services_map( } for service in services_in_registry: try: - service_data = ServiceMetaDataPublished.parse_obj(service) + service_data = ServiceMetaDataPublished.model_validate(service) services[(service_data.key, service_data.version)] = service_data except ValidationError: # noqa: PERF203 diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 032909f4853..4122a035b0f 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -1,7 +1,6 @@ import logging from models_library.api_schemas_catalog.services import ServiceGetV2, ServiceUpdateV2 -from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt from models_library.services_access import ServiceGroupAccessRightsV2 @@ -9,7 +8,7 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import HttpUrl, NonNegativeInt, parse_obj_as +from pydantic import HttpUrl, NonNegativeInt from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogItemNotFoundError, @@ -41,28 +40,20 @@ def _db_to_api_model( key=service_db.key, version=service_db.version, name=service_db.name, - thumbnail=( - parse_obj_as(HttpUrl, service_db.thumbnail) - if service_db.thumbnail - else None - ), + thumbnail=HttpUrl(service_db.thumbnail) if service_db.thumbnail else None, description=service_db.description, description_ui=service_db.description_ui, version_display=service_db.version_display, type=service_manifest.service_type, contact=service_manifest.contact, authors=service_manifest.authors, - owner=( - LowerCaseEmailStr(service_db.owner_email) - if service_db.owner_email - else None - ), + owner=(service_db.owner_email if service_db.owner_email else None), inputs=service_manifest.inputs or {}, outputs=service_manifest.outputs or {}, boot_options=service_manifest.boot_options, min_visible_inputs=service_manifest.min_visible_inputs, access_rights={ - a.gid: ServiceGroupAccessRightsV2.construct( + a.gid: ServiceGroupAccessRightsV2.model_construct( execute=a.execute_access, write=a.write_access, ) @@ -71,7 +62,7 @@ def _db_to_api_model( classifiers=service_db.classifiers, quality=service_db.quality, history=[ - ServiceRelease.construct( + ServiceRelease.model_construct( version=h.version, version_display=h.version_display, released=h.created, @@ -251,7 +242,7 @@ async def update_service( ServiceMetaDataAtDB( key=service_key, version=service_version, - **update.dict(exclude_unset=True), + **update.model_dump(exclude_unset=True), ) ) diff --git a/services/catalog/src/simcore_service_catalog/utils/service_resources.py b/services/catalog/src/simcore_service_catalog/utils/service_resources.py index 1b6b7ddcbc9..1e61dfffbe5 100644 --- a/services/catalog/src/simcore_service_catalog/utils/service_resources.py +++ b/services/catalog/src/simcore_service_catalog/utils/service_resources.py @@ -42,12 +42,15 @@ def merge_service_resources_with_user_specs( service_resources: ResourcesDict, user_specific_spec: ServiceSpec ) -> ResourcesDict: if ( - not user_specific_spec.TaskTemplate - or not user_specific_spec.TaskTemplate.Resources + not user_specific_spec.task_template + or not user_specific_spec.task_template.resources ): return service_resources - user_specific_resources = user_specific_spec.dict( - include={"TaskTemplate": {"Resources"}} + + assert "task_template" in user_specific_spec.model_fields # nosec + + user_specific_resources = user_specific_spec.model_dump( + include={"task_template": {"resources"}}, by_alias=True )["TaskTemplate"]["Resources"] merged_resources = deepcopy(service_resources) @@ -58,25 +61,29 @@ def merge_service_resources_with_user_specs( # res_name: NanoCPUs, MemoryBytes, Pids, GenericResources if res_value is None: continue + if res_name == "GenericResources": # special case here merged_resources |= parse_generic_resource(res_value) continue + if res_name not in _DOCKER_TO_OSPARC_RESOURCE_MAP: continue - if _DOCKER_TO_OSPARC_RESOURCE_MAP[res_name] in merged_resources: - # upgrade - merged_resources[_DOCKER_TO_OSPARC_RESOURCE_MAP[res_name]].__setattr__( - osparc_res_attr, - res_value * _DOCKER_TO_OSPARC_RESOURCE_CONVERTER[res_name], - ) + + scale = _DOCKER_TO_OSPARC_RESOURCE_CONVERTER[res_name] + key = _DOCKER_TO_OSPARC_RESOURCE_MAP[res_name] + if key in merged_resources: + # updates. + # NOTE: do not use assignment! + # SEE test_reservation_is_cap_by_limit_on_assigment_pydantic_2_bug + data = merged_resources[key].model_dump() + data[osparc_res_attr] = res_value * scale + merged_resources[key] = ResourceValue(**data) else: - merged_resources[ - _DOCKER_TO_OSPARC_RESOURCE_MAP[res_name] - ] = ResourceValue( - limit=res_value * _DOCKER_TO_OSPARC_RESOURCE_CONVERTER[res_name], - reservation=res_value - * _DOCKER_TO_OSPARC_RESOURCE_CONVERTER[res_name], + # constructs + merged_resources[key] = ResourceValue( + limit=res_value * scale, + reservation=res_value * scale, ) return merged_resources diff --git a/services/catalog/tests/unit/conftest.py b/services/catalog/tests/unit/conftest.py index 6069514b085..278af317091 100644 --- a/services/catalog/tests/unit/conftest.py +++ b/services/catalog/tests/unit/conftest.py @@ -31,6 +31,7 @@ from simcore_service_catalog.core.settings import ApplicationSettings pytest_plugins = [ + "pytest_simcore.cli_runner", "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", @@ -86,7 +87,12 @@ def app_environment( """ return setenvs_from_dict( monkeypatch, - {**docker_compose_service_environment_dict}, + { + "SC_BOOT_MODE": "production", + "SC_BOOT_TARGET": "null", + **docker_compose_service_environment_dict, + "CATALOG_TRACING": "null", + }, ) diff --git a/services/catalog/tests/unit/test__model_examples.py b/services/catalog/tests/unit/test__model_examples.py new file mode 100644 index 00000000000..7592b8d21f1 --- /dev/null +++ b/services/catalog/tests/unit/test__model_examples.py @@ -0,0 +1,28 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import json +from typing import Any + +import pytest +import simcore_service_catalog.models +from pydantic import BaseModel, ValidationError +from pytest_simcore.pydantic_models import walk_model_examples_in_package + + +@pytest.mark.parametrize( + "model_cls, example_name, example_data", + walk_model_examples_in_package(simcore_service_catalog.models), +) +def test_catalog_service_model_examples( + model_cls: type[BaseModel], example_name: int, example_data: Any +): + try: + assert model_cls.model_validate(example_data) is not None + except ValidationError as err: + pytest.fail( + f"\n{example_name}: {json.dumps(example_data, indent=1)}\nError: {err}" + ) diff --git a/services/catalog/tests/unit/test_cli.py b/services/catalog/tests/unit/test_cli.py new file mode 100644 index 00000000000..044cc6c5884 --- /dev/null +++ b/services/catalog/tests/unit/test_cli.py @@ -0,0 +1,34 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os + +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_catalog._meta import API_VERSION +from simcore_service_catalog.cli import main +from simcore_service_catalog.core.settings import ApplicationSettings +from typer.testing import CliRunner + + +def test_cli_help_and_version(cli_runner: CliRunner): + result = cli_runner.invoke(main, "--help") + assert result.exit_code == os.EX_OK, result.output + + result = cli_runner.invoke(main, "--version") + assert result.exit_code == os.EX_OK, result.output + assert result.stdout.strip() == API_VERSION + + +def test_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): + result = cli_runner.invoke(main, ["settings", "--show-secrets", "--as-json"]) + assert result.exit_code == os.EX_OK + + settings = ApplicationSettings.model_validate_json(result.output) + assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() + + +def test_run(cli_runner: CliRunner): + result = cli_runner.invoke(main, ["run"]) + assert result.exit_code == 0 + assert "disabled" in result.stdout diff --git a/services/catalog/tests/unit/test_core_settings.py b/services/catalog/tests/unit/test_core_settings.py new file mode 100644 index 00000000000..9f94c6c3588 --- /dev/null +++ b/services/catalog/tests/unit/test_core_settings.py @@ -0,0 +1,22 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + + +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_catalog.core.settings import ApplicationSettings + + +def test_valid_web_application_settings(app_environment: EnvVarsDict): + """ + We can validate actual .env files (also refered as `repo.config` files) by passing them via the CLI + + $ ln -s /path/to/osparc-config/deployments/mydeploy.com/repo.config .secrets + $ pytest --external-envfile=.secrets --pdb tests/unit/test_core_settings.py + + """ + settings = ApplicationSettings() # type: ignore + assert settings + + assert settings == ApplicationSettings.create_from_envs() diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_services_compatibility.py index 2fc9f8f06b5..04ef4bafd4d 100644 --- a/services/catalog/tests/unit/test_services_compatibility.py +++ b/services/catalog/tests/unit/test_services_compatibility.py @@ -160,9 +160,10 @@ def test_get_latest_compatible_version(versions_history: list[Version]): def _create_as(cls, **overrides): kwargs = { - "deprecated": None, - "created": arrow.now().datetime, "compatibility_policy": None, + "created": arrow.now().datetime, + "deprecated": None, + "version_display": None, } kwargs.update(overrides) return cls(**kwargs) @@ -265,7 +266,7 @@ async def test_evaluate_service_compatibility_map_with_deprecated_versions( ): service_release_history = [ _create_as(ReleaseFromDB, version="1.0.0"), - _create_as(ReleaseFromDB, version="1.0.1", deprecated=True), + _create_as(ReleaseFromDB, version="1.0.1", deprecated=arrow.now().datetime), _create_as(ReleaseFromDB, version="1.2.0"), _create_as(ReleaseFromDB, version="1.2.5"), ] diff --git a/services/catalog/tests/unit/test_utils_service_resources.py b/services/catalog/tests/unit/test_utils_service_resources.py index 1df8b18b896..3fc329d2f50 100644 --- a/services/catalog/tests/unit/test_utils_service_resources.py +++ b/services/catalog/tests/unit/test_utils_service_resources.py @@ -291,6 +291,6 @@ def test_merge_service_resources_with_user_specs( assert all(key in merged_resources for key in expected_resources) for resource_key, resource_value in merged_resources.items(): # NOTE: so that float values are compared correctly - assert resource_value.dict() == pytest.approx( - expected_resources[resource_key].dict() + assert resource_value.model_dump() == pytest.approx( + expected_resources[resource_key].model_dump() ) diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index e31913ab9bb..1bd0bb27e50 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -1,7 +1,6 @@ # pylint: disable=not-context-manager # pylint: disable=protected-access # pylint: disable=redefined-outer-name -# pylint: disable=too-many-positional-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable @@ -18,7 +17,7 @@ from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished from models_library.users import UserID -from pydantic import Extra, parse_obj_as +from pydantic import ConfigDict, TypeAdapter from pytest_simcore.helpers.faker_factories import ( random_service_access_rights, random_service_meta_data, @@ -122,13 +121,13 @@ async def product( @pytest.fixture def target_product(product: dict[str, Any], product_name: ProductName) -> ProductName: - assert product_name == parse_obj_as(ProductName, product["name"]) + assert product_name == TypeAdapter(ProductName).validate_python(product["name"]) return product_name @pytest.fixture def other_product(product: dict[str, Any]) -> ProductName: - other = parse_obj_as(ProductName, "osparc") + other = TypeAdapter(ProductName).validate_python("osparc") assert other != product["name"] return other @@ -347,7 +346,7 @@ def _fake_factory(**overrides): data = deepcopy(template) data.update(**overrides) - assert ServiceMetaDataPublished.parse_obj( + assert ServiceMetaDataPublished.model_validate( data ), "Invalid fake data. Out of sync!" return data @@ -454,9 +453,7 @@ def create_director_list_services_from() -> ( """ class _Loader(ServiceMetaDataPublished): - class Config: - extra = Extra.ignore - allow_population_by_field_name = True + model_config = ConfigDict(extra="ignore", populate_by_name=True) def _( expected_director_list_services: list[dict[str, Any]], @@ -464,7 +461,7 @@ def _( ): return [ jsonable_encoder( - _Loader.parse_obj( + _Loader.model_validate( { **next(itertools.cycle(expected_director_list_services)), **data[0], # service, **access_rights = data diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py index a3c85d3f31b..d4ca2539eb8 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py @@ -90,7 +90,7 @@ def test_get_service_with_details( assert response.status_code == 200 - got = ServiceGet.parse_obj(response.json()) + got = ServiceGet.model_validate(response.json()) assert got.key == service_key assert got.version == service_version diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py index 4b0bd5dceb6..8c2071fea23 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py @@ -13,7 +13,7 @@ from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from respx.router import MockRouter from starlette import status from starlette.testclient import TestClient @@ -56,9 +56,9 @@ async def test_list_services_with_details( url = URL("/v0/services").with_query({"user_id": user_id, "details": "true"}) # now fake the director such that it returns half the services - fake_registry_service_data = ServiceMetaDataPublished.Config.schema_extra[ - "examples" - ][0] + fake_registry_service_data = ServiceMetaDataPublished.model_config[ + "json_schema_extra" + ]["examples"][0] mocked_director_service_api_base.get("/services", name="list_services").respond( 200, @@ -255,16 +255,16 @@ async def test_list_services_that_are_deprecated( url = URL("/v0/services").with_query({"user_id": user_id, "details": "false"}) resp = client.get(f"{url}", headers={"x-simcore-products-name": target_product}) assert resp.status_code == status.HTTP_200_OK - list_of_services = parse_obj_as(list[ServiceGet], resp.json()) + list_of_services = TypeAdapter(list[ServiceGet]).validate_python(resp.json()) assert list_of_services assert len(list_of_services) == 1 received_service = list_of_services[0] assert received_service.deprecated == deprecation_date # for details, the director must return the same service - fake_registry_service_data = ServiceMetaDataPublished.Config.schema_extra[ - "examples" - ][0] + fake_registry_service_data = ServiceMetaDataPublished.model_config[ + "json_schema_extra" + ]["examples"][0] mocked_director_service_api_base.get("/services", name="list_services").respond( 200, json={ @@ -281,7 +281,7 @@ async def test_list_services_that_are_deprecated( url = URL("/v0/services").with_query({"user_id": user_id, "details": "true"}) resp = client.get(f"{url}", headers={"x-simcore-products-name": target_product}) assert resp.status_code == status.HTTP_200_OK - list_of_services = parse_obj_as(list[ServiceGet], resp.json()) + list_of_services = TypeAdapter(list[ServiceGet]).validate_python(resp.json()) assert list_of_services assert len(list_of_services) == 1 received_service = list_of_services[0] diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py index 87ac133a0df..425b1318950 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py @@ -14,7 +14,7 @@ ServiceAccessRightsGet, ) from models_library.products import ProductName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from respx.router import MockRouter from starlette.testclient import TestClient from yarl import URL @@ -66,7 +66,7 @@ async def test_get_service_access_rights( headers={"x-simcore-products-name": target_product}, ) assert response.status_code == 200 - data = parse_obj_as(ServiceAccessRightsGet, response.json()) + data = TypeAdapter(ServiceAccessRightsGet).validate_python(response.json()) assert data.service_key == service_to_check["key"] assert data.service_version == service_to_check["version"] assert data.gids_with_access_rights == { @@ -108,7 +108,7 @@ async def test_get_service_access_rights_with_more_gids( headers={"x-simcore-products-name": other_product}, ) assert response.status_code == 200 - data = parse_obj_as(ServiceAccessRightsGet, response.json()) + data = TypeAdapter(ServiceAccessRightsGet).validate_python(response.json()) assert data.service_key == service_to_check["key"] assert data.service_version == service_to_check["version"] assert data.gids_with_access_rights == { diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py index 1ea7e40f18f..d9ef5ea328f 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py @@ -6,13 +6,13 @@ from collections.abc import Callable from copy import deepcopy from dataclasses import dataclass -from random import choice, randint from typing import Any import httpx import pytest import respx from faker import Faker +from fastapi.encoders import jsonable_encoder from models_library.docker import DockerGenericTag from models_library.services_resources import ( BootMode, @@ -21,7 +21,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from respx.models import Route from simcore_service_catalog.core.settings import _DEFAULT_RESOURCES from starlette.testclient import TestClient @@ -58,13 +58,15 @@ def creator(): @pytest.fixture -def service_key() -> str: - return f"simcore/services/{choice(['comp', 'dynamic','frontend'])}/jupyter-math" +def service_key(faker: Faker) -> str: + return f"simcore/services/{faker.random_element(['comp', 'dynamic','frontend'])}/jupyter-math" @pytest.fixture -def service_version() -> str: - return f"{randint(0,100)}.{randint(0,100)}.{randint(0,100)}" +def service_version(faker: Faker) -> str: + return ( + f"{faker.random_int(0,100)}.{faker.random_int(0,100)}.{faker.random_int(0,100)}" + ) @pytest.fixture @@ -189,24 +191,27 @@ async def test_get_service_resources( mocked_director_service_labels: Route, client: TestClient, params: _ServiceResourceParams, + service_key: str, + service_version: str, ) -> None: - service_key = f"simcore/services/{choice(['comp', 'dynamic'])}/jupyter-math" - service_version = f"{randint(0,100)}.{randint(0,100)}.{randint(0,100)}" + mocked_director_service_labels.respond(json={"data": params.simcore_service_label}) url = URL(f"/v0/services/{service_key}/{service_version}/resources") response = client.get(f"{url}") assert response.status_code == 200, f"{response.text}" data = response.json() - received_resources: ServiceResourcesDict = parse_obj_as(ServiceResourcesDict, data) + received_resources: ServiceResourcesDict = ServiceResourcesDict(**data) assert isinstance(received_resources, dict) expected_service_resources = ServiceResourcesDictHelpers.create_from_single_service( - parse_obj_as(DockerGenericTag, f"{service_key}:{service_version}"), + TypeAdapter(DockerGenericTag).validate_python( + f"{service_key}:{service_version}" + ), params.expected_resources, boot_modes=params.expected_boot_modes, ) assert isinstance(expected_service_resources, dict) - assert received_resources == expected_service_resources + assert received_resources == jsonable_encoder(expected_service_resources) @pytest.fixture @@ -241,9 +246,10 @@ def factory(services_labels: dict[str, dict[str, Any]]) -> None: }, "sym-server": {"simcore.service.settings": "[]"}, }, - parse_obj_as( - ServiceResourcesDict, - ServiceResourcesDictHelpers.Config.schema_extra["examples"][1], + TypeAdapter(ServiceResourcesDict).validate_python( + ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][1] ), "simcore/services/dynamic/sim4life-dy", "3.0.0", @@ -257,16 +263,17 @@ def factory(services_labels: dict[str, dict[str, Any]]) -> None: }, "busybox": {"simcore.service.settings": "[]"}, }, - parse_obj_as( - ServiceResourcesDict, + TypeAdapter(ServiceResourcesDict).validate_python( { "jupyter-math": { "image": "simcore/services/dynamic/jupyter-math:2.0.5", "resources": { "CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), + "limit": TypeAdapter(ByteSize).validate_python("2Gib"), + "reservation": TypeAdapter(ByteSize).validate_python( + "2Gib" + ), }, }, }, @@ -275,8 +282,10 @@ def factory(services_labels: dict[str, dict[str, Any]]) -> None: "resources": { "CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": { - "limit": parse_obj_as(ByteSize, "2Gib"), - "reservation": parse_obj_as(ByteSize, "2Gib"), + "limit": TypeAdapter(ByteSize).validate_python("2Gib"), + "reservation": TypeAdapter(ByteSize).validate_python( + "2Gib" + ), }, }, }, @@ -304,7 +313,7 @@ async def test_get_service_resources_sim4life_case( response = client.get(f"{url}") assert response.status_code == 200, f"{response.text}" data = response.json() - received_service_resources = parse_obj_as(ServiceResourcesDict, data) + received_service_resources = TypeAdapter(ServiceResourcesDict).validate_python(data) assert received_service_resources == expected_service_resources @@ -314,10 +323,10 @@ async def test_get_service_resources_raises_errors( rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_labels: Route, client: TestClient, + service_key: str, + service_version: str, ) -> None: - service_key = f"simcore/services/{choice(['comp', 'dynamic'])}/jupyter-math" - service_version = f"{randint(0,100)}.{randint(0,100)}.{randint(0,100)}" url = URL(f"/v0/services/{service_key}/{service_version}/resources") # simulate a communication error mocked_director_service_labels.side_effect = httpx.HTTPError diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py index 394ea9123ad..e024a511ad4 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py @@ -4,10 +4,8 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable - import asyncio from collections.abc import AsyncIterator, Awaitable, Callable -from random import choice, randint from typing import Any import pytest @@ -16,7 +14,6 @@ from fastapi.encoders import jsonable_encoder from models_library.api_schemas_catalog.services_specifications import ( ServiceSpecifications, - ServiceSpecificationsGet, ) from models_library.generated_models.docker_rest_api import ( DiscreteResourceSpec, @@ -29,7 +26,7 @@ from models_library.generated_models.docker_rest_api import ( Resources1 as ServiceTaskResources, ) -from models_library.generated_models.docker_rest_api import ServiceSpec +from models_library.generated_models.docker_rest_api import ServiceSpec, TaskSpec from models_library.products import ProductName from models_library.users import UserID from simcore_postgres_database.models.groups import user_to_groups @@ -93,29 +90,33 @@ def _creator(service_key, service_version, gid) -> ServiceSpecificationsAtDB: service_key=service_key, service_version=service_version, gid=gid, - sidecar=ServiceSpec(Labels=faker.pydict(allowed_types=(str,))), # type: ignore - service=ServiceTaskResources( - Limits=Limit( - NanoCPUs=faker.pyint(), - MemoryBytes=faker.pyint(), - Pids=faker.pyint(), - ), - Reservations=ResourceObject( - NanoCPUs=faker.pyint(), - MemoryBytes=faker.pyint(), - GenericResources=GenericResources( - __root__=[ - GenericResource( - NamedResourceSpec=NamedResourceSpec( - Kind=faker.pystr(), Value=faker.pystr() - ), - DiscreteResourceSpec=DiscreteResourceSpec( - Kind=faker.pystr(), Value=faker.pyint() - ), - ) - ] - ), - ), + sidecar=ServiceSpec(Labels=faker.pydict(allowed_types=(str,))), + service=ServiceSpec( + TaskTemplate=TaskSpec( + Resources=ServiceTaskResources( + Limits=Limit( + NanoCPUs=faker.pyint(), + MemoryBytes=faker.pyint(), + Pids=faker.pyint(), + ), + Reservations=ResourceObject( + NanoCPUs=faker.pyint(), + MemoryBytes=faker.pyint(), + GenericResources=GenericResources( + root=[ + GenericResource( + NamedResourceSpec=NamedResourceSpec( + Kind=faker.pystr(), Value=faker.pystr() + ), + DiscreteResourceSpec=DiscreteResourceSpec( + Kind=faker.pystr(), Value=faker.pyint() + ), + ) + ] + ), + ), + ) + ) ), ) @@ -128,9 +129,14 @@ async def test_get_service_specifications_returns_403_if_user_does_not_exist( rabbitmq_and_rpc_setup_disabled: None, client: TestClient, user_id: UserID, + faker: Faker, ): - service_key = f"simcore/services/{choice(['comp', 'dynamic'])}/jupyter-math" - service_version = f"{randint(0,100)}.{randint(0,100)}.{randint(0,100)}" + service_key = ( + f"simcore/services/{faker.random_element(['comp', 'dynamic'])}/jupyter-math" + ) + service_version = ( + f"{faker.random_int(0,100)}.{faker.random_int(0,100)}.{faker.random_int(0,100)}" + ) url = URL( f"/v0/services/{service_key}/{service_version}/specifications" ).with_query(user_id=user_id) @@ -147,21 +153,21 @@ async def test_get_service_specifications_of_unknown_service_returns_default_spe user: dict[str, Any], faker: Faker, ): - service_key = ( - f"simcore/services/{choice(['comp', 'dynamic'])}/{faker.pystr().lower()}" + service_key = f"simcore/services/{faker.random_element(['comp', 'dynamic'])}/{faker.pystr().lower()}" + service_version = ( + f"{faker.random_int(0,100)}.{faker.random_int(0,100)}.{faker.random_int(0,100)}" ) - service_version = f"{randint(0,100)}.{randint(0,100)}.{randint(0,100)}" url = URL( f"/v0/services/{service_key}/{service_version}/specifications" ).with_query(user_id=user_id) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs assert ( - service_specs - == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + service_specs.model_dump() + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS.model_dump() ) @@ -201,11 +207,11 @@ async def test_get_service_specifications( # this should now return default specs since there are none in the db response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs assert ( - service_specs - == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + service_specs.model_dump() + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS.model_dump() ) everyone_gid, user_gid, team_gid = user_groups_ids @@ -216,10 +222,10 @@ async def test_get_service_specifications( await services_specifications_injector(everyone_service_specs) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs - assert service_specs == ServiceSpecifications.parse_obj( - everyone_service_specs.dict() + assert service_specs == ServiceSpecifications.model_validate( + everyone_service_specs.model_dump() ) # let's inject some rights in a standard group, user is not part of that group yet, so it should still return only everyone @@ -229,10 +235,10 @@ async def test_get_service_specifications( await services_specifications_injector(standard_group_service_specs) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs - assert service_specs == ServiceSpecifications.parse_obj( - everyone_service_specs.dict() + assert service_specs == ServiceSpecifications.model_validate( + everyone_service_specs.model_dump() ) # put the user in that group now and try again @@ -240,10 +246,10 @@ async def test_get_service_specifications( await conn.execute(user_to_groups.insert().values(uid=user_id, gid=team_gid)) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs - assert service_specs == ServiceSpecifications.parse_obj( - standard_group_service_specs.dict() + assert service_specs == ServiceSpecifications.model_validate( + standard_group_service_specs.model_dump() ) # now add some other spec in the primary gid, this takes precedence @@ -253,10 +259,10 @@ async def test_get_service_specifications( await services_specifications_injector(user_group_service_specs) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs - assert service_specs == ServiceSpecifications.parse_obj( - user_group_service_specs.dict() + assert service_specs == ServiceSpecifications.model_validate( + user_group_service_specs.model_dump() ) @@ -328,11 +334,11 @@ async def test_get_service_specifications_are_passed_to_newer_versions_of_servic ) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs assert ( - service_specs - == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + service_specs.model_dump() + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS.model_dump() ) # check version between first index and second all return the specs of the first @@ -344,10 +350,10 @@ async def test_get_service_specifications_are_passed_to_newer_versions_of_servic ) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs - assert service_specs == ServiceSpecifications.parse_obj( - version_speced[0].dict() + assert service_specs == ServiceSpecifications.model_validate( + version_speced[0].model_dump() ), f"specifications for {version=} are not passed down from {sorted_versions[INDEX_FIRST_SERVICE_VERSION_WITH_SPEC]}" # check version from second to last use the second version @@ -357,10 +363,10 @@ async def test_get_service_specifications_are_passed_to_newer_versions_of_servic ) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs - assert service_specs == ServiceSpecifications.parse_obj( - version_speced[1].dict() + assert service_specs == ServiceSpecifications.model_validate( + version_speced[1].model_dump() ), f"specifications for {version=} are not passed down from {sorted_versions[INDEX_SECOND_SERVICE_VERSION_WITH_SPEC]}" # if we call with the strict parameter set to true, then we should only get the specs for the one that were specified @@ -370,7 +376,7 @@ async def test_get_service_specifications_are_passed_to_newer_versions_of_servic ) response = client.get(f"{url}") assert response.status_code == status.HTTP_200_OK - service_specs = ServiceSpecificationsGet.parse_obj(response.json()) + service_specs = ServiceSpecifications.model_validate(response.json()) assert service_specs if version in versions_with_specs: assert ( diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 3aeaaf4ef73..16fb6adb4cb 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -161,7 +161,7 @@ async def test_rpc_catalog_client( "description": "bar", "version_display": "this is a nice version", "description_ui": True, # owner activates wiki view - }, + }, # type: ignore ) assert updated.key == got.key @@ -392,7 +392,7 @@ async def test_rpc_get_service_access_rights( service_key=service_key, service_version=service_version, ) - assert updated_service.dict(include={"name", "description"}) == { + assert updated_service.model_dump(include={"name", "description"}) == { "name": "foo", "description": "bar", } diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_db_repositories.py index 3438492f740..8c4053c4ca6 100644 --- a/services/catalog/tests/unit/with_dbs/test_db_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_db_repositories.py @@ -10,7 +10,7 @@ from models_library.users import UserID from packaging import version from packaging.version import Version -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ( ServiceAccessRightsAtDB, @@ -109,16 +109,18 @@ async def test_create_services( ) # validation - service = ServiceMetaDataAtDB.parse_obj(fake_service) + service = ServiceMetaDataAtDB.model_validate(fake_service) service_access_rights = [ - ServiceAccessRightsAtDB.parse_obj(a) for a in fake_access_rights + ServiceAccessRightsAtDB.model_validate(a) for a in fake_access_rights ] new_service = await services_repo.create_or_update_service( service, service_access_rights ) - assert new_service.dict(include=set(fake_service.keys())) == service.dict() + assert ( + new_service.model_dump(include=set(fake_service.keys())) == service.model_dump() + ) async def test_read_services( @@ -177,7 +179,7 @@ async def test_read_services( assert service access_rights = await services_repo.get_service_access_rights( - product_name=target_product, **service.dict(include={"key", "version"}) + product_name=target_product, **service.model_dump(include={"key", "version"}) ) assert { user_gid, @@ -190,7 +192,7 @@ async def test_read_services( assert service access_rights = await services_repo.get_service_access_rights( - product_name=target_product, **service.dict(include={"key", "version"}) + product_name=target_product, **service.model_dump(include={"key", "version"}) ) assert {user_gid, team_gid} == {a.gid for a in access_rights} @@ -347,7 +349,9 @@ async def test_list_all_services_and_history_with_pagination( for service in services_items: assert len(service.history) == num_versions_per_service - assert parse_obj_as(EmailStr, service.owner_email), "resolved own'es email" + assert TypeAdapter(EmailStr).validate_python( + service.owner_email + ), "resolved own'es email" expected_latest_version = service.history[0].version # latest service is first assert service.version == expected_latest_version @@ -382,13 +386,13 @@ async def test_get_and_update_service_meta_data( assert got.version == service_version await services_repo.update_service( - ServiceMetaDataAtDB.construct( + ServiceMetaDataAtDB.model_construct( key=service_key, version=service_version, name="foo" ), ) updated = await services_repo.get_service(service_key, service_version) - assert got.copy(update={"name": "foo"}) == updated + assert got.model_copy(update={"name": "foo"}) == updated assert await services_repo.get_service(service_key, service_version) == updated diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py index d4506855f6d..47d0dc201ac 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py @@ -8,7 +8,7 @@ from models_library.groups import GroupAtDB from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished, ServiceVersion -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ServiceAccessRightsAtDB from simcore_service_catalog.services.access_rights import ( @@ -27,7 +27,7 @@ def test_reduce_access_rights(): - sample = ServiceAccessRightsAtDB.parse_obj( + sample = ServiceAccessRightsAtDB.model_validate( { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", @@ -41,20 +41,20 @@ def test_reduce_access_rights(): # fixture with overrides and with other products reduced = reduce_access_rights( [ - sample.copy(deep=True), - sample.copy(deep=True), - sample.copy(update={"execute_access": False}, deep=True), - sample.copy(update={"product_name": "s4l"}, deep=True), + sample.model_copy(deep=True), + sample.model_copy(deep=True), + sample.model_copy(update={"execute_access": False}, deep=True), + sample.model_copy(update={"product_name": "s4l"}, deep=True), ] ) # two products with the same flags assert len(reduced) == 2 - assert reduced[0].dict(include={"execute_access", "write_access"}) == { + assert reduced[0].model_dump(include={"execute_access", "write_access"}) == { "execute_access": True, "write_access": True, } - assert reduced[1].dict(include={"execute_access", "write_access"}) == { + assert reduced[1].model_dump(include={"execute_access", "write_access"}) == { "execute_access": True, "write_access": True, } @@ -62,8 +62,8 @@ def test_reduce_access_rights(): # two gids with the different falgs reduced = reduce_access_rights( [ - sample.copy(deep=True), - sample.copy( + sample.model_copy(deep=True), + sample.model_copy( update={"gid": 1, "execute_access": True, "write_access": False}, deep=True, ), @@ -71,11 +71,11 @@ def test_reduce_access_rights(): ) assert len(reduced) == 2 - assert reduced[0].dict(include={"execute_access", "write_access"}) == { + assert reduced[0].model_dump(include={"execute_access", "write_access"}) == { "execute_access": True, "write_access": True, } - assert reduced[1].dict(include={"execute_access", "write_access"}) == { + assert reduced[1].model_dump(include={"execute_access", "write_access"}) == { "execute_access": True, "write_access": False, } @@ -98,11 +98,11 @@ async def test_auto_upgrade_policy( return_value=False, ) # Avoids creating a users + user_to_group table - data = GroupAtDB.Config.schema_extra["example"] + data = GroupAtDB.model_config["json_schema_extra"]["example"] data["gid"] = everyone_gid mocker.patch( "simcore_service_catalog.services.access_rights.GroupsRepository.get_everyone_group", - return_value=GroupAtDB.parse_obj(data), + return_value=GroupAtDB.model_validate(data), ) mocker.patch( "simcore_service_catalog.services.access_rights.GroupsRepository.get_user_gid_from_email", @@ -111,10 +111,12 @@ async def test_auto_upgrade_policy( # SETUP --- MOST_UPDATED_EXAMPLE = -1 - new_service_metadata = ServiceMetaDataPublished.parse_obj( - ServiceMetaDataPublished.Config.schema_extra["examples"][MOST_UPDATED_EXAMPLE] + new_service_metadata = ServiceMetaDataPublished.model_validate( + ServiceMetaDataPublished.model_config["json_schema_extra"]["examples"][ + MOST_UPDATED_EXAMPLE + ] ) - new_service_metadata.version = parse_obj_as(ServiceVersion, "1.0.11") + new_service_metadata.version = TypeAdapter(ServiceVersion).validate_python("1.0.11") # we have three versions of the service in the database for which the sorting matters: (1.0.11 should inherit from 1.0.10 not 1.0.9) await services_db_tables_injector( @@ -167,7 +169,7 @@ async def test_auto_upgrade_policy( assert owner_gid == user_gid assert len(service_access_rights) == 1 assert {a.gid for a in service_access_rights} == {owner_gid} - assert service_access_rights[0].dict() == { + assert service_access_rights[0].model_dump() == { "key": new_service_metadata.key, "version": new_service_metadata.version, "gid": user_gid, From 2486b38b16217f7764d97b7649710f8bdbf31d09 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:07:35 +0100 Subject: [PATCH 26/53] =?UTF-8?q?=E2=99=BB=EF=B8=8FPydantic=20V2=20migrati?= =?UTF-8?q?on:=20various=20fixes=20(#6647)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/aws-library/tests/test_s3_client.py | 2 +- .../src/models_library/docker.py | 2 +- .../tests/test_utils_projects.py | 4 +- .../rabbitmq/rpc_interfaces/agent/volumes.py | 10 +++-- .../dynamic_sidecar/disk_usage.py | 16 +++++-- .../resource_usage_tracker/service_runs.py | 4 +- .../node_ports_common/aws_s3_cli_utils.py | 4 +- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 43 +++++++++++++++---- .../simcore_sdk/node_ports_v2/port_utils.py | 8 ++-- .../simcore-sdk/tests/integration/conftest.py | 4 +- .../test_node_data_data_manager.py | 19 +++++--- .../test_node_ports_common_aws_s3_cli.py | 32 ++++++++------ .../test_node_ports_common_filemanager.py | 39 +++++++++++------ .../test_node_ports_common_r_clone.py | 36 +++++++++------- .../test_node_ports_common_r_clone_utils.py | 4 +- .../dynamic-scheduler/requirements/_base.txt | 17 +++++++- .../services/service_tracker/_models.py | 2 +- .../core/rabbitmq.py | 4 +- .../core/utils.py | 4 +- .../modules/outputs/_manager.py | 1 + .../tests/unit/test_modules_notifier.py | 4 +- .../osparc_gateway_server/backend/models.py | 10 ++--- .../services/payments_gateway.py | 4 +- .../api/rpc/_resource_tracker.py | 3 +- .../resource_tracker_utils.py | 6 +-- .../services/resource_tracker_service_runs.py | 18 ++++---- ...i_resource_tracker_service_runs__export.py | 15 ++++--- 27 files changed, 199 insertions(+), 116 deletions(-) diff --git a/packages/aws-library/tests/test_s3_client.py b/packages/aws-library/tests/test_s3_client.py index 5e5efc962a5..bd853f51860 100644 --- a/packages/aws-library/tests/test_s3_client.py +++ b/packages/aws-library/tests/test_s3_client.py @@ -696,7 +696,7 @@ async def test_create_single_presigned_download_link( dest_file = tmp_path / faker.file_name() async with ClientSession() as session: - response = await session.get(download_url) + response = await session.get(f"{download_url}") response.raise_for_status() with dest_file.open("wb") as fp: fp.write(await response.read()) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index b16c4ae13cc..b8134b3ec73 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -75,7 +75,7 @@ def to_simcore_runtime_docker_label_key(key: str) -> DockerLabelKey: class StandardSimcoreDockerLabels(BaseModel): """ Represents the standard label on oSparc created containers (not yet services) - In order to create this object in code, please use construct() method! + In order to create this object in code, please use model_construct() method! """ user_id: UserID = Field(..., alias=f"{_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX}user-id") # type: ignore[literal-required] diff --git a/packages/postgres-database/tests/test_utils_projects.py b/packages/postgres-database/tests/test_utils_projects.py index 9af318fca38..c0c00d271e6 100644 --- a/packages/postgres-database/tests/test_utils_projects.py +++ b/packages/postgres-database/tests/test_utils_projects.py @@ -12,7 +12,7 @@ from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy from faker import Faker -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.models.projects import projects from simcore_postgres_database.utils_projects import ( DBProjectNotFoundError, @@ -69,7 +69,7 @@ async def test_get_project_trashed_at_column_can_be_converted_to_datetime( row = result.fetchone() - trashed_at = parse_obj_as(datetime | None, row.trashed_at) + trashed_at = TypeAdapter(datetime | None).validate_python(row.trashed_at) assert trashed_at == expected diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py index d414cd6b979..043898dcb30 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py @@ -4,7 +4,7 @@ from models_library.projects_nodes_io import NodeID from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RabbitMQRPCClient @@ -29,7 +29,9 @@ async def remove_volumes_without_backup_for_service( "swarm_stack_name": swarm_stack_name, } ), - parse_obj_as(RPCMethodName, "remove_volumes_without_backup_for_service"), + TypeAdapter(RPCMethodName).validate_python( + "remove_volumes_without_backup_for_service" + ), node_id=node_id, timeout_s=_REQUEST_TIMEOUT, ) @@ -51,7 +53,9 @@ async def backup_and_remove_volumes_for_all_services( "swarm_stack_name": swarm_stack_name, } ), - parse_obj_as(RPCMethodName, "backup_and_remove_volumes_for_all_services"), + TypeAdapter(RPCMethodName).validate_python( + "backup_and_remove_volumes_for_all_services" + ), timeout_s=_REQUEST_TIMEOUT, ) assert result is None # nosec diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py index 5938ad871ff..dbace2f1f4b 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py @@ -1,14 +1,20 @@ import logging +from typing import Final from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage from models_library.projects_nodes_io import NodeID from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace -from pydantic import parse_obj_as -from servicelib.logging_utils import log_decorator -from servicelib.rabbitmq import RabbitMQRPCClient +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient _logger = logging.getLogger(__name__) +_UPDATE_DISK_USAGE: Final[RPCMethodName] = TypeAdapter(RPCMethodName).validate_python( + "update_disk_usage" +) + @log_decorator(_logger, level=logging.DEBUG) async def update_disk_usage( @@ -21,6 +27,8 @@ async def update_disk_usage( {"service": "dy-sidecar", "node_id": f"{node_id}"} ) result = await rabbitmq_rpc_client.request( - rpc_namespace, parse_obj_as(RPCMethodName, "update_disk_usage"), usage=usage + rpc_namespace, + _UPDATE_DISK_USAGE, + usage=usage, ) assert result is None # nosec diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py index ad7b2fd908b..9d4bd57204c 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py @@ -1,5 +1,5 @@ import logging -from typing import Final, cast +from typing import Final from models_library.api_schemas_resource_usage_tracker import ( RESOURCE_USAGE_TRACKER_RPC_NAMESPACE, @@ -115,5 +115,5 @@ async def export_service_runs( filters=filters, timeout_s=_DEFAULT_TIMEOUT_S, ) - assert cast(AnyUrl, isinstance(result, AnyUrl)) # nosec + assert isinstance(result, AnyUrl) # nosec return result diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli_utils.py index 5cfbb536583..3c7a8bd9d60 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli_utils.py @@ -1,7 +1,7 @@ import logging import re -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.logging_utils import log_catch from servicelib.progress_bar import ProgressBarData @@ -35,5 +35,5 @@ async def __call__(self, logs: str) -> None: _logger.debug("received logs: %s", logs) with log_catch(_logger, reraise=False): if _size := _parse_size(logs): - _bytes = parse_obj_as(ByteSize, _size) + _bytes = TypeAdapter(ByteSize).validate_python(_size) await self.progress_bar.set_(_bytes) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 2ecf1422866..bc44698a593 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -1,6 +1,7 @@ import logging +import traceback from abc import ABC, abstractmethod -from asyncio import CancelledError +from asyncio import CancelledError, Task from collections.abc import Callable, Coroutine from pathlib import Path from typing import Any @@ -12,6 +13,7 @@ from models_library.services_types import ServicePortKey from models_library.users import UserID from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic_core import InitErrorDetails from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather from settings_library.aws_s3_cli import AwsS3CliSettings @@ -28,6 +30,28 @@ log = logging.getLogger(__name__) +# -> @GitHK this looks very dangerous, using a lot of protected stuff, just checking the number of ignores shows it's a bad idea... +def _format_error(task: Task) -> str: + # pylint:disable=protected-access + assert task._exception # nosec # noqa: SLF001 + error_list = traceback.format_exception( + type(task._exception), # noqa: SLF001 + task._exception, # noqa: SLF001 + task._exception.__traceback__, # noqa: SLF001 + ) + return "\n".join(error_list) + + +def _get_error_details(task: Task, port_key: str) -> InitErrorDetails: + # pylint:disable=protected-access + return InitErrorDetails( + type="value_error", + loc=(f"{port_key}",), + input=_format_error(task), + ctx={"error": task._exception}, # noqa: SLF001 + ) + + class OutputsCallbacks(ABC): @abstractmethod async def aborted(self, key: ServicePortKey) -> None: @@ -72,9 +96,9 @@ def __init__(self, **data: Any): # let's pass ourselves down for input_key in self.internal_inputs: - self.internal_inputs[input_key]._node_ports = self + self.internal_inputs[input_key]._node_ports = self # noqa: SLF001 for output_key in self.internal_outputs: - self.internal_outputs[output_key]._node_ports = self + self.internal_outputs[output_key]._node_ports = self # noqa: SLF001 @property async def inputs(self) -> InputsList: @@ -132,10 +156,11 @@ async def set( async def set_file_by_keymap(self, item_value: Path) -> None: for output in (await self.outputs).values(): - if is_file_type(output.property_type) and output.file_to_key_map: - if item_value.name in output.file_to_key_map: - await output.set(item_value) - return + if (is_file_type(output.property_type) and output.file_to_key_map) and ( + item_value.name in output.file_to_key_map + ): + await output.set(item_value) + return raise PortNotFound(msg=f"output port for item {item_value} not found") async def _node_ports_creator_cb(self, node_uuid: NodeIDStr) -> type["Nodeports"]: @@ -152,9 +177,9 @@ async def _auto_update_from_db(self) -> None: # let's pass ourselves down # pylint: disable=protected-access for input_key in self.internal_inputs: - self.internal_inputs[input_key]._node_ports = self + self.internal_inputs[input_key]._node_ports = self # noqa: SLF001 for output_key in self.internal_outputs: - self.internal_outputs[output_key]._node_ports = self + self.internal_outputs[output_key]._node_ports = self # noqa: SLF001 async def set_multiple( self, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 655c9576408..3c1462d6fab 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -8,8 +8,7 @@ from models_library.basic_types import IDStr, SHA256Str from models_library.services_types import FileName, ServicePortKey from models_library.users import UserID -from pydantic import AnyUrl, ByteSize -from pydantic.tools import parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.progress_bar import ProgressBarData from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.r_clone import RCloneSettings @@ -64,6 +63,7 @@ async def get_value_from_link( file_name = other_value.name # move the file to the right final location # if a file alias is present use it + if file_to_key_map: file_name = next(iter(file_to_key_map)) @@ -101,7 +101,7 @@ async def get_download_link_from_storage( # could raise ValidationError but will never do it since assert isinstance(link, URL) # nosec - url: AnyUrl = parse_obj_as(AnyUrl, f"{link}") + url: AnyUrl = TypeAdapter(AnyUrl).validate_python(f"{link}") return url @@ -123,7 +123,7 @@ async def get_download_link_from_storage_overload( s3_object=s3_object, link_type=link_type, ) - url: AnyUrl = parse_obj_as(AnyUrl, f"{link}") + url: AnyUrl = TypeAdapter(AnyUrl).validate_python(f"{link}") return url diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index 92b6afaa81b..552a629565a 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -17,7 +17,7 @@ from models_library.generics import Envelope from models_library.projects_nodes_io import LocationID, NodeIDStr, SimcoreS3FileID from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.faker_factories import random_project, random_user from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.r_clone import RCloneSettings, S3Provider @@ -94,7 +94,7 @@ def create_valid_file_uuid( ) -> Callable[[str, Path], SimcoreS3FileID]: def _create(key: str, file_path: Path) -> SimcoreS3FileID: clean_path = Path(f"{project_id}/{node_uuid}/{key}/{file_path.name}") - return parse_obj_as(SimcoreS3FileID, f"{clean_path}") + return TypeAdapter(SimcoreS3FileID).validate_python(f"{clean_path}") return _create diff --git a/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py b/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py index ca7a81e6c17..ed2033813e1 100644 --- a/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py +++ b/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py @@ -14,10 +14,11 @@ import pytest from faker import Faker +from models_library.basic_types import IDStr from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.progress_bar import ProgressBarData from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.r_clone import RCloneSettings @@ -157,7 +158,9 @@ async def test_valid_upload_download( mock_io_log_redirect_cb: LogRedirectCB, faker: Faker, ): - async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: + async with ProgressBarData( + num_steps=2, description=IDStr(faker.pystr()) + ) as progress_bar: await data_manager._push_directory( # noqa: SLF001 user_id=user_id, project_id=project_id, @@ -203,7 +206,9 @@ async def test_valid_upload_download_saved_to( mock_io_log_redirect_cb: LogRedirectCB, faker: Faker, ): - async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: + async with ProgressBarData( + num_steps=2, description=IDStr(faker.pystr()) + ) as progress_bar: await data_manager._push_directory( # noqa: SLF001 user_id=user_id, project_id=project_id, @@ -251,7 +256,9 @@ async def test_delete_legacy_archive( temp_dir: Path, faker: Faker, ): - async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: + async with ProgressBarData( + num_steps=2, description=IDStr(faker.pystr()) + ) as progress_bar: # NOTE: legacy archives can no longer be crated # generating a "legacy style archive" archive_into_dir = temp_dir / f"legacy-archive-dir-{uuid4()}" @@ -263,8 +270,8 @@ async def test_delete_legacy_archive( user_id=user_id, store_id=SIMCORE_LOCATION, store_name=None, - s3_object=parse_obj_as( - SimcoreS3FileID, f"{project_id}/{node_uuid}/{legacy_archive_name.name}" + s3_object=TypeAdapter(SimcoreS3FileID).validate_python( + f"{project_id}/{node_uuid}/{legacy_archive_name.name}" ), path_to_upload=legacy_archive_name, io_log_redirect_cb=None, diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py index b20f280e291..0c0c03b0363 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py @@ -14,8 +14,9 @@ import aiofiles import pytest from faker import Faker +from models_library.basic_types import IDStr from models_library.progress_bar import ProgressReport -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.file_utils import remove_directory from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather @@ -74,8 +75,7 @@ async def cleanup_bucket_after_test( # put to shared config def _fake_s3_link(aws_s3_cli_settings: AwsS3CliSettings, s3_object: str) -> AnyUrl: - return parse_obj_as( - AnyUrl, + return TypeAdapter(AnyUrl).validate_python( f"s3://{aws_s3_cli_settings.AWS_S3_CLI_S3.S3_BUCKET_NAME}/{urllib.parse.quote(s3_object)}", ) @@ -85,7 +85,7 @@ async def _create_random_binary_file( file_path: Path, file_size: ByteSize, # NOTE: bigger files get created faster with bigger chunk_size - chunk_size: int = parse_obj_as(ByteSize, "1mib"), + chunk_size: int = TypeAdapter(ByteSize).validate_python("1mib"), ): async with aiofiles.open(file_path, mode="wb") as file: bytes_written = 0 @@ -148,7 +148,7 @@ async def _report_progress_upload(report: ProgressReport) -> None: async with ProgressBarData( num_steps=1, progress_report_cb=_report_progress_upload, - description=faker.pystr(), + description=IDStr(faker.pystr()), ) as progress_bar: await aws_s3_cli.sync_local_to_s3( aws_s3_cli_settings, @@ -175,7 +175,7 @@ async def _report_progress_download(report: ProgressReport) -> None: async with ProgressBarData( num_steps=1, progress_report_cb=_report_progress_download, - description=faker.pystr(), + description=IDStr(faker.pystr()), ) as progress_bar: await aws_s3_cli.sync_s3_to_local( aws_s3_cli_settings, @@ -246,15 +246,21 @@ async def dir_downloaded_files_2(tmp_path: Path, faker: Faker) -> AsyncIterator[ @pytest.mark.parametrize( "file_count, file_size, check_progress", [ - (0, parse_obj_as(ByteSize, "0"), False), - (1, parse_obj_as(ByteSize, "1mib"), False), - (2, parse_obj_as(ByteSize, "1mib"), False), - (1, parse_obj_as(ByteSize, "1Gib"), True), + (0, TypeAdapter(ByteSize).validate_python("0"), False), + (1, TypeAdapter(ByteSize).validate_python("1mib"), False), + (2, TypeAdapter(ByteSize).validate_python("1mib"), False), + (1, TypeAdapter(ByteSize).validate_python("1Gib"), True), pytest.param( - 4, parse_obj_as(ByteSize, "500Mib"), True, marks=pytest.mark.heavy_load + 4, + TypeAdapter(ByteSize).validate_python("500Mib"), + True, + marks=pytest.mark.heavy_load, ), pytest.param( - 100, parse_obj_as(ByteSize, "20mib"), True, marks=pytest.mark.heavy_load + 100, + TypeAdapter(ByteSize).validate_python("20mib"), + True, + marks=pytest.mark.heavy_load, ), ], ) @@ -372,7 +378,7 @@ async def test_overwrite_an_existing_file_and_sync_again( generated_file_names: set[str] = await _create_files_in_dir( dir_locally_created_files, 3, - parse_obj_as(ByteSize, "1kib"), + TypeAdapter(ByteSize).validate_python("1kib"), ) assert len(generated_file_names) > 0 diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py index 56f696bb46d..2e435d68a18 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py @@ -13,13 +13,14 @@ import pytest from aiohttp import ClientError from faker import Faker +from models_library.basic_types import IDStr from models_library.projects_nodes_io import ( LocationID, SimcoreS3DirectoryID, SimcoreS3FileID, ) from models_library.users import UserID -from pydantic import BaseModel, ByteSize, parse_obj_as +from pydantic import BaseModel, ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.parametrizations import byte_size_ids from servicelib.progress_bar import ProgressBarData @@ -70,7 +71,9 @@ def optional_sync_settings( def _file_size(size_str: str, **pytest_params): - return pytest.param(parse_obj_as(ByteSize, size_str), id=size_str, **pytest_params) + return pytest.param( + TypeAdapter(ByteSize).validate_python(size_str), id=size_str, **pytest_params + ) @pytest.mark.parametrize( @@ -99,7 +102,9 @@ async def test_valid_upload_download( file_path = create_file_of_size(file_size, "test.test") file_id = create_valid_file_uuid("", file_path) - async with ProgressBarData(num_steps=2, description=faker.pystr()) as progress_bar: + async with ProgressBarData( + num_steps=2, description=IDStr(faker.pystr()) + ) as progress_bar: upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( user_id=user_id, store_id=s3_simcore_location, @@ -187,7 +192,9 @@ async def test_valid_upload_download_using_file_object( assert file_metadata.etag == e_tag download_folder = Path(tmpdir) / "downloads" - async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: + async with ProgressBarData( + num_steps=1, description=IDStr(faker.pystr()) + ) as progress_bar: download_file_path = await filemanager.download_path_from_s3( user_id=user_id, store_id=s3_simcore_location, @@ -356,9 +363,9 @@ async def test_invalid_file_path( ) download_folder = Path(tmpdir) / "downloads" - with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 + with pytest.raises(exceptions.S3InvalidPathError): async with ProgressBarData( - num_steps=1, description=faker.pystr() + num_steps=1, description=IDStr(faker.pystr()) ) as progress_bar: await filemanager.download_path_from_s3( user_id=user_id, @@ -412,7 +419,7 @@ async def test_errors_upon_invalid_file_identifiers( download_folder = Path(tmpdir) / "downloads" with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 async with ProgressBarData( - num_steps=1, description=faker.pystr() + num_steps=1, description=IDStr(faker.pystr()) ) as progress_bar: invalid_s3_path = SimcoreS3FileID("") await filemanager.download_path_from_s3( @@ -427,15 +434,17 @@ async def test_errors_upon_invalid_file_identifiers( aws_s3_cli_settings=optional_sync_settings.aws_s3_cli_settings, ) - with pytest.raises(exceptions.S3InvalidPathError): # noqa: PT012 + with pytest.raises(exceptions.S3InvalidPathError): async with ProgressBarData( - num_steps=1, description=faker.pystr() + num_steps=1, description=IDStr(faker.pystr()) ) as progress_bar: await filemanager.download_path_from_s3( user_id=user_id, store_id=store, store_name=None, - s3_object=SimcoreS3FileID(f"{project_id}/{uuid4()}/invisible.txt"), + s3_object=TypeAdapter(SimcoreS3FileID).validate_python( + f"{project_id}/{uuid4()}/invisible.txt" + ), local_path=download_folder, io_log_redirect_cb=None, r_clone_settings=optional_sync_settings.r_clone_settings, @@ -469,9 +478,9 @@ async def test_invalid_store( ) download_folder = Path(tmpdir) / "downloads" - with pytest.raises(exceptions.S3InvalidStore): # noqa: PT012 + with pytest.raises(exceptions.S3InvalidStore): async with ProgressBarData( - num_steps=1, description=faker.pystr() + num_steps=1, description=IDStr(faker.pystr()) ) as progress_bar: await filemanager.download_path_from_s3( user_id=user_id, @@ -662,7 +671,7 @@ async def test_upload_path_source_is_a_folder( directory_id = SimcoreS3DirectoryID.from_simcore_s3_object( f"{project_id}/{faker.uuid4()}/some-dir-in-node-root/" ) - s3_object = SimcoreS3FileID(directory_id) + s3_object = TypeAdapter(SimcoreS3FileID).validate_python(directory_id) upload_result: UploadedFolder | UploadedFile = await filemanager.upload_path( user_id=user_id, @@ -677,7 +686,9 @@ async def test_upload_path_source_is_a_folder( assert isinstance(upload_result, UploadedFolder) assert source_dir.exists() - async with ProgressBarData(num_steps=1, description=faker.pystr()) as progress_bar: + async with ProgressBarData( + num_steps=1, description=IDStr(faker.pystr()) + ) as progress_bar: await filemanager.download_path_from_s3( user_id=user_id, store_name=None, diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py index 64e7d34ef1f..5d728aad51d 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py @@ -15,8 +15,9 @@ import aiofiles import pytest from faker import Faker +from models_library.basic_types import IDStr from models_library.progress_bar import ProgressReport -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.file_utils import remove_directory from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather @@ -74,9 +75,8 @@ async def cleanup_bucket_after_test( def _fake_s3_link(r_clone_settings: RCloneSettings, s3_object: str) -> AnyUrl: - return parse_obj_as( - AnyUrl, - f"s3://{r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME}/{urllib.parse.quote(s3_object)}", + return TypeAdapter(AnyUrl).validate_python( + f"s3://{r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME}/{urllib.parse.quote(s3_object)}" ) @@ -99,7 +99,7 @@ async def _create_random_binary_file( file_path: Path, file_size: ByteSize, # NOTE: bigger files get created faster with bigger chunk_size - chunk_size: int = parse_obj_as(ByteSize, "1mib"), + chunk_size: int = TypeAdapter(ByteSize).validate_python("1mib"), ): async with aiofiles.open(file_path, mode="wb") as file: bytes_written = 0 @@ -160,7 +160,7 @@ async def _report_progress_upload(report: ProgressReport) -> None: async with ProgressBarData( num_steps=1, progress_report_cb=_report_progress_upload, - description=faker.pystr(), + description=IDStr(faker.pystr()), ) as progress_bar: await r_clone.sync_local_to_s3( r_clone_settings, @@ -187,13 +187,13 @@ async def _report_progress_download(report: ProgressReport) -> None: async with ProgressBarData( num_steps=1, progress_report_cb=_report_progress_download, - description=faker.pystr(), + description=IDStr(faker.pystr()), ) as progress_bar: await r_clone.sync_s3_to_local( r_clone_settings, progress_bar, local_directory_path=destination_dir, - download_s3_link=s3_directory_link, + download_s3_link=f"{s3_directory_link}", debug_logs=True, ) @@ -258,15 +258,21 @@ async def dir_downloaded_files_2(tmp_path: Path, faker: Faker) -> AsyncIterator[ @pytest.mark.parametrize( "file_count, file_size, check_progress", [ - (0, parse_obj_as(ByteSize, "0"), False), - (1, parse_obj_as(ByteSize, "1mib"), False), - (2, parse_obj_as(ByteSize, "1mib"), False), - (1, parse_obj_as(ByteSize, "1Gib"), True), + (0, TypeAdapter(ByteSize).validate_python("0"), False), + (1, TypeAdapter(ByteSize).validate_python("1mib"), False), + (2, TypeAdapter(ByteSize).validate_python("1mib"), False), + (1, TypeAdapter(ByteSize).validate_python("1Gib"), True), pytest.param( - 4, parse_obj_as(ByteSize, "500Mib"), True, marks=pytest.mark.heavy_load + 4, + TypeAdapter(ByteSize).validate_python("500Mib"), + True, + marks=pytest.mark.heavy_load, ), pytest.param( - 100, parse_obj_as(ByteSize, "20mib"), True, marks=pytest.mark.heavy_load + 100, + TypeAdapter(ByteSize).validate_python("20mib"), + True, + marks=pytest.mark.heavy_load, ), ], ) @@ -384,7 +390,7 @@ async def test_overwrite_an_existing_file_and_sync_again( generated_file_names: set[str] = await _create_files_in_dir( dir_locally_created_files, r_clone_settings.R_CLONE_OPTION_TRANSFERS * 3, - parse_obj_as(ByteSize, "1kib"), + TypeAdapter(ByteSize).validate_python("1kib"), ) assert len(generated_file_names) > 0 diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_common_r_clone_utils.py b/packages/simcore-sdk/tests/unit/test_node_ports_common_r_clone_utils.py index 13d2bed6042..e2d9b890ba5 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_common_r_clone_utils.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_common_r_clone_utils.py @@ -2,7 +2,7 @@ from unittest.mock import AsyncMock import pytest -from pydantic import parse_raw_as +from pydantic import TypeAdapter from simcore_sdk.node_ports_common.r_clone_utils import ( SyncProgressLogParser, _RCloneSyncMessageBase, @@ -68,7 +68,7 @@ ], ) async def test_rclone_stbc_message_parsing_regression(log_message: str, expected: type): - parsed_log = parse_raw_as(_RCloneSyncMessages, log_message) # type: ignore[arg-type] + parsed_log = TypeAdapter(_RCloneSyncMessages).validate_json(log_message) assert isinstance(parsed_log, expected) progress_log_parser = SyncProgressLogParser(AsyncMock()) diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 7493081203d..4487b628b11 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -102,7 +102,6 @@ fastapi==0.115.2 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 @@ -255,8 +254,15 @@ orjson==3.10.7 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 @@ -314,8 +320,15 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in pydantic-settings==2.5.2 # via # -r requirements/../../../packages/models-library/requirements/_base.in @@ -462,7 +475,7 @@ typing-extensions==4.12.2 # typer u-msgpack-python==2.8.0 # via -r requirements/_base.in -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py index 6b1b5b1a75d..70e063462bc 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py @@ -89,7 +89,7 @@ def __setattr__(self, name, value): last_state_change: float = Field( default_factory=lambda: arrow.utcnow().timestamp(), - metadata={"description": "keeps track when the current_state was last updated"}, + description="keeps track when the current_state was last updated", ) ############################# diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/rabbitmq.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/rabbitmq.py index f33a43d33fb..88c77c84997 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/rabbitmq.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/rabbitmq.py @@ -46,7 +46,7 @@ async def post_log_message( app: FastAPI, log: LogMessageStr, *, log_level: LogLevelInt ) -> None: app_settings: ApplicationSettings = app.state.settings - message = LoggerRabbitMessage.construct( + message = LoggerRabbitMessage.model_construct( node_id=app_settings.DY_SIDECAR_NODE_ID, user_id=app_settings.DY_SIDECAR_USER_ID, project_id=app_settings.DY_SIDECAR_PROJECT_ID, @@ -61,7 +61,7 @@ async def post_progress_message( app: FastAPI, progress_type: ProgressType, report: ProgressReport ) -> None: app_settings: ApplicationSettings = app.state.settings - message = ProgressRabbitMessageNode.construct( + message = ProgressRabbitMessageNode.model_construct( node_id=app_settings.DY_SIDECAR_NODE_ID, user_id=app_settings.DY_SIDECAR_USER_ID, project_id=app_settings.DY_SIDECAR_PROJECT_ID, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py index 697dc673b8e..4e6f9ee0df5 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py @@ -7,7 +7,7 @@ from typing import NamedTuple import psutil -from models_library.error_codes import create_error_code +from common_library.error_codes import create_error_code from servicelib.logging_errors import create_troubleshotting_log_kwargs from ..modules.mounted_fs import MountedVolumes @@ -74,7 +74,7 @@ async def async_command( try: stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) - except asyncio.TimeoutError: + except TimeoutError: proc.terminate() _close_transport(proc) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py index fc33fe660ac..adf046dd468 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_manager.py @@ -6,6 +6,7 @@ from datetime import timedelta from functools import partial +from common_library.errors_classes import OsparcErrorMixin from fastapi import FastAPI from models_library.basic_types import IDStr from models_library.rabbitmq_messages import ProgressType diff --git a/services/dynamic-sidecar/tests/unit/test_modules_notifier.py b/services/dynamic-sidecar/tests/unit/test_modules_notifier.py index 51855ffd20f..380e6fa639c 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_notifier.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_notifier.py @@ -131,7 +131,7 @@ async def _assert_call_count(mock: AsyncMock, *, call_count: int) -> None: def _get_mocked_disk_usage(byte_size_str: str) -> DiskUsage: return DiskUsage( - total=ByteSize.validate(byte_size_str), + total=TypeAdapter(ByteSize).validate_python(byte_size_str), used=ByteSize(0), free=TypeAdapter(ByteSize).validate_python(byte_size_str), used_percent=0, @@ -222,7 +222,7 @@ async def test_notifier_publish_disk_usage( @pytest.fixture def port_key() -> ServicePortKey: - return ServicePortKey("test_port") + return TypeAdapter(ServicePortKey).validate_python("test_port") def _get_on_input_port_spy( diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py index 43e2240270d..9cdd2fc9edb 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py @@ -1,7 +1,8 @@ +from collections.abc import Mapping from ipaddress import IPv4Address -from typing import Any, Mapping, Union +from typing import Any, Union -from pydantic import BaseModel, ByteSize, Field, PositiveFloat, parse_obj_as +from pydantic import BaseModel, ByteSize, Field, PositiveFloat, TypeAdapter Hostname = str ResourceName = str @@ -26,8 +27,7 @@ class NodeInformation(BaseModel): def cluster_information_from_docker_nodes( nodes_list: list[Mapping[str, Any]] ) -> ClusterInformation: - return parse_obj_as( - ClusterInformation, + return TypeAdapter(ClusterInformation).validate_python( { node["Description"]["Hostname"]: { "docker_node_id": node["ID"], @@ -38,5 +38,5 @@ def cluster_information_from_docker_nodes( }, } for node in nodes_list - }, + } ) diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index a1ac9d0b061..f7989f6fa56 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -25,9 +25,6 @@ HealthMixinMixin, ) from servicelib.fastapi.httpx_utils import to_curl_command -from simcore_service_payments.models.schemas.acknowledgements import ( - AckPaymentWithPaymentMethod, -) from ..core.settings import ApplicationSettings from ..models.payments_gateway import ( @@ -41,6 +38,7 @@ PaymentMethodInitiated, PaymentMethodsBatch, ) +from ..models.schemas.acknowledgements import AckPaymentWithPaymentMethod _logger = logging.getLogger(__name__) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py index 9da69bad6f3..cae70b1152c 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py @@ -24,6 +24,7 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID from models_library.wallets import WalletID +from pydantic import AnyUrl from servicelib.rabbitmq import RPCRouter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, @@ -78,7 +79,7 @@ async def export_service_runs( access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, filters: ServiceResourceUsagesFilters | None = None, -) -> str: +) -> AnyUrl: app_settings: ApplicationSettings = app.state.settings s3_settings = app_settings.RESOURCE_USAGE_TRACKER_S3 assert s3_settings # nosec diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py index 4466fc5e7de..73aa7416244 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py @@ -1,6 +1,6 @@ import asyncio import logging -from datetime import datetime, timezone +from datetime import UTC, datetime from decimal import Decimal from models_library.api_schemas_resource_usage_tracker.credit_transactions import ( @@ -41,9 +41,9 @@ async def sum_credit_transactions_and_publish_to_rabbitmq( wallet_id, ) ) - publish_message = WalletCreditsMessage.construct( + publish_message = WalletCreditsMessage.model_construct( wallet_id=wallet_id, - created_at=datetime.now(tz=timezone.utc), + created_at=datetime.now(tz=UTC), credits=wallet_total_credits.available_osparc_credits, product_name=product_name, ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index e5145f8f6bd..31171ea4104 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta, timezone import shortuuid from aws_library.s3 import SimcoreS3API @@ -18,7 +18,7 @@ from models_library.rest_ordering import OrderBy from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import PositiveInt, TypeAdapter +from pydantic import AnyUrl, PositiveInt, TypeAdapter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, ) @@ -116,7 +116,7 @@ async def list_service_runs( service_runs_api_model: list[ServiceRunGet] = [] for service in service_runs_db_model: service_runs_api_model.append( - ServiceRunGet.construct( + ServiceRunGet.model_construct( service_run_id=service.service_run_id, wallet_id=service.wallet_id, wallet_name=service.wallet_name, @@ -144,6 +144,7 @@ async def list_service_runs( async def export_service_runs( s3_client: SimcoreS3API, + *, bucket_name: str, s3_region: str, user_id: UserID, @@ -153,7 +154,7 @@ async def export_service_runs( access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, filters: ServiceResourceUsagesFilters | None = None, -) -> str: +) -> AnyUrl: started_from = filters.started_at.from_ if filters else None started_until = filters.started_at.until if filters else None @@ -161,7 +162,9 @@ async def export_service_runs( s3_bucket_name = TypeAdapter(S3BucketName).validate_python(bucket_name) # NOTE: su stands for "service usage" file_name = f"su_{shortuuid.uuid()}.csv" - s3_object_key = f"resource-usage-tracker-service-runs/{datetime.now(tz=timezone.utc).date()}/{file_name}" + s3_object_key = ( + f"resource-usage-tracker-service-runs/{datetime.now(tz=UTC).date()}/{file_name}" + ) # Export CSV to S3 await resource_tracker_repo.export_service_runs_table_to_s3( @@ -177,12 +180,11 @@ async def export_service_runs( ) # Create presigned S3 link - generated_url = await s3_client.create_single_presigned_download_link( + return await s3_client.create_single_presigned_download_link( bucket=s3_bucket_name, object_key=s3_object_key, expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC, ) - return f"{generated_url}" async def get_osparc_credits_aggregated_usages_page( @@ -216,7 +218,7 @@ async def get_osparc_credits_aggregated_usages_page( output_api_model: list[OsparcCreditsAggregatedByServiceGet] = [] for item in output_list_db: output_api_model.append( - OsparcCreditsAggregatedByServiceGet.construct( + OsparcCreditsAggregatedByServiceGet.model_construct( osparc_credits=item.osparc_credits, service_key=item.service_key, running_time_in_hours=item.running_time_in_hours, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py index e433d8c13e6..0cb3db3733e 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py @@ -1,3 +1,8 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments + import os from unittest.mock import Mock @@ -25,25 +30,21 @@ @pytest.fixture async def mocked_export(mocker: MockerFixture): - mock_export = mocker.patch( + return mocker.patch( "simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.ResourceTrackerRepository.export_service_runs_table_to_s3", autospec=True, ) - return mock_export - @pytest.fixture async def mocked_presigned_link(mocker: MockerFixture): - mock_presigned_link = mocker.patch( + return mocker.patch( "simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.SimcoreS3API.create_single_presigned_download_link", return_value=TypeAdapter(AnyUrl).validate_python( "https://www.testing.com/", ), ) - return mock_presigned_link - @pytest.fixture async def enable_resource_usage_tracker_s3( @@ -76,6 +77,6 @@ async def test_rpc_list_service_runs_which_was_billed( user_id=_USER_ID, product_name="osparc", ) - assert isinstance(download_url, AnyUrl) + assert isinstance(download_url, AnyUrl) # nosec assert mocked_export.called assert mocked_presigned_link.called From 24218c1f2aa42b4ee4664ce6af552b607967435a Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 1 Nov 2024 13:34:22 +0100 Subject: [PATCH 27/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Pydantic=20V2:=20aft?= =?UTF-8?q?er=20upstream=20merge=20->=20fixes=20(#6651)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent/src/simcore_service_agent/core/settings.py | 4 +++- .../src/simcore_service_api_server/core/settings.py | 4 +++- .../src/simcore_service_autoscaling/core/settings.py | 4 +++- .../src/simcore_service_catalog/core/settings.py | 4 +++- .../simcore_service_clusters_keeper/core/settings.py | 4 +++- .../src/simcore_service_dask_sidecar/settings.py | 2 +- .../simcore_service_datcore_adapter/core/settings.py | 4 +++- .../src/simcore_service_director_v2/core/settings.py | 12 +++++++----- .../core/settings.py | 4 +++- .../simcore_service_dynamic_sidecar/core/settings.py | 4 +++- .../simcore_service_efs_guardian/core/settings.py | 4 +++- .../src/simcore_service_invitations/core/settings.py | 4 +++- .../src/simcore_service_payments/core/settings.py | 4 +++- .../core/settings.py | 4 +++- .../storage/src/simcore_service_storage/settings.py | 4 +++- .../application_settings.py | 4 +++- 16 files changed, 50 insertions(+), 20 deletions(-) diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index bb7e8765188..53cdb199f5e 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -35,7 +35,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) AGENT_VOLUMES_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["AGENT_VOLUMES_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "AGENT_VOLUMES_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field( diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index d452b82c5e9..b492ec79eff 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -70,7 +70,9 @@ class BasicSettings(BaseCustomSettings, MixinLoggingSettings): ) API_SERVER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["API_SERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "API_SERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index cca295a0c91..51e7a06e7d5 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -248,7 +248,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) AUTOSCALING_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["AUTOSCALING_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "AUTOSCALING_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index 4aad3157427..da8a71fca36 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -58,7 +58,9 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ) CATALOG_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["CATALOG_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "CATALOG_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) CATALOG_DEV_FEATURES_ENABLED: bool = Field( diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index b7bf1de9245..6595827ea6b 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -283,7 +283,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): LoggerName, list[MessageSubstring] ] = Field( default_factory=dict, - env=["CLUSTERS_KEEPER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "CLUSTERS_KEEPER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py index aae311e54c7..75ac1fb6cb8 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py @@ -48,7 +48,7 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): ) DASK_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["DASK_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices("DASK_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py index 4bf1cd5a3a1..e2b2bc8791b 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py @@ -49,7 +49,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): LoggerName, list[MessageSubstring] ] = Field( default_factory=dict, - env=["DATCORE_ADAPTER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "DATCORE_ADAPTER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) DATCORE_ADAPTER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 9bd5201055b..717694f7f25 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -20,7 +20,7 @@ ClusterTypeInModel, NoAuthentication, ) -from pydantic import AnyHttpUrl, AnyUrl, Field, NonNegativeInt, validator +from pydantic import AliasChoices, AnyHttpUrl, AnyUrl, Field, NonNegativeInt, validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings @@ -123,19 +123,21 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): LOG_LEVEL: LogLevel = Field( LogLevel.INFO.value, - env=["DIRECTOR_V2_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices("DIRECTOR_V2_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DIRECTOR_V2_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["DIRECTOR_V2_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "DIRECTOR_V2_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) DIRECTOR_V2_DEV_FEATURES_ENABLED: bool = False diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index ea475e0aec0..d1471052a9f 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -41,7 +41,9 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): LoggerName, list[MessageSubstring] ] = Field( default_factory=dict, - env=["DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index b45ea1118fa..e78abbfeba4 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -154,7 +154,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) DY_SIDECAR_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default={}, - env=["DY_SIDECAR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "DY_SIDECAR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) DY_SIDECAR_USER_ID: UserID diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index bd2e711d6ac..39e0b0aa813 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -96,7 +96,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) EFS_GUARDIAN_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["EFS_GUARDIAN_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "EFS_GUARDIAN_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/invitations/src/simcore_service_invitations/core/settings.py b/services/invitations/src/simcore_service_invitations/core/settings.py index 20b04cd830e..6d9b1ec3a25 100644 --- a/services/invitations/src/simcore_service_invitations/core/settings.py +++ b/services/invitations/src/simcore_service_invitations/core/settings.py @@ -60,7 +60,9 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) INVITATIONS_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["INVITATIONS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "INVITATIONS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index f6af9d6a8fc..7d33beff83f 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -48,7 +48,9 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ) PAYMENTS_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["PAYMENTS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "PAYMENTS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index d321f733c3c..3a534b692dc 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -69,7 +69,9 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): LoggerName, list[MessageSubstring] ] = Field( default_factory=dict, - env=["RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py index 80ea8634cfb..1931bdb79c2 100644 --- a/services/storage/src/simcore_service_storage/settings.py +++ b/services/storage/src/simcore_service_storage/settings.py @@ -90,7 +90,9 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): ) STORAGE_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 9fa0ac5de7c..a105dcdcc1a 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -117,7 +117,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) WEBSERVER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["WEBSERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "WEBSERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) # TODO: find a better name!? From b01c4a79b09a4b036c3233bd44f72eb738c8dbb7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:23:51 +0100 Subject: [PATCH 28/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Fix=20custom=20setti?= =?UTF-8?q?ngs=20class=20with=20`pydantic-settings~=3D2.6`=20(#6644)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/aws-library/requirements/_base.txt | 62 +++- .../common-library/requirements/_test.txt | 2 +- .../requirements/_base.txt | 29 +- .../models-library/requirements/_base.txt | 10 +- .../requirements/_base.txt | 37 +- .../postgres-database/requirements/_base.txt | 14 + .../src/pytest_simcore/environment_configs.py | 1 + .../requirements/_base.txt | 23 +- .../service-library/requirements/_aiohttp.txt | 15 + .../service-library/requirements/_base.txt | 35 +- .../service-library/requirements/_fastapi.txt | 12 + .../settings-library/requirements/_base.txt | 2 +- .../src/settings_library/base.py | 76 +++- packages/settings-library/tests/test_base.py | 18 +- .../tests/test_base_w_postgres.py | 19 +- packages/simcore-sdk/requirements/_base.txt | 86 ++++- services/agent/requirements/_base.txt | 15 +- services/api-server/requirements/_base.txt | 31 +- services/autoscaling/requirements/_base.txt | 158 ++++++++- services/catalog/requirements/_base.txt | 16 +- .../simcore_service_catalog/core/settings.py | 1 + .../clusters-keeper/requirements/_base.txt | 158 ++++++++- services/dask-sidecar/requirements/_base.txt | 20 +- .../datcore-adapter/requirements/_base.txt | 14 +- services/director-v2/requirements/_base.txt | 331 ++++++++++++++++-- .../dynamic-scheduler/requirements/_base.txt | 2 +- .../dynamic-sidecar/requirements/_base.txt | 30 +- services/efs-guardian/requirements/_base.txt | 185 +++++++++- services/efs-guardian/requirements/_test.txt | 1 + services/invitations/requirements/_base.txt | 80 ++++- services/payments/requirements/_base.txt | 16 +- .../requirements/_base.txt | 184 +++++++++- .../requirements/_test.txt | 4 + services/storage/requirements/_base.txt | 28 +- tests/public-api/requirements/_base.txt | 12 +- tests/swarm-deploy/requirements/_test.txt | 26 +- 36 files changed, 1680 insertions(+), 73 deletions(-) diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 96524ffbfff..c09c18f30e3 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -20,10 +20,16 @@ aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore @@ -62,10 +68,16 @@ botocore-stubs==1.35.25 # via types-aiobotocore certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -164,14 +176,26 @@ opentelemetry-util-http==0.48b0 # via opentelemetry-instrumentation-requests orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in pamqp==3.3.0 # via aiormq protobuf==4.25.5 @@ -182,16 +206,28 @@ psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in pydantic==2.9.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends @@ -201,9 +237,15 @@ pydantic-core==2.23.3 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -221,19 +263,31 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -303,10 +357,16 @@ typing-extensions==4.12.2 # types-aiobotocore-ssm urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/packages/common-library/requirements/_test.txt b/packages/common-library/requirements/_test.txt index 89b9a19eca6..abf7c70b23c 100644 --- a/packages/common-library/requirements/_test.txt +++ b/packages/common-library/requirements/_test.txt @@ -29,7 +29,7 @@ pydantic-core==2.23.4 # via # -c requirements/_base.txt # pydantic -pydantic-settings==2.5.2 +pydantic-settings==2.6.1 # via -r requirements/_test.in pytest==8.3.3 # via diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt index eb111f70fb1..2fa04d4835b 100644 --- a/packages/dask-task-models-library/requirements/_base.txt +++ b/packages/dask-task-models-library/requirements/_base.txt @@ -33,7 +33,10 @@ importlib-metadata==8.5.0 # via dask jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # distributed @@ -55,10 +58,16 @@ msgpack==1.1.0 # via distributed orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via # dask @@ -69,10 +78,16 @@ psutil==6.0.0 # via distributed pydantic==2.9.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # pydantic-extra-types @@ -80,8 +95,12 @@ pydantic==2.9.1 pydantic-core==2.23.3 # via pydantic pydantic-extra-types==2.9.0 - # via -r requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -93,7 +112,10 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # dask @@ -136,7 +158,10 @@ typing-extensions==4.12.2 # typer urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # distributed diff --git a/packages/models-library/requirements/_base.txt b/packages/models-library/requirements/_base.txt index e80a9bc5fd9..b4bda5f971d 100644 --- a/packages/models-library/requirements/_base.txt +++ b/packages/models-library/requirements/_base.txt @@ -18,19 +18,25 @@ jsonschema-specifications==2023.12.1 # via jsonschema orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in # pydantic-extra-types # pydantic-settings pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 - # via -r requirements/_base.in -pydantic-settings==2.4.0 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/_base.in +pydantic-settings==2.6.1 # via -r requirements/_base.in python-dateutil==2.9.0.post0 # via arrow diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt index 08807d95984..f8088ba40d6 100644 --- a/packages/notifications-library/requirements/_base.txt +++ b/packages/notifications-library/requirements/_base.txt @@ -30,8 +30,12 @@ idna==3.10 # yarl jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -41,8 +45,12 @@ jsonschema-specifications==2023.12.1 # via jsonschema mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -58,29 +66,50 @@ multidict==6.1.0 # via yarl orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy pydantic==2.9.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # pydantic-extra-types # pydantic-settings pydantic-core==2.23.3 # via pydantic pydantic-extra-types==2.9.0 - # via -r requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -108,8 +137,12 @@ six==1.16.0 # via python-dateutil sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index bf02ea152a8..40aa6138e86 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -12,22 +12,34 @@ idna==3.10 # via yarl mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic markupsafe==2.1.5 # via mako multidict==6.1.0 # via yarl +orjson==3.10.11 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy pydantic==2.9.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in + # pydantic-extra-types pydantic-core==2.23.3 # via pydantic +pydantic-extra-types==2.10.0 + # via -r requirements/../../../packages/common-library/requirements/_base.in sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in # alembic @@ -35,5 +47,7 @@ typing-extensions==4.12.2 # via # alembic # pydantic + # pydantic-core + # pydantic-extra-types yarl==1.12.1 # via -r requirements/_base.in diff --git a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py index 33a38db4c7e..52d245c6210 100644 --- a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py +++ b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py @@ -38,6 +38,7 @@ def external_envfile_dict(request: pytest.FixtureRequest) -> EnvVarsDict: print("🚨 EXTERNAL `envfile` option detected. Loading", envfile, "...") assert isinstance(envfile, Path) + assert envfile.exists() assert envfile.is_file() envs = load_dotenv(envfile) diff --git a/packages/service-integration/requirements/_base.txt b/packages/service-integration/requirements/_base.txt index a465920e7ba..810abab5e48 100644 --- a/packages/service-integration/requirements/_base.txt +++ b/packages/service-integration/requirements/_base.txt @@ -13,6 +13,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -42,6 +44,8 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # cookiecutter @@ -64,8 +68,12 @@ multidict==6.1.0 # via yarl orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in packaging==24.1 # via pytest @@ -73,16 +81,23 @@ pluggy==1.5.0 # via pytest pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # pydantic-extra-types # pydantic-settings pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 - # via -r requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.6.1 # via -r requirements/../../../packages/models-library/requirements/_base.in pygments==2.18.0 # via rich @@ -96,6 +111,8 @@ python-slugify==8.0.4 # via cookiecutter pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -133,6 +150,8 @@ typing-extensions==4.12.2 # typer urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # docker diff --git a/packages/service-library/requirements/_aiohttp.txt b/packages/service-library/requirements/_aiohttp.txt index 1536a562252..d3e27ed894b 100644 --- a/packages/service-library/requirements/_aiohttp.txt +++ b/packages/service-library/requirements/_aiohttp.txt @@ -2,7 +2,10 @@ aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_aiohttp.in @@ -20,7 +23,10 @@ attrs==24.2.0 # referencing certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -119,7 +125,10 @@ psycopg2-binary==2.9.9 # sqlalchemy pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # jsonschema-path @@ -145,13 +154,19 @@ six==1.16.0 # rfc3339-validator sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiopg urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index 7118a1dba61..a0fea278ddb 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -12,7 +12,10 @@ aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -37,7 +40,10 @@ attrs==24.2.0 # referencing certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -130,10 +136,16 @@ opentelemetry-util-http==0.48b0 # via opentelemetry-instrumentation-requests orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in pamqp==3.3.0 # via aiormq protobuf==4.25.5 @@ -144,10 +156,16 @@ psutil==6.0.0 # via -r requirements/_base.in pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends @@ -156,8 +174,12 @@ pydantic==2.9.2 pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 - # via -r requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -171,13 +193,19 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -226,7 +254,10 @@ typing-extensions==4.12.2 # typer urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests diff --git a/packages/service-library/requirements/_fastapi.txt b/packages/service-library/requirements/_fastapi.txt index cc0209d5676..42dd6b7a550 100644 --- a/packages/service-library/requirements/_fastapi.txt +++ b/packages/service-library/requirements/_fastapi.txt @@ -8,7 +8,10 @@ asgiref==3.8.1 # via opentelemetry-instrumentation-asgi certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -31,7 +34,10 @@ httpcore==1.0.5 # via httpx httpx==0.27.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_fastapi.in @@ -71,7 +77,10 @@ prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/_fastapi.in pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -85,7 +94,10 @@ sniffio==1.3.1 # httpx starlette==0.38.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi diff --git a/packages/settings-library/requirements/_base.txt b/packages/settings-library/requirements/_base.txt index 3d57d1af6c2..2e155733652 100644 --- a/packages/settings-library/requirements/_base.txt +++ b/packages/settings-library/requirements/_base.txt @@ -23,7 +23,7 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via -r requirements/../../../packages/common-library/requirements/_base.in -pydantic-settings==2.5.2 +pydantic-settings==2.6.1 # via -r requirements/_base.in pygments==2.18.0 # via rich diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index 7bdd7d2cd00..74d2a3fa1b8 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -6,7 +6,12 @@ from pydantic import ValidationInfo, field_validator from pydantic.fields import FieldInfo from pydantic_core import ValidationError -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import ( + BaseSettings, + EnvSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, +) _logger = logging.getLogger(__name__) @@ -17,7 +22,7 @@ class DefaultFromEnvFactoryError(ValueError): def __init__(self, errors): - super().__init__() + super().__init__("Default could not be constructed") self.errors = errors @@ -46,6 +51,47 @@ def _default_factory(): return _default_factory +def _is_auto_default_from_env_enabled(field: FieldInfo) -> bool: + return bool( + field.json_schema_extra is not None + and field.json_schema_extra.get("auto_default_from_env", False) + ) + + +class EnvSettingsWithAutoDefaultSource(EnvSettingsSource): + def __init__( + self, settings_cls: type[BaseSettings], env_settings: EnvSettingsSource + ): + super().__init__( + settings_cls, + env_settings.case_sensitive, + env_settings.env_prefix, + env_settings.env_nested_delimiter, + env_settings.env_ignore_empty, + env_settings.env_parse_none_str, + env_settings.env_parse_enums, + ) + + def prepare_field_value( + self, + field_name: str, + field: FieldInfo, + value: Any, + value_is_complex: bool, # noqa: FBT001 + ) -> Any: + prepared_value = super().prepare_field_value( + field_name, field, value, value_is_complex + ) + if ( + _is_auto_default_from_env_enabled(field) + and field.default_factory + and field.default is None + and prepared_value == {} + ): + prepared_value = field.default_factory() + return prepared_value + + class BaseCustomSettings(BaseSettings): """ - Customized configuration for all settings @@ -82,12 +128,7 @@ def __pydantic_init_subclass__(cls, **kwargs: Any): super().__pydantic_init_subclass__(**kwargs) for name, field in cls.model_fields.items(): - auto_default_from_env = ( - field.json_schema_extra is not None - and field.json_schema_extra.get( # type: ignore[union-attr] - "auto_default_from_env", False - ) - ) + auto_default_from_env = _is_auto_default_from_env_enabled(field) field_type = get_type(field) # Avoids issubclass raising TypeError. SEE test_issubclass_type_error_with_pydantic_models @@ -102,7 +143,7 @@ def __pydantic_init_subclass__(cls, **kwargs: Any): and issubclass(field_type, BaseCustomSettings) ): if auto_default_from_env: - # Transform it into something like `Field(default_factory=create_settings_from_env(field))` + # Builds a default factory `Field(default_factory=create_settings_from_env(field))` field.default_factory = _create_settings_from_env(name, field) field.default = None @@ -126,3 +167,20 @@ def create_from_envs(cls, **overrides): # Optional to use to make the code more readable # More explicit and pylance seems to get less confused return cls(**overrides) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: BaseSettings, + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + assert env_settings # nosec + return ( + init_settings, + EnvSettingsWithAutoDefaultSource(settings_cls, env_settings=env_settings), + dotenv_settings, + file_secret_settings, + ) diff --git a/packages/settings-library/tests/test_base.py b/packages/settings-library/tests/test_base.py index 3344aa6b35a..24b0fbbaa0b 100644 --- a/packages/settings-library/tests/test_base.py +++ b/packages/settings-library/tests/test_base.py @@ -12,14 +12,16 @@ import settings_library.base from pydantic import BaseModel, ValidationError from pydantic.fields import Field -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_envfile +from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.base import ( _DEFAULTS_TO_NONE_MSG, BaseCustomSettings, DefaultFromEnvFactoryError, ) +from settings_library.email import SMTPSettings S2 = json.dumps({"S_VALUE": 2}) S3 = json.dumps({"S_VALUE": 3}) @@ -336,3 +338,17 @@ class SettingsClassThatFailed(BaseCustomSettings): SettingsClassThatFailed(FOO={}) assert SettingsClassThatFailed(FOO=None) == SettingsClassThatFailed() + + +def test_upgrade_failure_to_pydantic_settings_2_6( + mock_env_devel_environment: EnvVarsDict, +): + class ProblematicSettings(BaseCustomSettings): + WEBSERVER_EMAIL: SMTPSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) + + model_config = SettingsConfigDict(nested_model_default_partial_update=True) + + settings = ProblematicSettings() + assert settings.WEBSERVER_EMAIL is not None diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index 85fd98c7522..c2c877ca703 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -120,16 +120,24 @@ def test_parse_from_empty_envs( S1, S2, S3, S4, S5 = model_classes_factory() - with pytest.raises(ValidationError): + with pytest.raises(ValidationError, match="WEBSERVER_POSTGRES") as exc_info: S1() + validation_error = exc_info.value + assert validation_error.error_count() == 1 + error = validation_error.errors()[0] + assert error["type"] == "missing" + assert error["input"] == {} + s2 = S2() assert s2.WEBSERVER_POSTGRES_NULLABLE_OPTIONAL is None - with pytest.raises(DefaultFromEnvFactoryError): + with pytest.raises(DefaultFromEnvFactoryError) as exc_info: # NOTE: cannot have a default or assignment S3() + assert len(exc_info.value.errors) == 4, "Default could not be constructed" + # auto default factory resolves to None (because is nullable) s4 = S4() assert s4.WEBSERVER_POSTGRES_NULLABLE_DEFAULT_ENV is None @@ -357,7 +365,7 @@ def test_parse_from_mixed_envs( "POSTGRES_DB": "db2", "POSTGRES_MAXSIZE": 50, "POSTGRES_MINSIZE": 1, - "POSTGRES_CLIENT_NAME": "client-name", + "POSTGRES_CLIENT_NAME": None, } } # NOTE how unset marks also applies to embedded fields @@ -369,7 +377,6 @@ def test_parse_from_mixed_envs( "POSTGRES_USER": "test2", # <- (1) "POSTGRES_PASSWORD": "shh2", # <- (1) "POSTGRES_DB": "db2", # <- (1) - "POSTGRES_CLIENT_NAME": "client-name", # <- (2) } } @@ -386,7 +393,6 @@ def test_parse_from_mixed_envs( "POSTGRES_USER": "test2", "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", - "POSTGRES_CLIENT_NAME": "client-name", } } @@ -403,7 +409,6 @@ def test_parse_from_mixed_envs( "POSTGRES_USER": "test2", "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", - "POSTGRES_CLIENT_NAME": "client-name", } } @@ -420,7 +425,6 @@ def test_parse_from_mixed_envs( "POSTGRES_USER": "test2", "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", - "POSTGRES_CLIENT_NAME": "client-name", } } @@ -437,7 +441,6 @@ def test_parse_from_mixed_envs( "POSTGRES_USER": "test2", "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", - "POSTGRES_CLIENT_NAME": "client-name", } } diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index c5841c8dd74..005ab923cfd 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -16,11 +16,18 @@ aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -59,11 +66,18 @@ attrs==24.2.0 # referencing certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -117,11 +131,18 @@ jsonschema-specifications==2023.7.1 # via jsonschema mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -176,15 +197,29 @@ opentelemetry-util-http==0.48b0 # via opentelemetry-instrumentation-requests orjson==3.10.7 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 @@ -203,18 +238,32 @@ psycopg2-binary==2.9.9 # sqlalchemy pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends @@ -224,9 +273,16 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -242,21 +298,35 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -288,11 +358,18 @@ sniffio==1.3.1 # via anyio sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -328,11 +405,18 @@ typing-extensions==4.12.2 # typer urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 1f4c324cbec..ccdcbdb0474 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -90,7 +90,6 @@ fastapi==0.115.2 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator faststream==0.5.28 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.4.1 @@ -214,8 +213,14 @@ orjson==3.10.7 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 @@ -268,9 +273,15 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index c3bfeb2b28d..712567f425e 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -433,10 +433,24 @@ orjson==3.10.0 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in # fastapi packaging==24.0 @@ -536,12 +550,26 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # fastapi -pydantic-settings==2.5.2 +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -750,7 +778,6 @@ typer==0.12.3 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fastapi-cli - # faststream types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.10.0 diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index fbf37db9739..29aa4ae7a5e 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -28,16 +28,28 @@ aiofiles==23.2.1 # aioboto3 aiohttp==3.9.5 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore @@ -83,16 +95,28 @@ botocore-stubs==1.34.94 # via types-aiobotocore certifi==2024.2.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -164,16 +188,28 @@ httpcore==1.0.5 # via httpx httpx==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -191,16 +227,28 @@ importlib-metadata==7.1.0 # opentelemetry-api jinja2==3.1.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -299,22 +347,46 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -345,28 +417,52 @@ psutil==6.0.0 # distributed pydantic==2.9.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi @@ -376,11 +472,23 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -404,16 +512,28 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -423,16 +543,28 @@ pyyaml==6.0.1 # distributed redis==5.0.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -480,16 +612,28 @@ sortedcontainers==2.4.0 # distributed starlette==0.38.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -551,16 +695,28 @@ typing-extensions==4.11.0 # types-aiobotocore-ssm urllib3==2.2.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index e33aee23b8f..9004709650e 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -281,8 +281,15 @@ orjson==3.10.0 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # fastapi packaging==24.0 # via -r requirements/_base.in @@ -340,10 +347,17 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # fastapi -pydantic-settings==2.6.0 +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index da8a71fca36..dc49cbbf68e 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -9,6 +9,7 @@ from models_library.services_resources import ResourcesDict, ResourceValue from pydantic import AliasChoices, ByteSize, Field, PositiveInt, TypeAdapter from servicelib.logging_utils_filtering import LoggerName, MessageSubstring +from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.http_client_request import ClientRequestSettings from settings_library.postgres import PostgresSettings diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 508174d8d15..806dca3beb0 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -26,16 +26,28 @@ aiofiles==23.2.1 # aioboto3 aiohttp==3.9.5 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore @@ -81,16 +93,28 @@ botocore-stubs==1.34.94 # via types-aiobotocore certifi==2024.2.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -162,16 +186,28 @@ httpcore==1.0.5 # via httpx httpx==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -189,16 +225,28 @@ importlib-metadata==7.1.0 # opentelemetry-api jinja2==3.1.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -297,22 +345,46 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -343,28 +415,52 @@ psutil==6.0.0 # distributed pydantic==2.9.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi @@ -374,11 +470,23 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -402,16 +510,28 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -421,16 +541,28 @@ pyyaml==6.0.1 # distributed redis==5.0.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -478,16 +610,28 @@ sortedcontainers==2.4.0 # distributed starlette==0.38.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -549,16 +693,28 @@ typing-extensions==4.11.0 # types-aiobotocore-ssm urllib3==2.2.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 159f18f1ad2..01fd1d54565 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -269,9 +269,18 @@ orjson==3.10.3 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via # bokeh @@ -340,10 +349,19 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.6.0 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index 1608048db0f..04e4eed01d9 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -236,8 +236,14 @@ orjson==3.10.0 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in pamqp==3.3.0 # via aiormq prometheus-client==0.20.0 @@ -287,9 +293,15 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.6.0 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 7e548a9fdcb..1ca7e9a0e9a 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -25,20 +25,37 @@ aiofiles==23.2.1 # -r requirements/../../../packages/simcore-sdk/requirements/_base.in aiohttp==3.9.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -57,6 +74,8 @@ alembic==1.13.1 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -94,20 +113,37 @@ blosc==1.11.1 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt certifi==2024.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -156,28 +192,13 @@ email-validator==2.1.1 # pydantic fast-depends==2.4.12 # via faststream -fastapi==0.99.1 +fastapi==0.115.4 # via - # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator +fastapi-cli==0.0.5 + # via fastapi faststream==0.5.28 # via # -r requirements/../../../packages/service-library/requirements/_base.in @@ -213,20 +234,37 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -249,20 +287,37 @@ itsdangerous==2.2.0 # via fastapi jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -287,20 +342,37 @@ lz4==4.3.3 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt mako==1.3.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -390,28 +462,62 @@ ordered-set==4.1.0 # via -r requirements/_base.in orjson==3.10.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in # fastapi packaging==24.0 @@ -448,45 +554,120 @@ psycopg2-binary==2.9.9 # via # aiopg # sqlalchemy -pydantic==1.10.15 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.10.0 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # fastapi +pydantic-settings==2.6.1 + # via + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in + # fastapi pygments==2.18.0 # via rich pyinstrument==4.6.2 @@ -497,7 +678,7 @@ python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 # via - # pydantic + # pydantic-settings # uvicorn python-engineio==4.9.1 # via python-socketio @@ -507,20 +688,37 @@ python-socketio==5.11.2 # via -r requirements/_base.in pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -534,20 +732,37 @@ pyyaml==6.0.1 # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -598,20 +813,37 @@ sortedcontainers==2.4.0 # distributed sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -619,22 +851,39 @@ sqlalchemy==1.4.52 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # aiopg # alembic -starlette==0.27.0 +starlette==0.41.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -674,6 +923,7 @@ typer==0.12.3 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in + # fastapi-cli types-python-dateutil==2.9.0.20240316 # via arrow typing-extensions==4.11.0 @@ -688,43 +938,79 @@ typing-extensions==4.11.0 # opentelemetry-sdk # pint # pydantic + # pydantic-core + # pydantic-extra-types # typer ujson==5.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi urllib3==2.2.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -735,6 +1021,7 @@ uvicorn==0.29.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 4487b628b11..86558012e81 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -329,7 +329,7 @@ pydantic-extra-types==2.9.0 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -pydantic-settings==2.5.2 +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index e531f0ac0ba..a1cfdbe68da 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -363,10 +363,24 @@ orjson==3.10.0 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in pamqp==3.3.0 @@ -458,11 +472,25 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 9caea149b29..6a909af84e5 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -28,17 +28,30 @@ aiohappyeyeballs==2.4.3 # via aiohttp aiohttp==3.10.10 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore @@ -90,17 +103,30 @@ botocore-stubs==1.35.43 # via types-aiobotocore certifi==2024.8.30 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -153,17 +179,30 @@ httpcore==1.0.6 # via httpx httpx==0.27.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -190,17 +229,30 @@ jsonschema-specifications==2023.7.1 # via jsonschema mako==1.3.5 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -275,23 +327,49 @@ opentelemetry-util-http==0.48b0 # opentelemetry-instrumentation-requests orjson==3.10.7 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 @@ -316,30 +394,56 @@ psycopg2-binary==2.9.10 # via sqlalchemy pydantic==2.9.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi @@ -349,11 +453,24 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -377,34 +494,60 @@ python-dotenv==1.0.1 # via pydantic-settings pyyaml==6.0.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in redis==5.1.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -448,34 +591,60 @@ sniffio==1.3.1 # httpx sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic starlette==0.38.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -512,7 +681,6 @@ types-python-dateutil==2.9.0.20241003 typing-extensions==4.12.2 # via # aiodebug - # aiodocker # alembic # fastapi # faststream @@ -526,17 +694,30 @@ typing-extensions==4.12.2 # types-aiobotocore-ssm urllib3==2.2.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/services/efs-guardian/requirements/_test.txt b/services/efs-guardian/requirements/_test.txt index 3a082363fd5..c2b186a1310 100644 --- a/services/efs-guardian/requirements/_test.txt +++ b/services/efs-guardian/requirements/_test.txt @@ -319,6 +319,7 @@ typing-extensions==4.12.2 # cfn-lint # faker # pydantic + # pydantic-core urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 3f4569b50b9..f1a478c30be 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -10,10 +10,16 @@ aiofiles==23.2.1 # via -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.9.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -44,10 +50,16 @@ attrs==23.2.0 # referencing certifi==2024.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -63,10 +75,16 @@ click==8.1.7 # uvicorn cryptography==42.0.5 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in @@ -109,10 +127,16 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -194,14 +218,26 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via -r requirements/_base.in pamqp==3.3.0 @@ -222,16 +258,28 @@ pycparser==2.21 # via cffi pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi @@ -241,9 +289,15 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -261,20 +315,32 @@ python-dotenv==1.0.1 # uvicorn pyyaml==6.0.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -308,10 +374,16 @@ sniffio==1.3.1 # httpx starlette==0.38.6 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -340,10 +412,16 @@ typing-extensions==4.10.0 # typer urllib3==2.2.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index f4beebc5341..e5c6a6edb48 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -296,8 +296,15 @@ orjson==3.10.6 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 @@ -359,9 +366,16 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 6124ad0de12..3587be89a74 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -27,17 +27,30 @@ aiofiles==23.2.1 # aioboto3 aiohttp==3.9.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiobotocore @@ -90,17 +103,30 @@ botocore-stubs==1.34.69 # via types-aiobotocore certifi==2024.2.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -165,17 +191,30 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -204,17 +243,30 @@ kiwisolver==1.4.5 # via matplotlib mako==1.3.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -297,23 +349,49 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via # -r requirements/_base.in @@ -344,30 +422,56 @@ psycopg2-binary==2.9.9 # via sqlalchemy pydantic==2.9.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi @@ -377,11 +481,24 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -416,17 +533,30 @@ pytz==2024.1 # pandas pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -434,17 +564,30 @@ pyyaml==6.0.1 # uvicorn redis==5.0.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -495,34 +638,60 @@ sniffio==1.3.1 # httpx sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic starlette==0.38.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -578,17 +747,30 @@ tzlocal==5.2 # via dateparser urllib3==2.0.7 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/services/resource-usage-tracker/requirements/_test.txt b/services/resource-usage-tracker/requirements/_test.txt index 6ddd9949286..2a0809dfa17 100644 --- a/services/resource-usage-tracker/requirements/_test.txt +++ b/services/resource-usage-tracker/requirements/_test.txt @@ -81,6 +81,10 @@ flask-cors==5.0.0 # via moto graphql-core==3.2.4 # via moto +greenlet==3.0.3 + # via + # -c requirements/_base.txt + # sqlalchemy h11==0.14.0 # via # -c requirements/_base.txt diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index c2766e7dc10..701d9577d62 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -382,10 +382,23 @@ orjson==3.10.0 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.0 # via -r requirements/_base.in pamqp==3.3.0 @@ -468,11 +481,24 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.6.0 + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in diff --git a/tests/public-api/requirements/_base.txt b/tests/public-api/requirements/_base.txt index 836be727d0c..753d75bafbe 100644 --- a/tests/public-api/requirements/_base.txt +++ b/tests/public-api/requirements/_base.txt @@ -30,6 +30,11 @@ mdurl==0.1.2 # via markdown-it-py nest-asyncio==1.6.0 # via osparc +orjson==3.10.11 + # via + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in osparc==0.6.6 # via -r requirements/_base.in osparc-client==0.6.6 @@ -42,10 +47,13 @@ pydantic==2.9.2 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # pydantic-extra-types # pydantic-settings pydantic-core==2.23.4 # via pydantic -pydantic-settings==2.5.2 +pydantic-extra-types==2.10.0 + # via -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich @@ -77,8 +85,10 @@ typing-extensions==4.12.2 # via # pydantic # pydantic-core + # pydantic-extra-types # typer urllib3==2.2.3 # via + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # osparc-client diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 0a0e6a66fc1..5aeffad83c4 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -300,10 +300,22 @@ orjson==3.10.7 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in packaging==24.1 # via # -r requirements/../../../packages/simcore-sdk/requirements/_base.in @@ -385,11 +397,23 @@ pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -pydantic-settings==2.5.2 + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.6.1 # via # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in From 840227049a13e6fd045245c107958bc192ad093b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 6 Nov 2024 09:14:22 +0100 Subject: [PATCH 29/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Upgrade=20Web=20Serv?= =?UTF-8?q?er=20service=20(Pydantic=20v2)=20(#6584)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sylvain <35365065+sanderegg@users.noreply.github.com> --- .../src/common_library/pydantic_validators.py | 27 +- .../tests/test_pydantic_validators.py | 36 ++- .../api_schemas_directorv2/clusters.py | 25 +- .../api_schemas_webserver/auth.py | 3 +- .../api_schemas_webserver/clusters.py | 2 +- .../api_schemas_webserver/folders_v2.py | 2 +- .../api_schemas_webserver/groups.py | 9 +- .../api_schemas_webserver/projects.py | 20 +- .../api_schemas_webserver/resource_usage.py | 6 +- .../api_schemas_webserver/wallets.py | 12 +- .../src/models_library/clusters.py | 3 +- .../src/models_library/projects_ui.py | 2 +- .../src/models_library/resource_tracker.py | 2 +- .../models_library/rest_pagination_utils.py | 11 +- .../src/models_library/user_preferences.py | 12 +- .../models/products.py | 3 +- .../hypothesis_type_strategies.py | 16 +- .../servicelib/aiohttp/requests_validation.py | 2 +- .../tests/test_logging_errors.py | 1 + services/web/server/requirements/_base.in | 1 + services/web/server/requirements/_base.txt | 241 +++++++++++++++++- services/web/server/requirements/_test.txt | 4 +- services/web/server/requirements/ci.txt | 1 + services/web/server/requirements/dev.txt | 1 + services/web/server/requirements/prod.txt | 1 + .../activity/_handlers.py | 4 +- .../announcements/_models.py | 17 +- .../announcements/_redis.py | 2 +- .../api_keys/_api.py | 2 +- .../simcore_service_webserver/api_keys/_db.py | 6 +- .../api_keys/_handlers.py | 6 +- .../application_settings.py | 143 +++++++---- .../application_settings_utils.py | 7 +- .../simcore_service_webserver/catalog/_api.py | 24 +- .../catalog/_handlers.py | 34 +-- .../catalog/client.py | 6 +- .../src/simcore_service_webserver/cli.py | 4 +- .../clusters/_handlers.py | 24 +- .../diagnostics/_handlers.py | 8 +- .../diagnostics/settings.py | 19 +- .../director_v2/_api_utils.py | 6 +- .../director_v2/_core_computations.py | 44 ++-- .../director_v2/_core_dynamic_services.py | 6 +- .../director_v2/_handlers.py | 19 +- .../director_v2/_models.py | 15 +- .../director_v2/settings.py | 10 +- .../dynamic_scheduler/settings.py | 17 +- .../simcore_service_webserver/email/_core.py | 2 +- .../email/_handlers.py | 4 +- .../src/simcore_service_webserver/errors.py | 5 +- .../exporter/_formatter/_sds.py | 7 +- .../folders/_folders_db.py | 10 +- .../folders/_folders_handlers.py | 35 ++- .../groups/_classifiers.py | 32 ++- .../simcore_service_webserver/groups/_db.py | 5 +- .../groups/_handlers.py | 54 ++-- .../simcore_service_webserver/groups/api.py | 4 +- .../invitations/_client.py | 6 +- .../invitations/_core.py | 6 +- .../invitations/settings.py | 4 +- .../login/_auth_handlers.py | 10 +- .../login/_models.py | 21 +- .../login/_registration.py | 13 +- .../login/_registration_api.py | 8 +- .../login/_registration_handlers.py | 2 +- .../login/handlers_change.py | 4 +- .../login/handlers_confirmation.py | 16 +- .../login/handlers_registration.py | 27 +- .../simcore_service_webserver/login/plugin.py | 2 +- .../login/settings.py | 21 +- .../long_running_tasks.py | 2 +- .../meta_modeling/_function_nodes.py | 2 +- .../meta_modeling/_handlers.py | 12 +- .../meta_modeling/_iterations.py | 8 +- .../meta_modeling/_results.py | 16 +- .../_rabbitmq_exclusive_queue_consumers.py | 20 +- .../_rabbitmq_nonexclusive_queue_consumers.py | 6 +- .../payments/_autorecharge_db.py | 10 +- .../payments/_methods_api.py | 8 +- .../payments/_methods_db.py | 19 +- .../payments/_onetime_api.py | 16 +- .../payments/_onetime_db.py | 14 +- .../payments/_rpc.py | 25 +- .../payments/_tasks.py | 5 +- .../payments/settings.py | 24 +- .../simcore_service_webserver/products/_db.py | 2 +- .../products/_events.py | 2 +- .../products/_handlers.py | 14 +- .../products/_invitations_handlers.py | 8 +- .../products/_model.py | 68 ++--- .../projects/_comments_api.py | 7 +- .../projects/_comments_db.py | 10 +- .../projects/_comments_handlers.py | 32 +-- .../projects/_common_models.py | 8 +- .../projects/_crud_api_create.py | 16 +- .../projects/_crud_api_read.py | 4 +- .../projects/_crud_handlers.py | 69 +++-- .../projects/_crud_handlers_models.py | 46 ++-- .../projects/_db_utils.py | 4 +- .../projects/_folders_db.py | 6 +- .../projects/_folders_handlers.py | 12 +- .../projects/_groups_api.py | 10 +- .../projects/_groups_db.py | 10 +- .../projects/_groups_handlers.py | 18 +- .../projects/_metadata_api.py | 4 +- .../projects/_metadata_db.py | 6 +- .../projects/_metadata_handlers.py | 4 +- .../projects/_nodes_api.py | 13 +- .../projects/_nodes_handlers.py | 34 +-- .../projects/_ports_api.py | 7 +- .../projects/_ports_handlers.py | 18 +- .../_projects_nodes_pricing_unit_handlers.py | 16 +- .../projects/_states_handlers.py | 8 +- .../projects/_tags_api.py | 6 +- .../projects/_wallets_api.py | 4 +- .../projects/_wallets_handlers.py | 10 +- .../projects/_workspaces_handlers.py | 15 +- .../simcore_service_webserver/projects/db.py | 16 +- .../projects/lock.py | 4 +- .../projects/models.py | 29 +-- .../projects/projects_api.py | 41 +-- .../projects/settings.py | 4 +- .../projects/utils.py | 6 +- .../resource_usage/_client.py | 8 +- .../_pricing_plans_admin_handlers.py | 40 ++- .../resource_usage/_pricing_plans_handlers.py | 10 +- .../resource_usage/_service_runs_handlers.py | 49 ++-- .../rest/_handlers.py | 4 +- .../scicrunch/_resolver.py | 6 +- .../scicrunch/_rest.py | 10 +- .../simcore_service_webserver/scicrunch/db.py | 6 +- .../scicrunch/models.py | 11 +- .../scicrunch/service_client.py | 8 +- .../scicrunch/settings.py | 8 +- .../security/_authz_db.py | 10 +- .../session/access_policies.py | 4 +- .../session/settings.py | 16 +- .../socketio/models.py | 12 +- .../statics/_events.py | 2 +- .../statics/settings.py | 10 +- .../storage/_handlers.py | 12 +- .../simcore_service_webserver/storage/api.py | 18 +- .../storage/schemas.py | 36 +-- .../storage/settings.py | 5 +- .../studies_dispatcher/_catalog.py | 10 +- .../studies_dispatcher/_core.py | 6 +- .../studies_dispatcher/_errors.py | 8 +- .../studies_dispatcher/_models.py | 6 +- .../studies_dispatcher/_projects.py | 38 +-- .../_projects_permalinks.py | 5 +- .../studies_dispatcher/_redirects_handlers.py | 21 +- .../studies_dispatcher/_rest_handlers.py | 29 ++- .../studies_dispatcher/_users.py | 8 +- .../studies_dispatcher/settings.py | 22 +- .../simcore_service_webserver/tags/_api.py | 4 +- .../tags/_handlers.py | 12 +- .../simcore_service_webserver/tags/schemas.py | 8 +- .../simcore_service_webserver/users/_api.py | 6 +- .../simcore_service_webserver/users/_db.py | 2 +- .../users/_handlers.py | 12 +- .../users/_notifications.py | 12 +- .../users/_notifications_handlers.py | 18 +- .../users/_preferences_api.py | 6 +- .../users/_preferences_db.py | 2 +- .../users/_preferences_handlers.py | 2 +- .../users/_preferences_models.py | 2 +- .../users/_schemas.py | 30 +-- .../users/_tokens_handlers.py | 8 +- .../simcore_service_webserver/users/api.py | 6 +- .../users/exceptions.py | 12 +- .../users/schemas.py | 27 +- .../src/simcore_service_webserver/utils.py | 5 +- .../utils_aiohttp.py | 5 +- .../version_control/_core.py | 18 +- .../version_control/_handlers.py | 44 ++-- .../version_control/_handlers_base.py | 2 +- .../version_control/models.py | 12 +- .../version_control/vc_changes.py | 2 +- .../simcore_service_webserver/wallets/_api.py | 20 +- .../simcore_service_webserver/wallets/_db.py | 11 +- .../wallets/_groups_api.py | 20 +- .../wallets/_groups_db.py | 10 +- .../wallets/_groups_handlers.py | 18 +- .../wallets/_handlers.py | 10 +- .../wallets/_payments_handlers.py | 30 +-- .../workspaces/_groups_api.py | 16 +- .../workspaces/_groups_db.py | 14 +- .../workspaces/_groups_handlers.py | 18 +- .../workspaces/_workspaces_db.py | 8 +- .../workspaces/_workspaces_handlers.py | 26 +- services/web/server/tests/conftest.py | 6 +- .../notifications/test_rabbitmq_consumers.py | 4 +- .../02/scicrunch/test_scicrunch__rest.py | 6 +- .../server/tests/unit/isolated/conftest.py | 5 +- .../notifications/test_rabbitmq_consumers.py | 4 +- .../isolated/test_application_settings.py | 9 +- .../test_application_settings_utils.py | 2 +- .../unit/isolated/test_catalog_api_units.py | 24 +- .../isolated/test_diagnostics_healthcheck.py | 13 +- .../unit/isolated/test_dynamic_scheduler.py | 26 +- .../isolated/test_garbage_collector_core.py | 4 +- .../tests/unit/isolated/test_groups_models.py | 4 +- ...st_statics.py => test_isolated_statics.py} | 0 .../unit/isolated/test_login_settings.py | 8 +- .../unit/isolated/test_products_model.py | 12 +- .../unit/isolated/test_projects__nodes_api.py | 6 +- .../test_projects__nodes_resources.py | 24 +- .../unit/isolated/test_projects_utils.py | 4 +- .../tests/unit/isolated/test_security_api.py | 6 +- .../unit/isolated/test_statics_settings.py | 6 +- .../unit/isolated/test_storage_schemas.py | 2 +- .../isolated/test_studies_dispatcher_core.py | 16 +- .../test_studies_dispatcher_models.py | 46 ++-- ..._studies_dispatcher_projects_permalinks.py | 5 + .../test_studies_dispatcher_settings.py | 16 +- .../unit/isolated/test_user_notifications.py | 16 +- .../tests/unit/isolated/test_users_models.py | 22 +- .../unit/isolated/test_utils_rate_limiting.py | 5 +- .../01/clusters/test_clusters_handlers.py | 62 ++++- .../unit/with_dbs/01/folders/test_folders.py | 34 +-- .../with_dbs/01/products/test_products_db.py | 4 +- .../with_dbs/01/products/test_products_rpc.py | 12 +- .../01/studies_dispatcher/conftest.py | 2 +- .../test_studies_dispatcher_handlers.py | 14 +- .../test_studies_dispatcher_projects.py | 2 +- .../unit/with_dbs/01/test_api_keys_rpc.py | 12 +- .../01/test_catalog_handlers__pricing_plan.py | 6 +- .../01/test_catalog_handlers__services.py | 22 +- ...st_catalog_handlers__services_resources.py | 7 +- .../tests/unit/with_dbs/01/test_groups.py | 2 +- .../01/test_resource_manager_user_sessions.py | 10 +- .../tests/unit/with_dbs/01/test_statics.py | 2 +- .../unit/with_dbs/01/wallets/conftest.py | 2 + .../with_dbs/01/wallets/payments/conftest.py | 5 +- .../01/wallets/payments/test_payments.py | 18 +- .../wallets/payments/test_payments_methods.py | 28 +- .../01/wallets/payments/test_payments_rpc.py | 6 +- .../with_dbs/01/workspaces/test_workspaces.py | 6 +- ...t_workspaces__folders_and_projects_crud.py | 23 +- ...t_workspaces__list_projects_full_search.py | 16 +- ...ces__moving_projects_between_workspaces.py | 24 +- .../server/tests/unit/with_dbs/02/conftest.py | 11 +- .../unit/with_dbs/02/test_announcements.py | 6 +- .../unit/with_dbs/02/test_project_lock.py | 12 +- .../02/test_projects_cancellations.py | 12 +- .../02/test_projects_comments_handlers.py | 32 +-- .../02/test_projects_crud_handlers.py | 22 +- .../02/test_projects_crud_handlers__clone.py | 10 +- ...handlers__clone_in_workspace_and_folder.py | 6 +- .../02/test_projects_crud_handlers__list.py | 10 +- ...s_crud_handlers__list_with_query_params.py | 30 +-- .../02/test_projects_metadata_handlers.py | 40 +-- .../02/test_projects_nodes_handler.py | 52 ++-- ...st_projects_nodes_pricing_unit_handlers.py | 11 +- .../02/test_projects_ports_handlers.py | 10 +- .../02/test_projects_states_handlers.py | 16 +- .../02/test_projects_wallet_handlers.py | 19 +- .../unit/with_dbs/03/invitations/conftest.py | 10 +- ...login_handlers_registration_invitations.py | 12 +- .../test_products__invitations_handlers.py | 16 +- .../tests/unit/with_dbs/03/login/conftest.py | 3 +- .../unit/with_dbs/03/login/test_login_2fa.py | 4 +- .../03/login/test_login_2fa_resend.py | 5 +- .../unit/with_dbs/03/login/test_login_auth.py | 2 +- .../03/login/test_login_change_password.py | 2 +- .../03/login/test_login_registration.py | 8 +- .../login/test_login_registration_handlers.py | 4 +- .../test_meta_modeling_iterations.py | 10 +- .../with_dbs/03/resource_usage/conftest.py | 2 +- .../test_admin_pricing_plans.py | 43 ++-- .../03/resource_usage/test_pricing_plans.py | 10 +- .../test_usage_services__export.py | 8 +- .../test_usage_services__list.py | 8 +- .../unit/with_dbs/03/test__openapi_specs.py | 2 +- .../tests/unit/with_dbs/03/test_email.py | 6 +- .../tests/unit/with_dbs/03/test_socketio.py | 2 +- .../unit/with_dbs/03/test_storage_handlers.py | 33 +-- .../tests/unit/with_dbs/03/test_users.py | 28 +- .../with_dbs/03/test_users__notifications.py | 16 +- .../03/test_users__preferences_api.py | 9 +- .../with_dbs/03/version_control/conftest.py | 2 +- .../version_control/test_version_control.py | 13 +- .../test_version_control_core.py | 4 +- .../test_version_control_handlers.py | 38 +-- .../server/tests/unit/with_dbs/conftest.py | 4 +- 285 files changed, 2261 insertions(+), 1800 deletions(-) rename services/web/server/tests/unit/isolated/{test_statics.py => test_isolated_statics.py} (100%) diff --git a/packages/common-library/src/common_library/pydantic_validators.py b/packages/common-library/src/common_library/pydantic_validators.py index 60f6219fb13..471ba1a4bf3 100644 --- a/packages/common-library/src/common_library/pydantic_validators.py +++ b/packages/common-library/src/common_library/pydantic_validators.py @@ -1,10 +1,35 @@ import datetime +import re import warnings from datetime import timedelta from pydantic import TypeAdapter, field_validator +def _validate_legacy_timedelta_str(time_str: str | timedelta) -> str | timedelta: + if not isinstance(time_str, str): + return time_str + + # Match the format [-][DD ][HH:MM]SS[.ffffff] + match = re.match( + r"^(?P-)?(?:(?P\d+)\s)?(?:(?P\d+):)?(?:(?P\d+):)?(?P\d+)(?P\.\d+)?$", + time_str, + ) + if not match: + return time_str + + # Extract components with defaults if not present + sign = match.group("sign") or "" + days = match.group("days") or "0" + hours = match.group("hours") or "0" + minutes = match.group("minutes") or "0" + seconds = match.group("seconds") + fraction = match.group("fraction") or "" + + # Convert to the format [-][DD]D[,][HH:MM:]SS[.ffffff] + return f"{sign}{int(days)}D,{int(hours):02}:{int(minutes):02}:{seconds}{fraction}" + + def validate_numeric_string_as_timedelta(field: str): """Transforms a float/int number into a valid datetime as it used to work in the past""" @@ -29,7 +54,7 @@ def _numeric_string_as_timedelta( return converted_value except ValueError: # returns format like "1:00:00" - return v + return _validate_legacy_timedelta_str(v) return v return field_validator(field, mode="before")(_numeric_string_as_timedelta) diff --git a/packages/common-library/tests/test_pydantic_validators.py b/packages/common-library/tests/test_pydantic_validators.py index da1ccf95adb..89e57bc0999 100644 --- a/packages/common-library/tests/test_pydantic_validators.py +++ b/packages/common-library/tests/test_pydantic_validators.py @@ -1,13 +1,45 @@ from datetime import timedelta +from typing import Annotated import pytest -from common_library.pydantic_validators import validate_numeric_string_as_timedelta +from common_library.pydantic_validators import ( + validate_legacy_timedelta_str, + validate_numeric_string_as_timedelta, +) from faker import Faker -from pydantic import Field +from pydantic import BeforeValidator, Field from pydantic_settings import BaseSettings, SettingsConfigDict from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +def test_validate_legacy_timedelta(monkeypatch: pytest.MonkeyPatch, faker: Faker): + class Settings(BaseSettings): + APP_NAME: str + REQUEST_TIMEOUT: Annotated[ + timedelta, BeforeValidator(validate_legacy_timedelta_str) + ] = Field(default=timedelta(hours=1)) + + model_config = SettingsConfigDict() + + app_name = faker.pystr() + env_vars: dict[str, str | bool] = {"APP_NAME": app_name} + + # without timedelta + setenvs_from_dict(monkeypatch, env_vars) + settings = Settings() + print(settings.model_dump()) + assert app_name == settings.APP_NAME + assert timedelta(hours=1) == settings.REQUEST_TIMEOUT + + # with timedelta in seconds + env_vars["REQUEST_TIMEOUT"] = "2 1:10:00" + setenvs_from_dict(monkeypatch, env_vars) + settings = Settings() + print(settings.model_dump()) + assert app_name == settings.APP_NAME + assert timedelta(days=2, hours=1, minutes=10) == settings.REQUEST_TIMEOUT + + def test_validate_timedelta_in_legacy_mode( monkeypatch: pytest.MonkeyPatch, faker: Faker ): diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py index 1c9892a7201..9c4698a5dc3 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py @@ -7,6 +7,7 @@ Field, HttpUrl, NonNegativeFloat, + ValidationInfo, field_validator, model_validator, ) @@ -96,7 +97,7 @@ class ClusterGet(Cluster): alias="accessRights", default_factory=dict ) - model_config = ConfigDict(populate_by_name=True) + model_config = ConfigDict(extra="allow", populate_by_name=True) @model_validator(mode="before") @classmethod @@ -112,7 +113,7 @@ class ClusterDetailsGet(ClusterDetails): class ClusterCreate(BaseCluster): - owner: GroupID | None # type: ignore[assignment] + owner: GroupID | None authentication: ExternalClusterAuthentication access_rights: dict[GroupID, ClusterAccessRights] = Field( alias="accessRights", default_factory=dict @@ -154,9 +155,9 @@ class ClusterCreate(BaseCluster): @field_validator("thumbnail", mode="before") @classmethod - def set_default_thumbnail_if_empty(cls, v, values): + def set_default_thumbnail_if_empty(cls, v, info: ValidationInfo): if v is None: - cluster_type = values["type"] + cluster_type = info.data["type"] default_thumbnails = { ClusterTypeInModel.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", ClusterTypeInModel.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", @@ -167,15 +168,15 @@ def set_default_thumbnail_if_empty(cls, v, values): class ClusterPatch(BaseCluster): - name: str | None # type: ignore[assignment] - description: str | None - type: ClusterTypeInModel | None # type: ignore[assignment] - owner: GroupID | None # type: ignore[assignment] - thumbnail: HttpUrl | None - endpoint: AnyUrl | None # type: ignore[assignment] - authentication: ExternalClusterAuthentication | None # type: ignore[assignment] + name: str | None = None # type: ignore[assignment] + description: str | None = None + type: ClusterTypeInModel | None = None # type: ignore[assignment] + owner: GroupID | None = None # type: ignore[assignment] + thumbnail: HttpUrl | None = None + endpoint: AnyUrl | None = None # type: ignore[assignment] + authentication: ExternalClusterAuthentication | None = None # type: ignore[assignment] access_rights: dict[GroupID, ClusterAccessRights] | None = Field( # type: ignore[assignment] - alias="accessRights" + default=None, alias="accessRights" ) model_config = ConfigDict( diff --git a/packages/models-library/src/models_library/api_schemas_webserver/auth.py b/packages/models-library/src/models_library/api_schemas_webserver/auth.py index b0b11661cb3..c841056d40c 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/auth.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/auth.py @@ -83,9 +83,10 @@ class ApiKeyGet(BaseModel): api_secret: str model_config = ConfigDict( + from_attributes=True, json_schema_extra={ "examples": [ {"display_name": "myapi", "api_key": "key", "api_secret": "secret"}, ] - } + }, ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/clusters.py b/packages/models-library/src/models_library/api_schemas_webserver/clusters.py index 109e0618b98..17232a8b482 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/clusters.py @@ -8,7 +8,7 @@ class ClusterPathParams(BaseModel): cluster_id: ClusterID model_config = ConfigDict( - populate_by_name=True, + populate_by_name=True, extra="forbid", ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py b/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py index ee44a9b18d4..85f3604381a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py @@ -45,7 +45,7 @@ class CreateFolderBodyParams(InputSchema): class PutFolderBodyParams(InputSchema): name: IDStr - parent_folder_id: FolderID | None + parent_folder_id: FolderID | None = None model_config = ConfigDict(extra="forbid") _null_or_none_str_to_none_validator = field_validator( diff --git a/packages/models-library/src/models_library/api_schemas_webserver/groups.py b/packages/models-library/src/models_library/api_schemas_webserver/groups.py index 46e9da3dc52..d77c04288c9 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/groups.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/groups.py @@ -1,6 +1,7 @@ from contextlib import suppress from pydantic import ( + AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, @@ -90,7 +91,7 @@ def _sanitize_legacy_data(cls, v): if v: # Enforces null if thumbnail is not valid URL or empty with suppress(ValidationError): - return TypeAdapter(AnyUrl).validate_python(v) + return TypeAdapter(AnyHttpUrl).validate_python(v) return None @@ -143,12 +144,14 @@ class AllUsersGroups(BaseModel): class GroupUserGet(BaseModel): - id: str | None = Field(None, description="the user id") + id: str | None = Field(None, description="the user id", coerce_numbers_to_str=True) login: LowerCaseEmailStr | None = Field(None, description="the user login email") first_name: str | None = Field(None, description="the user first name") last_name: str | None = Field(None, description="the user last name") gravatar_id: str | None = Field(None, description="the user gravatar id hash") - gid: str | None = Field(None, description="the user primary gid") + gid: str | None = Field( + None, description="the user primary gid", coerce_numbers_to_str=True + ) access_rights: GroupAccessRights = Field(..., alias="accessRights") model_config = ConfigDict( diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 976ee74a712..0ae2bdee311 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -6,9 +6,11 @@ """ from datetime import datetime -from typing import Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias -from pydantic import ConfigDict, Field, HttpUrl, field_validator +from models_library.folders import FolderID +from models_library.workspaces import WorkspaceID +from pydantic import BeforeValidator, ConfigDict, Field, HttpUrl, field_validator from ..api_schemas_long_running_tasks.tasks import TaskGet from ..basic_types import LongTruncatedStr, ShortTruncatedStr @@ -105,7 +107,7 @@ class ProjectReplace(InputSchema): uuid: ProjectID name: ShortTruncatedStr description: LongTruncatedStr - thumbnail: HttpUrl | None + thumbnail: Annotated[HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)] = Field(default=None) creation_date: DateTimeStr last_change_date: DateTimeStr workbench: NodesDict @@ -121,25 +123,17 @@ class ProjectReplace(InputSchema): default_factory=dict, json_schema_extra={"default": {}} ) - _empty_is_none = field_validator("thumbnail", mode="before")( - empty_str_to_none_pre_validator - ) - class ProjectPatch(InputSchema): name: ShortTruncatedStr | None = Field(default=None) description: LongTruncatedStr | None = Field(default=None) - thumbnail: HttpUrl | None = Field(default=None) + thumbnail: Annotated[HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)] = Field(default=None) access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None) classifiers: list[ClassifierID] | None = Field(default=None) dev: dict | None = Field(default=None) ui: StudyUI | None = Field(default=None) quality: dict[str, Any] | None = Field(default=None) - - _empty_is_none = field_validator("thumbnail", mode="before")( - empty_str_to_none_pre_validator - ) - + __all__: tuple[str, ...] = ( "EmptyModel", diff --git a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py index 8242105f55a..3eea55c6d67 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py @@ -1,7 +1,7 @@ from datetime import datetime from decimal import Decimal -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from ..projects import ProjectID from ..projects_nodes_io import NodeID @@ -48,7 +48,7 @@ class ServiceRunGet( class PricingUnitGet(OutputSchema): pricing_unit_id: PricingUnitId unit_name: str - unit_extra_info: dict + unit_extra_info: UnitExtraInfo current_cost_per_unit: Decimal default: bool @@ -131,7 +131,7 @@ class UpdatePricingUnitBodyParams(InputSchema): unit_extra_info: UnitExtraInfo default: bool specific_info: SpecificInfo - pricing_unit_cost_update: PricingUnitCostUpdate | None + pricing_unit_cost_update: PricingUnitCostUpdate | None = Field(default=None) model_config = ConfigDict( str_strip_whitespace=True, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index 3ba88301f88..ae20bee1e62 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -2,7 +2,7 @@ from decimal import Decimal from typing import Annotated, Literal, TypeAlias -from pydantic import ConfigDict, Field, HttpUrl, PlainSerializer, field_validator +from pydantic import ConfigDict, Field, HttpUrl, PlainSerializer, ValidationInfo, field_validator from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID @@ -20,7 +20,10 @@ class WalletGet(OutputSchema): created: datetime modified: datetime - model_config = ConfigDict(frozen=False) + model_config = ConfigDict( + from_attributes=True, + frozen=False + ) class WalletGetWithAvailableCredits(WalletGet): @@ -139,6 +142,7 @@ class PaymentMethodGet(OutputSchema): ) model_config = ConfigDict( + frozen=False, json_schema_extra={ "examples": [ { @@ -200,8 +204,8 @@ class ReplaceWalletAutoRecharge(InputSchema): @field_validator("monthly_limit_in_usd") @classmethod - def _monthly_limit_greater_than_top_up(cls, v, values): - top_up = values["top_up_amount_in_usd"] + def _monthly_limit_greater_than_top_up(cls, v, info: ValidationInfo): + top_up = info.data["top_up_amount_in_usd"] if v is not None and v < top_up: msg = "Monthly limit ({v} USD) should be greater than top up amount ({top_up} USD)" raise ValueError(msg) diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index c98ea29757a..09540b8c16e 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -159,6 +159,7 @@ class Cluster(BaseCluster): id: ClusterID = Field(..., description="The cluster ID") model_config = ConfigDict( + extra="allow", json_schema_extra={ "examples": [ { @@ -217,7 +218,7 @@ class Cluster(BaseCluster): }, }, ] - } + }, ) @model_validator(mode="before") diff --git a/packages/models-library/src/models_library/projects_ui.py b/packages/models-library/src/models_library/projects_ui.py index 93aa68d628b..0e914de9dba 100644 --- a/packages/models-library/src/models_library/projects_ui.py +++ b/packages/models-library/src/models_library/projects_ui.py @@ -68,7 +68,7 @@ class StudyUI(BaseModel): current_node_id: NodeID | None = Field(default=None, alias="currentNodeId") annotations: dict[NodeIDStr, Annotation] | None = None - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", populate_by_name=True) _empty_is_none = field_validator("*", mode="before")( empty_str_to_none_pre_validator diff --git a/packages/models-library/src/models_library/resource_tracker.py b/packages/models-library/src/models_library/resource_tracker.py index c3b42a08795..53e370913a9 100644 --- a/packages/models-library/src/models_library/resource_tracker.py +++ b/packages/models-library/src/models_library/resource_tracker.py @@ -259,7 +259,7 @@ class PricingUnitWithCostUpdate(BaseModel): unit_extra_info: UnitExtraInfo default: bool specific_info: SpecificInfo - pricing_unit_cost_update: None | PricingUnitCostUpdate + pricing_unit_cost_update: PricingUnitCostUpdate | None model_config = ConfigDict( json_schema_extra={ diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index 41899acd8cf..d4027583ae7 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -1,5 +1,6 @@ from math import ceil -from typing import Any, Protocol, TypedDict, Union, runtime_checkable +from typing import Any, Protocol, runtime_checkable +from typing_extensions import TypedDict from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from pydantic import TypeAdapter @@ -29,7 +30,7 @@ def replace_query_params(self, **kwargs: Any) -> "_StarletteURL": ... -_URLType = Union[_YarlURL, _StarletteURL] +_URLType = _YarlURL | _StarletteURL def _replace_query(url: _URLType, query: dict[str, Any]) -> str: @@ -69,9 +70,11 @@ def paginate_data( """ last_page = ceil(total / limit) - 1 + data = [item.model_dump() if hasattr(item, "model_dump") else item for item in chunk] + return PageDict( _meta=PageMetaInfoLimitOffset( - total=total, count=len(chunk), limit=limit, offset=offset + total=total, count=len(data), limit=limit, offset=offset ), _links=PageLinks( self=_replace_query(request_url, {"offset": offset, "limit": limit}), @@ -91,5 +94,5 @@ def paginate_data( request_url, {"offset": last_page * limit, "limit": limit} ), ), - data=chunk, + data=data, ) diff --git a/packages/models-library/src/models_library/user_preferences.py b/packages/models-library/src/models_library/user_preferences.py index 2680c10223d..f16c934b2da 100644 --- a/packages/models-library/src/models_library/user_preferences.py +++ b/packages/models-library/src/models_library/user_preferences.py @@ -1,6 +1,7 @@ from enum import auto -from typing import Annotated, Any, ClassVar, Literal, TypeAlias, get_args +from typing import Annotated, Any, ClassVar, Literal, TypeAlias +from common_library.pydantic_fields_extension import get_type from pydantic import BaseModel, Field from pydantic._internal._model_construction import ModelMetaclass @@ -94,11 +95,7 @@ def to_db(self) -> dict: @classmethod def update_preference_default_value(cls, new_default: Any) -> None: - expected_type = ( - t[0] - if (t := get_args(cls.model_fields["value"].annotation)) - else cls.model_fields["value"].annotation - ) + expected_type = get_type(cls.model_fields["value"]) detected_type = type(new_default) if expected_type != detected_type: msg = ( @@ -110,6 +107,9 @@ def update_preference_default_value(cls, new_default: Any) -> None: cls.model_fields["value"].default_factory = lambda: new_default else: cls.model_fields["value"].default = new_default + cls.model_fields["value"].default_factory = None + + cls.model_rebuild(force=True) class UserServiceUserPreference(_BaseUserPreferenceModel): diff --git a/packages/postgres-database/src/simcore_postgres_database/models/products.py b/packages/postgres-database/src/simcore_postgres_database/models/products.py index 03e137528ec..a71906ace59 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/products.py @@ -6,11 +6,12 @@ """ import json -from typing import Literal, TypedDict +from typing import Literal import sqlalchemy as sa from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.sql import func +from typing_extensions import TypedDict from .base import metadata from .groups import groups diff --git a/packages/pytest-simcore/src/pytest_simcore/hypothesis_type_strategies.py b/packages/pytest-simcore/src/pytest_simcore/hypothesis_type_strategies.py index ad80ab57774..788a0d36dab 100644 --- a/packages/pytest-simcore/src/pytest_simcore/hypothesis_type_strategies.py +++ b/packages/pytest-simcore/src/pytest_simcore/hypothesis_type_strategies.py @@ -1,9 +1,13 @@ from hypothesis import provisional from hypothesis import strategies as st -from pydantic import AnyHttpUrl, AnyUrl, HttpUrl +from hypothesis.strategies import composite +from pydantic import TypeAdapter +from pydantic_core import Url -# FIXME: For now it seems the pydantic hypothesis plugin does not provide strategies for these types. -# therefore we currently provide it -st.register_type_strategy(AnyUrl, provisional.urls()) -st.register_type_strategy(HttpUrl, provisional.urls()) -st.register_type_strategy(AnyHttpUrl, provisional.urls()) + +@composite +def url_strategy(draw): + return TypeAdapter(Url).validate_python(draw(provisional.urls())) + + +st.register_type_strategy(Url, url_strategy()) diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index 5a588041682..0dd40949ba8 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -224,7 +224,7 @@ async def parse_request_body_as( except json.decoder.JSONDecodeError as err: raise web.HTTPBadRequest(reason=f"Invalid json in body: {err}") from err - if hasattr(model_schema_cls, "parse_obj"): + if hasattr(model_schema_cls, "model_validate"): # NOTE: model_schema can be 'list[T]' or 'dict[T]' which raise TypeError # with issubclass(model_schema, BaseModel) assert issubclass(model_schema_cls, BaseModel) # nosec diff --git a/packages/service-library/tests/test_logging_errors.py b/packages/service-library/tests/test_logging_errors.py index b6b652a46d7..8bbbee60d40 100644 --- a/packages/service-library/tests/test_logging_errors.py +++ b/packages/service-library/tests/test_logging_errors.py @@ -3,6 +3,7 @@ import logging import pytest + from common_library.error_codes import create_error_code from common_library.errors_classes import OsparcErrorMixin from servicelib.logging_errors import ( diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in index 8d5ba7d34d8..308a1604cb3 100644 --- a/services/web/server/requirements/_base.in +++ b/services/web/server/requirements/_base.in @@ -9,6 +9,7 @@ # - Added as constraints instead of requirements in order to avoid polluting base.txt # - Will be installed when prod.txt or dev.txt # +--requirement ../../../../packages/common-library/requirements/_base.in --requirement ../../../../packages/models-library/requirements/_base.in --requirement ../../../../packages/postgres-database/requirements/_base.in --requirement ../../../../packages/settings-library/requirements/_base.in diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index d566b8d9112..9f0548262a8 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -26,17 +26,31 @@ aiofiles==0.8.0 # -r requirements/_base.in aiohttp==3.8.5 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -71,6 +85,8 @@ alembic==1.8.1 # via # -r requirements/../../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -105,17 +121,31 @@ captcha==0.5.0 # via -r requirements/_base.in certifi==2023.7.22 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -130,17 +160,31 @@ click==8.1.3 # via typer cryptography==41.0.7 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -154,7 +198,7 @@ deprecated==1.2.14 # opentelemetry-semantic-conventions dnspython==2.2.1 # via email-validator -email-validator==1.2.1 +email-validator==2.2.0 # via pydantic et-xmlfile==1.1.0 # via openpyxl @@ -199,17 +243,31 @@ jinja-app-loader==1.0.2 # via -r requirements/_base.in jinja2==3.1.2 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -232,17 +290,31 @@ lazy-object-proxy==1.7.1 # via openapi-core mako==1.2.2 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -341,24 +413,52 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt + # -r requirements/../../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in packaging==24.1 # via @@ -393,39 +493,101 @@ pycountry==23.12.11 # via -r requirements/_base.in pycparser==2.21 # via cffi -pydantic==1.10.17 +pydantic==2.9.2 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt # -c requirements/./constraints.txt + # -r requirements/../../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../../packages/service-library/requirements/_base.in + # -r requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -c requirements/./constraints.txt + # -r requirements/../../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in pygments==2.15.1 # via rich pyinstrument==4.6.1 @@ -440,6 +602,8 @@ python-dateutil==2.8.2 # via # arrow # faker +python-dotenv==1.0.1 + # via pydantic-settings python-engineio==4.3.4 # via python-socketio python-magic==0.4.25 @@ -450,17 +614,31 @@ pytz==2022.1 # via twilio pyyaml==6.0.1 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -470,17 +648,31 @@ pyyaml==6.0.1 # openapi-spec-validator redis==5.0.4 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -521,17 +713,31 @@ sniffio==1.3.1 # via anyio sqlalchemy==1.4.47 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt @@ -574,37 +780,66 @@ typing-extensions==4.12.0 # opentelemetry-sdk # pint # pydantic + # pydantic-core # typer ujson==5.5.0 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt # aiohttp-swagger urllib3==1.26.11 # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index 3aab7cde47d..d787c281d34 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -173,7 +173,9 @@ python-dateutil==2.8.2 # -c requirements/_base.txt # faker python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.1 # via # -c requirements/../../../../requirements/constraints.txt diff --git a/services/web/server/requirements/ci.txt b/services/web/server/requirements/ci.txt index f9917eb3748..bf55a2ed211 100644 --- a/services/web/server/requirements/ci.txt +++ b/services/web/server/requirements/ci.txt @@ -14,6 +14,7 @@ --requirement _tools.txt # installs this repo's packages +simcore-common-library @ ../../../packages/common-library simcore-models-library @ ../../../packages/models-library simcore-postgres-database @ ../../../packages/postgres-database simcore-settings-library @ ../../../packages/settings-library diff --git a/services/web/server/requirements/dev.txt b/services/web/server/requirements/dev.txt index b62c7127482..fdc9cb27429 100644 --- a/services/web/server/requirements/dev.txt +++ b/services/web/server/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../../packages/common-library/ --editable ../../../packages/models-library/ --editable ../../../packages/postgres-database/ --editable ../../../packages/settings-library/ diff --git a/services/web/server/requirements/prod.txt b/services/web/server/requirements/prod.txt index 9494dd12c30..2ccad765e49 100644 --- a/services/web/server/requirements/prod.txt +++ b/services/web/server/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +simcore-common-library @ ../../../packages/common-library simcore-models-library @ ../../../packages/models-library simcore-postgres-database @ ../../../packages/postgres-database simcore-settings-library @ ../../../packages/settings-library diff --git a/services/web/server/src/simcore_service_webserver/activity/_handlers.py b/services/web/server/src/simcore_service_webserver/activity/_handlers.py index ba7ec32557a..4e87c8f3bc0 100644 --- a/services/web/server/src/simcore_service_webserver/activity/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/activity/_handlers.py @@ -4,7 +4,7 @@ import aiohttp import aiohttp.web from models_library.api_schemas_webserver.activity import ActivityStatusDict -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.aiohttp.client_session import get_client_session from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.request_keys import RQT_USERID_KEY @@ -73,5 +73,5 @@ async def get_activity_status(request: aiohttp.web.Request): if not res: raise aiohttp.web.HTTPNoContent(content_type=MIMETYPE_APPLICATION_JSON) - assert parse_obj_as(ActivityStatusDict, res) is not None # nosec + assert TypeAdapter(ActivityStatusDict).validate_python(res) is not None # nosec return dict(res) diff --git a/services/web/server/src/simcore_service_webserver/announcements/_models.py b/services/web/server/src/simcore_service_webserver/announcements/_models.py index 4edb7c8d20a..7b48d86e2b2 100644 --- a/services/web/server/src/simcore_service_webserver/announcements/_models.py +++ b/services/web/server/src/simcore_service_webserver/announcements/_models.py @@ -1,15 +1,15 @@ from datetime import datetime -from typing import Any, ClassVar, Literal +from typing import Literal import arrow -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator # NOTE: this model is used for BOTH # - parse+validate from redis # - schema in the response class Announcement(BaseModel): - id: str # noqa: A003 + id: str products: list[str] start: datetime end: datetime @@ -18,10 +18,10 @@ class Announcement(BaseModel): link: str widgets: list[Literal["login", "ribbon", "user-menu"]] - @validator("end") + @field_validator("end") @classmethod - def check_start_before_end(cls, v, values): - if start := values.get("start"): + def _check_start_before_end(cls, v, info: ValidationInfo): + if start := info.data.get("start"): end = v if end <= start: msg = f"end={end!r} is not before start={start!r}" @@ -31,8 +31,8 @@ def check_start_before_end(cls, v, values): def expired(self) -> bool: return self.end <= arrow.utcnow().datetime - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "id": "Student_Competition_2023", @@ -56,3 +56,4 @@ class Config: }, ] } + ) diff --git a/services/web/server/src/simcore_service_webserver/announcements/_redis.py b/services/web/server/src/simcore_service_webserver/announcements/_redis.py index aad45ea8fee..785f954521f 100644 --- a/services/web/server/src/simcore_service_webserver/announcements/_redis.py +++ b/services/web/server/src/simcore_service_webserver/announcements/_redis.py @@ -37,7 +37,7 @@ async def list_announcements( announcements = [] for i, item in enumerate(items): try: - model = Announcement.parse_raw(item) + model = Announcement.model_validate_json(item) # filters if include_product not in model.products: continue diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_api.py b/services/web/server/src/simcore_service_webserver/api_keys/_api.py index 9a46ad9f512..9bbe56f7c6f 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_api.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_api.py @@ -70,7 +70,7 @@ async def get_api_key( ) -> ApiKeyGet | None: repo = ApiKeyRepo.create_from_app(app) row = await repo.get(display_name=name, user_id=user_id, product_name=product_name) - return ApiKeyGet.parse_obj(row) if row else None + return ApiKeyGet.model_validate(row) if row else None async def get_or_create_api_key( diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_db.py b/services/web/server/src/simcore_service_webserver/api_keys/_db.py index 4a51464e1a9..ec08ce5dd67 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_db.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_db.py @@ -79,7 +79,7 @@ async def get( result: ResultProxy = await conn.execute(stmt) row: RowProxy | None = await result.fetchone() - return ApiKeyInDB.from_orm(row) if row else None + return ApiKeyInDB.model_validate(row) if row else None async def get_or_create( self, @@ -116,7 +116,7 @@ async def get_or_create( result = await conn.execute(insert_stmt) row = await result.fetchone() assert row # nosec - return ApiKeyInDB.from_orm(row) + return ApiKeyInDB.model_validate(row) async def delete_by_name( self, *, display_name: str, user_id: UserID, product_name: ProductName @@ -145,7 +145,7 @@ async def prune_expired(self) -> list[str]: stmt = ( api_keys.delete() .where( - (api_keys.c.expires_at != None) # noqa: E711 + (api_keys.c.expires_at.is_not(None)) & (api_keys.c.expires_at < sa.func.now()) ) .returning(api_keys.c.display_name) diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_handlers.py b/services/web/server/src/simcore_service_webserver/api_keys/_handlers.py index 627d733d9c7..6327a57ca54 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_handlers.py @@ -32,7 +32,7 @@ class _RequestContext(RequestParams): @login_required @permission_required("user.apikey.*") async def list_api_keys(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) api_keys_names = await _api.list_api_keys( request.app, user_id=req_ctx.user_id, @@ -45,7 +45,7 @@ async def list_api_keys(request: web.Request): @login_required @permission_required("user.apikey.*") async def create_api_key(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) new = await parse_request_body_as(ApiKeyCreate, request) try: data = await _api.create_api_key( @@ -67,7 +67,7 @@ async def create_api_key(request: web.Request): @login_required @permission_required("user.apikey.*") async def delete_api_key(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) # NOTE: SEE https://github.com/ITISFoundation/osparc-simcore/issues/4920 body = await request.json() diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index a105dcdcc1a..94d1be5358d 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -3,6 +3,7 @@ from typing import Any, Final from aiohttp import web +from common_library.pydantic_fields_extension import is_nullable from models_library.basic_types import ( BootModeEnum, BuildTargetEnum, @@ -11,9 +12,17 @@ VersionTag, ) from models_library.utils.change_case import snake_to_camel -from pydantic import AnyHttpUrl, parse_obj_as, root_validator, validator -from pydantic.fields import Field, ModelField +from pydantic import ( + AliasChoices, + AnyHttpUrl, + TypeAdapter, + ValidationInfo, + field_validator, + model_validator, +) +from pydantic.fields import Field from pydantic.types import PositiveInt +from pydantic_settings import SettingsConfigDict from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.email import SMTPSettings @@ -54,7 +63,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # CODE STATICS --------------------------------------------------------- API_VERSION: str = API_VERSION APP_NAME: str = APP_NAME - API_VTAG: VersionTag = parse_obj_as(VersionTag, API_VTAG) + API_VTAG: VersionTag = TypeAdapter(VersionTag).validate_python(API_VTAG) # IMAGE BUILDTIME ------------------------------------------------------ # @Makefile @@ -84,13 +93,15 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): SIMCORE_VCS_RELEASE_TAG: str | None = Field( default=None, description="Name of the tag that marks this release, or None if undefined", - example="ResistanceIsFutile10", + examples=["ResistanceIsFutile10"], ) SIMCORE_VCS_RELEASE_URL: AnyHttpUrl | None = Field( default=None, description="URL to release notes", - example="https://github.com/ITISFoundation/osparc-simcore/releases/tag/staging_ResistanceIsFutile10", + examples=[ + "https://github.com/ITISFoundation/osparc-simcore/releases/tag/staging_ResistanceIsFutile10" + ], ) SWARM_STACK_NAME: str | None = Field( @@ -106,13 +117,14 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) WEBSERVER_LOGLEVEL: LogLevel = Field( default=LogLevel.WARNING.value, - env=["WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices("WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), # NOTE: suffix '_LOGLEVEL' is used overall ) - WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=["WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + validation_alias=AliasChoices( + "WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED" + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) WEBSERVER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( @@ -128,100 +140,117 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="host name to serve within the container." "NOTE that this different from WEBSERVER_HOST env which is the host seen outside the container", ) - WEBSERVER_HOST: str | None = Field(None, env=["WEBSERVER_HOST", "HOST", "HOSTNAME"]) - WEBSERVER_PORT: PortInt = parse_obj_as(PortInt, DEFAULT_AIOHTTP_PORT) + WEBSERVER_HOST: str | None = Field( + None, validation_alias=AliasChoices("WEBSERVER_HOST", "HOST", "HOSTNAME") + ) + WEBSERVER_PORT: PortInt = TypeAdapter(PortInt).validate_python(DEFAULT_AIOHTTP_PORT) WEBSERVER_FRONTEND: FrontEndAppSettings | None = Field( - auto_default_from_env=True, description="front-end static settings" + json_schema_extra={"auto_default_from_env": True}, + description="front-end static settings", ) # PLUGINS ---------------- WEBSERVER_ACTIVITY: PrometheusSettings | None = Field( - auto_default_from_env=True, + json_schema_extra={"auto_default_from_env": True}, description="activity plugin", ) WEBSERVER_CATALOG: CatalogSettings | None = Field( - auto_default_from_env=True, description="catalog service client's plugin" + json_schema_extra={"auto_default_from_env": True}, + description="catalog service client's plugin", ) # TODO: Shall be required WEBSERVER_DB: PostgresSettings | None = Field( - auto_default_from_env=True, description="database plugin" + json_schema_extra={"auto_default_from_env": True}, description="database plugin" ) WEBSERVER_DIAGNOSTICS: DiagnosticsSettings | None = Field( - auto_default_from_env=True, description="diagnostics plugin" + json_schema_extra={"auto_default_from_env": True}, + description="diagnostics plugin", ) WEBSERVER_DIRECTOR_V2: DirectorV2Settings | None = Field( - auto_default_from_env=True, description="director-v2 service client's plugin" + json_schema_extra={"auto_default_from_env": True}, + description="director-v2 service client's plugin", ) WEBSERVER_EMAIL: SMTPSettings | None = Field( - auto_default_from_env=True, description="email plugin" + json_schema_extra={"auto_default_from_env": True}, description="email plugin" ) WEBSERVER_EXPORTER: ExporterSettings | None = Field( - auto_default_from_env=True, description="exporter plugin" + json_schema_extra={"auto_default_from_env": True}, description="exporter plugin" ) WEBSERVER_GARBAGE_COLLECTOR: GarbageCollectorSettings | None = Field( - auto_default_from_env=True, description="garbage collector plugin" + json_schema_extra={"auto_default_from_env": True}, + description="garbage collector plugin", ) WEBSERVER_INVITATIONS: InvitationsSettings | None = Field( - auto_default_from_env=True, description="invitations plugin" + json_schema_extra={"auto_default_from_env": True}, + description="invitations plugin", ) WEBSERVER_LOGIN: LoginSettings | None = Field( - auto_default_from_env=True, description="login plugin" + json_schema_extra={"auto_default_from_env": True}, description="login plugin" ) WEBSERVER_PAYMENTS: PaymentsSettings | None = Field( - auto_default_from_env=True, description="payments plugin settings" + json_schema_extra={"auto_default_from_env": True}, + description="payments plugin settings", ) WEBSERVER_DYNAMIC_SCHEDULER: DynamicSchedulerSettings | None = Field( - auto_default_from_env=True, description="dynamic-scheduler plugin settings" + description="dynamic-scheduler plugin settings", + json_schema_extra={"auto_default_from_env": True}, ) - WEBSERVER_REDIS: RedisSettings | None = Field(auto_default_from_env=True) + WEBSERVER_REDIS: RedisSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) WEBSERVER_REST: RestSettings | None = Field( - auto_default_from_env=True, description="rest api plugin" + description="rest api plugin", json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_RESOURCE_MANAGER: ResourceManagerSettings = Field( - auto_default_from_env=True, description="resource_manager plugin" + description="resource_manager plugin", + json_schema_extra={"auto_default_from_env": True}, ) WEBSERVER_RESOURCE_USAGE_TRACKER: ResourceUsageTrackerSettings | None = Field( - auto_default_from_env=True, description="resource usage tracker service client's plugin", + json_schema_extra={"auto_default_from_env": True}, ) WEBSERVER_SCICRUNCH: SciCrunchSettings | None = Field( - auto_default_from_env=True, description="scicrunch plugin" + description="scicrunch plugin", + json_schema_extra={"auto_default_from_env": True}, ) WEBSERVER_SESSION: SessionSettings = Field( - auto_default_from_env=True, description="session plugin" + description="session plugin", json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_STATICWEB: StaticWebserverModuleSettings | None = Field( - auto_default_from_env=True, description="static-webserver service plugin" + description="static-webserver service plugin", + json_schema_extra={"auto_default_from_env": True}, ) WEBSERVER_STORAGE: StorageSettings | None = Field( - auto_default_from_env=True, description="storage service client's plugin" + description="storage service client's plugin", + json_schema_extra={"auto_default_from_env": True}, ) WEBSERVER_STUDIES_DISPATCHER: StudiesDispatcherSettings | None = Field( - auto_default_from_env=True, description="studies dispatcher plugin" + description="studies dispatcher plugin", + json_schema_extra={"auto_default_from_env": True}, ) WEBSERVER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="tracing plugin" + description="tracing plugin", json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_PROJECTS: ProjectsSettings | None = Field( - auto_default_from_env=True, description="projects plugin" + description="projects plugin", json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_RABBITMQ: RabbitSettings | None = Field( - auto_default_from_env=True, description="rabbitmq plugin" + description="rabbitmq plugin", json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_USERS: UsersSettings | None = Field( - auto_default_from_env=True, description="users plugin" + description="users plugin", json_schema_extra={"auto_default_from_env": True} ) # These plugins only require (for the moment) an entry to toggle between enabled/disabled @@ -250,57 +279,59 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): "Currently this is a system plugin and cannot be disabled", ) - @root_validator() + @model_validator(mode="after") @classmethod - def build_vcs_release_url_if_unset(cls, values): - release_url = values.get("SIMCORE_VCS_RELEASE_URL") + def build_vcs_release_url_if_unset(cls, v): + release_url = v.SIMCORE_VCS_RELEASE_URL - if release_url is None and ( - vsc_release_tag := values.get("SIMCORE_VCS_RELEASE_TAG") - ): + if release_url is None and (vsc_release_tag := v.SIMCORE_VCS_RELEASE_TAG): if vsc_release_tag == "latest": release_url = ( "https://github.com/ITISFoundation/osparc-simcore/commits/master/" ) else: release_url = f"https://github.com/ITISFoundation/osparc-simcore/releases/tag/{vsc_release_tag}" - values["SIMCORE_VCS_RELEASE_URL"] = release_url + v.SIMCORE_VCS_RELEASE_URL = release_url - return values + return v - @validator( + @field_validator( # List of plugins under-development (keep up-to-date) # TODO: consider mark as dev-feature in field extras of Config attr. # Then they can be automtically advertised "WEBSERVER_META_MODELING", "WEBSERVER_VERSION_CONTROL", - pre=True, - always=True, + mode="before", ) @classmethod - def enable_only_if_dev_features_allowed(cls, v, values, field: ModelField): + def enable_only_if_dev_features_allowed(cls, v, info: ValidationInfo): """Ensures that plugins 'under development' get programatically disabled if WEBSERVER_DEV_FEATURES_ENABLED=False """ - if values["WEBSERVER_DEV_FEATURES_ENABLED"]: + if info.data["WEBSERVER_DEV_FEATURES_ENABLED"]: return v if v: _logger.warning( - "%s still under development and will be disabled.", field.name + "%s still under development and will be disabled.", info.field_name ) - return None if field.allow_none else False + + return ( + None + if info.field_name and is_nullable(cls.model_fields[info.field_name]) + else False + ) @cached_property def log_level(self) -> int: level: int = getattr(logging, self.WEBSERVER_LOGLEVEL.upper()) return level - @validator("WEBSERVER_LOGLEVEL", pre=True) + @field_validator("WEBSERVER_LOGLEVEL") @classmethod - def valid_log_level(cls, value: str) -> str: + def valid_log_level(cls, value): return cls.validate_log_level(value) - @validator("SC_HEALTHCHECK_TIMEOUT", pre=True) + @field_validator("SC_HEALTHCHECK_TIMEOUT", mode="before") @classmethod def get_healthcheck_timeout_in_seconds(cls, v): # Ex. HEALTHCHECK --interval=5m --timeout=3s @@ -356,7 +387,7 @@ def _export_by_alias(self, **kwargs) -> dict[str, Any]: def config_alias_generator(s): return s.lower() - data: dict[str, Any] = self.dict(**kwargs) + data: dict[str, Any] = self.model_dump(**kwargs) current_keys = list(data.keys()) for key in current_keys: @@ -418,7 +449,7 @@ def setup_settings(app: web.Application) -> ApplicationSettings: app[APP_SETTINGS_KEY] = settings _logger.debug( "Captured app settings:\n%s", - app[APP_SETTINGS_KEY].json(indent=1, sort_keys=True), + app[APP_SETTINGS_KEY].model_dump_json(indent=1), ) return settings diff --git a/services/web/server/src/simcore_service_webserver/application_settings_utils.py b/services/web/server/src/simcore_service_webserver/application_settings_utils.py index 9123c8ad574..9843e84afdd 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings_utils.py +++ b/services/web/server/src/simcore_service_webserver/application_settings_utils.py @@ -10,6 +10,7 @@ from typing import Any from aiohttp import web +from common_library.pydantic_fields_extension import get_type, is_nullable from pydantic.types import SecretStr from servicelib.aiohttp.typing_extension import Handler @@ -200,10 +201,10 @@ def convert_to_environ_vars( # noqa: C901, PLR0915, PLR0912 def _set_if_disabled(field_name, section): # Assumes that by default is enabled enabled = section.get("enabled", True) - field = ApplicationSettings.__fields__[field_name] + field = ApplicationSettings.model_fields[field_name] if not enabled: - envs[field_name] = "null" if field.allow_none else "0" - elif field.type_ == bool: + envs[field_name] = "null" if is_nullable(field) else "0" + elif get_type(field) == bool: envs[field_name] = "1" if main := cfg.get("main"): diff --git a/services/web/server/src/simcore_service_webserver/catalog/_api.py b/services/web/server/src/simcore_service_webserver/catalog/_api.py index 9bbbae4e43c..f2fc9be73a9 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_api.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_api.py @@ -23,7 +23,7 @@ from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from pint import UnitRegistry -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from servicelib.aiohttp.requests_validation import handle_validation_as_http_error from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc from servicelib.rest_constants import RESPONSE_MODEL_POLICY @@ -42,9 +42,7 @@ class CatalogRequestContext(BaseModel): user_id: UserID product_name: str unit_registry: UnitRegistry - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) @classmethod def create(cls, request: Request) -> "CatalogRequestContext": @@ -157,7 +155,7 @@ async def update_service_v2( user_id=user_id, service_key=service_key, service_version=service_version, - update=ServiceUpdateV2.parse_obj(update_data), + update=ServiceUpdateV2.model_validate(update_data), ) data = jsonable_encoder(service, exclude_unset=True) @@ -286,8 +284,8 @@ async def get_compatible_inputs_given_source_output( from_service_key, from_service_version, from_output_key, ctx ) - from_output: ServiceOutput = ServiceOutput.construct( - **service_output.dict(include=ServiceOutput.__fields__.keys()) + from_output: ServiceOutput = ServiceOutput.model_construct( + **service_output.model_dump(include=ServiceOutput.model_fields.keys()) # type: ignore[arg-type] ) # N inputs @@ -295,8 +293,8 @@ async def get_compatible_inputs_given_source_output( def iter_service_inputs() -> Iterator[tuple[ServiceInputKey, ServiceInput]]: for service_input in service_inputs: - yield service_input.key_id, ServiceInput.construct( - **service_input.dict(include=ServiceInput.__fields__.keys()) + yield service_input.key_id, ServiceInput.model_construct( + **service_input.model_dump(include=ServiceInput.model_fields.keys()) # type: ignore[arg-type] ) # check @@ -354,16 +352,16 @@ async def get_compatible_outputs_given_target_input( def iter_service_outputs() -> Iterator[tuple[ServiceOutputKey, ServiceOutput]]: for service_output in service_outputs: - yield service_output.key_id, ServiceOutput.construct( - **service_output.dict(include=ServiceOutput.__fields__.keys()) + yield service_output.key_id, ServiceOutput.model_construct( + **service_output.model_dump(include=ServiceOutput.model_fields.keys()) # type: ignore[arg-type] ) # 1 input service_input = await get_service_input( to_service_key, to_service_version, to_input_key, ctx ) - to_input: ServiceInput = ServiceInput.construct( - **service_input.dict(include=ServiceInput.__fields__.keys()) + to_input: ServiceInput = ServiceInput.model_construct( + **service_input.model_dump(include=ServiceInput.model_fields.keys()) # type: ignore[arg-type] ) # check diff --git a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_handlers.py index 02e21f37e29..a9ba40b5378 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_handlers.py @@ -26,7 +26,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import BaseModel, Extra, Field, parse_obj_as, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -54,12 +54,12 @@ class ServicePathParams(BaseModel): service_key: ServiceKey service_version: ServiceVersion + model_config = ConfigDict( + populate_by_name=True, + extra="forbid", + ) - class Config: - allow_population_by_field_name = True - extra = Extra.forbid - - @validator("service_key", pre=True) + @field_validator("service_key", mode="before") @classmethod def ensure_unquoted(cls, v): # NOTE: this is needed as in pytest mode, the aiohttp server does not seem to unquote automatically @@ -90,7 +90,7 @@ async def list_services_latest(request: Request): user_id=request_ctx.user_id, product_name=request_ctx.product_name, unit_registry=request_ctx.unit_registry, - page_params=PageQueryParameters.construct( + page_params=PageQueryParameters.model_construct( offset=query_params.offset, limit=query_params.limit ), ) @@ -98,7 +98,7 @@ async def list_services_latest(request: Request): assert page_meta.limit == query_params.limit # nosec assert page_meta.offset == query_params.offset # nosec - page = Page[CatalogServiceGet].parse_obj( + page = Page[CatalogServiceGet].model_validate( paginate_data( chunk=page_items, request_url=request.url, @@ -133,7 +133,7 @@ async def get_service(request: Request): service_version=path_params.service_version, ) - return envelope_json_response(CatalogServiceGet.parse_obj(service)) + return envelope_json_response(CatalogServiceGet.model_validate(service)) @routes.patch( @@ -160,11 +160,11 @@ async def update_service(request: Request): product_name=request_ctx.product_name, service_key=path_params.service_key, service_version=path_params.service_version, - update_data=update.dict(exclude_unset=True), + update_data=update.model_dump(exclude_unset=True), unit_registry=request_ctx.unit_registry, ) - return envelope_json_response(CatalogServiceGet.parse_obj(updated)) + return envelope_json_response(CatalogServiceGet.model_validate(updated)) @routes.get( @@ -182,7 +182,7 @@ async def list_service_inputs(request: Request): path_params.service_key, path_params.service_version, ctx ) - data = [m.dict(**RESPONSE_MODEL_POLICY) for m in response_model] + data = [m.model_dump(**RESPONSE_MODEL_POLICY) for m in response_model] return await asyncio.get_event_loop().run_in_executor( None, envelope_json_response, data ) @@ -210,7 +210,7 @@ async def get_service_input(request: Request): ctx, ) - data = response_model.dict(**RESPONSE_MODEL_POLICY) + data = response_model.model_dump(**RESPONSE_MODEL_POLICY) return await asyncio.get_event_loop().run_in_executor( None, envelope_json_response, data ) @@ -265,7 +265,7 @@ async def list_service_outputs(request: Request): path_params.service_key, path_params.service_version, ctx ) - data = [m.dict(**RESPONSE_MODEL_POLICY) for m in response_model] + data = [m.model_dump(**RESPONSE_MODEL_POLICY) for m in response_model] return await asyncio.get_event_loop().run_in_executor( None, envelope_json_response, data ) @@ -293,7 +293,7 @@ async def get_service_output(request: Request): ctx, ) - data = response_model.dict(**RESPONSE_MODEL_POLICY) + data = response_model.model_dump(**RESPONSE_MODEL_POLICY) return await asyncio.get_event_loop().run_in_executor( None, envelope_json_response, data ) @@ -387,4 +387,6 @@ async def get_service_pricing_plan(request: Request): service_version=f"{path_params.service_version}", ) - return envelope_json_response(parse_obj_as(PricingPlanGet, pricing_plan)) + return envelope_json_response( + PricingPlanGet.model_validate(pricing_plan.model_dump()) + ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/client.py b/services/web/server/src/simcore_service_webserver/catalog/client.py index 8a8f6083252..386ae811da0 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/client.py +++ b/services/web/server/src/simcore_service_webserver/catalog/client.py @@ -19,7 +19,7 @@ ) from models_library.services_resources import ServiceResourcesDict from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.client_session import get_client_session from servicelib.rest_constants import X_PRODUCT_NAME_HEADER @@ -146,7 +146,7 @@ async def get_service_resources( async with session.get(url) as resp: resp.raise_for_status() dict_response = await resp.json() - return parse_obj_as(ServiceResourcesDict, dict_response) + return TypeAdapter(ServiceResourcesDict).validate_python(dict_response) async def get_service_access_rights( @@ -168,7 +168,7 @@ async def get_service_access_rights( ) as resp: resp.raise_for_status() body = await resp.json() - return ServiceAccessRightsGet.parse_obj(body) + return ServiceAccessRightsGet.model_validate(body) async def update_service( diff --git a/services/web/server/src/simcore_service_webserver/cli.py b/services/web/server/src/simcore_service_webserver/cli.py index ca85670ad74..b09f4eca0c4 100644 --- a/services/web/server/src/simcore_service_webserver/cli.py +++ b/services/web/server/src/simcore_service_webserver/cli.py @@ -19,6 +19,7 @@ import typer from aiohttp import web +from common_library.json_serialization import json_dumps from settings_library.utils_cli import create_settings_command from typing_extensions import Annotated @@ -68,7 +69,8 @@ async def app_factory() -> web.Application: assert app_settings.SC_BUILD_TARGET # nosec _logger.info( - "Application settings: %s", app_settings.json(indent=2, sort_keys=True) + "Application settings: %s", + json_dumps(app_settings, indent=2, sort_keys=True), ) app, _ = _setup_app_from_settings(app_settings) diff --git a/services/web/server/src/simcore_service_webserver/clusters/_handlers.py b/services/web/server/src/simcore_service_webserver/clusters/_handlers.py index 70752da883b..41d64a2d5ef 100644 --- a/services/web/server/src/simcore_service_webserver/clusters/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/clusters/_handlers.py @@ -11,7 +11,7 @@ ClusterPing, ) from models_library.users import UserID -from pydantic import BaseModel, Field, parse_obj_as +from pydantic import BaseModel, Field, TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -78,7 +78,7 @@ class _RequestContext(BaseModel): @permission_required("clusters.create") @_handle_cluster_exceptions async def create_cluster(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) new_cluster = await parse_request_body_as(ClusterCreate, request) created_cluster = await director_v2_api.create_cluster( @@ -94,13 +94,13 @@ async def create_cluster(request: web.Request) -> web.Response: @permission_required("clusters.read") @_handle_cluster_exceptions async def list_clusters(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) clusters = await director_v2_api.list_clusters( app=request.app, user_id=req_ctx.user_id, ) - assert parse_obj_as(list[ClusterGet], clusters) is not None # nosec + assert TypeAdapter(list[ClusterGet]).validate_python(clusters) is not None # nosec return envelope_json_response(clusters) @@ -109,7 +109,7 @@ async def list_clusters(request: web.Request) -> web.Response: @permission_required("clusters.read") @_handle_cluster_exceptions async def get_cluster(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ClusterPathParams, request) cluster = await director_v2_api.get_cluster( @@ -117,7 +117,7 @@ async def get_cluster(request: web.Request) -> web.Response: user_id=req_ctx.user_id, cluster_id=path_params.cluster_id, ) - assert parse_obj_as(ClusterGet, cluster) is not None # nosec + assert ClusterGet.model_validate(cluster) is not None # nosec return envelope_json_response(cluster) @@ -126,7 +126,7 @@ async def get_cluster(request: web.Request) -> web.Response: @permission_required("clusters.write") @_handle_cluster_exceptions async def update_cluster(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ClusterPathParams, request) cluster_patch = await parse_request_body_as(ClusterPatch, request) @@ -137,7 +137,7 @@ async def update_cluster(request: web.Request) -> web.Response: cluster_patch=cluster_patch, ) - assert parse_obj_as(ClusterGet, updated_cluster) is not None # nosec + assert ClusterGet.model_validate(updated_cluster) is not None # nosec return envelope_json_response(updated_cluster) @@ -146,7 +146,7 @@ async def update_cluster(request: web.Request) -> web.Response: @permission_required("clusters.delete") @_handle_cluster_exceptions async def delete_cluster(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ClusterPathParams, request) await director_v2_api.delete_cluster( @@ -165,7 +165,7 @@ async def delete_cluster(request: web.Request) -> web.Response: @permission_required("clusters.read") @_handle_cluster_exceptions async def get_cluster_details(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ClusterPathParams, request) cluster_details = await director_v2_api.get_cluster_details( @@ -173,7 +173,7 @@ async def get_cluster_details(request: web.Request) -> web.Response: user_id=req_ctx.user_id, cluster_id=path_params.cluster_id, ) - assert parse_obj_as(ClusterDetails, cluster_details) is not None # nosec + assert ClusterDetails.model_validate(cluster_details) is not None # nosec return envelope_json_response(cluster_details) @@ -199,7 +199,7 @@ async def ping_cluster(request: web.Request) -> web.Response: @permission_required("clusters.read") @_handle_cluster_exceptions async def ping_cluster_cluster_id(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ClusterPathParams, request) await director_v2_api.ping_specific_cluster( diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py index 3171d0a80fb..bed2f77f7f2 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py @@ -9,7 +9,7 @@ from aiohttp import ClientError, ClientSession, web from models_library.app_diagnostics import AppStatusCheck -from pydantic import BaseModel, Field, parse_obj_as +from pydantic import BaseModel, Field from servicelib.aiohttp.client_session import get_client_session from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.utils import logged_gather @@ -61,7 +61,7 @@ async def get_app_diagnostics(request: web.Request): top_tracemalloc=get_tracemalloc_info(top=query_params.top_tracemalloc) ) - assert parse_obj_as(StatusDiagnosticsGet, data) is not None # nosec + assert StatusDiagnosticsGet.model_validate(data) is not None # nosec return envelope_json_response(data) @@ -98,7 +98,7 @@ def _get_client_session_info(): return info - check = AppStatusCheck.parse_obj( + check = AppStatusCheck.model_validate( { "app_name": APP_NAME, "version": API_VERSION, @@ -149,7 +149,7 @@ async def _check_resource_usage_tracker(): reraise=False, ) - return envelope_json_response(check.dict(exclude_unset=True)) + return envelope_json_response(check.model_dump(exclude_unset=True)) @routes.get(f"/{api_version_prefix}/status/{{service_name}}", name="get_service_status") diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/settings.py b/services/web/server/src/simcore_service_webserver/diagnostics/settings.py index 5c82496ad34..b9557f6d231 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/settings.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/settings.py @@ -1,5 +1,12 @@ from aiohttp.web import Application -from pydantic import Field, NonNegativeFloat, PositiveFloat, validator +from pydantic import ( + AliasChoices, + Field, + NonNegativeFloat, + PositiveFloat, + ValidationInfo, + field_validator, +) from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from settings_library.base import BaseCustomSettings @@ -11,7 +18,9 @@ class DiagnosticsSettings(BaseCustomSettings): "Any task blocked more than slow_duration_secs is logged as WARNING" "Aims to identify possible blocking calls" ), - env=["DIAGNOSTICS_SLOW_DURATION_SECS", "AIODEBUG_SLOW_DURATION_SECS"], + validation_alias=AliasChoices( + "DIAGNOSTICS_SLOW_DURATION_SECS", "AIODEBUG_SLOW_DURATION_SECS" + ), ) DIAGNOSTICS_HEALTHCHECK_ENABLED: bool = Field( @@ -32,13 +41,13 @@ class DiagnosticsSettings(BaseCustomSettings): DIAGNOSTICS_START_SENSING_DELAY: NonNegativeFloat = 60.0 - @validator("DIAGNOSTICS_MAX_TASK_DELAY", pre=True) + @field_validator("DIAGNOSTICS_MAX_TASK_DELAY", mode="before") @classmethod - def _validate_max_task_delay(cls, v, values): + def _validate_max_task_delay(cls, v, info: ValidationInfo): # Sets an upper threshold for blocking functions, i.e. # settings.DIAGNOSTICS_SLOW_DURATION_SECS < settings.DIAGNOSTICS_MAX_TASK_DELAY # - slow_duration_secs = float(values["DIAGNOSTICS_SLOW_DURATION_SECS"]) + slow_duration_secs = float(info.data["DIAGNOSTICS_SLOW_DURATION_SECS"]) return max( 10 * slow_duration_secs, float(v), diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_api_utils.py b/services/web/server/src/simcore_service_webserver/director_v2/_api_utils.py index e9bbca91c50..74bc8e8ee14 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_api_utils.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_api_utils.py @@ -2,7 +2,7 @@ from models_library.projects import ProjectID from models_library.users import UserID from models_library.wallets import WalletID, WalletInfo -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..application_settings import get_application_settings from ..products.api import Product @@ -35,7 +35,9 @@ async def get_wallet_info( ) if user_default_wallet_preference is None: raise UserDefaultWalletNotFoundError(uid=user_id) - project_wallet_id = parse_obj_as(WalletID, user_default_wallet_preference.value) + project_wallet_id = TypeAdapter(WalletID).validate_python( + user_default_wallet_preference.value + ) await projects_api.connect_wallet_to_project( app, product_name=product_name, diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_core_computations.py b/services/web/server/src/simcore_service_webserver/director_v2/_core_computations.py index 5b8a69e33c3..c034f93a660 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_core_computations.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_core_computations.py @@ -4,12 +4,12 @@ """ -import json import logging from typing import Any from uuid import UUID from aiohttp import web +from common_library.serialization import model_dump_with_secrets from models_library.api_schemas_directorv2.clusters import ( ClusterCreate, ClusterDetails, @@ -26,11 +26,10 @@ from models_library.projects_pipeline import ComputationTask from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.types import PositiveInt from servicelib.aiohttp import status from servicelib.logging_utils import log_decorator -from settings_library.utils_encoders import create_json_encoder_wo_secrets from ..products.api import get_product from ._api_utils import get_wallet_info @@ -182,7 +181,7 @@ async def get_computation_task( computation_task_out_dict = await request_director_v2( app, "GET", backend_url, expected_status=web.HTTPOk ) - task_out = ComputationTask.parse_obj(computation_task_out_dict) + task_out = ComputationTask.model_validate(computation_task_out_dict) _logger.debug("found computation task: %s", f"{task_out=}") return task_out except DirectorServiceError as exc: @@ -245,16 +244,12 @@ async def create_cluster( "POST", url=(settings.base_url / "clusters").update_query(user_id=int(user_id)), expected_status=web.HTTPCreated, - data=json.loads( - new_cluster.json( - by_alias=True, - exclude_unset=True, - encoder=create_json_encoder_wo_secrets(ClusterCreate), - ) + data=model_dump_with_secrets( + new_cluster, show_secrets=True, by_alias=True, exclude_unset=True ), ) assert isinstance(cluster, dict) # nosec - assert parse_obj_as(ClusterGet, cluster) is not None # nosec + assert ClusterGet.model_validate(cluster) is not None # nosec return cluster @@ -268,7 +263,7 @@ async def list_clusters(app: web.Application, user_id: UserID) -> list[DataType] ) assert isinstance(clusters, list) # nosec - assert parse_obj_as(list[ClusterGet], clusters) is not None # nosec + assert TypeAdapter(list[ClusterGet]).validate_python(clusters) is not None # nosec return clusters @@ -296,7 +291,7 @@ async def get_cluster( ) assert isinstance(cluster, dict) # nosec - assert parse_obj_as(ClusterGet, cluster) is not None # nosec + assert ClusterGet.model_validate(cluster) is not None # nosec return cluster @@ -324,7 +319,7 @@ async def get_cluster_details( }, ) assert isinstance(cluster, dict) # nosec - assert parse_obj_as(ClusterDetails, cluster) is not None # nosec + assert ClusterDetails.model_validate(cluster) is not None # nosec return cluster @@ -342,12 +337,8 @@ async def update_cluster( user_id=int(user_id) ), expected_status=web.HTTPOk, - data=json.loads( - cluster_patch.json( - by_alias=True, - exclude_unset=True, - encoder=create_json_encoder_wo_secrets(ClusterPatch), - ) + data=model_dump_with_secrets( + cluster_patch, show_secrets=True, by_alias=True, exclude_none=True ), on_error={ status.HTTP_404_NOT_FOUND: ( @@ -362,7 +353,7 @@ async def update_cluster( ) assert isinstance(cluster, dict) # nosec - assert parse_obj_as(ClusterGet, cluster) is not None # nosec + assert ClusterGet.model_validate(cluster) is not None # nosec return cluster @@ -397,12 +388,11 @@ async def ping_cluster(app: web.Application, cluster_ping: ClusterPing) -> None: "POST", url=settings.base_url / "clusters:ping", expected_status=web.HTTPNoContent, - data=json.loads( - cluster_ping.json( - by_alias=True, - exclude_unset=True, - encoder=create_json_encoder_wo_secrets(ClusterPing), - ) + data=model_dump_with_secrets( + cluster_ping, + show_secrets=True, + by_alias=True, + exclude_unset=True, ), on_error={ status.HTTP_422_UNPROCESSABLE_ENTITY: ( diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py index 20cf772075e..21793b79376 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_core_dynamic_services.py @@ -10,7 +10,7 @@ from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.projects import ProjectID from models_library.services import ServicePortKey -from pydantic import BaseModel, NonNegativeInt, parse_obj_as +from pydantic import BaseModel, NonNegativeInt, TypeAdapter from pydantic.types import PositiveInt from servicelib.logging_utils import log_decorator from yarl import URL @@ -33,7 +33,7 @@ async def list_dynamic_services( project_id: str | None = None, ) -> list[DynamicServiceGet]: params = _Params(user_id=user_id, project_id=project_id) - params_dict = params.dict(exclude_none=True) + params_dict = params.model_dump(exclude_none=True) settings: DirectorV2Settings = get_plugin_settings(app) if params_dict: # Update query doesnt work with no params to unwrap backend_url = (settings.base_url / "dynamic_services").update_query( @@ -49,7 +49,7 @@ async def list_dynamic_services( if services is None: services = [] assert isinstance(services, list) # nosec - return parse_obj_as(list[DynamicServiceGet], services) + return TypeAdapter(list[DynamicServiceGet]).validate_python(services) # NOTE: ANE https://github.com/ITISFoundation/osparc-simcore/issues/3191 diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py b/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py index daffd3be369..b7d973937ef 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_handlers.py @@ -8,7 +8,7 @@ from models_library.clusters import ClusterID from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import BaseModel, Field, ValidationError, parse_obj_as +from pydantic import BaseModel, Field, TypeAdapter, ValidationError from pydantic.types import NonNegativeInt from servicelib.aiohttp import status from servicelib.aiohttp.rest_responses import create_http_error, exception_to_response @@ -64,7 +64,7 @@ class _ComputationStarted(BaseModel): async def start_computation(request: web.Request) -> web.Response: # pylint: disable=too-many-statements try: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) computations = ComputationsApi(request.app) run_policy = get_project_run_policy(request.app) @@ -78,7 +78,9 @@ async def start_computation(request: web.Request) -> web.Response: if request.can_read_body: body = await request.json() - assert parse_obj_as(ComputationStart, body) is not None # nosec + assert ( + TypeAdapter(ComputationStart).validate_python(body) is not None + ) # nosec subgraph = body.get("subgraph", []) force_restart = bool(body.get("force_restart", force_restart)) @@ -158,7 +160,9 @@ async def start_computation(request: web.Request) -> web.Response: if project_vc_commits: data["ref_ids"] = project_vc_commits - assert parse_obj_as(_ComputationStarted, data) is not None # nosec + assert ( + TypeAdapter(_ComputationStarted).validate_python(data) is not None + ) # nosec return envelope_json_response(data, status_cls=web.HTTPCreated) @@ -186,7 +190,7 @@ async def start_computation(request: web.Request) -> web.Response: @permission_required("services.pipeline.*") @permission_required("project.read") async def stop_computation(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) computations = ComputationsApi(request.app) run_policy = get_project_run_policy(request.app) assert run_policy # nosec @@ -234,8 +238,7 @@ async def get_computation(request: web.Request) -> web.Response: request, project_id ) _logger.debug("Project %s will get %d variants", project_id, len(project_ids)) - list_computation_tasks = parse_obj_as( - list[ComputationTaskGet], + list_computation_tasks = TypeAdapter(list[ComputationTaskGet]).validate_python( await asyncio.gather( *[ computations.get(project_id=pid, user_id=user_id) @@ -251,7 +254,7 @@ async def get_computation(request: web.Request) -> web.Response: for c in list_computation_tasks ) return web.json_response( - data={"data": list_computation_tasks[0].dict(by_alias=True)}, + data={"data": list_computation_tasks[0].model_dump(by_alias=True)}, dumps=json_dumps, ) except DirectorServiceError as exc: diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_models.py b/services/web/server/src/simcore_service_webserver/director_v2/_models.py index 70dd53ff5fd..6d14d4b709d 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_models.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_models.py @@ -1,5 +1,3 @@ -from typing import Any, ClassVar - from models_library.clusters import ( CLUSTER_ADMIN_RIGHTS, CLUSTER_MANAGER_RIGHTS, @@ -10,7 +8,7 @@ ExternalClusterAuthentication, ) from models_library.users import GroupID -from pydantic import AnyHttpUrl, BaseModel, Field, validator +from pydantic import AnyHttpUrl, BaseModel, ConfigDict, Field, field_validator from pydantic.networks import AnyUrl, HttpUrl from simcore_postgres_database.models.clusters import ClusterType @@ -33,7 +31,7 @@ class ClusterCreate(BaseCluster): alias="accessRights", default_factory=dict ) - @validator("thumbnail", always=True, pre=True) + @field_validator("thumbnail", mode="before") @classmethod def set_default_thumbnail_if_empty(cls, v, values): if v is None and ( @@ -42,12 +40,12 @@ def set_default_thumbnail_if_empty(cls, v, values): return _DEFAULT_THUMBNAILS[f"{cluster_type}"] return v - class Config(BaseCluster.Config): - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "name": "My awesome cluster", - "type": ClusterType.ON_PREMISE, # can use also values from equivalent enum + "type": f"{ClusterType.ON_PREMISE}", # can use also values from equivalent enum "endpoint": "https://registry.osparc-development.fake.dev", "authentication": { "type": "simple", @@ -58,7 +56,7 @@ class Config(BaseCluster.Config): { "name": "My AWS cluster", "description": "a AWS cluster administered by me", - "type": ClusterType.AWS, + "type": f"{ClusterType.AWS}", "owner": 154, "endpoint": "https://registry.osparc-development.fake.dev", "authentication": { @@ -74,6 +72,7 @@ class Config(BaseCluster.Config): }, ] } + ) class ClusterPatch(BaseCluster): diff --git a/services/web/server/src/simcore_service_webserver/director_v2/settings.py b/services/web/server/src/simcore_service_webserver/director_v2/settings.py index d182ad6df28..21cb368ff50 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/settings.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/settings.py @@ -6,7 +6,7 @@ from aiohttp import ClientSession, ClientTimeout, web from models_library.basic_types import VersionTag -from pydantic import Field, PositiveInt +from pydantic import AliasChoices, Field, PositiveInt from servicelib.aiohttp.application_keys import APP_CLIENT_SESSION_KEY from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt @@ -36,9 +36,9 @@ def base_url(self) -> URL: DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT: PositiveInt = Field( 1 * _MINUTE, description="timeout of containers restart", - envs=[ + validation_alias=AliasChoices( "DIRECTOR_V2_RESTART_DYNAMIC_SERVICE_TIMEOUT", - ], + ), ) DIRECTOR_V2_STORAGE_SERVICE_UPLOAD_DOWNLOAD_TIMEOUT: PositiveInt = Field( @@ -49,9 +49,9 @@ def base_url(self) -> URL: "such payloads it is required to have long timeouts which " "allow the service to finish the operation." ), - envs=[ + validation_alias=AliasChoices( "DIRECTOR_V2_DYNAMIC_SERVICE_DATA_UPLOAD_DOWNLOAD_TIMEOUT", - ], + ), ) def get_service_retrieve_timeout(self) -> ClientTimeout: diff --git a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py index b92a0e2d432..91dac1317b6 100644 --- a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py +++ b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/settings.py @@ -1,21 +1,18 @@ -from typing import Final +import datetime from aiohttp import web -from pydantic import Field, NonNegativeInt +from pydantic import AliasChoices, Field from settings_library.base import BaseCustomSettings from settings_library.utils_service import MixinServiceSettings from .._constants import APP_SETTINGS_KEY -_MINUTE: Final[NonNegativeInt] = 60 -_HOUR: Final[NonNegativeInt] = 60 * _MINUTE - class DynamicSchedulerSettings(BaseCustomSettings, MixinServiceSettings): - DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: NonNegativeInt = Field( - _HOUR + 10, + DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: datetime.timedelta = Field( + datetime.timedelta(hours=1, seconds=10), description=( - "Timeout on stop service request (seconds)" + "Timeout on stop service request" "ANE: The below will try to help explaining what is happening: " "webserver -(stop_service)-> dynamic-scheduler -(relays the stop)-> " "director-v* -(save_state)-> service_x" @@ -23,10 +20,10 @@ class DynamicSchedulerSettings(BaseCustomSettings, MixinServiceSettings): "- director-v* requests save_state and uses a 01:00:00 timeout" "The +10 seconds is used to make sure the director replies" ), - envs=[ + validation_alias=AliasChoices( "DIRECTOR_V2_STOP_SERVICE_TIMEOUT", "DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT", - ], + ), ) diff --git a/services/web/server/src/simcore_service_webserver/email/_core.py b/services/web/server/src/simcore_service_webserver/email/_core.py index 0c2329cac54..269687a95a7 100644 --- a/services/web/server/src/simcore_service_webserver/email/_core.py +++ b/services/web/server/src/simcore_service_webserver/email/_core.py @@ -36,7 +36,7 @@ async def _do_send_mail( WARNING: _do_send_mail is mocked so be careful when changing the signature or name !! """ - _logger.debug("Email configuration %s", settings.json(indent=1)) + _logger.debug("Email configuration %s", settings.model_dump_json(indent=1)) if settings.SMTP_PORT == 587: # NOTE: aiosmtplib does not handle port 587 correctly this is a workaround diff --git a/services/web/server/src/simcore_service_webserver/email/_handlers.py b/services/web/server/src/simcore_service_webserver/email/_handlers.py index 829f58e4a0b..84126852347 100644 --- a/services/web/server/src/simcore_service_webserver/email/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/email/_handlers.py @@ -100,7 +100,7 @@ async def test_email(request: web.Request): return envelope_json_response( EmailTestPassed( - fixtures=body.dict(), + fixtures=body.model_dump(), info={ "email-server": info, "email-headers": message.items(), @@ -111,7 +111,7 @@ async def test_email(request: web.Request): except Exception as err: # pylint: disable=broad-except logger.exception( "test_email failed for %s", - f"{settings.json(indent=1)}", + f"{settings.model_dump_json(indent=1)}", ) return envelope_json_response( EmailTestFailed.create_from_exception(error=err, test_name="test_email") diff --git a/services/web/server/src/simcore_service_webserver/errors.py b/services/web/server/src/simcore_service_webserver/errors.py index 1bc48eda031..ac21f882297 100644 --- a/services/web/server/src/simcore_service_webserver/errors.py +++ b/services/web/server/src/simcore_service_webserver/errors.py @@ -1,8 +1,5 @@ -from typing import Any - from common_library.errors_classes import OsparcErrorMixin class WebServerBaseError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + """WebServer base error.""" diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index 994d06690de..65749210e77 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -5,7 +5,6 @@ from typing import Any, Final from aiohttp import web -from pydantic import parse_obj_as from servicelib.pools import non_blocking_process_pool_executor from ...catalog.client import get_service @@ -79,8 +78,7 @@ async def _add_rrid_entries( continue rrid_entires.append( - parse_obj_as( - RRIDEntry, + RRIDEntry.model_validate( { "rrid_term": scicrunch_resource.name, "rrid_identifier": scicrunch_resource.rrid, @@ -158,8 +156,7 @@ async def create_sds_directory( _logger.debug("Project data: %s", project_data) # assemble params here - dataset_description_params = parse_obj_as( - DatasetDescriptionParams, + dataset_description_params = DatasetDescriptionParams.model_validate( {"name": project_data["name"], "description": project_data["description"]}, ) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py index 5c1dcf4d47f..d2992ed30d5 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_db.py @@ -70,7 +70,7 @@ async def create( .returning(*_SELECTION_ARGS) ) row = await result.first() - return FolderDB.from_orm(row) + return FolderDB.model_validate(row) async def list_( @@ -123,7 +123,7 @@ async def list_( result = await conn.execute(list_query) rows = await result.fetchall() or [] - results: list[FolderDB] = [FolderDB.from_orm(row) for row in rows] + results: list[FolderDB] = [FolderDB.model_validate(row) for row in rows] return cast(int, total_count), results @@ -149,7 +149,7 @@ async def get( raise FolderAccessForbiddenError( reason=f"Folder {folder_id} does not exist.", ) - return FolderDB.from_orm(row) + return FolderDB.model_validate(row) async def get_for_user_or_workspace( @@ -185,7 +185,7 @@ async def get_for_user_or_workspace( raise FolderAccessForbiddenError( reason=f"User does not have access to the folder {folder_id}. Or folder does not exist.", ) - return FolderDB.from_orm(row) + return FolderDB.model_validate(row) async def update( @@ -213,7 +213,7 @@ async def update( row = await result.first() if row is None: raise FolderNotFoundError(reason=f"Folder {folder_id} not found.") - return FolderDB.from_orm(row) + return FolderDB.model_validate(row) async def delete_recursively( diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py index f331c98da4a..857eb56ef57 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py @@ -16,7 +16,7 @@ from models_library.users import UserID from models_library.utils.common_validators import null_or_none_str_to_none_validator from models_library.workspaces import WorkspaceID -from pydantic import Extra, Field, Json, parse_obj_as, validator +from pydantic import ConfigDict, Field, Json, TypeAdapter, field_validator from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( RequestParams, @@ -94,7 +94,7 @@ class FolderListWithJsonStrQueryParams(PageQueryParameters): order_by: Json[OrderBy] = Field( default=OrderBy(field=IDStr("modified"), direction=OrderDirection.DESC), description="Order by field (modified_at|name|description) and direction (asc|desc). The default sorting order is ascending.", - example='{"field": "name", "direction": "desc"}', + examples=['{"field": "name", "direction": "desc"}'], alias="order_by", ) folder_id: FolderID | None = Field( @@ -106,7 +106,7 @@ class FolderListWithJsonStrQueryParams(PageQueryParameters): description="List folders in specific workspace. By default, list in the user private workspace", ) - @validator("order_by", check_fields=False) + @field_validator("order_by", check_fields=False) @classmethod def validate_order_by_field(cls, v): if v.field not in { @@ -120,16 +120,15 @@ def validate_order_by_field(cls, v): v.field = "modified" return v - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") # validators - _null_or_none_str_to_none_validator = validator( - "folder_id", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) + _null_or_none_str_to_none_validator = field_validator("folder_id", mode="before")( + null_or_none_str_to_none_validator + ) - _null_or_none_str_to_none_validator2 = validator( - "workspace_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator2 = field_validator( + "workspace_id", mode="before" )(null_or_none_str_to_none_validator) @@ -138,7 +137,7 @@ class Config: @permission_required("folder.create") @handle_folders_exceptions async def create_folder(request: web.Request): - req_ctx = FoldersRequestContext.parse_obj(request) + req_ctx = FoldersRequestContext.model_validate(request) body_params = await parse_request_body_as(CreateFolderBodyParams, request) folder = await _folders_api.create_folder( @@ -158,7 +157,7 @@ async def create_folder(request: web.Request): @permission_required("folder.read") @handle_folders_exceptions async def list_folders(request: web.Request): - req_ctx = FoldersRequestContext.parse_obj(request) + req_ctx = FoldersRequestContext.model_validate(request) query_params: FolderListWithJsonStrQueryParams = parse_request_query_parameters_as( FolderListWithJsonStrQueryParams, request ) @@ -171,10 +170,10 @@ async def list_folders(request: web.Request): workspace_id=query_params.workspace_id, offset=query_params.offset, limit=query_params.limit, - order_by=parse_obj_as(OrderBy, query_params.order_by), + order_by=TypeAdapter(OrderBy).validate_python(query_params.order_by), ) - page = Page[FolderGet].parse_obj( + page = Page[FolderGet].model_validate( paginate_data( chunk=folders.items, request_url=request.url, @@ -184,7 +183,7 @@ async def list_folders(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -194,7 +193,7 @@ async def list_folders(request: web.Request): @permission_required("folder.read") @handle_folders_exceptions async def get_folder(request: web.Request): - req_ctx = FoldersRequestContext.parse_obj(request) + req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) folder: FolderGet = await _folders_api.get_folder( @@ -215,7 +214,7 @@ async def get_folder(request: web.Request): @permission_required("folder.update") @handle_folders_exceptions async def replace_folder(request: web.Request): - req_ctx = FoldersRequestContext.parse_obj(request) + req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) body_params = await parse_request_body_as(PutFolderBodyParams, request) @@ -238,7 +237,7 @@ async def replace_folder(request: web.Request): @permission_required("folder.delete") @handle_folders_exceptions async def delete_folder_group(request: web.Request): - req_ctx = FoldersRequestContext.parse_obj(request) + req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) await _folders_api.delete_folder( diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers.py index 5ac89e0ee94..68b9e2a5fdb 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers.py @@ -9,20 +9,19 @@ """ import logging -import re -from typing import Any, Final, Literal +from typing import Annotated, Any, Final, Literal, TypeAlias import sqlalchemy as sa from aiohttp import web from aiopg.sa.result import RowProxy from pydantic import ( BaseModel, - ConstrainedStr, Field, HttpUrl, + StringConstraints, + TypeAdapter, ValidationError, - parse_obj_as, - validator, + field_validator, ) from simcore_postgres_database.models.classifiers import group_classifiers @@ -37,8 +36,11 @@ # DOMAIN MODELS --- -class TreePath(ConstrainedStr): - regex = re.compile(r"[\w:]+") # Examples 'a::b::c +TreePath: TypeAlias = Annotated[ + # Examples 'a::b::c + str, + StringConstraints(pattern=r"[\w:]+"), +] class ClassifierItem(BaseModel): @@ -50,10 +52,10 @@ class ClassifierItem(BaseModel): url: HttpUrl | None = Field( None, description="Link to more information", - example="https://scicrunch.org/resources/Any/search?q=osparc&l=osparc", + examples=["https://scicrunch.org/resources/Any/search?q=osparc&l=osparc"], ) - @validator("short_description", pre=True) + @field_validator("short_description", mode="before") @classmethod def truncate_to_short(cls, v): if v and len(v) >= MAX_SIZE_SHORT_MSG: @@ -91,7 +93,9 @@ async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any]: if bundle: try: # truncate bundle to what is needed and drop the rest - return Classifiers(**bundle).dict(exclude_unset=True, exclude_none=True) + return Classifiers(**bundle).model_dump( + exclude_unset=True, exclude_none=True + ) except ValidationError as err: _logger.error( "DB corrupt data in 'groups_classifiers' table. " @@ -136,7 +140,9 @@ async def build_rrids_tree_view( url=scicrunch.get_resolver_web_url(resource.rrid), ) - node = parse_obj_as(TreePath, validated_item.display_name.replace(":", " ")) + node = TypeAdapter(TreePath).validate_python( + validated_item.display_name.replace(":", " ") + ) flat_tree_view[node] = validated_item except ValidationError as err: @@ -144,4 +150,6 @@ async def build_rrids_tree_view( "Cannot convert RRID into a classifier item. Skipping. Details: %s", err ) - return Classifiers.construct(classifiers=flat_tree_view).dict(exclude_unset=True) + return Classifiers.model_construct(classifiers=flat_tree_view).model_dump( + exclude_unset=True + ) diff --git a/services/web/server/src/simcore_service_webserver/groups/_db.py b/services/web/server/src/simcore_service_webserver/groups/_db.py index 38bbb4e7d7c..4eda0ff3e65 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_db.py +++ b/services/web/server/src/simcore_service_webserver/groups/_db.py @@ -6,7 +6,6 @@ from aiopg.sa.result import ResultProxy, RowProxy from models_library.groups import GroupAtDB from models_library.users import GroupID, UserID -from pydantic import parse_obj_as from simcore_postgres_database.utils_products import get_or_create_product_group from sqlalchemy import and_, literal_column from sqlalchemy.dialects.postgresql import insert @@ -117,7 +116,7 @@ async def get_all_user_groups(conn: SAConnection, user_id: UserID) -> list[Group .where(user_to_groups.c.uid == user_id) ) rows = await result.fetchall() or [] - return [parse_obj_as(GroupAtDB, row) for row in rows] + return [GroupAtDB.model_validate(row) for row in rows] async def get_user_group( @@ -409,5 +408,5 @@ async def get_group_from_gid(conn: SAConnection, gid: GroupID) -> GroupAtDB | No row: ResultProxy = await conn.execute(groups.select().where(groups.c.gid == gid)) result = await row.first() if result: - return GroupAtDB.from_orm(result) + return GroupAtDB.model_validate(result) return None diff --git a/services/web/server/src/simcore_service_webserver/groups/_handlers.py b/services/web/server/src/simcore_service_webserver/groups/_handlers.py index ba366aaef46..9baeb984d9f 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_handlers.py @@ -12,7 +12,7 @@ ) from models_library.emails import LowerCaseEmailStr from models_library.users import GroupID, UserID -from pydantic import BaseModel, Extra, Field, parse_obj_as +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_path_parameters_as, @@ -83,7 +83,7 @@ async def list_groups(request: web.Request): """ product: Product = get_current_product(request) - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) primary_group, user_groups, all_group = await api.list_user_groups_with_read_access( request.app, req_ctx.user_id @@ -104,15 +104,13 @@ async def list_groups(request: web.Request): product_gid=product.group_id, ) - assert parse_obj_as(AllUsersGroups, result) is not None # nosec + assert AllUsersGroups.model_validate(result) is not None # nosec return result class _GroupPathParams(BaseModel): gid: GroupID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.get(f"/{API_VTAG}/groups/{{gid}}", name="get_group") @@ -121,11 +119,11 @@ class Config: @_handle_groups_exceptions async def get_group(request: web.Request): """Get one group details""" - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) group = await api.get_user_group(request.app, req_ctx.user_id, path_params.gid) - assert parse_obj_as(UsersGroup, group) is not None # nosec + assert UsersGroup.model_validate(group) is not None # nosec return group @@ -135,11 +133,11 @@ async def get_group(request: web.Request): @_handle_groups_exceptions async def create_group(request: web.Request): """Creates organization groups""" - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) new_group = await request.json() created_group = await api.create_user_group(request.app, req_ctx.user_id, new_group) - assert parse_obj_as(UsersGroup, created_group) is not None # nosec + assert UsersGroup.model_validate(created_group) is not None # nosec raise web.HTTPCreated( text=json_dumps({"data": created_group}), content_type=MIMETYPE_APPLICATION_JSON ) @@ -150,14 +148,14 @@ async def create_group(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def update_group(request: web.Request): - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) new_group_values = await request.json() updated_group = await api.update_user_group( request.app, req_ctx.user_id, path_params.gid, new_group_values ) - assert parse_obj_as(UsersGroup, updated_group) is not None # nosec + assert UsersGroup.model_validate(updated_group) is not None # nosec return envelope_json_response(updated_group) @@ -166,7 +164,7 @@ async def update_group(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def delete_group(request: web.Request): - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) await api.delete_user_group(request.app, req_ctx.user_id, path_params.gid) @@ -178,13 +176,15 @@ async def delete_group(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def get_group_users(request: web.Request): - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) group_user = await api.list_users_in_group( request.app, req_ctx.user_id, path_params.gid ) - assert parse_obj_as(list[GroupUserGet], group_user) is not None # nosec + assert ( + TypeAdapter(list[GroupUserGet]).validate_python(group_user) is not None + ) # nosec return envelope_json_response(group_user) @@ -196,7 +196,7 @@ async def add_group_user(request: web.Request): """ Adds a user in an organization group """ - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) new_user_in_group = await request.json() @@ -204,7 +204,7 @@ async def add_group_user(request: web.Request): new_user_id = new_user_in_group["uid"] if "uid" in new_user_in_group else None new_user_email = ( - parse_obj_as(LowerCaseEmailStr, new_user_in_group["email"]) + TypeAdapter(LowerCaseEmailStr).validate_python(new_user_in_group["email"]) if "email" in new_user_in_group else None ) @@ -222,9 +222,7 @@ async def add_group_user(request: web.Request): class _GroupUserPathParams(BaseModel): gid: GroupID uid: UserID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.get(f"/{API_VTAG}/groups/{{gid}}/users/{{uid}}", name="get_group_user") @@ -235,12 +233,12 @@ async def get_group_user(request: web.Request): """ Gets specific user in group """ - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupUserPathParams, request) user = await api.get_user_in_group( request.app, req_ctx.user_id, path_params.gid, path_params.uid ) - assert parse_obj_as(GroupUserGet, user) is not None # nosec + assert GroupUserGet.model_validate(user) is not None # nosec return envelope_json_response(user) @@ -252,7 +250,7 @@ async def update_group_user(request: web.Request): """ Modify specific user in group """ - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupUserPathParams, request) new_values_for_user_in_group = await request.json() user = await api.update_user_in_group( @@ -262,7 +260,7 @@ async def update_group_user(request: web.Request): path_params.uid, new_values_for_user_in_group, ) - assert parse_obj_as(GroupUserGet, user) is not None # nosec + assert GroupUserGet.model_validate(user) is not None # nosec return envelope_json_response(user) @@ -271,7 +269,7 @@ async def update_group_user(request: web.Request): @permission_required("groups.*") @_handle_groups_exceptions async def delete_group_user(request: web.Request): - req_ctx = _GroupsRequestContext.parse_obj(request) + req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupUserPathParams, request) await api.delete_user_in_group( request.app, req_ctx.user_id, path_params.gid, path_params.uid @@ -352,7 +350,7 @@ async def get_scicrunch_resource(request: web.Request): scicrunch = SciCrunch.get_instance(request.app) resource = await scicrunch.get_resource_fields(rrid) - return envelope_json_response(resource.dict()) + return envelope_json_response(resource.model_dump()) @routes.post( @@ -376,7 +374,7 @@ async def add_scicrunch_resource(request: web.Request): # insert new or if exists, then update await repo.upsert(resource) - return envelope_json_response(resource.dict()) + return envelope_json_response(resource.model_dump()) @routes.get( @@ -392,4 +390,4 @@ async def search_scicrunch_resources(request: web.Request): scicrunch = SciCrunch.get_instance(request.app) hits: list[ResourceHit] = await scicrunch.search_resource(guess_name) - return envelope_json_response([hit.dict() for hit in hits]) + return envelope_json_response([hit.model_dump() for hit in hits]) diff --git a/services/web/server/src/simcore_service_webserver/groups/api.py b/services/web/server/src/simcore_service_webserver/groups/api.py index e0a837f8399..ec0a615874e 100644 --- a/services/web/server/src/simcore_service_webserver/groups/api.py +++ b/services/web/server/src/simcore_service_webserver/groups/api.py @@ -32,7 +32,7 @@ async def list_all_user_groups(app: web.Application, user_id: UserID) -> list[Gr async with get_database_engine(app).acquire() as conn: groups_db = await _db.get_all_user_groups(conn, user_id=user_id) - return [Group.construct(**group.dict()) for group in groups_db] + return [Group.construct(**group.model_dump()) for group in groups_db] async def get_user_group( @@ -199,5 +199,5 @@ async def get_group_from_gid(app: web.Application, gid: GroupID) -> Group | None group_db = await _db.get_group_from_gid(conn, gid=gid) if group_db: - return Group.construct(**group_db.dict()) + return Group.construct(**group_db.model_dump()) return None diff --git a/services/web/server/src/simcore_service_webserver/invitations/_client.py b/services/web/server/src/simcore_service_webserver/invitations/_client.py index cc427cf28ce..4ca20894b0e 100644 --- a/services/web/server/src/simcore_service_webserver/invitations/_client.py +++ b/services/web/server/src/simcore_service_webserver/invitations/_client.py @@ -10,7 +10,7 @@ ApiInvitationInputs, ) from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl from yarl import URL from .._constants import APP_SETTINGS_KEY @@ -86,7 +86,7 @@ async def extract_invitation( url=self._url_vtag("/invitations:extract"), json={"invitation_url": invitation_url}, ) - return parse_obj_as(ApiInvitationContent, await response.json()) + return ApiInvitationContent.model_validate(await response.json()) async def generate_invitation( self, params: ApiInvitationInputs @@ -95,7 +95,7 @@ async def generate_invitation( url=self._url_vtag("/invitations"), json=jsonable_encoder(params), ) - return parse_obj_as(ApiInvitationContentAndLink, await response.json()) + return ApiInvitationContentAndLink.model_validate(await response.json()) # diff --git a/services/web/server/src/simcore_service_webserver/invitations/_core.py b/services/web/server/src/simcore_service_webserver/invitations/_core.py index 2bf18487638..fcd9e619742 100644 --- a/services/web/server/src/simcore_service_webserver/invitations/_core.py +++ b/services/web/server/src/simcore_service_webserver/invitations/_core.py @@ -9,7 +9,7 @@ ApiInvitationInputs, ) from models_library.emails import LowerCaseEmailStr -from pydantic import AnyHttpUrl, ValidationError, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter, ValidationError from servicelib.aiohttp import status from ..groups.api import is_user_by_email_in_group @@ -92,7 +92,7 @@ async def validate_invitation_url( with _handle_exceptions_as_invitations_errors(): try: - valid_url = parse_obj_as(AnyHttpUrl, invitation_url) + valid_url = TypeAdapter(AnyHttpUrl).validate_python(invitation_url) except ValidationError as err: raise InvalidInvitationError(reason=MSG_INVALID_INVITATION_URL) from err @@ -143,7 +143,7 @@ async def extract_invitation( with _handle_exceptions_as_invitations_errors(): try: - valid_url = parse_obj_as(AnyHttpUrl, invitation_url) + valid_url = TypeAdapter(AnyHttpUrl).validate_python(invitation_url) except ValidationError as err: raise InvalidInvitationError(reason=MSG_INVALID_INVITATION_URL) from err diff --git a/services/web/server/src/simcore_service_webserver/invitations/settings.py b/services/web/server/src/simcore_service_webserver/invitations/settings.py index 025f89955ff..02755291910 100644 --- a/services/web/server/src/simcore_service_webserver/invitations/settings.py +++ b/services/web/server/src/simcore_service_webserver/invitations/settings.py @@ -8,7 +8,7 @@ from typing import Final from aiohttp import web -from pydantic import Field, SecretStr, parse_obj_as +from pydantic import Field, SecretStr, TypeAdapter from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -19,7 +19,7 @@ from .._constants import APP_SETTINGS_KEY -_INVITATION_VTAG_V1: Final[VersionTag] = parse_obj_as(VersionTag, "v1") +_INVITATION_VTAG_V1: Final[VersionTag] = TypeAdapter(VersionTag).validate_python("v1") class InvitationsSettings(BaseCustomSettings, MixinServiceSettings): diff --git a/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py b/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py index ca1e1a3a18d..db8ee3421e3 100644 --- a/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/_auth_handlers.py @@ -4,7 +4,7 @@ from aiohttp.web import RouteTableDef from models_library.authentification import TwoFactorAuthentificationMethod from models_library.emails import LowerCaseEmailStr -from pydantic import BaseModel, Field, PositiveInt, SecretStr, parse_obj_as +from pydantic import BaseModel, Field, PositiveInt, SecretStr, TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_body_as from servicelib.logging_utils import get_log_record_extra, log_context @@ -137,9 +137,9 @@ async def login(request: web.Request): value=user_2fa_authentification_method, ) else: - user_2fa_authentification_method = parse_obj_as( - TwoFactorAuthentificationMethod, user_2fa_preference.value - ) + user_2fa_authentification_method = TypeAdapter( + TwoFactorAuthentificationMethod + ).validate_python(user_2fa_preference.value) if user_2fa_authentification_method == TwoFactorAuthentificationMethod.DISABLED: return await login_granted_response(request, user=user) @@ -275,7 +275,7 @@ async def login_2fa(request: web.Request): class LogoutBody(InputSchema): client_session_id: str | None = Field( - None, example="5ac57685-c40f-448f-8711-70be1936fd63" + None, examples=["5ac57685-c40f-448f-8711-70be1936fd63"] ) diff --git a/services/web/server/src/simcore_service_webserver/login/_models.py b/services/web/server/src/simcore_service_webserver/login/_models.py index 2ac7b94f11a..c0aef7a6015 100644 --- a/services/web/server/src/simcore_service_webserver/login/_models.py +++ b/services/web/server/src/simcore_service_webserver/login/_models.py @@ -1,25 +1,26 @@ -from typing import Any, Callable +from typing import Callable -from pydantic import BaseModel, Extra, SecretStr +from pydantic import BaseModel, ConfigDict, SecretStr, ValidationInfo from ._constants import MSG_PASSWORD_MISMATCH class InputSchema(BaseModel): - class Config: - allow_population_by_field_name = False - extra = Extra.forbid - allow_mutations = False + model_config = ConfigDict( + populate_by_name=False, + extra="forbid", + frozen=True, + ) def create_password_match_validator( reference_field: str, -) -> Callable[[SecretStr, dict[str, Any]], SecretStr]: - def _check(v: SecretStr, values: dict[str, Any]): +) -> Callable[[SecretStr, ValidationInfo], SecretStr]: + def _check(v: SecretStr, info: ValidationInfo): if ( v is not None - and reference_field in values - and v.get_secret_value() != values[reference_field].get_secret_value() + and reference_field in info.data + and v.get_secret_value() != info.data[reference_field].get_secret_value() ): raise ValueError(MSG_PASSWORD_MISMATCH) return v diff --git a/services/web/server/src/simcore_service_webserver/login/_registration.py b/services/web/server/src/simcore_service_webserver/login/_registration.py index 1cfc53396d2..1a66b5ba4e5 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration.py @@ -19,9 +19,9 @@ Field, Json, PositiveInt, + TypeAdapter, ValidationError, - parse_obj_as, - validator, + field_validator, ) from servicelib.logging_errors import create_troubleshotting_log_kwargs from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON @@ -78,7 +78,7 @@ class _InvitationValidator(BaseModel): action: ConfirmationAction data: Json[InvitationData] # pylint: disable=unsubscriptable-object - @validator("action", pre=True) + @field_validator("action", mode="before") @classmethod def ensure_enum(cls, v): if isinstance(v, ConfirmationAction): @@ -256,7 +256,7 @@ async def extract_email_from_invitation( """Returns associated email""" with _invitations_request_context(invitation_code=invitation_code) as url: content = await extract_invitation(app, invitation_url=f"{url}") - return parse_obj_as(LowerCaseEmailStr, content.guest) + return TypeAdapter(LowerCaseEmailStr).validate_python(content.guest) async def check_and_consume_invitation( @@ -286,7 +286,8 @@ async def check_and_consume_invitation( ) _logger.info( - "Consuming invitation from service:\n%s", content.json(indent=1) + "Consuming invitation from service:\n%s", + content.model_dump_json(indent=1), ) return InvitationData( issuer=content.issuer, @@ -299,7 +300,7 @@ async def check_and_consume_invitation( # database-type invitations if confirmation_token := await validate_confirmation_code(invitation_code, db, cfg): try: - invitation_data: InvitationData = _InvitationValidator.parse_obj( + invitation_data: InvitationData = _InvitationValidator.model_validate( confirmation_token ).data return invitation_data diff --git a/services/web/server/src/simcore_service_webserver/login/_registration_api.py b/services/web/server/src/simcore_service_webserver/login/_registration_api.py index b11dca73662..22252f2dc86 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration_api.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration_api.py @@ -9,7 +9,7 @@ from models_library.emails import LowerCaseEmailStr from models_library.utils.fastapi_encoders import jsonable_encoder from PIL.Image import Image -from pydantic import EmailStr, PositiveInt, ValidationError, parse_obj_as +from pydantic import EmailStr, PositiveInt, TypeAdapter, ValidationError from servicelib.utils_secrets import generate_passcode from ..email.utils import send_email_from_template @@ -66,7 +66,9 @@ async def send_account_request_email_to_support( support_email = product.support_email email_template_path = await get_product_template_path(request, template_name) try: - user_email = parse_obj_as(LowerCaseEmailStr, request_form.get("email", None)) + user_email = TypeAdapter(LowerCaseEmailStr).validate_python( + request_form.get("email", None) + ) except ValidationError: user_email = None @@ -80,7 +82,7 @@ async def send_account_request_email_to_support( context={ "host": request.host, "name": "support-team", - "product": product.dict( + "product": product.model_dump( include={ "name", "display_name", diff --git a/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py b/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py index 869fa7a2973..42e8229e7a6 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration_handlers.py @@ -98,7 +98,7 @@ class _AuthenticatedContext(BaseModel): @login_required @permission_required("user.profile.delete") async def unregister_account(request: web.Request): - req_ctx = _AuthenticatedContext.parse_obj(request) + req_ctx = _AuthenticatedContext.model_validate(request) body = await parse_request_body_as(UnregisterCheck, request) product: Product = get_current_product(request) diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_change.py b/services/web/server/src/simcore_service_webserver/login/handlers_change.py index f8b71ce8763..75c93ff990e 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_change.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_change.py @@ -3,7 +3,7 @@ from aiohttp import web from aiohttp.web import RouteTableDef from models_library.emails import LowerCaseEmailStr -from pydantic import SecretStr, validator +from pydantic import SecretStr, field_validator from servicelib.aiohttp.requests_validation import parse_request_body_as from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.request_keys import RQT_USERID_KEY @@ -188,7 +188,7 @@ class ChangePasswordBody(InputSchema): new: SecretStr confirm: SecretStr - _password_confirm_match = validator("confirm", allow_reuse=True)( + _password_confirm_match = field_validator("confirm")( create_password_match_validator(reference_field="new") ) diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py b/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py index 63c4505c647..2fe63036378 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_confirmation.py @@ -12,9 +12,9 @@ Field, PositiveInt, SecretStr, + TypeAdapter, ValidationError, - parse_obj_as, - validator, + field_validator, ) from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( @@ -73,7 +73,7 @@ def _parse_extra_credits_in_usd_or_none( ) -> PositiveInt | None: with suppress(ValidationError, JSONDecodeError): confirmation_data = confirmation.get("data", "EMPTY") or "EMPTY" - invitation = InvitationData.parse_raw(confirmation_data) + invitation = InvitationData.model_validate_json(confirmation_data) return invitation.extra_credits_in_usd return None @@ -110,7 +110,11 @@ async def _handle_confirm_change_email( # update and consume confirmation token await db.delete_confirmation_and_update_user( user_id=user_id, - updates={"email": parse_obj_as(LowerCaseEmailStr, confirmation["data"])}, + updates={ + "email": TypeAdapter(LowerCaseEmailStr).validate_python( + confirmation["data"] + ) + }, confirmation=confirmation, ) @@ -265,9 +269,7 @@ class ResetPasswordConfirmation(InputSchema): password: SecretStr confirm: SecretStr - _password_confirm_match = validator("confirm", allow_reuse=True)( - check_confirm_password_match - ) + _password_confirm_match = field_validator("confirm")(check_confirm_password_match) @routes.post("/v0/auth/reset-password/{code}", name="auth_reset_password_allowed") diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_registration.py b/services/web/server/src/simcore_service_webserver/login/handlers_registration.py index 7c140184767..3d00ab57c03 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_registration.py @@ -1,12 +1,19 @@ import logging -from datetime import datetime, timedelta -from typing import Any, ClassVar, Literal +from datetime import UTC, datetime, timedelta +from typing import Literal from aiohttp import web from aiohttp.web import RouteTableDef from common_library.error_codes import create_error_code from models_library.emails import LowerCaseEmailStr -from pydantic import BaseModel, Field, PositiveInt, SecretStr, validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PositiveInt, + SecretStr, + field_validator, +) from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_body_as from servicelib.logging_errors import create_troubleshotting_log_kwargs @@ -115,12 +122,9 @@ class RegisterBody(InputSchema): confirm: SecretStr | None = Field(None, description="Password confirmation") invitation: str | None = Field(None, description="Invitation code") - _password_confirm_match = validator("confirm", allow_reuse=True)( - check_confirm_password_match - ) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + _password_confirm_match = field_validator("confirm")(check_confirm_password_match) + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "email": "foo@mymail.com", @@ -130,6 +134,7 @@ class Config: } ] } + ) @routes.post(f"/{API_VTAG}/auth/register", name="auth_register") @@ -204,7 +209,7 @@ async def register(request: web.Request): app=request.app, ) if invitation.trial_account_days: - expires_at = datetime.utcnow() + timedelta(invitation.trial_account_days) + expires_at = datetime.now(UTC) + timedelta(invitation.trial_account_days) # get authorized user or create new user = await _auth_api.get_user_by_email(request.app, email=registration.email) @@ -244,7 +249,7 @@ async def register(request: web.Request): if settings.LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: # Confirmation required: send confirmation email _confirmation: ConfirmationTokenDict = await db.create_confirmation( - user["id"], REGISTRATION, data=invitation.json() if invitation else None + user["id"], REGISTRATION, data=invitation.model_dump_json() if invitation else None ) try: diff --git a/services/web/server/src/simcore_service_webserver/login/plugin.py b/services/web/server/src/simcore_service_webserver/login/plugin.py index 174bcd55f4a..ef0c77c2f18 100644 --- a/services/web/server/src/simcore_service_webserver/login/plugin.py +++ b/services/web/server/src/simcore_service_webserver/login/plugin.py @@ -68,7 +68,7 @@ def setup_login_storage(app: web.Application): def _setup_login_options(app: web.Application): settings: SMTPSettings = get_email_plugin_settings(app) - cfg = settings.dict() + cfg = settings.model_dump() if INDEX_RESOURCE_NAME in app.router: cfg["LOGIN_REDIRECT"] = f"{app.router[INDEX_RESOURCE_NAME].url_for()}" diff --git a/services/web/server/src/simcore_service_webserver/login/settings.py b/services/web/server/src/simcore_service_webserver/login/settings.py index c32ce319c7f..91ee1041889 100644 --- a/services/web/server/src/simcore_service_webserver/login/settings.py +++ b/services/web/server/src/simcore_service_webserver/login/settings.py @@ -2,7 +2,7 @@ from typing import Final, Literal from aiohttp import web -from pydantic import BaseModel, validator +from pydantic import BaseModel, ValidationInfo, field_validator from pydantic.fields import Field from pydantic.types import PositiveFloat, PositiveInt, SecretStr from settings_library.base import BaseCustomSettings @@ -36,7 +36,7 @@ class LoginSettings(BaseCustomSettings): ) LOGIN_TWILIO: TwilioSettings | None = Field( - auto_default_from_env=True, + json_schema_extra={"auto_default_from_env": True}, description="Twilio service settings. Used to send SMS for 2FA", ) @@ -54,19 +54,19 @@ class LoginSettings(BaseCustomSettings): description="Minimum length of password", ) - @validator("LOGIN_2FA_REQUIRED") + @field_validator("LOGIN_2FA_REQUIRED") @classmethod - def login_2fa_needs_email_registration(cls, v, values): + def _login_2fa_needs_email_registration(cls, v, info: ValidationInfo): # NOTE: this constraint ensures that a phone is registered in current workflow - if v and not values.get("LOGIN_REGISTRATION_CONFIRMATION_REQUIRED", False): + if v and not info.data.get("LOGIN_REGISTRATION_CONFIRMATION_REQUIRED", False): msg = "Cannot enable 2FA w/o email confirmation" raise ValueError(msg) return v - @validator("LOGIN_2FA_REQUIRED") + @field_validator("LOGIN_2FA_REQUIRED") @classmethod - def login_2fa_needs_sms_service(cls, v, values): - if v and values.get("LOGIN_TWILIO") is None: + def _login_2fa_needs_sms_service(cls, v, info: ValidationInfo): + if v and info.data.get("LOGIN_TWILIO") is None: msg = "Cannot enable 2FA w/o twilio settings which is used to send SMS" raise ValueError(msg) return v @@ -94,7 +94,10 @@ def create_from_composition( """ For the LoginSettings, product-specific settings override app-specifics settings """ - composed_settings = {**app_login_settings.dict(), **product_login_settings} + composed_settings = { + **app_login_settings.model_dump(), + **product_login_settings, + } if "two_factor_enabled" in composed_settings: # legacy safe diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks.py b/services/web/server/src/simcore_service_webserver/long_running_tasks.py index a7e4e8c725b..2f42f18927e 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks.py @@ -28,7 +28,7 @@ async def _test_task_context_decorator( request: web.Request, ) -> web.StreamResponse: """this task context callback tries to get the user_id from the query if available""" - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] = jsonable_encoder(req_ctx) return await handler(request) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_function_nodes.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_function_nodes.py index c51f3374510..3e0e3a630f7 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_function_nodes.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_function_nodes.py @@ -37,7 +37,7 @@ def create_param_node_from_iterator_with_outputs(iterator_node: Node) -> Node: label=iterator_node.label, inputs={}, inputNodes=[], - thumbnail="", # type: ignore[arg-type] # NOTE: hack due to issue in projects json-schema + thumbnail="", # NOTE: hack due to issue in projects json-schema outputs=deepcopy(iterator_node.outputs), ) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_handlers.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_handlers.py index 35244dc5363..847395e6acd 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_handlers.py @@ -9,7 +9,7 @@ from models_library.projects import ProjectID from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel, ValidationError, validator +from pydantic import BaseModel, ValidationError, field_validator from pydantic.fields import Field from pydantic.networks import HttpUrl from servicelib.rest_constants import RESPONSE_MODEL_POLICY @@ -33,7 +33,7 @@ class ParametersModel(PageQueryParameters): project_uuid: ProjectID ref_id: CommitID - @validator("ref_id", pre=True) + @field_validator("ref_id", mode="before") @classmethod def tags_as_refid_not_implemented(cls, v): try: @@ -292,7 +292,7 @@ async def list_project_iterations(request: web.Request) -> web.Response: for item in iterations_range.items ] - page = Page[ProjectIterationItem].parse_obj( + page = Page[ProjectIterationItem].model_validate( paginate_data( chunk=page_items, request_url=request.url, @@ -302,7 +302,7 @@ async def list_project_iterations(request: web.Request) -> web.Response: ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type="application/json", ) @@ -395,7 +395,7 @@ def _get_project_results(project_id) -> ExtractedResults: for item in iterations_range.items ] - page = Page[ProjectIterationResultItem].parse_obj( + page = Page[ProjectIterationResultItem].model_validate( paginate_data( chunk=page_items, request_url=request.url, @@ -405,6 +405,6 @@ def _get_project_results(project_id) -> ExtractedResults: ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type="application/json", ) diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py index 4cfd2bfca24..b8adef9bcc4 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py @@ -156,7 +156,7 @@ def from_tag_name( ) -> Optional["ProjectIteration"]: """Parses iteration info from tag name""" try: - return cls.parse_obj(parse_iteration_tag_name(tag_name)) + return cls.model_validate(parse_iteration_tag_name(tag_name)) except ValidationError as err: if return_none_if_fails: _logger.debug("%s", f"{err=}") @@ -218,7 +218,7 @@ async def get_or_create_runnable_projects( raise web.HTTPForbidden(reason="Unauthenticated request") from err project_nodes: dict[NodeID, Node] = { - nid: Node.parse_obj(n) for nid, n in project["workbench"].items() + nid: Node.model_validate(n) for nid, n in project["workbench"].items() } # init returns @@ -280,7 +280,7 @@ async def get_or_create_runnable_projects( project["workbench"].update( { # converts model in dict patching first thumbnail - nid: n.copy(update={"thumbnail": n.thumbnail or ""}).dict( + nid: n.copy(update={"thumbnail": n.thumbnail or ""}).model_dump( by_alias=True, exclude_unset=True ) for nid, n in updated_nodes.items() @@ -326,7 +326,7 @@ async def get_runnable_projects_ids( project: ProjectDict = await vc_repo.get_project(str(project_uuid)) assert project["uuid"] == str(project_uuid) # nosec project_nodes: dict[NodeID, Node] = { - nid: Node.parse_obj(n) for nid, n in project["workbench"].items() + nid: Node.model_validate(n) for nid, n in project["workbench"].items() } # init returns diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_results.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_results.py index 68829e3489a..150c2b8f680 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_results.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_results.py @@ -7,18 +7,16 @@ import logging -from typing import Any +from typing import Annotated, Any from models_library.projects_nodes import OutputsDict from models_library.projects_nodes_io import NodeIDStr -from pydantic import BaseModel, ConstrainedInt, Field +from pydantic import BaseModel, ConfigDict, Field _logger = logging.getLogger(__name__) -class ProgressInt(ConstrainedInt): - ge = 0 - le = 100 +ProgressInt = Annotated[int, Field(ge=0, le=100)] class ExtractedResults(BaseModel): @@ -31,9 +29,8 @@ class ExtractedResults(BaseModel): values: dict[NodeIDStr, OutputsDict] = Field( ..., description="Captured outputs per node" ) - - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { # sample with 2 computational services, 2 data sources (iterator+parameter) and 2 observers (probes) "progress": { @@ -57,6 +54,7 @@ class Config: }, } } + ) def extract_project_results(workbench: dict[str, Any]) -> ExtractedResults: @@ -112,5 +110,5 @@ def extract_project_results(workbench: dict[str, Any]) -> ExtractedResults: values = node["outputs"] results[noid], labels[noid] = values, label - res = ExtractedResults(progress=progress, labels=labels, values=results) # type: ignore[arg-type] + res = ExtractedResults(progress=progress, labels=labels, values=results) return res diff --git a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py index d9a6b1f0861..67c5d39b65b 100644 --- a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py +++ b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py @@ -13,7 +13,7 @@ ) from models_library.socketio import SocketMessageDict from models_library.users import GroupID -from pydantic import parse_raw_as +from pydantic import TypeAdapter from servicelib.logging_utils import log_catch, log_context from servicelib.rabbitmq import RabbitMQClient from servicelib.utils import logged_gather @@ -58,17 +58,19 @@ async def _convert_to_node_update_event( "data": project["workbench"][f"{message.node_id}"], }, ) - _logger.warning("node not found: '%s'", message.dict()) + _logger.warning("node not found: '%s'", message.model_dump()) except ProjectNotFoundError: - _logger.warning("project not found: '%s'", message.dict()) + _logger.warning("project not found: '%s'", message.model_dump()) return None async def _progress_message_parser(app: web.Application, data: bytes) -> bool: rabbit_message: ( ProgressRabbitMessageNode | ProgressRabbitMessageProject - ) = parse_raw_as( - ProgressRabbitMessageNode | ProgressRabbitMessageProject, data # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 + ) = TypeAdapter( + ProgressRabbitMessageNode | ProgressRabbitMessageProject + ).validate_json( + data ) message: SocketMessageDict | None = None if isinstance(rabbit_message, ProgressRabbitMessageProject): @@ -95,13 +97,13 @@ async def _progress_message_parser(app: web.Application, data: bytes) -> bool: async def _log_message_parser(app: web.Application, data: bytes) -> bool: - rabbit_message = LoggerRabbitMessage.parse_raw(data) + rabbit_message = LoggerRabbitMessage.model_validate_json(data) await send_message_to_user( app, rabbit_message.user_id, message=SocketMessageDict( event_type=SOCKET_IO_LOG_EVENT, - data=rabbit_message.dict(exclude={"user_id", "channel_name"}), + data=rabbit_message.model_dump(exclude={"user_id", "channel_name"}), ), ignore_queue=True, ) @@ -109,7 +111,7 @@ async def _log_message_parser(app: web.Application, data: bytes) -> bool: async def _events_message_parser(app: web.Application, data: bytes) -> bool: - rabbit_message = EventRabbitMessage.parse_raw(data) + rabbit_message = EventRabbitMessage.model_validate_json(data) await send_message_to_user( app, rabbit_message.user_id, @@ -126,7 +128,7 @@ async def _events_message_parser(app: web.Application, data: bytes) -> bool: async def _osparc_credits_message_parser(app: web.Application, data: bytes) -> bool: - rabbit_message = parse_raw_as(WalletCreditsMessage, data) + rabbit_message = TypeAdapter(WalletCreditsMessage).validate_json(data) wallet_groups = await wallets_api.list_wallet_groups_with_read_access_by_wallet( app, wallet_id=rabbit_message.wallet_id ) diff --git a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_nonexclusive_queue_consumers.py b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_nonexclusive_queue_consumers.py index 0a8e04b5e7f..b6271be822a 100644 --- a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_nonexclusive_queue_consumers.py +++ b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_nonexclusive_queue_consumers.py @@ -23,12 +23,12 @@ async def _instrumentation_message_parser(app: web.Application, data: bytes) -> bool: - rabbit_message = InstrumentationRabbitMessage.parse_raw(data) + rabbit_message = InstrumentationRabbitMessage.model_validate_json(data) if rabbit_message.metrics == "service_started": service_started( app, **{ - key: rabbit_message.dict()[key] + key: rabbit_message.model_dump()[key] for key in MONITOR_SERVICE_STARTED_LABELS }, ) @@ -36,7 +36,7 @@ async def _instrumentation_message_parser(app: web.Application, data: bytes) -> service_stopped( app, **{ - key: rabbit_message.dict()[key] + key: rabbit_message.model_dump()[key] for key in MONITOR_SERVICE_STOPPED_LABELS }, ) diff --git a/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py b/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py index 8aec3e45359..813fa6b9eb1 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py +++ b/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py @@ -6,7 +6,7 @@ from models_library.basic_types import NonNegativeDecimal from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, PositiveInt +from pydantic import BaseModel, ConfigDict, PositiveInt from simcore_postgres_database.utils_payments_autorecharge import AutoRechargeStmts from ..db.plugin import get_database_engine @@ -24,9 +24,7 @@ class PaymentsAutorechargeDB(BaseModel): primary_payment_method_id: PaymentMethodID top_up_amount_in_usd: NonNegativeDecimal monthly_limit_in_usd: NonNegativeDecimal | None - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) async def get_wallet_autorecharge( @@ -38,7 +36,7 @@ async def get_wallet_autorecharge( stmt = AutoRechargeStmts.get_wallet_autorecharge(wallet_id) result = await conn.execute(stmt) row = await result.first() - return PaymentsAutorechargeDB.from_orm(row) if row else None + return PaymentsAutorechargeDB.model_validate(row) if row else None async def replace_wallet_autorecharge( @@ -75,4 +73,4 @@ async def replace_wallet_autorecharge( result = await conn.execute(stmt) row = await result.first() assert row # nosec - return PaymentsAutorechargeDB.from_orm(row) + return PaymentsAutorechargeDB.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/payments/_methods_api.py b/services/web/server/src/simcore_service_webserver/payments/_methods_api.py index a1eac2b440d..d19313d5bcc 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_methods_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_methods_api.py @@ -15,7 +15,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from servicelib.logging_utils import log_decorator from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState from yarl import URL @@ -56,7 +56,7 @@ def _to_api_model( ) -> PaymentMethodGet: assert entry.completed_at # nosec - return PaymentMethodGet.parse_obj( + return PaymentMethodGet.model_validate( { **payment_method_details_from_gateway, "idr": entry.payment_method_id, @@ -79,7 +79,7 @@ async def _fake_init_creation_of_wallet_payment_method( await asyncio.sleep(1) payment_method_id = PaymentMethodID(f"{_FAKE_PAYMENT_METHOD_ID_PREFIX}_{uuid4()}") form_link = ( - URL(settings.PAYMENTS_FAKE_GATEWAY_URL) + URL(f"{settings.PAYMENTS_FAKE_GATEWAY_URL}") .with_path("/payment-methods/form") .with_query(id=payment_method_id) ) @@ -97,7 +97,7 @@ async def _fake_init_creation_of_wallet_payment_method( return PaymentMethodInitiated( wallet_id=wallet_id, payment_method_id=payment_method_id, - payment_method_form_url=parse_obj_as(HttpUrl, f"{form_link}"), + payment_method_form_url=TypeAdapter(HttpUrl).validate_python(f"{form_link}"), ) diff --git a/services/web/server/src/simcore_service_webserver/payments/_methods_db.py b/services/web/server/src/simcore_service_webserver/payments/_methods_db.py index b5838eb171c..3b2bcf8ede8 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_methods_db.py +++ b/services/web/server/src/simcore_service_webserver/payments/_methods_db.py @@ -8,7 +8,7 @@ from models_library.api_schemas_webserver.wallets import PaymentMethodID from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, ConfigDict, TypeAdapter from simcore_postgres_database.models.payments_methods import ( InitPromptAckFlowState, payments_methods, @@ -35,9 +35,7 @@ class PaymentsMethodsDB(BaseModel): completed_at: datetime.datetime | None state: InitPromptAckFlowState state_message: str | None - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) async def insert_init_payment_method( @@ -81,7 +79,7 @@ async def list_successful_payment_methods( .order_by(payments_methods.c.created.desc()) ) # newest first rows = await result.fetchall() or [] - return parse_obj_as(list[PaymentsMethodsDB], rows) + return TypeAdapter(list[PaymentsMethodsDB]).validate_python(rows) async def get_successful_payment_method( @@ -104,7 +102,7 @@ async def get_successful_payment_method( if row is None: raise PaymentMethodNotFoundError(payment_method_id=payment_method_id) - return PaymentsMethodsDB.from_orm(row) + return PaymentsMethodsDB.model_validate(row) async def get_pending_payment_methods_ids( @@ -113,11 +111,14 @@ async def get_pending_payment_methods_ids( async with get_database_engine(app).acquire() as conn: result = await conn.execute( sa.select(payments_methods.c.payment_method_id) - .where(payments_methods.c.completed_at == None) # noqa: E711 + .where(payments_methods.c.completed_at.is_(None)) .order_by(payments_methods.c.initiated_at.asc()) # oldest first ) rows = await result.fetchall() or [] - return [parse_obj_as(PaymentMethodID, row.payment_method_id) for row in rows] + return [ + TypeAdapter(PaymentMethodID).validate_python(row.payment_method_id) + for row in rows + ] async def udpate_payment_method( @@ -168,7 +169,7 @@ async def udpate_payment_method( row = await result.first() assert row, "execute above should have caught this" # nosec - return PaymentsMethodsDB.from_orm(row) + return PaymentsMethodsDB.model_validate(row) async def delete_payment_method( diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index f54f48403bb..903e14ad002 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -1,6 +1,6 @@ import logging from decimal import Decimal -from typing import Any, cast +from typing import Any from uuid import uuid4 import arrow @@ -15,7 +15,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from servicelib.logging_utils import log_decorator from simcore_postgres_database.models.payments_transactions import ( PaymentTransactionState, @@ -49,7 +49,7 @@ def _to_api_model( "osparc_credits": transaction.osparc_credits, "wallet_id": transaction.wallet_id, "created_at": transaction.initiated_at, - "state": transaction.state, + "state": f"{transaction.state}", "completed_at": transaction.completed_at, } @@ -62,7 +62,7 @@ def _to_api_model( if transaction.invoice_url: data["invoice_url"] = transaction.invoice_url - return PaymentTransaction.parse_obj(data) + return PaymentTransaction.model_validate(data) @log_decorator(_logger, level=logging.INFO) @@ -81,7 +81,7 @@ async def _fake_init_payment( # get_form_payment_url settings: PaymentsSettings = get_plugin_settings(app) external_form_link = ( - URL(settings.PAYMENTS_FAKE_GATEWAY_URL) + URL(f"{settings.PAYMENTS_FAKE_GATEWAY_URL}") .with_path("/pay") .with_query(id=payment_id) ) @@ -128,7 +128,7 @@ async def _ack_creation_of_wallet_payment( assert transaction.completed_at is not None # nosec assert transaction.initiated_at < transaction.completed_at # nosec - _logger.info("Transaction completed: %s", transaction.json(indent=1)) + _logger.info("Transaction completed: %s", transaction.model_dump_json(indent=1)) payment = _to_api_model(transaction) @@ -235,8 +235,8 @@ async def _fake_get_payment_invoice_url( assert user_id # nosec assert wallet_id # nosec - return cast( - HttpUrl, parse_obj_as(HttpUrl, f"https://fake-invoice.com/?id={payment_id}") + return TypeAdapter(HttpUrl).validate_python( + f"https://fake-invoice.com/?id={payment_id}" ) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py index 9f94d46b707..d6146cd0f81 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py @@ -9,7 +9,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, HttpUrl, PositiveInt, parse_obj_as +from pydantic import BaseModel, ConfigDict, HttpUrl, PositiveInt, TypeAdapter from simcore_postgres_database.models.payments_transactions import ( PaymentTransactionState, payments_transactions, @@ -44,9 +44,7 @@ class PaymentsTransactionsDB(BaseModel): completed_at: datetime.datetime | None state: PaymentTransactionState state_message: str | None - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) async def list_user_payment_transactions( @@ -64,7 +62,7 @@ async def list_user_payment_transactions( total_number_of_items, rows = await get_user_payments_transactions( conn, user_id=user_id, offset=offset, limit=limit ) - page = parse_obj_as(list[PaymentsTransactionsDB], rows) + page = TypeAdapter(list[PaymentsTransactionsDB]).validate_python(rows) return total_number_of_items, page @@ -76,7 +74,7 @@ async def get_pending_payment_transactions_ids(app: web.Application) -> list[Pay .order_by(payments_transactions.c.initiated_at.asc()) # oldest first ) rows = await result.fetchall() or [] - return [parse_obj_as(PaymentID, row.payment_id) for row in rows] + return [TypeAdapter(PaymentID).validate_python(row.payment_id) for row in rows] async def complete_payment_transaction( @@ -103,7 +101,7 @@ async def complete_payment_transaction( payment_id=payment_id, completion_state=completion_state, state_message=state_message, - **optional_kwargs, + **optional_kwargs, # type: ignore[arg-type] ) if isinstance(row, PaymentNotFound): @@ -113,4 +111,4 @@ async def complete_payment_transaction( raise PaymentCompletedError(payment_id=row.payment_id) assert row # nosec - return PaymentsTransactionsDB.from_orm(row) + return PaymentsTransactionsDB.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index f2b88bc1765..5401568953c 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -21,7 +21,7 @@ from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import EmailStr, HttpUrl, parse_obj_as +from pydantic import EmailStr, HttpUrl, TypeAdapter from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RPC_REQUEST_DEFAULT_TIMEOUT_S @@ -52,7 +52,7 @@ async def init_payment( # pylint: disable=too-many-arguments # NOTE: remote errors are aio_pika.MessageProcessError result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment"), + TypeAdapter(RPCMethodName).validate_python("init_payment"), amount_dollars=amount_dollars, target_credits=target_credits, product_name=product_name, @@ -83,7 +83,7 @@ async def cancel_payment( await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "cancel_payment"), + TypeAdapter(RPCMethodName).validate_python("cancel_payment"), payment_id=payment_id, user_id=user_id, wallet_id=wallet_id, @@ -104,7 +104,7 @@ async def get_payments_page( result: tuple[int, list[PaymentTransaction]] = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_payments_page"), + TypeAdapter(RPCMethodName).validate_python("get_payments_page"), user_id=user_id, product_name=product_name, limit=limit, @@ -112,7 +112,8 @@ async def get_payments_page( timeout_s=2 * RPC_REQUEST_DEFAULT_TIMEOUT_S, ) assert ( # nosec - parse_obj_as(tuple[int, list[PaymentTransaction]], result) is not None + TypeAdapter(tuple[int, list[PaymentTransaction]]).validate_python(result) + is not None ) return result @@ -129,7 +130,7 @@ async def get_payment_invoice_url( result: HttpUrl = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_payment_invoice_url"), + TypeAdapter(RPCMethodName).validate_python("get_payment_invoice_url"), user_id=user_id, wallet_id=wallet_id, payment_id=payment_id, @@ -152,7 +153,7 @@ async def init_creation_of_payment_method( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_creation_of_payment_method"), + TypeAdapter(RPCMethodName).validate_python("init_creation_of_payment_method"), wallet_id=wallet_id, wallet_name=wallet_name, user_id=user_id, @@ -176,7 +177,7 @@ async def cancel_creation_of_payment_method( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "cancel_creation_of_payment_method"), + TypeAdapter(RPCMethodName).validate_python("cancel_creation_of_payment_method"), payment_method_id=payment_method_id, user_id=user_id, wallet_id=wallet_id, @@ -196,7 +197,7 @@ async def list_payment_methods( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "list_payment_methods"), + TypeAdapter(RPCMethodName).validate_python("list_payment_methods"), user_id=user_id, wallet_id=wallet_id, timeout_s=2 * RPC_REQUEST_DEFAULT_TIMEOUT_S, @@ -217,7 +218,7 @@ async def get_payment_method( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_payment_method"), + TypeAdapter(RPCMethodName).validate_python("get_payment_method"), payment_method_id=payment_method_id, user_id=user_id, wallet_id=wallet_id, @@ -239,7 +240,7 @@ async def delete_payment_method( result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "delete_payment_method"), + TypeAdapter(RPCMethodName).validate_python("delete_payment_method"), payment_method_id=payment_method_id, user_id=user_id, wallet_id=wallet_id, @@ -270,7 +271,7 @@ async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-a result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "pay_with_payment_method"), + TypeAdapter(RPCMethodName).validate_python("pay_with_payment_method"), payment_method_id=payment_method_id, amount_dollars=amount_dollars, target_credits=target_credits, diff --git a/services/web/server/src/simcore_service_webserver/payments/_tasks.py b/services/web/server/src/simcore_service_webserver/payments/_tasks.py index d6c8a5719fb..b87465f5f3e 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_tasks.py +++ b/services/web/server/src/simcore_service_webserver/payments/_tasks.py @@ -6,7 +6,7 @@ from aiohttp import web from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from servicelib.aiohttp.typing_extension import CleanupContextFunc from servicelib.logging_utils import log_decorator from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState @@ -51,8 +51,7 @@ def _create_possible_outcomes(accepted, rejected): accepted={ "completion_state": PaymentTransactionState.SUCCESS, "message": "Succesful payment (fake)", - "invoice_url": parse_obj_as( - HttpUrl, + "invoice_url": TypeAdapter(HttpUrl).validate_python( "https://assets.website-files.com/63206faf68ab2dc3ee3e623b/634ea60a9381021f775e7a28_Placeholder%20PDF.pdf", ), }, diff --git a/services/web/server/src/simcore_service_webserver/payments/settings.py b/services/web/server/src/simcore_service_webserver/payments/settings.py index 846e2b1e9f9..8553e508b76 100644 --- a/services/web/server/src/simcore_service_webserver/payments/settings.py +++ b/services/web/server/src/simcore_service_webserver/payments/settings.py @@ -3,7 +3,15 @@ from aiohttp import web from models_library.basic_types import NonNegativeDecimal -from pydantic import Field, HttpUrl, PositiveInt, SecretStr, parse_obj_as, validator +from pydantic import ( + Field, + HttpUrl, + PositiveInt, + SecretStr, + TypeAdapter, + ValidationInfo, + field_validator, +) from settings_library.base import BaseCustomSettings from settings_library.basic_types import PortInt, VersionTag from settings_library.utils_service import ( @@ -18,7 +26,7 @@ class PaymentsSettings(BaseCustomSettings, MixinServiceSettings): PAYMENTS_HOST: str = "payments" PAYMENTS_PORT: PortInt = DEFAULT_FASTAPI_PORT - PAYMENTS_VTAG: VersionTag = parse_obj_as(VersionTag, "v1") + PAYMENTS_VTAG: VersionTag = TypeAdapter(VersionTag).validate_python("v1") PAYMENTS_USERNAME: str = Field( ..., @@ -42,7 +50,9 @@ class PaymentsSettings(BaseCustomSettings, MixinServiceSettings): ) PAYMENTS_FAKE_GATEWAY_URL: HttpUrl = Field( - default=parse_obj_as(HttpUrl, "https://fake-payment-gateway.com"), + default=TypeAdapter(HttpUrl).validate_python( + "https://fake-payment-gateway.com" + ), description="FAKE Base url to the payment gateway", ) @@ -82,7 +92,7 @@ def base_url(self) -> str: ) return base_url_without_vtag - @validator("PAYMENTS_FAKE_COMPLETION") + @field_validator("PAYMENTS_FAKE_COMPLETION") @classmethod def _payments_cannot_be_faken_in_production(cls, v): if v is True and "production" in os.environ.get("SWARM_STACK_NAME", ""): @@ -90,10 +100,10 @@ def _payments_cannot_be_faken_in_production(cls, v): raise ValueError(msg) return v - @validator("PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT") + @field_validator("PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT") @classmethod - def _monthly_limit_greater_than_top_up(cls, v, values): - top_up = values["PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT"] + def _monthly_limit_greater_than_top_up(cls, v, info: ValidationInfo): + top_up = info.data["PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT"] if v < 2 * top_up: msg = "PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT (={v}) should be at least twice PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT ({top_up})" raise ValueError(msg) diff --git a/services/web/server/src/simcore_service_webserver/products/_db.py b/services/web/server/src/simcore_service_webserver/products/_db.py index 37a960bf9a4..d0317fdc61b 100644 --- a/services/web/server/src/simcore_service_webserver/products/_db.py +++ b/services/web/server/src/simcore_service_webserver/products/_db.py @@ -100,7 +100,7 @@ async def get_product(self, product_name: str) -> Product | None: return Product( **dict(row.items()), is_payment_enabled=payments.enabled, - credits_per_usd=payments.credits_per_usd, # type: ignore[arg-type] + credits_per_usd=payments.credits_per_usd, ) return None diff --git a/services/web/server/src/simcore_service_webserver/products/_events.py b/services/web/server/src/simcore_service_webserver/products/_events.py index f1e4601d7c7..836e43a902f 100644 --- a/services/web/server/src/simcore_service_webserver/products/_events.py +++ b/services/web/server/src/simcore_service_webserver/products/_events.py @@ -90,7 +90,7 @@ async def load_products_on_startup(app: web.Application): app_products[name] = Product( **dict(row.items()), is_payment_enabled=payments.enabled, - credits_per_usd=payments.credits_per_usd, # type: ignore[arg-type] + credits_per_usd=payments.credits_per_usd, ) assert name in FRONTEND_APPS_AVAILABLE # nosec diff --git a/services/web/server/src/simcore_service_webserver/products/_handlers.py b/services/web/server/src/simcore_service_webserver/products/_handlers.py index bfdabef6d6f..3f9db2d2cab 100644 --- a/services/web/server/src/simcore_service_webserver/products/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/products/_handlers.py @@ -5,7 +5,7 @@ from models_library.api_schemas_webserver.product import GetCreditPrice, GetProduct from models_library.basic_types import IDStr from models_library.users import UserID -from pydantic import Extra, Field +from pydantic import Field from servicelib.aiohttp.requests_validation import ( RequestParams, StrictRequestParams, @@ -36,12 +36,12 @@ class _ProductsRequestContext(RequestParams): @login_required @permission_required("product.price.read") async def _get_current_product_price(request: web.Request): - req_ctx = _ProductsRequestContext.parse_obj(request) + req_ctx = _ProductsRequestContext.model_validate(request) price_info = await _api.get_current_product_credit_price_info(request) credit_price = GetCreditPrice( product_name=req_ctx.product_name, - usd_per_credit=price_info.usd_per_credit if price_info else None, # type: ignore[arg-type] + usd_per_credit=price_info.usd_per_credit if price_info else None, min_payment_amount_usd=price_info.min_payment_amount_usd # type: ignore[arg-type] if price_info else None, @@ -57,7 +57,7 @@ class _ProductsRequestParams(StrictRequestParams): @login_required @permission_required("product.details.*") async def _get_product(request: web.Request): - req_ctx = _ProductsRequestContext.parse_obj(request) + req_ctx = _ProductsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProductsRequestParams, request) if path_params.product_name == "current": @@ -70,8 +70,8 @@ async def _get_product(request: web.Request): except KeyError as err: raise web.HTTPNotFound(reason=f"{product_name=} not found") from err - assert GetProduct.Config.extra == Extra.ignore # nosec - data = GetProduct(**product.dict(), templates=[]) + assert GetProduct.model_config["extra"] == "ignore" # nosec + data = GetProduct(**product.model_dump(), templates=[]) return envelope_json_response(data) @@ -86,7 +86,7 @@ class _ProductTemplateParams(_ProductsRequestParams): @login_required @permission_required("product.details.*") async def update_product_template(request: web.Request): - req_ctx = _ProductsRequestContext.parse_obj(request) + req_ctx = _ProductsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProductTemplateParams, request) assert req_ctx # nosec diff --git a/services/web/server/src/simcore_service_webserver/products/_invitations_handlers.py b/services/web/server/src/simcore_service_webserver/products/_invitations_handlers.py index a300a4c43e9..fa2bb927405 100644 --- a/services/web/server/src/simcore_service_webserver/products/_invitations_handlers.py +++ b/services/web/server/src/simcore_service_webserver/products/_invitations_handlers.py @@ -35,7 +35,7 @@ class _ProductsRequestContext(RequestParams): @login_required @permission_required("product.invitations.create") async def generate_invitation(request: web.Request): - req_ctx = _ProductsRequestContext.parse_obj(request) + req_ctx = _ProductsRequestContext.model_validate(request) body = await parse_request_body_as(GenerateInvitation, request) _, user_email = await get_user_name_and_email(request.app, user_id=req_ctx.user_id) @@ -55,16 +55,16 @@ async def generate_invitation(request: web.Request): assert generated.product == req_ctx.product_name # nosec assert generated.guest == body.guest # nosec - url = URL(generated.invitation_url) + url = URL(f"{generated.invitation_url}") invitation_link = request.url.with_path(url.path).with_fragment(url.raw_fragment) invitation = InvitationGenerated( product_name=generated.product, issuer=generated.issuer, - guest=generated.guest, # type: ignore[arg-type] + guest=generated.guest, trial_account_days=generated.trial_account_days, extra_credits_in_usd=generated.extra_credits_in_usd, created=generated.created, invitation_link=f"{invitation_link}", # type: ignore[arg-type] ) - return envelope_json_response(invitation.dict(exclude_none=True)) + return envelope_json_response(invitation.model_dump(exclude_none=True)) diff --git a/services/web/server/src/simcore_service_webserver/products/_model.py b/services/web/server/src/simcore_service_webserver/products/_model.py index 82c4a3b64aa..48f9f20fa01 100644 --- a/services/web/server/src/simcore_service_webserver/products/_model.py +++ b/services/web/server/src/simcore_service_webserver/products/_model.py @@ -1,9 +1,9 @@ import logging +import re import string from typing import ( # noqa: UP035 # pydantic does not validate with re.Pattern + Annotated, Any, - ClassVar, - Pattern, ) from models_library.basic_regex import ( @@ -14,7 +14,15 @@ from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from models_library.utils.change_case import snake_to_camel -from pydantic import BaseModel, Extra, Field, PositiveInt, validator +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + Field, + PositiveInt, + field_serializer, + field_validator, +) from simcore_postgres_database.models.products import ( EmailFeedback, Forum, @@ -40,19 +48,20 @@ class Product(BaseModel): SEE descriptions in packages/postgres-database/src/simcore_postgres_database/models/products.py """ - name: ProductName = Field(regex=PUBLIC_VARIABLE_NAME_RE) + name: ProductName = Field(pattern=PUBLIC_VARIABLE_NAME_RE, validate_default=True) display_name: str = Field(..., description="Long display name") short_name: str | None = Field( None, - regex=TWILIO_ALPHANUMERIC_SENDER_ID_RE, + pattern=re.compile(TWILIO_ALPHANUMERIC_SENDER_ID_RE), min_length=2, max_length=11, description="Short display name for SMS", ) - host_regex: Pattern = Field(..., description="Host regex") - # NOTE: typing.Pattern is supported but not re.Pattern (SEE https://github.com/pydantic/pydantic/pull/4366) + host_regex: Annotated[re.Pattern, BeforeValidator(str.strip)] = Field( + ..., description="Host regex" + ) support_email: LowerCaseEmailStr = Field( ..., @@ -82,7 +91,7 @@ class Product(BaseModel): ) registration_email_template: str | None = Field( - None, x_template_name="registration_email" + None, json_schema_extra={"x_template_name": "registration_email"} ) max_open_studies_per_user: PositiveInt | None = Field( @@ -109,7 +118,7 @@ class Product(BaseModel): description="Price of the credits in this product given in credit/USD. None for free product.", ) - @validator("*", pre=True) + @field_validator("*", mode="before") @classmethod def _parse_empty_string_as_null(cls, v): """Safe measure: database entries are sometimes left blank instead of null""" @@ -117,7 +126,7 @@ def _parse_empty_string_as_null(cls, v): return None return v - @validator("name", pre=True, always=True) + @field_validator("name", mode="before") @classmethod def _validate_name(cls, v): if v not in FRONTEND_APPS_AVAILABLE: @@ -125,27 +134,22 @@ def _validate_name(cls, v): raise ValueError(msg) return v - @validator("host_regex", pre=True) - @classmethod - def _strip_whitespaces(cls, v): - if v and isinstance(v, str): - # Prevents unintended leading & trailing spaces when added - # manually in the database - return v.strip() + @field_serializer("issues", "vendor") + def _preserve_snake_case(self, v: Any) -> Any: return v @property def twilio_alpha_numeric_sender_id(self) -> str: return self.short_name or self.display_name.replace(string.punctuation, "")[:11] - class Config: - alias_generator = snake_to_camel # to export - allow_population_by_field_name = True - anystr_strip_whitespace = True - extra = Extra.ignore - frozen = True # read-only - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + alias_generator=snake_to_camel, + populate_by_name=True, + str_strip_whitespace=True, + frozen=True, + from_attributes=True, + extra="ignore", + json_schema_extra={ "examples": [ { # fake mandatory @@ -234,7 +238,8 @@ class Config: "is_payment_enabled": False, }, ] - } + }, + ) # helpers ---- @@ -247,7 +252,7 @@ def to_statics(self) -> dict[str, Any]: # SECURITY WARNING: do not expose sensitive information here # keys will be named as e.g. displayName, supportEmail, ... - return self.dict( + return self.model_dump( include={ "display_name": True, "support_email": True, @@ -266,8 +271,11 @@ def to_statics(self) -> dict[str, Any]: def get_template_name_for(self, filename: str) -> str | None: """Checks for field marked with 'x_template_name' that fits the argument""" template_name = filename.removesuffix(".jinja2") - for field in self.__fields__.values(): - if field.field_info.extra.get("x_template_name") == template_name: - template_name_attribute: str = getattr(self, field.name) + for name, field in self.model_fields.items(): + if ( + field.json_schema_extra + and field.json_schema_extra.get("x_template_name") == template_name # type: ignore[union-attr] + ): + template_name_attribute: str = getattr(self, name) return template_name_attribute return None diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_api.py b/services/web/server/src/simcore_service_webserver/projects/_comments_api.py index a3626d099bb..55cfedac30c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_api.py @@ -43,7 +43,8 @@ async def list_project_comments( ProjectsCommentsDB ] = await db.list_project_comments(project_uuid, offset, limit) projects_comments_api_model = [ - ProjectsCommentsAPI(**comment.dict()) for comment in projects_comments_db_model + ProjectsCommentsAPI(**comment.model_dump()) + for comment in projects_comments_db_model ] return projects_comments_api_model @@ -70,7 +71,7 @@ async def update_project_comment( comment_id, project_uuid, contents ) projects_comments_api_model = ProjectsCommentsAPI( - **projects_comments_db_model.dict() + **projects_comments_db_model.model_dump() ) return projects_comments_api_model @@ -90,6 +91,6 @@ async def get_project_comment( comment_id ) projects_comments_api_model = ProjectsCommentsAPI( - **projects_comments_db_model.dict() + **projects_comments_db_model.model_dump() ) return projects_comments_api_model diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_db.py b/services/web/server/src/simcore_service_webserver/projects/_comments_db.py index 102e43971da..0cc52bea1e7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_db.py @@ -9,7 +9,7 @@ from models_library.projects import ProjectID from models_library.projects_comments import CommentID, ProjectsCommentsDB from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.types import PositiveInt from simcore_postgres_database.models.projects_comments import projects_comments from sqlalchemy import func, literal_column @@ -32,7 +32,7 @@ async def create_project_comment( .returning(projects_comments.c.comment_id) ) result: tuple[PositiveInt] = await project_comment_id.first() - return parse_obj_as(CommentID, result[0]) + return TypeAdapter(CommentID).validate_python(result[0]) async def list_project_comments( @@ -50,7 +50,7 @@ async def list_project_comments( .limit(limit) ) result = [ - parse_obj_as(ProjectsCommentsDB, row) + ProjectsCommentsDB.model_validate(row) for row in await project_comment_result.fetchall() ] return result @@ -86,7 +86,7 @@ async def update_project_comment( .returning(literal_column("*")) ) result = await project_comment_result.first() - return parse_obj_as(ProjectsCommentsDB, result) + return ProjectsCommentsDB.model_validate(result) async def delete_project_comment(conn, comment_id: CommentID) -> None: @@ -100,4 +100,4 @@ async def get_project_comment(conn, comment_id: CommentID) -> ProjectsCommentsDB projects_comments.select().where(projects_comments.c.comment_id == comment_id) ) result = await project_comment_result.first() - return parse_obj_as(ProjectsCommentsDB, result) + return ProjectsCommentsDB.model_validate(result) diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py index 5325f389e9a..6ad8b290ba0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py @@ -15,7 +15,7 @@ Page, ) from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel, Extra, Field, NonNegativeInt +from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -60,24 +60,18 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _ProjectCommentsPathParams(BaseModel): project_uuid: ProjectID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _ProjectCommentsWithCommentPathParams(BaseModel): project_uuid: ProjectID comment_id: CommentID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _ProjectCommentsBodyParams(BaseModel): contents: str - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.post( @@ -87,7 +81,7 @@ class Config: @permission_required("project.read") @_handle_project_comments_exceptions async def create_project_comment(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectCommentsPathParams, request) body_params = await parse_request_body_as(_ProjectCommentsBodyParams, request) @@ -119,9 +113,7 @@ class _ListProjectCommentsQueryParams(BaseModel): offset: NonNegativeInt = Field( default=0, description="index to the first item to return (pagination)" ) - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.get(f"/{VTAG}/projects/{{project_uuid}}/comments", name="list_project_comments") @@ -129,7 +121,7 @@ class Config: @permission_required("project.read") @_handle_project_comments_exceptions async def list_project_comments(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectCommentsPathParams, request) query_params: _ListProjectCommentsQueryParams = parse_request_query_parameters_as( _ListProjectCommentsQueryParams, request @@ -155,7 +147,7 @@ async def list_project_comments(request: web.Request): limit=query_params.limit, ) - page = Page[dict[str, Any]].parse_obj( + page = Page[dict[str, Any]].model_validate( paginate_data( chunk=project_comments, request_url=request.url, @@ -165,7 +157,7 @@ async def list_project_comments(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -177,7 +169,7 @@ async def list_project_comments(request: web.Request): @login_required @permission_required("project.read") async def update_project_comment(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _ProjectCommentsWithCommentPathParams, request ) @@ -207,7 +199,7 @@ async def update_project_comment(request: web.Request): @permission_required("project.read") @_handle_project_comments_exceptions async def delete_project_comment(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _ProjectCommentsWithCommentPathParams, request ) @@ -235,7 +227,7 @@ async def delete_project_comment(request: web.Request): @permission_required("project.read") @_handle_project_comments_exceptions async def get_project_comment(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _ProjectCommentsWithCommentPathParams, request ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_common_models.py b/services/web/server/src/simcore_service_webserver/projects/_common_models.py index 073c012a8ac..aacec646f05 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_common_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_common_models.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import BaseModel, Extra, Field +from pydantic import ConfigDict, BaseModel, Field from servicelib.request_keys import RQT_USERID_KEY from .._constants import RQ_PRODUCT_KEY @@ -19,11 +19,7 @@ class RequestContext(BaseModel): class ProjectPathParams(BaseModel): project_id: ProjectID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid - + model_config = ConfigDict(populate_by_name=True, extra="forbid") class RemoveQueryParams(BaseModel): force: bool = Field( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 4886b3b6242..d26a63a9cf8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -15,7 +15,7 @@ from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.workspaces import UserWorkspaceAccessRightsDB -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.aiohttp.long_running_tasks.server import TaskProgress from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from simcore_postgres_database.utils_projects_nodes import ( @@ -139,7 +139,7 @@ def _mapped_node_id(node: ProjectNode) -> NodeID: node_id=_mapped_node_id(node), **{ k: v - for k, v in node.dict().items() + for k, v in node.model_dump().items() if k in ProjectNodeCreate.get_field_names(exclude={"node_id"}) }, ) @@ -157,7 +157,9 @@ async def _copy_files_from_source_project( ): db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(app) needs_lock_source_project: bool = ( - await db.get_project_type(parse_obj_as(ProjectID, source_project["uuid"])) + await db.get_project_type( + TypeAdapter(ProjectID).validate_python(source_project["uuid"]) + ) != ProjectTypeDB.TEMPLATE ) @@ -178,8 +180,7 @@ async def _copy_files_from_source_project( ): task_progress.update( message=long_running_task.progress.message, - percent=parse_obj_as( - ProgressPercent, + percent=TypeAdapter(ProgressPercent).validate_python( ( starting_value + long_running_task.progress.percent * (1.0 - starting_value) @@ -416,11 +417,12 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche ) ) new_project["accessRights"] = { - gid: access.dict() for gid, access in workspace_db.access_rights.items() + f"{gid}": access.model_dump() + for gid, access in workspace_db.access_rights.items() } # Ensures is like ProjectGet - data = ProjectGet.parse_obj(new_project).data(exclude_unset=True) + data = ProjectGet.model_validate(new_project).data(exclude_unset=True) raise web.HTTPCreated( text=json_dumps({"data": data}), diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index f6c98c6e08e..793dd19b083 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -53,11 +53,11 @@ async def _append_fields( # replace project access rights (if project is in workspace) if workspace_access_rights: project["accessRights"] = { - gid: access.dict() for gid, access in workspace_access_rights.items() + gid: access.model_dump() for gid, access in workspace_access_rights.items() } # validate - return model_schema_cls.parse_obj(project).data(exclude_unset=True) + return model_schema_cls.model_validate(project).data(exclude_unset=True) async def list_projects( # pylint: disable=too-many-arguments diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index adbf8b0682e..a6a32fbb0bc 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -8,6 +8,7 @@ import logging from aiohttp import web +from common_library.json_serialization import json_dumps from models_library.api_schemas_webserver.projects import ( EmptyModel, ProjectCopyOverride, @@ -21,7 +22,6 @@ from models_library.rest_pagination import Page from models_library.rest_pagination_utils import paginate_data from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import parse_obj_as from servicelib.aiohttp import status from servicelib.aiohttp.long_running_tasks.server import start_long_running_task from servicelib.aiohttp.requests_validation import ( @@ -109,17 +109,15 @@ async def _wrapper(request: web.Request) -> web.StreamResponse: routes = web.RouteTableDef() -# -# - Create https://google.aip.dev/133 -# - - @routes.post(f"/{VTAG}/projects", name="create_project") @login_required @permission_required("project.create") @permission_required("services.pipeline.*") # due to update_pipeline_db async def create_project(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + # + # - Create https://google.aip.dev/133 + # + req_ctx = RequestContext.model_validate(request) query_params: ProjectCreateParams = parse_request_query_parameters_as( ProjectCreateParams, request ) @@ -143,7 +141,7 @@ async def create_project(request: web.Request): ProjectCreateNew | ProjectCopyOverride | EmptyModel, request # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 ) predefined_project = ( - project_create.dict( + project_create.model_dump( exclude_unset=True, by_alias=True, exclude_none=True, @@ -171,8 +169,6 @@ async def create_project(request: web.Request): ) -# - List https://google.aip.dev/132 -# @routes.get(f"/{VTAG}/projects", name="list_projects") @@ -180,13 +176,16 @@ async def create_project(request: web.Request): @permission_required("project.read") @_handle_projects_exceptions async def list_projects(request: web.Request): + # + # - List https://google.aip.dev/132 + # """ Raises: web.HTTPUnprocessableEntity: (422) if validation of request parameters fail """ - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) query_params: ProjectListWithJsonStrParams = parse_request_query_parameters_as( ProjectListWithJsonStrParams, request ) @@ -206,12 +205,12 @@ async def list_projects(request: web.Request): limit=query_params.limit, offset=query_params.offset, search=query_params.search, - order_by=parse_obj_as(OrderBy, query_params.order_by), + order_by=OrderBy.model_validate(query_params.order_by), folder_id=query_params.folder_id, workspace_id=query_params.workspace_id, ) - page = Page[ProjectDict].parse_obj( + page = Page[ProjectDict].model_validate( paginate_data( chunk=projects, request_url=request.url, @@ -221,7 +220,7 @@ async def list_projects(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -231,7 +230,7 @@ async def list_projects(request: web.Request): @permission_required("project.read") @_handle_projects_exceptions async def list_projects_full_search(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) query_params: ProjectListFullSearchWithJsonStrParams = ( parse_request_query_parameters_as( ProjectListFullSearchWithJsonStrParams, request @@ -250,7 +249,7 @@ async def list_projects_full_search(request: web.Request): tag_ids_list=tag_ids_list, ) - page = Page[ProjectDict].parse_obj( + page = Page[ProjectDict].model_validate( paginate_data( chunk=projects, request_url=request.url, @@ -260,28 +259,26 @@ async def list_projects_full_search(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) -# -# - Get https://google.aip.dev/131 -# - Get active project: Singleton per-session resources https://google.aip.dev/156 -# - - @routes.get(f"/{VTAG}/projects/active", name="get_active_project") @login_required @permission_required("project.read") async def get_active_project(request: web.Request) -> web.Response: + # + # - Get https://google.aip.dev/131 + # - Get active project: Singleton per-session resources https://google.aip.dev/156 + # """ Raises: web.HTTPUnprocessableEntity: (422) if validation of request parameters fail web.HTTPNotFound: If active project is not found """ - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) query_params: ProjectActiveParams = parse_request_query_parameters_as( ProjectActiveParams, request ) @@ -306,7 +303,7 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.parse_obj(project).data(exclude_unset=True) + data = ProjectGet.model_validate(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) @@ -327,7 +324,7 @@ async def get_project(request: web.Request): web.HTTPNotFound: This project was not found """ - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) user_available_services: list[dict] = await get_services_for_user_in_product( @@ -362,7 +359,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.parse_obj(project).data(exclude_unset=True) + data = ProjectGet.model_validate(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) except ProjectInvalidRightsError as exc: @@ -389,18 +386,16 @@ async def get_project_inactivity(request: web.Request): return web.json_response(Envelope(data=project_inactivity), dumps=json_dumps) -# -# - Update https://google.aip.dev/134 -# - - @routes.patch(f"/{VTAG}/projects/{{project_id}}", name="patch_project") @login_required @permission_required("project.update") @permission_required("services.pipeline.*") @_handle_projects_exceptions async def patch_project(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + # + # Update https://google.aip.dev/134 + # + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) project_patch = await parse_request_body_as(ProjectPatch, request) @@ -416,7 +411,7 @@ async def patch_project(request: web.Request): # -# - Delete https://google.aip.dev/135 + # @@ -424,6 +419,7 @@ async def patch_project(request: web.Request): @login_required @permission_required("project.delete") async def delete_project(request: web.Request): + # Delete https://google.aip.dev/135 """ Raises: @@ -436,8 +432,7 @@ async def delete_project(request: web.Request): web.HTTPConflict: Somethine went wrong while deleting web.HTTPNoContent: Sucess """ - - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) try: @@ -512,7 +507,7 @@ async def delete_project(request: web.Request): @permission_required("project.create") @permission_required("services.pipeline.*") # due to update_pipeline_db async def clone_project(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) return await start_long_running_task( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py index b1c499fd3a9..1f67baeabc0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py @@ -20,12 +20,12 @@ from models_library.workspaces import WorkspaceID from pydantic import ( BaseModel, - Extra, + ConfigDict, Field, Json, - parse_obj_as, - root_validator, - validator, + TypeAdapter, + field_validator, + model_validator, ) from servicelib.common_headers import ( UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, @@ -57,7 +57,7 @@ class ProjectCreateHeaders(BaseModel): alias=X_SIMCORE_PARENT_NODE_ID, ) - @root_validator + @model_validator(mode="before") @classmethod def check_parent_valid(cls, values: dict[str, Any]) -> dict[str, Any]: if ( @@ -71,8 +71,7 @@ def check_parent_valid(cls, values: dict[str, Any]) -> dict[str, Any]: raise ValueError(msg) return values - class Config: - allow_population_by_field_name = False + model_config = ConfigDict(populate_by_name=False) class ProjectCreateParams(BaseModel): @@ -92,9 +91,7 @@ class ProjectCreateParams(BaseModel): default=False, description="Enables/disables hidden flag. Hidden projects are by default unlisted", ) - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class ProjectFilters(Filters): @@ -113,7 +110,7 @@ class ProjectListParams(PageQueryParameters): default=None, description="Multi column full text search", max_length=100, - example="My Project", + examples=["My Project"], ) folder_id: FolderID | None = Field( default=None, @@ -124,19 +121,19 @@ class ProjectListParams(PageQueryParameters): description="Filter projects in specific workspace. Default filtering is a private workspace.", ) - @validator("search", pre=True) + @field_validator("search", mode="before") @classmethod def search_check_empty_string(cls, v): if not v: return None return v - _null_or_none_str_to_none_validator = validator( - "folder_id", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) + _null_or_none_str_to_none_validator = field_validator("folder_id", mode="before")( + null_or_none_str_to_none_validator + ) - _null_or_none_str_to_none_validator2 = validator( - "workspace_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator2 = field_validator( + "workspace_id", mode="before" )(null_or_none_str_to_none_validator) @@ -144,11 +141,11 @@ class ProjectListSortParams(BaseModel): order_by: Json[OrderBy] = Field( # pylint: disable=unsubscriptable-object default=OrderBy(field=IDStr("last_change_date"), direction=OrderDirection.DESC), description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.", - example='{"field": "prj_owner", "direction": "desc"}', + examples=['{"field": "prj_owner", "direction": "desc"}'], alias="order_by", ) - @validator("order_by", check_fields=False) + @field_validator("order_by", check_fields=False) @classmethod def validate_order_by_field(cls, v): if v.field not in { @@ -164,8 +161,7 @@ def validate_order_by_field(cls, v): raise ValueError(msg) return v - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class ProjectListWithJsonStrParams( @@ -183,15 +179,15 @@ class ProjectListFullSearchParams(PageQueryParameters): default=None, description="Multi column full text search, across all folders and workspaces", max_length=100, - example="My Project", + examples=["My Project"], ) tag_ids: str | None = Field( default=None, description="Search by tag ID (multiple tag IDs may be provided separated by column)", - example="1,3", + examples=["1,3"], ) - _empty_is_none = validator("text", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("text", mode="before")( empty_str_to_none_pre_validator ) @@ -205,7 +201,7 @@ def tag_ids_list(self) -> list[int]: if self.tag_ids: tag_ids_list = list(map(int, self.tag_ids.split(","))) # Validate that the tag_ids_list is indeed a list of integers - parse_obj_as(list[int], tag_ids_list) + TypeAdapter(list[int]).validate_python(tag_ids_list) else: tag_ids_list = [] except ValueError as exc: diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py index 35be5c4056c..6a5dab54cf9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py @@ -191,7 +191,7 @@ async def _execute_without_permission_check( assert isinstance(row, RowProxy) # nosec try: await asyncio.get_event_loop().run_in_executor( - None, ProjectAtDB.from_orm, row + None, ProjectAtDB.model_validate, row ) except ProjectInvalidRightsError: @@ -380,7 +380,7 @@ def patch_workbench( raise ProjectInvalidUsageError # if it's a new node, let's check that it validates try: - Node.parse_obj(new_node_data) + Node.model_validate(new_node_data) patched_project["workbench"][node_key] = new_node_data changed_entries.update({node_key: new_node_data}) except ValidationError as err: diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py index 1ac57057c53..59ea8ebe282 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_db.py @@ -11,7 +11,7 @@ from models_library.folders import FolderID from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel from simcore_postgres_database.models.projects_to_folders import projects_to_folders from sqlalchemy import func, literal_column from sqlalchemy.sql import select @@ -56,7 +56,7 @@ async def insert_project_to_folder( .returning(literal_column("*")) ) row = await result.first() - return parse_obj_as(ProjectToFolderDB, row) + return ProjectToFolderDB.model_validate(row) async def get_project_to_folder( @@ -81,7 +81,7 @@ async def get_project_to_folder( row = await result.first() if row is None: return None - return parse_obj_as(ProjectToFolderDB, row) + return ProjectToFolderDB.model_validate(row) async def delete_project_to_folder( diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py index 0e22c5970b9..2e644a4d598 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py @@ -5,7 +5,7 @@ from models_library.folders import FolderID from models_library.projects import ProjectID from models_library.utils.common_validators import null_or_none_str_to_none_validator -from pydantic import BaseModel, Extra, validator +from pydantic import ConfigDict, BaseModel, field_validator from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -41,13 +41,11 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _ProjectsFoldersPathParams(BaseModel): project_id: ProjectID folder_id: FolderID | None - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") # validators - _null_or_none_str_to_none_validator = validator( - "folder_id", allow_reuse=True, pre=True + _null_or_none_str_to_none_validator = field_validator( + "folder_id", mode="before" )(null_or_none_str_to_none_validator) @@ -59,7 +57,7 @@ class Config: @permission_required("project.folders.*") @_handle_projects_folders_exceptions async def replace_project_folder(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectsFoldersPathParams, request) await _folders_api.move_project_into_folder( diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_api.py b/services/web/server/src/simcore_service_webserver/projects/_groups_api.py index 2477c36ecfc..7ae45f0f90c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_api.py @@ -5,7 +5,7 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.users import GroupID, UserID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel from ..users import api as users_api from . import _groups_db as projects_groups_db @@ -53,7 +53,9 @@ async def create_project_group( write=write, delete=delete, ) - project_group_api: ProjectGroupGet = ProjectGroupGet(**project_group_db.dict()) + project_group_api: ProjectGroupGet = ProjectGroupGet( + **project_group_db.model_dump() + ) return project_group_api @@ -78,7 +80,7 @@ async def list_project_groups_by_user_and_project( ] = await projects_groups_db.list_project_groups(app=app, project_id=project_id) project_groups_api: list[ProjectGroupGet] = [ - parse_obj_as(ProjectGroupGet, group) for group in project_groups_db + ProjectGroupGet.model_validate(group.model_dump()) for group in project_groups_db ] return project_groups_api @@ -127,7 +129,7 @@ async def replace_project_group( ) ) - project_api: ProjectGroupGet = ProjectGroupGet(**project_group_db.dict()) + project_api: ProjectGroupGet = ProjectGroupGet(**project_group_db.model_dump()) return project_api diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_db.py b/services/web/server/src/simcore_service_webserver/projects/_groups_db.py index 8420d71ef7a..5b963b90cdb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_db.py @@ -9,7 +9,7 @@ from aiohttp import web from models_library.projects import ProjectID from models_library.users import GroupID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, TypeAdapter from simcore_postgres_database.models.project_to_groups import project_to_groups from sqlalchemy import func, literal_column from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -59,7 +59,7 @@ async def create_project_group( .returning(literal_column("*")) ) row = await result.first() - return parse_obj_as(ProjectGroupGetDB, row) + return ProjectGroupGetDB.model_validate(row) async def list_project_groups( @@ -82,7 +82,7 @@ async def list_project_groups( async with get_database_engine(app).acquire() as conn: result = await conn.execute(stmt) rows = await result.fetchall() or [] - return parse_obj_as(list[ProjectGroupGetDB], rows) + return TypeAdapter(list[ProjectGroupGetDB]).validate_python(rows) async def get_project_group( @@ -113,7 +113,7 @@ async def get_project_group( raise ProjectGroupNotFoundError( reason=f"Project {project_id} group {group_id} not found" ) - return parse_obj_as(ProjectGroupGetDB, row) + return ProjectGroupGetDB.model_validate(row) async def replace_project_group( @@ -144,7 +144,7 @@ async def replace_project_group( raise ProjectGroupNotFoundError( reason=f"Project {project_id} group {group_id} not found" ) - return parse_obj_as(ProjectGroupGetDB, row) + return ProjectGroupGetDB.model_validate(row) async def update_or_insert_project_group( diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py index 85c71d0d62d..a747798100e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py @@ -8,7 +8,7 @@ from aiohttp import web from models_library.projects import ProjectID from models_library.users import GroupID -from pydantic import BaseModel, Extra +from pydantic import ConfigDict, BaseModel from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -53,18 +53,14 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _ProjectsGroupsPathParams(BaseModel): project_id: ProjectID group_id: GroupID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _ProjectsGroupsBodyParams(BaseModel): read: bool write: bool delete: bool - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.post( @@ -74,7 +70,7 @@ class Config: @permission_required("project.access_rights.update") @_handle_projects_groups_exceptions async def create_project_group(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectsGroupsPathParams, request) body_params = await parse_request_body_as(_ProjectsGroupsBodyParams, request) @@ -97,7 +93,7 @@ async def create_project_group(request: web.Request): @permission_required("project.read") @_handle_projects_groups_exceptions async def list_project_groups(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) project_groups: list[ @@ -120,7 +116,7 @@ async def list_project_groups(request: web.Request): @permission_required("project.access_rights.update") @_handle_projects_groups_exceptions async def replace_project_group(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectsGroupsPathParams, request) body_params = await parse_request_body_as(_ProjectsGroupsBodyParams, request) @@ -144,7 +140,7 @@ async def replace_project_group(request: web.Request): @permission_required("project.access_rights.update") @_handle_projects_groups_exceptions async def delete_project_group(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectsGroupsPathParams, request) await _groups_api.delete_project_group( diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py index db27c3359bd..f17c7941a1d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_api.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..db.plugin import get_database_engine from . import _metadata_db @@ -67,7 +67,7 @@ async def set_project_ancestors_from_custom_metadata( if parent_node_idstr := custom_metadata.get("node_id"): # NOTE: backward compatibility with S4l old client - parent_node_id = parse_obj_as(NodeID, parent_node_idstr) + parent_node_id = TypeAdapter(NodeID).validate_python(parent_node_idstr) if parent_node_id == _NIL_NODE_UUID: return diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py index 6a511a8ba4c..2c72a395a5a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_db.py @@ -6,7 +6,7 @@ from models_library.api_schemas_webserver.projects_metadata import MetadataDict from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database import utils_projects_metadata from simcore_postgres_database.utils_projects_metadata import ( DBProjectInvalidAncestorsError, @@ -84,7 +84,7 @@ async def get_project_custom_metadata( connection, project_uuid=project_uuid ) # NOTE: if no metadata in table, it returns None -- which converts here to --> {} - return parse_obj_as(MetadataDict, metadata.custom or {}) + return TypeAdapter(MetadataDict).validate_python(metadata.custom or {}) @_handle_projects_metadata_exceptions @@ -104,7 +104,7 @@ async def set_project_custom_metadata( custom_metadata=custom_metadata, ) - return parse_obj_as(MetadataDict, metadata.custom) + return TypeAdapter(MetadataDict).validate_python(metadata.custom) @_handle_projects_metadata_exceptions diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 614d0ba03b9..802c13f7937 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -79,7 +79,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @permission_required("project.read") @_handle_project_exceptions async def get_project_metadata(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) custom_metadata = await _metadata_api.get_project_custom_metadata( @@ -99,7 +99,7 @@ async def get_project_metadata(request: web.Request) -> web.Response: @permission_required("project.update") @_handle_project_exceptions async def update_project_metadata(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) update = await parse_request_body_as(ProjectMetadataUpdate, request) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py index ab6ba4b7d93..4815ae19d03 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_api.py @@ -20,8 +20,7 @@ NonNegativeFloat, NonNegativeInt, ValidationError, - parse_obj_as, - root_validator, + model_validator, ) from servicelib.utils import logged_gather @@ -96,10 +95,10 @@ class NodeScreenshot(BaseModel): mimetype: str | None = Field( default=None, description="File's media type or None if unknown. SEE https://www.iana.org/assignments/media-types/media-types.xhtml", - example="image/jpeg", + examples=["image/jpeg"], ) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def guess_mimetype_if_undefined(cls, values): mimetype = values.get("mimetype") @@ -173,7 +172,7 @@ async def __get_link( return __get_search_key(file_meta_data), await get_download_link( app, user_id, - parse_obj_as(SimCoreFileLink, {"store": "0", "path": file_meta_data.file_id}), + SimCoreFileLink.model_validate({"store": "0", "path": file_meta_data.file_id}), ) @@ -228,7 +227,7 @@ async def get_node_screenshots( assert node.outputs is not None # nosec - filelink = parse_obj_as(SimCoreFileLink, node.outputs[KeyIDStr("outFile")]) + filelink = SimCoreFileLink.model_validate(node.outputs[KeyIDStr("outFile")]) file_url = await get_download_link(app, user_id, filelink) screenshots.append( @@ -240,7 +239,7 @@ async def get_node_screenshots( except (KeyError, ValidationError, ClientError) as err: _logger.warning( "Skipping fake node. Unable to create link from file-picker %s: %s", - node.json(indent=1), + node.model_dump_json(indent=1), err, ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index daeb1e454e0..b1088b67873 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -32,7 +32,7 @@ from models_library.services_resources import ServiceResourcesDict from models_library.users import GroupID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import BaseModel, Field, parse_obj_as +from pydantic import BaseModel, Field from servicelib.aiohttp import status from servicelib.aiohttp.long_running_tasks.server import ( TaskProgress, @@ -145,7 +145,7 @@ class NodePathParams(ProjectPathParams): @permission_required("project.node.create") @_handle_project_nodes_exceptions async def create_node(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) body = await parse_request_body_as(NodeCreate, request) @@ -177,7 +177,7 @@ async def create_node(request: web.Request) -> web.Response: body.service_id, ) } - assert parse_obj_as(NodeCreated, data) is not None # nosec + assert NodeCreated.model_validate(data) is not None # nosec return envelope_json_response(data, status_cls=web.HTTPCreated) @@ -188,7 +188,7 @@ async def create_node(request: web.Request) -> web.Response: @_handle_project_nodes_exceptions # NOTE: Careful, this endpoint is actually "get_node_state," and it doesn't return a Node resource. async def get_node(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists @@ -226,7 +226,7 @@ async def get_node(request: web.Request) -> web.Response: @permission_required("project.node.update") @_handle_project_nodes_exceptions async def patch_project_node(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) node_patch = await parse_request_body_as(NodePatch, request) @@ -247,7 +247,7 @@ async def patch_project_node(request: web.Request) -> web.Response: @permission_required("project.node.delete") @_handle_project_nodes_exceptions async def delete_node(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists @@ -295,7 +295,7 @@ async def retrieve_node(request: web.Request) -> web.Response: @permission_required("project.node.update") @_handle_project_nodes_exceptions async def update_node_outputs(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) node_outputs = await parse_request_body_as(NodeOutputs, request) @@ -323,7 +323,7 @@ async def update_node_outputs(request: web.Request) -> web.Response: @_handle_project_nodes_exceptions async def start_node(request: web.Request) -> web.Response: """Has only effect on nodes associated to dynamic services""" - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) await projects_api.start_project_node( @@ -367,7 +367,7 @@ async def _stop_dynamic_service_task( @_handle_project_nodes_exceptions async def stop_node(request: web.Request) -> web.Response: """Has only effect on nodes associated to dynamic services""" - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) save_state = await has_user_project_access_rights( @@ -430,7 +430,7 @@ async def restart_node(request: web.Request) -> web.Response: @permission_required("project.node.read") @_handle_project_nodes_exceptions async def get_node_resources(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists @@ -463,7 +463,7 @@ async def get_node_resources(request: web.Request) -> web.Response: @permission_required("project.node.update") @_handle_project_nodes_exceptions async def replace_node_resources(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) body = await parse_request_body_as(ServiceResourcesDict, request) @@ -524,7 +524,7 @@ class _ProjectGroupAccess(BaseModel): async def get_project_services_access_for_gid( request: web.Request, ) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) query_params: _ServicesAccessQuery = parse_request_query_parameters_as( _ServicesAccessQuery, request @@ -623,7 +623,7 @@ async def get_project_services_access_for_gid( inaccessible_services=project_inaccessible_services, ) - return envelope_json_response(project_group_access.dict(exclude_none=True)) + return envelope_json_response(project_group_access.model_dump(exclude_none=True)) class _ProjectNodePreview(BaseModel): @@ -640,7 +640,7 @@ class _ProjectNodePreview(BaseModel): @permission_required("project.read") @_handle_project_nodes_exceptions async def list_project_nodes_previews(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) assert req_ctx # nosec @@ -650,7 +650,7 @@ async def list_project_nodes_previews(request: web.Request) -> web.Response: project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, ) - project = Project.parse_obj(project_data) + project = Project.model_validate(project_data) for node_id, node in project.workbench.items(): screenshots = await get_node_screenshots( @@ -680,7 +680,7 @@ async def list_project_nodes_previews(request: web.Request) -> web.Response: @permission_required("project.read") @_handle_project_nodes_exceptions async def get_project_node_preview(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) assert req_ctx # nosec @@ -690,7 +690,7 @@ async def get_project_node_preview(request: web.Request) -> web.Response: user_id=req_ctx.user_id, ) - project = Project.parse_obj(project_data) + project = Project.model_validate(project_data) node = project.workbench.get(NodeIDStr(path_params.node_id)) if node is None: diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_api.py b/services/web/server/src/simcore_service_webserver/projects/_ports_api.py index 95a42f32046..9ae42c397c8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_api.py @@ -24,7 +24,7 @@ jsonschema_validate_data, ) from models_library.utils.services_io import JsonSchemaDict, get_service_io_json_schema -from pydantic import ValidationError +from pydantic import ConfigDict, ValidationError from ..director_v2.api import get_batch_tasks_outputs from .exceptions import InvalidInputValue @@ -163,8 +163,7 @@ def set_inputs_in_project( class _NonStrictPortLink(PortLink): - class Config(PortLink.Config): - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True, from_attributes=True) class _OutputPortInfo(NamedTuple): @@ -181,7 +180,7 @@ def _get_outputs_in_workbench(workbench: dict[NodeID, Node]) -> dict[NodeID, Any if port.node.inputs: try: # Every port is associated to the output of a task - port_link = _NonStrictPortLink.parse_obj( + port_link = _NonStrictPortLink.model_validate( port.node.inputs[KeyIDStr("in_1")] ) # Here we resolve which task and which tasks' output is associated to this port? diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index 40e33893428..eaacd9c1aa3 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -22,7 +22,7 @@ from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.utils.services_io import JsonSchemaDict -from pydantic import BaseModel, Field, parse_obj_as +from pydantic import BaseModel, Field, TypeAdapter from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -88,7 +88,7 @@ async def _get_validated_workbench_model( include_state=False, ) - return parse_obj_as(dict[NodeID, Node], project["workbench"]) + return TypeAdapter(dict[NodeID, Node]).validate_python(project["workbench"]) routes = web.RouteTableDef() @@ -103,7 +103,7 @@ async def _get_validated_workbench_model( @permission_required("project.read") @_handle_project_exceptions async def get_project_inputs(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) assert request.app # nosec @@ -129,7 +129,7 @@ async def get_project_inputs(request: web.Request) -> web.Response: @_handle_project_exceptions async def update_project_inputs(request: web.Request) -> web.Response: db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) inputs_updates = await parse_request_body_as(list[ProjectInputUpdate], request) @@ -148,7 +148,7 @@ async def update_project_inputs(request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason=f"Invalid input key [{node_id}]") workbench[node_id].outputs = {KeyIDStr("out_1"): input_update.value} - partial_workbench_data[node_id] = workbench[node_id].dict( + partial_workbench_data[node_id] = workbench[node_id].model_dump( include={"outputs"}, exclude_unset=True ) @@ -169,7 +169,9 @@ async def update_project_inputs(request: web.Request) -> web.Response: partial_workbench_data=jsonable_encoder(partial_workbench_data), ) - workbench = parse_obj_as(dict[NodeID, Node], updated_project["workbench"]) + workbench = TypeAdapter(dict[NodeID, Node]).validate_python( + updated_project["workbench"] + ) inputs: dict[NodeID, Any] = _ports_api.get_project_inputs(workbench) return _web_json_response_enveloped( @@ -192,7 +194,7 @@ async def update_project_inputs(request: web.Request) -> web.Response: @permission_required("project.read") @_handle_project_exceptions async def get_project_outputs(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) assert request.app # nosec @@ -239,7 +241,7 @@ class ProjectMetadataPortGet(BaseModel): @permission_required("project.read") @_handle_project_exceptions async def list_project_metadata_ports(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) assert request.app # nosec diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py index 552869a0404..05bb2f8e767 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py @@ -6,12 +6,12 @@ import logging from aiohttp import web +from common_library.errors_classes import OsparcErrorMixin from models_library.api_schemas_webserver.resource_usage import PricingUnitGet from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.resource_tracker import PricingPlanId, PricingUnitId -from pydantic import BaseModel, Extra -from pydantic.errors import PydanticErrorMixin +from pydantic import BaseModel, ConfigDict from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -29,7 +29,7 @@ _logger = logging.getLogger(__name__) -class PricingUnitError(PydanticErrorMixin, ValueError): +class PricingUnitError(OsparcErrorMixin, ValueError): ... @@ -64,7 +64,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @_handle_projects_nodes_pricing_unit_exceptions async def get_project_node_pricing_unit(request: web.Request): db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists @@ -87,7 +87,7 @@ async def get_project_node_pricing_unit(request: web.Request): webserver_pricing_unit_get = PricingUnitGet( pricing_unit_id=pricing_unit_get.pricing_unit_id, unit_name=pricing_unit_get.unit_name, - unit_extra_info=pricing_unit_get.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit_get.unit_extra_info, current_cost_per_unit=pricing_unit_get.current_cost_per_unit, default=pricing_unit_get.default, ) @@ -99,9 +99,7 @@ class _ProjectNodePricingUnitPathParams(BaseModel): node_id: NodeID pricing_plan_id: PricingPlanId pricing_unit_id: PricingUnitId - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.put( @@ -113,7 +111,7 @@ class Config: @_handle_projects_nodes_pricing_unit_exceptions async def connect_pricing_unit_to_project_node(request: web.Request): db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _ProjectNodePricingUnitPathParams, request ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index ca0725b37b9..b2f5e46381c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -93,7 +93,7 @@ class _OpenProjectQuery(BaseModel): @permission_required("project.open") @_handle_project_exceptions async def open_project(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) query_params: _OpenProjectQuery = parse_request_query_parameters_as( _OpenProjectQuery, request @@ -196,7 +196,7 @@ async def open_project(request: web.Request) -> web.Response: @permission_required("project.close") @_handle_project_exceptions async def close_project(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) try: @@ -234,7 +234,7 @@ async def close_project(request: web.Request) -> web.Response: @login_required @permission_required("project.read") async def get_project_state(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) # check that project exists and queries state @@ -245,4 +245,4 @@ async def get_project_state(request: web.Request) -> web.Response: include_state=True, ) project_state = ProjectState(**validated_project["state"]) - return envelope_json_response(project_state.dict()) + return envelope_json_response(project_state.model_dump()) diff --git a/services/web/server/src/simcore_service_webserver/projects/_tags_api.py b/services/web/server/src/simcore_service_webserver/projects/_tags_api.py index ba4be3c5fb4..c8e0937dbdb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_tags_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_tags_api.py @@ -45,7 +45,8 @@ async def add_tag( ) ) project["accessRights"] = { - gid: access.dict() for gid, access in workspace_db.access_rights.items() + gid: access.model_dump() + for gid, access in workspace_db.access_rights.items() } return project @@ -79,7 +80,8 @@ async def remove_tag( ) ) project["accessRights"] = { - gid: access.dict() for gid, access in workspace_db.access_rights.items() + gid: access.model_dump() + for gid, access in workspace_db.access_rights.items() } return project diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_api.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_api.py index 6c5a27bed9e..85bc9cf43a3 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_api.py @@ -12,7 +12,9 @@ async def get_project_wallet(app, project_id: ProjectID): db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(app) wallet_db: WalletDB | None = await db.get_project_wallet(project_uuid=project_id) - wallet: WalletGet | None = WalletGet(**wallet_db.dict()) if wallet_db else None + wallet: WalletGet | None = ( + WalletGet(**wallet_db.model_dump()) if wallet_db else None + ) return wallet diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py index 04c6fd3f218..56e7136d299 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py @@ -9,7 +9,7 @@ from models_library.api_schemas_webserver.wallets import WalletGet from models_library.projects import ProjectID from models_library.wallets import WalletID -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler from simcore_service_webserver.utils_aiohttp import envelope_json_response @@ -49,7 +49,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @permission_required("project.wallet.*") @_handle_project_wallet_exceptions async def get_project_wallet(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) # ensure the project exists @@ -69,9 +69,7 @@ async def get_project_wallet(request: web.Request): class _ProjectWalletPathParams(BaseModel): project_id: ProjectID wallet_id: WalletID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.put( @@ -82,7 +80,7 @@ class Config: @permission_required("project.wallet.*") @_handle_project_wallet_exceptions async def connect_wallet_to_project(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_ProjectWalletPathParams, request) # ensure the project exists diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py index 6b553a6d3ba..ff881b418af 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py @@ -1,11 +1,12 @@ import functools import logging +from typing import Annotated from aiohttp import web from models_library.projects import ProjectID from models_library.utils.common_validators import null_or_none_str_to_none_validator from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, Extra, validator +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -50,15 +51,9 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _ProjectWorkspacesPathParams(BaseModel): project_id: ProjectID - workspace_id: WorkspaceID | None + workspace_id: Annotated[WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator)] = Field(default=None) - class Config: - extra = Extra.forbid - - # validators - _null_or_none_str_to_none_validator = validator( - "workspace_id", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) + model_config = ConfigDict(extra="forbid") @routes.put( @@ -69,7 +64,7 @@ class Config: @permission_required("project.workspaces.*") @_handle_projects_workspaces_exceptions async def replace_project_workspace(request: web.Request): - req_ctx = RequestContext.parse_obj(request) + req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _ProjectWorkspacesPathParams, request ) diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index 6cbe059dfb7..a74375640fd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -32,7 +32,7 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.wallets import WalletDB, WalletID from models_library.workspaces import WorkspaceID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pydantic.types import PositiveInt from servicelib.aiohttp.application_keys import APP_AIOPG_ENGINE_KEY from servicelib.logging_utils import get_log_record_extra, log_context @@ -254,7 +254,9 @@ async def insert_project( """ # NOTE: tags are removed in convert_to_db_names so we keep it - project_tag_ids = parse_obj_as(list[int], project.get("tags", []).copy()) + project_tag_ids = TypeAdapter(list[int]).validate_python( + project.get("tags", []).copy() + ) insert_values = convert_to_db_names(project) insert_values.update( { @@ -757,7 +759,7 @@ async def get_project_db(self, project_uuid: ProjectID) -> ProjectDB: row = await result.fetchone() if row is None: raise ProjectNotFoundError(project_uuid=project_uuid) - return ProjectDB.from_orm(row) + return ProjectDB.model_validate(row) async def get_user_specific_project_data_db( self, project_uuid: ProjectID, private_workspace_user_id_or_none: UserID | None @@ -785,7 +787,7 @@ async def get_user_specific_project_data_db( row = await result.fetchone() if row is None: raise ProjectNotFoundError(project_uuid=project_uuid) - return UserSpecificProjectDataDB.from_orm(row) + return UserSpecificProjectDataDB.model_validate(row) async def get_pure_project_access_rights_without_workspace( self, user_id: UserID, project_uuid: ProjectID @@ -831,7 +833,7 @@ async def get_pure_project_access_rights_without_workspace( raise ProjectInvalidRightsError( user_id=user_id, project_uuid=project_uuid ) - return UserProjectAccessRightsDB.from_orm(row) + return UserProjectAccessRightsDB.model_validate(row) async def replace_project( self, @@ -926,7 +928,7 @@ async def patch_project( row = await result.fetchone() if row is None: raise ProjectNotFoundError(project_uuid=project_uuid) - return ProjectDB.from_orm(row) + return ProjectDB.model_validate(row) async def get_project_product(self, project_uuid: ProjectID) -> ProductName: async with self.engine.acquire() as conn: @@ -1368,7 +1370,7 @@ async def get_project_wallet( .where(projects_to_wallet.c.project_uuid == f"{project_uuid}") ) row = await result.fetchone() - return parse_obj_as(WalletDB, row) if row else None + return WalletDB.model_validate(row) if row else None async def connect_wallet_to_project( self, diff --git a/services/web/server/src/simcore_service_webserver/projects/lock.py b/services/web/server/src/simcore_service_webserver/projects/lock.py index 3141b7bca8d..84b24c087e7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/lock.py +++ b/services/web/server/src/simcore_service_webserver/projects/lock.py @@ -33,7 +33,7 @@ async def lock_project( PROJECT_REDIS_LOCK_KEY.format(project_uuid), timeout=PROJECT_LOCK_TIMEOUT.total_seconds(), ) - owner = Owner(user_id=user_id, **user_fullname) # type: ignore[arg-type] + owner = Owner(user_id=user_id, **user_fullname) async with common_lock_project( redis_lock, project_uuid=project_uuid, status=status, owner=owner @@ -63,5 +63,5 @@ async def get_project_locked_state( if lock_value := await redis_locks_client.get( PROJECT_REDIS_LOCK_KEY.format(project_uuid) ): - return ProjectLocked.parse_raw(lock_value) + return ProjectLocked.model_validate_json(lock_value) return None diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index 37961a9aff4..5b3e900b531 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -4,7 +4,6 @@ from aiopg.sa.result import RowProxy from models_library.api_schemas_webserver.projects import ProjectPatch -from models_library.basic_types import HttpUrlWithCustomMinLength from models_library.folders import FolderID from models_library.projects import ClassifierID, ProjectID from models_library.projects_ui import StudyUI @@ -14,7 +13,7 @@ none_to_empty_str_pre_validator, ) from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, Extra, validator +from pydantic import BaseModel, ConfigDict, HttpUrl, field_validator from simcore_postgres_database.models.projects import ProjectType, projects ProjectDict: TypeAlias = dict[str, Any] @@ -41,7 +40,7 @@ class ProjectDB(BaseModel): uuid: ProjectID name: str description: str - thumbnail: HttpUrlWithCustomMinLength | None + thumbnail: HttpUrl | None prj_owner: UserID creation_date: datetime last_change_date: datetime @@ -54,14 +53,13 @@ class ProjectDB(BaseModel): workspace_id: WorkspaceID | None trashed_at: datetime | None - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True, arbitrary_types_allowed=True) # validators - _empty_thumbnail_is_none = validator("thumbnail", allow_reuse=True, pre=True)( + _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")( empty_str_to_none_pre_validator ) - _none_description_is_empty = validator("description", allow_reuse=True, pre=True)( + _none_description_is_empty = field_validator("description", mode="before")( none_to_empty_str_pre_validator ) @@ -69,11 +67,10 @@ class Config: class UserSpecificProjectDataDB(ProjectDB): folder_id: FolderID | None - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) -assert set(ProjectDB.__fields__.keys()).issubset( # nosec +assert set(ProjectDB.model_fields.keys()).issubset( # nosec {c.name for c in projects.columns if c.name not in ["access_rights"]} ) @@ -83,9 +80,7 @@ class UserProjectAccessRightsDB(BaseModel): read: bool write: bool delete: bool - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class UserProjectAccessRightsWithWorkspace(BaseModel): @@ -94,18 +89,14 @@ class UserProjectAccessRightsWithWorkspace(BaseModel): read: bool write: bool delete: bool - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class ProjectPatchExtended(ProjectPatch): # Only used internally trashed_at: datetime | None = None - class Config: - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict(populate_by_name=True, extra="forbid") __all__: tuple[str, ...] = ( diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index d1c97856c54..d86be96058d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -63,7 +63,7 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.wallets import ZERO_CREDITS, WalletID, WalletInfo from models_library.workspaces import UserWorkspaceAccessRightsDB -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.common_headers import ( UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, @@ -216,10 +216,11 @@ async def get_project_for_user( ) ) project["accessRights"] = { - gid: access.dict() for gid, access in workspace_db.access_rights.items() + f"{gid}": access.model_dump() + for gid, access in workspace_db.access_rights.items() } - Project.parse_obj(project) # NOTE: only validates + Project.model_validate(project) # NOTE: only validates return project @@ -379,7 +380,7 @@ async def _get_default_pricing_and_hardware_info( _MACHINE_TOTAL_RAM_SAFE_MARGIN_RATIO: Final[ float ] = 0.1 # NOTE: machines always have less available RAM than advertised -_SIDECARS_OPS_SAFE_RAM_MARGIN: Final[ByteSize] = parse_obj_as(ByteSize, "1GiB") +_SIDECARS_OPS_SAFE_RAM_MARGIN: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("1GiB") _CPUS_SAFE_MARGIN: Final[float] = 1.4 _MIN_NUM_CPUS: Final[float] = 0.5 @@ -636,8 +637,8 @@ async def _start_dynamic_service( ) if user_default_wallet_preference is None: raise UserDefaultWalletNotFoundError(uid=user_id) - project_wallet_id = parse_obj_as( - WalletID, user_default_wallet_preference.value + project_wallet_id = TypeAdapter(WalletID).validate_python( + user_default_wallet_preference.value ) await connect_wallet_to_project( request.app, @@ -793,7 +794,7 @@ async def add_project_node( ProjectNodeCreate( node_id=node_uuid, required_resources=jsonable_encoder(default_resources) ), - Node.parse_obj( + Node.model_validate( { "key": service_key, "version": service_version, @@ -1419,7 +1420,7 @@ async def _get_project_lock_state( ) return ProjectLocked( value=False, - owner=Owner(user_id=list(set_user_ids)[0], **usernames[0]), # type: ignore[arg-type] + owner=Owner(user_id=next(iter(set_user_ids)), **usernames[0]), status=ProjectStatus.OPENED, ) # the project is opened in another tab or browser, or by another user, both case resolves to the project being locked, and opened @@ -1430,7 +1431,7 @@ async def _get_project_lock_state( ) return ProjectLocked( value=True, - owner=Owner(user_id=list(set_user_ids)[0], **usernames[0]), # type: ignore[arg-type] + owner=Owner(user_id=next(iter(set_user_ids)), **usernames[0]), status=ProjectStatus.OPENED, ) @@ -1484,7 +1485,7 @@ async def add_project_states_for_user( if prj_node is None: continue node_state_dict = json.loads( - node_state.json(by_alias=True, exclude_unset=True) + node_state.model_dump_json(by_alias=True, exclude_unset=True) ) prj_node.setdefault("state", {}).update(node_state_dict) prj_node_progress = node_state_dict.get("progress", None) or 0 @@ -1492,7 +1493,7 @@ async def add_project_states_for_user( project["state"] = ProjectState( locked=lock_state, state=ProjectRunningState(value=running_state) - ).dict(by_alias=True, exclude_unset=True) + ).model_dump(by_alias=True, exclude_unset=True) return project @@ -1510,8 +1511,8 @@ async def is_service_deprecated( app, user_id, service_key, service_version, product_name ) if deprecation_date := service.get("deprecated"): - deprecation_date = parse_obj_as(datetime.datetime, deprecation_date) - deprecation_date_bool: bool = datetime.datetime.utcnow() > deprecation_date + deprecation_date_bool: bool = datetime.datetime.now(datetime.UTC) > datetime.datetime.fromisoformat(deprecation_date).replace(tzinfo=datetime.UTC) + return deprecation_date_bool return False @@ -1546,8 +1547,8 @@ async def get_project_node_resources( db = ProjectDBAPI.get_from_app_context(app) try: project_node = await db.get_project_node(project_id, node_id) - node_resources = parse_obj_as( - ServiceResourcesDict, project_node.required_resources + node_resources = TypeAdapter(ServiceResourcesDict).validate_python( + project_node.required_resources ) if not node_resources: # get default resources @@ -1576,8 +1577,8 @@ async def update_project_node_resources( try: # validate the resource are applied to the same container names current_project_node = await db.get_project_node(project_id, node_id) - current_resources = parse_obj_as( - ServiceResourcesDict, current_project_node.required_resources + current_resources = TypeAdapter(ServiceResourcesDict).validate_python( + current_project_node.required_resources ) if not current_resources: # NOTE: this can happen after the migration @@ -1597,7 +1598,9 @@ async def update_project_node_resources( required_resources=jsonable_encoder(resources), check_update_allowed=True, ) - return parse_obj_as(ServiceResourcesDict, project_node.required_resources) + return TypeAdapter(ServiceResourcesDict).validate_python( + project_node.required_resources + ) except ProjectNodesNodeNotFoundError as exc: raise NodeNotFoundError( project_uuid=f"{project_id}", node_uuid=f"{node_id}" @@ -1866,4 +1869,4 @@ async def get_project_inactivity( project_settings.PROJECTS_INACTIVITY_INTERVAL.total_seconds() ), ) - return parse_obj_as(GetProjectInactivityResponse, project_inactivity) + return GetProjectInactivityResponse.model_validate(project_inactivity) diff --git a/services/web/server/src/simcore_service_webserver/projects/settings.py b/services/web/server/src/simcore_service_webserver/projects/settings.py index 6f422e5fc33..7490c87ff55 100644 --- a/services/web/server/src/simcore_service_webserver/projects/settings.py +++ b/services/web/server/src/simcore_service_webserver/projects/settings.py @@ -2,7 +2,7 @@ from aiohttp import web from common_library.pydantic_validators import validate_numeric_string_as_timedelta -from pydantic import ByteSize, Field, NonNegativeInt, parse_obj_as +from pydantic import ByteSize, Field, NonNegativeInt, TypeAdapter from settings_library.base import BaseCustomSettings from .._constants import APP_SETTINGS_KEY @@ -10,7 +10,7 @@ class ProjectsSettings(BaseCustomSettings): PROJECTS_MAX_COPY_SIZE_BYTES: ByteSize = Field( - default=parse_obj_as(ByteSize, "30Gib"), + default=TypeAdapter(ByteSize).validate_python("30Gib"), description="defines the maximum authorized project data size" " when copying a project (disable with 0)", ) diff --git a/services/web/server/src/simcore_service_webserver/projects/utils.py b/services/web/server/src/simcore_service_webserver/projects/utils.py index d54bc2b433d..18a02a5fb3c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/utils.py @@ -7,7 +7,7 @@ from models_library.projects_nodes_io import NodeIDStr from models_library.services import ServiceKey -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.decorators import safe_return from yarl import URL @@ -378,7 +378,9 @@ def default_copy_project_name(name: str) -> str: new_copy_index = 1 if current_copy_index := match.group(2): # we receive something of type "(23)" - new_copy_index = parse_obj_as(int, current_copy_index.strip("()")) + 1 + new_copy_index = ( + TypeAdapter(int).validate_python(current_copy_index.strip("()")) + 1 + ) return f"{match.group(1)}({new_copy_index})" return f"{name} (Copy)" diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_client.py b/services/web/server/src/simcore_service_webserver/resource_usage/_client.py index eb616b5d209..63d5187a7d5 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_client.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_client.py @@ -22,7 +22,7 @@ from models_library.resource_tracker import PricingPlanId, PricingUnitId from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt from servicelib.aiohttp import status from servicelib.aiohttp.client_session import get_client_session from settings_library.resource_usage_tracker import ResourceUsageTrackerSettings @@ -101,7 +101,7 @@ async def get_default_service_pricing_plan( async with session.get(url) as response: response.raise_for_status() body: dict = await response.json() - return parse_obj_as(PricingPlanGet, body) + return PricingPlanGet.model_validate(body) except ClientResponseError as e: if e.status == status.HTTP_404_NOT_FOUND: raise DefaultPricingPlanNotFoundError from e @@ -130,7 +130,7 @@ async def get_pricing_plan_unit( async with session.get(url) as response: response.raise_for_status() body: dict = await response.json() - return parse_obj_as(PricingUnitGet, body) + return PricingUnitGet.model_validate(body) async def sum_total_available_credits_in_the_wallet( @@ -151,7 +151,7 @@ async def sum_total_available_credits_in_the_wallet( async with session.post(url) as response: response.raise_for_status() body: dict = await response.json() - return WalletTotalCredits.construct(**body) + return WalletTotalCredits.model_construct(**body) async def add_credits_to_wallet( diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py index b71317b1aab..762796afc07 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_handlers.py @@ -20,7 +20,7 @@ PricingUnitWithCostUpdate, ) from models_library.users import UserID -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -72,9 +72,7 @@ class _RequestContext(BaseModel): class _GetPricingPlanPathParams(BaseModel): pricing_plan_id: PricingPlanId - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.get( @@ -85,7 +83,7 @@ class Config: @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def list_pricing_plans(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) pricing_plans_list = await admin_api.list_pricing_plans( app=request.app, @@ -116,7 +114,7 @@ async def list_pricing_plans(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def get_pricing_plan(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingPlanPathParams, request) pricing_plan_get = await admin_api.get_pricing_plan( @@ -138,7 +136,7 @@ async def get_pricing_plan(request: web.Request): PricingUnitAdminGet( pricing_unit_id=pricing_unit.pricing_unit_id, unit_name=pricing_unit.unit_name, - unit_extra_info=pricing_unit.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit.unit_extra_info, specific_info=pricing_unit.specific_info, current_cost_per_unit=pricing_unit.current_cost_per_unit, default=pricing_unit.default, @@ -159,7 +157,7 @@ async def get_pricing_plan(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def create_pricing_plan(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) body_params = await parse_request_body_as(CreatePricingPlanBodyParams, request) _data = PricingPlanCreate( @@ -187,7 +185,7 @@ async def create_pricing_plan(request: web.Request): PricingUnitAdminGet( pricing_unit_id=pricing_unit.pricing_unit_id, unit_name=pricing_unit.unit_name, - unit_extra_info=pricing_unit.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit.unit_extra_info, specific_info=pricing_unit.specific_info, current_cost_per_unit=pricing_unit.current_cost_per_unit, default=pricing_unit.default, @@ -208,7 +206,7 @@ async def create_pricing_plan(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def update_pricing_plan(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingPlanPathParams, request) body_params = await parse_request_body_as(UpdatePricingPlanBodyParams, request) @@ -237,7 +235,7 @@ async def update_pricing_plan(request: web.Request): PricingUnitAdminGet( pricing_unit_id=pricing_unit.pricing_unit_id, unit_name=pricing_unit.unit_name, - unit_extra_info=pricing_unit.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit.unit_extra_info, specific_info=pricing_unit.specific_info, current_cost_per_unit=pricing_unit.current_cost_per_unit, default=pricing_unit.default, @@ -256,9 +254,7 @@ async def update_pricing_plan(request: web.Request): class _GetPricingUnitPathParams(BaseModel): pricing_plan_id: PricingPlanId pricing_unit_id: PricingUnitId - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.get( @@ -269,7 +265,7 @@ class Config: @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def get_pricing_unit(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingUnitPathParams, request) pricing_unit_get = await admin_api.get_pricing_unit( @@ -282,7 +278,7 @@ async def get_pricing_unit(request: web.Request): webserver_pricing_unit_get = PricingUnitAdminGet( pricing_unit_id=pricing_unit_get.pricing_unit_id, unit_name=pricing_unit_get.unit_name, - unit_extra_info=pricing_unit_get.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit_get.unit_extra_info, specific_info=pricing_unit_get.specific_info, current_cost_per_unit=pricing_unit_get.current_cost_per_unit, default=pricing_unit_get.default, @@ -299,7 +295,7 @@ async def get_pricing_unit(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def create_pricing_unit(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingPlanPathParams, request) body_params = await parse_request_body_as(CreatePricingUnitBodyParams, request) @@ -321,7 +317,7 @@ async def create_pricing_unit(request: web.Request): webserver_pricing_unit_get = PricingUnitAdminGet( pricing_unit_id=pricing_unit_get.pricing_unit_id, unit_name=pricing_unit_get.unit_name, - unit_extra_info=pricing_unit_get.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit_get.unit_extra_info, specific_info=pricing_unit_get.specific_info, current_cost_per_unit=pricing_unit_get.current_cost_per_unit, default=pricing_unit_get.default, @@ -338,7 +334,7 @@ async def create_pricing_unit(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def update_pricing_unit(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingUnitPathParams, request) body_params = await parse_request_body_as(UpdatePricingUnitBodyParams, request) @@ -360,7 +356,7 @@ async def update_pricing_unit(request: web.Request): webserver_pricing_unit_get = PricingUnitAdminGet( pricing_unit_id=pricing_unit_get.pricing_unit_id, unit_name=pricing_unit_get.unit_name, - unit_extra_info=pricing_unit_get.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit_get.unit_extra_info, specific_info=pricing_unit_get.specific_info, current_cost_per_unit=pricing_unit_get.current_cost_per_unit, default=pricing_unit_get.default, @@ -380,7 +376,7 @@ async def update_pricing_unit(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def list_connected_services_to_pricing_plan(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingPlanPathParams, request) connected_services_list = await admin_api.list_connected_services_to_pricing_plan( @@ -409,7 +405,7 @@ async def list_connected_services_to_pricing_plan(request: web.Request): @permission_required("resource-usage.write") @_handle_pricing_plan_admin_exceptions async def connect_service_to_pricing_plan(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GetPricingPlanPathParams, request) body_params = await parse_request_body_as( ConnectServiceToPricingPlanBodyParams, request diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py index 86072f00e5e..41d9be1ce09 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_handlers.py @@ -4,7 +4,7 @@ from models_library.api_schemas_webserver.resource_usage import PricingUnitGet from models_library.resource_tracker import PricingPlanId, PricingUnitId from models_library.users import UserID -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler from servicelib.request_keys import RQT_USERID_KEY @@ -49,9 +49,7 @@ class _RequestContext(BaseModel): class _GetPricingPlanUnitPathParams(BaseModel): pricing_plan_id: PricingPlanId pricing_unit_id: PricingUnitId - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.get( @@ -62,7 +60,7 @@ class Config: @permission_required("resource-usage.read") @_handle_resource_usage_exceptions async def get_pricing_plan_unit(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as( _GetPricingPlanUnitPathParams, request ) @@ -77,7 +75,7 @@ async def get_pricing_plan_unit(request: web.Request): webserver_pricing_unit_get = PricingUnitGet( pricing_unit_id=pricing_unit_get.pricing_unit_id, unit_name=pricing_unit_get.unit_name, - unit_extra_info=pricing_unit_get.unit_extra_info, # type: ignore[arg-type] + unit_extra_info=pricing_unit_get.unit_extra_info, current_cost_per_unit=pricing_unit_get.current_cost_per_unit, default=pricing_unit_get.default, ) diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py index f265e45faf1..5a9dbf05849 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py @@ -24,12 +24,12 @@ from models_library.wallets import WalletID from pydantic import ( BaseModel, - Extra, + ConfigDict, Field, Json, NonNegativeInt, - parse_obj_as, - validator, + TypeAdapter, + field_validator, ) from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -74,7 +74,7 @@ class _ListServicesResourceUsagesQueryParams(BaseModel): order_by: Json[OrderBy] = Field( # pylint: disable=unsubscriptable-object default=OrderBy(field=IDStr("started_at"), direction=OrderDirection.DESC), description=ORDER_BY_DESCRIPTION, - example='{"field": "started_at", "direction": "desc"}', + examples=['{"field": "started_at", "direction": "desc"}'], ) filters: ( Json[ServiceResourceUsagesFilters] # pylint: disable=unsubscriptable-object @@ -82,10 +82,10 @@ class _ListServicesResourceUsagesQueryParams(BaseModel): ) = Field( default=None, description="Filters to process on the resource usages list, encoded as JSON. Currently supports the filtering of 'started_at' field with 'from' and 'until' parameters in ISO 8601 format. The date range specified is inclusive.", - example='{"started_at": {"from": "yyyy-mm-dd", "until": "yyyy-mm-dd"}}', + examples=['{"started_at": {"from": "yyyy-mm-dd", "until": "yyyy-mm-dd"}}'], ) - @validator("order_by", allow_reuse=True) + @field_validator("order_by") @classmethod def validate_order_by_field(cls, v): if v.field not in { @@ -113,8 +113,7 @@ def validate_order_by_field(cls, v): v.field = "osparc_credits" return v - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _ListServicesResourceUsagesQueryParamsWithPagination( @@ -129,18 +128,14 @@ class _ListServicesResourceUsagesQueryParamsWithPagination( offset: NonNegativeInt = Field( default=0, description="index to the first item to return (pagination)" ) - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _ListServicesAggregatedUsagesQueryParams(PageQueryParameters): aggregated_by: ServicesAggregatedUsagesType time_period: ServicesAggregatedUsagesTimePeriod wallet_id: WalletID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") # @@ -155,7 +150,7 @@ class Config: @permission_required("resource-usage.read") @_handle_resource_usage_exceptions async def list_resource_usage_services(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) query_params: _ListServicesResourceUsagesQueryParamsWithPagination = ( parse_request_query_parameters_as( _ListServicesResourceUsagesQueryParamsWithPagination, request @@ -169,11 +164,11 @@ async def list_resource_usage_services(request: web.Request): wallet_id=query_params.wallet_id, offset=query_params.offset, limit=query_params.limit, - order_by=parse_obj_as(OrderBy, query_params.order_by), - filters=parse_obj_as(ServiceResourceUsagesFilters | None, query_params.filters), # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 + order_by=OrderBy.model_validate(query_params.order_by), + filters=TypeAdapter(ServiceResourceUsagesFilters | None).validate_python(query_params.filters), ) - page = Page[dict[str, Any]].parse_obj( + page = Page[dict[str, Any]].model_validate( paginate_data( chunk=services.items, request_url=request.url, @@ -183,7 +178,7 @@ async def list_resource_usage_services(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -196,7 +191,7 @@ async def list_resource_usage_services(request: web.Request): @permission_required("resource-usage.read") @_handle_resource_usage_exceptions async def list_osparc_credits_aggregated_usages(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) query_params: _ListServicesAggregatedUsagesQueryParams = ( parse_request_query_parameters_as( _ListServicesAggregatedUsagesQueryParams, request @@ -216,9 +211,9 @@ async def list_osparc_credits_aggregated_usages(request: web.Request): ) ) - page = Page[dict[str, Any]].parse_obj( + page = Page[dict[str, Any]].model_validate( paginate_data( - chunk=aggregated_services.items, + chunk=[item.model_dump() for item in aggregated_services.items], request_url=request.url, total=aggregated_services.total, limit=query_params.limit, @@ -226,7 +221,7 @@ async def list_osparc_credits_aggregated_usages(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -236,7 +231,7 @@ async def list_osparc_credits_aggregated_usages(request: web.Request): @permission_required("resource-usage.read") @_handle_resource_usage_exceptions async def export_resource_usage_services(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) query_params: _ListServicesResourceUsagesQueryParams = ( parse_request_query_parameters_as( _ListServicesResourceUsagesQueryParams, request @@ -247,7 +242,9 @@ async def export_resource_usage_services(request: web.Request): user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=query_params.wallet_id, - order_by=parse_obj_as(OrderBy | None, query_params.order_by), # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 - filters=parse_obj_as(ServiceResourceUsagesFilters | None, query_params.filters), # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 + order_by=TypeAdapter(OrderBy | None).validate_python(query_params.order_by), + filters=TypeAdapter(ServiceResourceUsagesFilters | None).validate_python( + query_params.filters + ), ) raise web.HTTPFound(location=f"{download_url}") diff --git a/services/web/server/src/simcore_service_webserver/rest/_handlers.py b/services/web/server/src/simcore_service_webserver/rest/_handlers.py index 085b69cb0f6..b874d441db0 100644 --- a/services/web/server/src/simcore_service_webserver/rest/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/rest/_handlers.py @@ -8,7 +8,7 @@ from typing import Any from aiohttp import web -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel from servicelib.aiohttp import status from .._constants import APP_PUBLIC_CONFIG_PER_PRODUCT, APP_SETTINGS_KEY @@ -104,7 +104,7 @@ async def get_scheduled_maintenance(request: web.Request): if maintenance_data := await redis_client.get(hash_key): assert ( # nosec - parse_obj_as(_ScheduledMaintenanceGet, maintenance_data) is not None + _ScheduledMaintenanceGet.model_validate(maintenance_data) is not None ) return envelope_json_response(maintenance_data) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_resolver.py b/services/web/server/src/simcore_service_webserver/scicrunch/_resolver.py index efef7f77668..2b02576e980 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_resolver.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_resolver.py @@ -41,7 +41,7 @@ class HitSource(BaseModel): def flatten_dict(self) -> dict[str, Any]: """Used as an output""" - return {**self.item.dict(), **self.rrid.dict()} + return {**self.item.model_dump(), **self.rrid.model_dump()} class HitDetail(BaseModel): @@ -93,7 +93,7 @@ async def resolve_rrid( body = await resp.json() # process and simplify response - resolved = ResolverResponseBody.parse_obj(body) + resolved = ResolverResponseBody.model_validate(body) if resolved.hits.total == 0: return [] @@ -113,7 +113,7 @@ async def resolve_rrid( items = [] for hit in resolved.hits.hits: try: - items.append(ResolvedItem.parse_obj(hit.source.flatten_dict())) + items.append(ResolvedItem.model_validate(hit.source.flatten_dict())) except ValidationError as err: logger.warning("Skipping unexpected response %s: %s", url, err) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py b/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py index 70e4963fc68..fd2f3d243a1 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py @@ -18,7 +18,7 @@ from typing import Any from aiohttp import ClientSession -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, RootModel from yarl import URL from .models import ResourceHit @@ -49,7 +49,7 @@ class ResourceView(BaseModel): @classmethod def from_response_payload(cls, payload: dict): - assert payload["success"] == True # nosec + assert payload["success"] is True # nosec return cls(**payload["data"]) @property @@ -72,8 +72,8 @@ def get_resource_url(self): return URL(str(self._get_field("Resource URL"))) -class ListOfResourceHits(BaseModel): - __root__: list[ResourceHit] +class ListOfResourceHits(RootModel[list[ResourceHit]]): + ... # REQUESTS @@ -120,4 +120,4 @@ async def autocomplete_by_name( ) as resp: body = await resp.json() assert body.get("success") # nosec - return ListOfResourceHits.parse_obj(body.get("data", [])) + return ListOfResourceHits.model_validate(body.get("data", [])) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/db.py b/services/web/server/src/simcore_service_webserver/scicrunch/db.py index 476e320f73d..57e19bbed35 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/db.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/db.py @@ -39,7 +39,7 @@ async def list_resources(self) -> list[ResearchResource]: ) res: ResultProxy = await conn.execute(stmt) rows: list[RowProxy] = await res.fetchall() - return [ResearchResource.from_orm(row) for row in rows] if rows else [] + return [ResearchResource.model_validate(row) for row in rows] if rows else [] async def get(self, rrid: str) -> ResearchResourceAtdB | None: async with self._engine.acquire() as conn: @@ -53,12 +53,12 @@ async def get(self, rrid: str) -> ResearchResourceAtdB | None: async def get_resource(self, rrid: str) -> ResearchResource | None: resource: ResearchResourceAtdB | None = await self.get(rrid) if resource: - return ResearchResource(**resource.dict()) + return ResearchResource(**resource.model_dump()) return resource async def upsert(self, resource: ResearchResource): async with self._engine.acquire() as conn: - values = resource.dict(exclude_unset=True) + values = resource.model_dump(exclude_unset=True) stmt = ( sa_pg_insert(scicrunch_resources) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/models.py b/services/web/server/src/simcore_service_webserver/scicrunch/models.py index 743f4bd8211..2140f88ea33 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/models.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/models.py @@ -6,7 +6,7 @@ import re from datetime import datetime -from pydantic import BaseModel, Field, validator +from pydantic import field_validator, ConfigDict, BaseModel, Field logger = logging.getLogger(__name__) @@ -58,19 +58,16 @@ class ResearchResource(BaseModel): rrid: str = Field( ..., description="Unique identifier used as classifier, i.e. to tag studies and services", - regex=STRICT_RRID_PATTERN, + pattern=STRICT_RRID_PATTERN, ) name: str description: str - @validator("rrid", pre=True) + @field_validator("rrid", mode="before") @classmethod def format_rrid(cls, v): return normalize_rrid_tags(v, with_prefix=True) - - class Config: - orm_mode = True - anystr_strip_whitespace = True + model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) # postgres_database.scicrunch_resources ORM -------------------- diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py b/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py index bcaf413b4db..ec8f43283b3 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py @@ -8,7 +8,7 @@ import logging from aiohttp import ClientSession, client_exceptions, web -from pydantic import HttpUrl, ValidationError, parse_obj_as +from pydantic import HttpUrl, TypeAdapter, ValidationError from servicelib.aiohttp.client_session import get_client_session from yarl import URL @@ -90,8 +90,8 @@ def get_search_web_url(self, rrid: str) -> str: def get_resolver_web_url(self, rrid: str) -> HttpUrl: # example https://scicrunch.org/resolver/RRID:AB_90755 - output: HttpUrl = parse_obj_as( - HttpUrl, f"{self.settings.SCICRUNCH_RESOLVER_BASE_URL}/{rrid}" + output: HttpUrl = TypeAdapter(HttpUrl).validate_python( + f"{self.settings.SCICRUNCH_RESOLVER_BASE_URL}/{rrid}" ) return output @@ -171,4 +171,4 @@ async def search_resource(self, name_as: str) -> list[ResourceHit]: # Might be slow and timeout! # Might be good to know that scicrunch.org is not reachable and cannot perform search now? hits = await autocomplete_by_name(name_as, self.client, self.settings) - return hits.__root__ + return hits.root diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/settings.py b/services/web/server/src/simcore_service_webserver/scicrunch/settings.py index ecc027374c0..0bf88e69b05 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/settings.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/settings.py @@ -1,5 +1,5 @@ from aiohttp import web -from pydantic import Field, HttpUrl, SecretStr, parse_obj_as +from pydantic import Field, HttpUrl, SecretStr, TypeAdapter from settings_library.base import BaseCustomSettings from .._constants import APP_SETTINGS_KEY @@ -11,7 +11,7 @@ class SciCrunchSettings(BaseCustomSettings): SCICRUNCH_API_BASE_URL: HttpUrl = Field( - default=parse_obj_as(HttpUrl, f"{SCICRUNCH_DEFAULT_URL}/api/1"), + default=TypeAdapter(HttpUrl).validate_python(f"{SCICRUNCH_DEFAULT_URL}/api/1"), description="Base url to scicrunch API's entrypoint", ) @@ -20,7 +20,9 @@ class SciCrunchSettings(BaseCustomSettings): SCICRUNCH_API_KEY: SecretStr SCICRUNCH_RESOLVER_BASE_URL: HttpUrl = Field( - default=parse_obj_as(HttpUrl, f"{SCICRUNCH_DEFAULT_URL}/resolver"), + default=TypeAdapter(HttpUrl).validate_python( + f"{SCICRUNCH_DEFAULT_URL}/resolver" + ), description="Base url to scicrunch resolver entrypoint", ) diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_db.py b/services/web/server/src/simcore_service_webserver/security/_authz_db.py index dbb04f7943c..300130b6f82 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_db.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_db.py @@ -7,7 +7,7 @@ from models_library.basic_types import IdInt from models_library.products import ProductName from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.models.groups import user_to_groups from simcore_postgres_database.models.products import products from simcore_postgres_database.models.users import UserRole @@ -35,8 +35,12 @@ async def get_active_user_or_none(engine: Engine, email: str) -> AuthInfoDict | ) ) row = await result.fetchone() - assert row is None or parse_obj_as(IdInt, row.id) is not None # nosec - assert row is None or parse_obj_as(UserRole, row.role) is not None # nosec + assert ( + row is None or TypeAdapter(IdInt).validate_python(row.id) is not None # nosec + ) + assert ( + row is None or TypeAdapter(UserRole).validate_python(row.role) is not None # nosec + ) return AuthInfoDict(id=row.id, role=row.role) if row else None diff --git a/services/web/server/src/simcore_service_webserver/session/access_policies.py b/services/web/server/src/simcore_service_webserver/session/access_policies.py index 05736703563..8fd72f71caf 100644 --- a/services/web/server/src/simcore_service_webserver/session/access_policies.py +++ b/services/web/server/src/simcore_service_webserver/session/access_policies.py @@ -7,7 +7,7 @@ from aiohttp import web from aiohttp_session import Session -from pydantic import PositiveInt, validate_arguments +from pydantic import PositiveInt, validate_call from servicelib.aiohttp import status from servicelib.aiohttp.typing_extension import Handler @@ -64,7 +64,7 @@ def _access_tokens_cleanup_ctx(session: Session) -> Iterator[dict[str, _AccessTo session[_SESSION_GRANTED_ACCESS_TOKENS_KEY] = pruned_access_tokens -@validate_arguments +@validate_call def on_success_grant_session_access_to( name: str, *, diff --git a/services/web/server/src/simcore_service_webserver/session/settings.py b/services/web/server/src/simcore_service_webserver/session/settings.py index b5f3c333fa8..f6fec9a878e 100644 --- a/services/web/server/src/simcore_service_webserver/session/settings.py +++ b/services/web/server/src/simcore_service_webserver/session/settings.py @@ -1,10 +1,10 @@ from typing import Final from aiohttp import web -from pydantic import PositiveInt -from pydantic.class_validators import validator +from pydantic import AliasChoices, PositiveInt, field_validator from pydantic.fields import Field from pydantic.types import SecretStr +from pydantic_settings import SettingsConfigDict from settings_library.base import BaseCustomSettings from settings_library.utils_session import MixinSessionSettings @@ -22,7 +22,9 @@ class SessionSettings(BaseCustomSettings, MixinSessionSettings): description="Secret key to encrypt cookies. " 'TIP: python3 -c "from cryptography.fernet import *; print(Fernet.generate_key())"', min_length=44, - env=["SESSION_SECRET_KEY", "WEBSERVER_SESSION_SECRET_KEY"], + validation_alias=AliasChoices( + "SESSION_SECRET_KEY", "WEBSERVER_SESSION_SECRET_KEY" + ), ) SESSION_ACCESS_TOKENS_EXPIRATION_INTERVAL_SECS: int = Field( @@ -52,13 +54,17 @@ class SessionSettings(BaseCustomSettings, MixinSessionSettings): default=True, description="This prevents JavaScript from accessing the session cookie", ) + + model_config = SettingsConfigDict( + extra="allow" + ) - @validator("SESSION_SECRET_KEY") + @field_validator("SESSION_SECRET_KEY") @classmethod def check_valid_fernet_key(cls, v): return cls.do_check_valid_fernet_key(v) - @validator("SESSION_COOKIE_SAMESITE") + @field_validator("SESSION_COOKIE_SAMESITE") @classmethod def check_valid_samesite_attribute(cls, v): # NOTE: Replacement to `Literal["Strict", "Lax"] | None` due to bug in settings_library/base.py:93: in prepare_field diff --git a/services/web/server/src/simcore_service_webserver/socketio/models.py b/services/web/server/src/simcore_service_webserver/socketio/models.py index 06e5b9014cb..37bb942298b 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/models.py +++ b/services/web/server/src/simcore_service_webserver/socketio/models.py @@ -12,23 +12,21 @@ from models_library.socketio import SocketMessageDict from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import BaseModel, Field +from pydantic import ConfigDict, BaseModel, Field class WebSocketMessageBase(BaseModel): - event_type: str = Field(..., const=True) + event_type: str = Field(frozen=True) @classmethod def get_event_type(cls) -> str: - _event_type: str = cls.__fields__["event_type"].default + _event_type: str = cls.model_fields["event_type"].default return _event_type @abstractmethod def to_socket_dict(self) -> SocketMessageDict: ... - - class Config: - frozen = True + model_config = ConfigDict(frozen=True) class _WebSocketProjectMixin(BaseModel): @@ -87,7 +85,7 @@ class WebSocketNodeProgress( def from_rabbit_message( cls, message: ProgressRabbitMessageNode ) -> "WebSocketNodeProgress": - return cls.construct( + return cls.model_construct( user_id=message.user_id, project_id=message.project_id, node_id=message.node_id, diff --git a/services/web/server/src/simcore_service_webserver/statics/_events.py b/services/web/server/src/simcore_service_webserver/statics/_events.py index 4348fa5ef3d..b34f7e8948a 100644 --- a/services/web/server/src/simcore_service_webserver/statics/_events.py +++ b/services/web/server/src/simcore_service_webserver/statics/_events.py @@ -64,7 +64,7 @@ async def create_cached_indexes(app: web.Application) -> None: session: ClientSession = get_client_session(app) for frontend_name in FRONTEND_APPS_AVAILABLE: - url = URL(settings.STATIC_WEBSERVER_URL) / frontend_name + url = URL(f"{settings.STATIC_WEBSERVER_URL}") / frontend_name _logger.info("Fetching index from %s", url) try: body = "" diff --git a/services/web/server/src/simcore_service_webserver/statics/settings.py b/services/web/server/src/simcore_service_webserver/statics/settings.py index 275def8154b..7a9c13f4e33 100644 --- a/services/web/server/src/simcore_service_webserver/statics/settings.py +++ b/services/web/server/src/simcore_service_webserver/statics/settings.py @@ -7,7 +7,7 @@ import pycountry from aiohttp import web from models_library.utils.change_case import snake_to_camel -from pydantic import AnyHttpUrl, Field, parse_obj_as +from pydantic import AliasChoices, AnyHttpUrl, Field, TypeAdapter from settings_library.base import BaseCustomSettings from .._constants import APP_SETTINGS_KEY @@ -93,7 +93,7 @@ class FrontEndAppSettings(BaseCustomSettings): # NOTE: for the moment, None but left here for future use def to_statics(self) -> dict[str, Any]: - data = self.dict( + data = self.model_dump( exclude_none=True, by_alias=True, ) @@ -121,12 +121,12 @@ def to_statics(self) -> dict[str, Any]: class StaticWebserverModuleSettings(BaseCustomSettings): STATIC_WEBSERVER_URL: AnyHttpUrl = Field( - default=parse_obj_as(AnyHttpUrl, "http://static-webserver:8000"), + default=TypeAdapter(AnyHttpUrl).validate_python("http://static-webserver:8000"), description="url fort static content", - env=[ + validation_alias=AliasChoices( "STATIC_WEBSERVER_URL", "WEBSERVER_STATIC_MODULE_STATIC_WEB_SERVER_URL", # legacy - ], + ), ) diff --git a/services/web/server/src/simcore_service_webserver/storage/_handlers.py b/services/web/server/src/simcore_service_webserver/storage/_handlers.py index f5acb0171b1..83372296dd2 100644 --- a/services/web/server/src/simcore_service_webserver/storage/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/storage/_handlers.py @@ -14,7 +14,7 @@ ) from models_library.projects_nodes_io import LocationID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyUrl, BaseModel, ByteSize, parse_obj_as +from pydantic import AnyUrl, BaseModel, ByteSize, TypeAdapter from servicelib.aiohttp.client_session import get_client_session from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -74,7 +74,7 @@ def _from_storage_url(request: web.Request, storage_url: AnyUrl) -> AnyUrl: f"/v0/storage{storage_url.path.removeprefix(prefix)}", encoded=True ).with_scheme(request.headers.get(X_FORWARDED_PROTO, request.url.scheme)) - webserver_url: AnyUrl = parse_obj_as(AnyUrl, f"{converted_url}") + webserver_url: AnyUrl = TypeAdapter(AnyUrl).validate_python(f"{converted_url}") return webserver_url @@ -229,7 +229,7 @@ class _PathParams(BaseModel): parse_request_path_parameters_as(_PathParams, request) class _QueryParams(BaseModel): - file_size: ByteSize | None + file_size: ByteSize | None = None link_type: LinkType = LinkType.PRESIGNED is_directory: bool = False @@ -237,7 +237,7 @@ class _QueryParams(BaseModel): payload, status = await _forward_request_to_storage(request, "PUT", body=None) data, _ = unwrap_envelope(payload) - file_upload_schema = FileUploadSchema.parse_obj(data) + file_upload_schema = FileUploadSchema.model_validate(data) file_upload_schema.links.complete_upload = _from_storage_url( request, file_upload_schema.links.complete_upload ) @@ -262,10 +262,10 @@ class _PathParams(BaseModel): body_item = await parse_request_body_as(FileUploadCompletionBody, request) payload, status = await _forward_request_to_storage( - request, "POST", body=body_item.dict() + request, "POST", body=body_item.model_dump() ) data, _ = unwrap_envelope(payload) - file_upload_complete = FileUploadCompleteResponse.parse_obj(data) + file_upload_complete = FileUploadCompleteResponse.model_validate(data) file_upload_complete.links.state = _from_storage_url( request, file_upload_complete.links.state ) diff --git a/services/web/server/src/simcore_service_webserver/storage/api.py b/services/web/server/src/simcore_service_webserver/storage/api.py index 2ddf66d8907..8e1ad334beb 100644 --- a/services/web/server/src/simcore_service_webserver/storage/api.py +++ b/services/web/server/src/simcore_service_webserver/storage/api.py @@ -20,7 +20,7 @@ from models_library.projects_nodes_io import LocationID, NodeID, SimCoreFileLink from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, HttpUrl, parse_obj_as +from pydantic import ByteSize, HttpUrl, TypeAdapter from servicelib.aiohttp.client_session import get_client_session from servicelib.aiohttp.long_running_tasks.client import ( LRTask, @@ -56,7 +56,7 @@ async def get_storage_locations( locations_url = (api_endpoint / "locations").with_query(user_id=user_id) async with session.get(f"{locations_url}") as response: response.raise_for_status() - locations_enveloped = Envelope[FileLocationArray].parse_obj( + locations_enveloped = Envelope[FileLocationArray].model_validate( await response.json() ) assert locations_enveloped.data # nosec @@ -89,15 +89,15 @@ async def get_project_total_size_simcore_s3( ).with_query(user_id=user_id, project_id=f"{project_uuid}") async with session.get(f"{files_metadata_url}") as response: response.raise_for_status() - list_of_files_enveloped = Envelope[list[FileMetaDataGet]].parse_obj( - await response.json() - ) + list_of_files_enveloped = Envelope[ + list[FileMetaDataGet] + ].model_validate(await response.json()) assert list_of_files_enveloped.data is not None # nosec project_size_bytes += sum( file_metadata.file_size for file_metadata in list_of_files_enveloped.data ) - return parse_obj_as(ByteSize, project_size_bytes) + return TypeAdapter(ByteSize).validate_python(project_size_bytes) async def copy_data_folders_from_project( @@ -204,10 +204,10 @@ async def get_download_link( async with session.get(f"{url}") as response: response.raise_for_status() download: PresignedLink | None = ( - Envelope[PresignedLink].parse_obj(await response.json()).data + Envelope[PresignedLink].model_validate(await response.json()).data ) assert download is not None # nosec - link: HttpUrl = parse_obj_as(HttpUrl, download.link) + link: HttpUrl = TypeAdapter(HttpUrl).validate_python(download.link) return link @@ -227,7 +227,7 @@ async def get_files_in_node_folder( async with session.get(f"{files_metadata_url}") as response: response.raise_for_status() - list_of_files_enveloped = Envelope[list[FileMetaDataGet]].parse_obj( + list_of_files_enveloped = Envelope[list[FileMetaDataGet]].model_validate( await response.json() ) assert list_of_files_enveloped.data is not None # nosec diff --git a/services/web/server/src/simcore_service_webserver/storage/schemas.py b/services/web/server/src/simcore_service_webserver/storage/schemas.py index 4c47c99a8ff..26381218c0e 100644 --- a/services/web/server/src/simcore_service_webserver/storage/schemas.py +++ b/services/web/server/src/simcore_service_webserver/storage/schemas.py @@ -1,8 +1,8 @@ from enum import Enum -from typing import Any, ClassVar, TypeAlias +from typing import Any, TypeAlias from models_library.api_schemas_storage import TableSynchronisation -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, RootModel # NOTE: storage generates URLs that contain double encoded # slashes, and when applying validation via `StorageFileID` @@ -14,18 +14,18 @@ class FileLocation(BaseModel): name: str | None = None id: float | None = None - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "simcore.s3", "id": 0, }, } + ) -class FileLocationArray(BaseModel): - __root__: list[FileLocation] +class FileLocationArray(RootModel[list[FileLocation]]): + ... class Links(BaseModel): @@ -60,18 +60,18 @@ class FileUploadCompleteFuture(BaseModel): class DatasetMetaData(BaseModel): dataset_id: str | None = None display_name: str | None = None - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "dataset_id": "N:id-aaaa", "display_name": "simcore-testing", }, } + ) -class DatasetMetaDataArray(BaseModel): - __root__: list[DatasetMetaData] +class DatasetMetaDataArray(RootModel[list[DatasetMetaData]]): + ... class FileLocationEnveloped(BaseModel): @@ -122,8 +122,8 @@ class FileMetaData(BaseModel): entity_tag: str | None = None is_directory: bool | None = None - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "file_uuid": "simcore-testing/105/1000/3", "location_id": "0", @@ -138,10 +138,11 @@ class Config: "is_directory": False, } } + ) -class FileMetaDataArray(BaseModel): - __root__: list[FileMetaData] +class FileMetaDataArray(RootModel[list[FileMetaData]]): + ... class FileMetaEnvelope(BaseModel): @@ -152,8 +153,7 @@ class FileMetaEnvelope(BaseModel): class PresignedLink(BaseModel): link: str | None = None - class Config: - schema_extra: ClassVar[dict[str, Any]] = {"example": {"link": "example_link"}} + model_config = ConfigDict(json_schema_extra={"example": {"link": "example_link"}}) class PresignedLinkEnveloped(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/storage/settings.py b/services/web/server/src/simcore_service_webserver/storage/settings.py index e49e652699d..04ac00f61c3 100644 --- a/services/web/server/src/simcore_service_webserver/storage/settings.py +++ b/services/web/server/src/simcore_service_webserver/storage/settings.py @@ -2,7 +2,6 @@ from aiohttp import web from models_library.basic_types import PortInt, VersionTag -from pydantic import parse_obj_as from settings_library.base import BaseCustomSettings from settings_library.utils_service import DEFAULT_AIOHTTP_PORT, MixinServiceSettings from yarl import URL @@ -12,8 +11,8 @@ class StorageSettings(BaseCustomSettings, MixinServiceSettings): STORAGE_HOST: str = "storage" - STORAGE_PORT: PortInt = parse_obj_as(PortInt, DEFAULT_AIOHTTP_PORT) - STORAGE_VTAG: VersionTag = parse_obj_as(VersionTag, "v0") + STORAGE_PORT: PortInt = DEFAULT_AIOHTTP_PORT + STORAGE_VTAG: VersionTag = "v0" @cached_property def base_url(self) -> URL: diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py index 3df62ebd379..e9250b18a01 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py @@ -9,7 +9,7 @@ from aiopg.sa.engine import Engine from models_library.groups import EVERYONE_GROUP_ID from models_library.services import ServiceKey, ServiceVersion -from pydantic import HttpUrl, PositiveInt, ValidationError, parse_obj_as +from pydantic import HttpUrl, PositiveInt, TypeAdapter, ValidationError from servicelib.logging_utils import log_decorator from simcore_postgres_database.models.services import ( services_access_rights, @@ -97,7 +97,7 @@ async def iter_latest_product_services( ) & (services_meta_data.c.deprecated.is_(None)) & (services_access_rights.c.gid == EVERYONE_GROUP_ID) - & (services_access_rights.c.execute_access == True) + & (services_access_rights.c.execute_access == sa.true()) & (services_access_rights.c.product_name == product_name) ) ) @@ -114,7 +114,7 @@ async def iter_latest_product_services( version=row.version, title=row.name, description=row.description, - thumbnail=row.thumbnail or settings.STUDIES_DEFAULT_SERVICE_THUMBNAIL, + thumbnail=row.thumbnail or f"{settings.STUDIES_DEFAULT_SERVICE_THUMBNAIL}", file_extensions=service_filetypes.get(row.key, []), ) @@ -161,7 +161,7 @@ async def validate_requested_service( sa.select(services_consume_filetypes.c.is_guest_allowed) .where( (services_consume_filetypes.c.service_key == service_key) - & (services_consume_filetypes.c.is_guest_allowed == True) + & (services_consume_filetypes.c.is_guest_allowed == sa.true()) ) .limit(1) ) @@ -171,7 +171,7 @@ async def validate_requested_service( thumbnail_or_none = None if row.thumbnail is not None: with suppress(ValidationError): - thumbnail_or_none = parse_obj_as(HttpUrl, row.thumbnail) + thumbnail_or_none = TypeAdapter(HttpUrl).validate_python(row.thumbnail) return ValidService( key=service_key, diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py index dcafdf528de..fe76e1a2855 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py @@ -7,7 +7,7 @@ from aiohttp import web from models_library.services import ServiceVersion from models_library.utils.pydantic_tools_extension import parse_obj_or_none -from pydantic import ByteSize, ValidationError, parse_obj_as +from pydantic import ByteSize, TypeAdapter, ValidationError from servicelib.logging_utils import log_decorator from simcore_postgres_database.models.services_consume_filetypes import ( services_consume_filetypes, @@ -138,7 +138,9 @@ def _version(column_or_value): row = await result.first() if row: view = ViewerInfo.create_from_db(row) - view.version = parse_obj_as(ServiceVersion, service_version) + view.version = TypeAdapter(ServiceVersion).validate_python( + service_version + ) return view raise IncompatibleService(file_type=file_type) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py index d68cc284190..5cff412f415 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py @@ -6,22 +6,22 @@ class StudyDispatcherError(WebServerBaseError, ValueError): class IncompatibleService(StudyDispatcherError): - code = "studies_dispatcher.incompatible_service" + code = "studies_dispatcher.incompatible_service" # type: ignore msg_template = "None of the registered services can handle '{file_type}'" class FileToLarge(StudyDispatcherError): - code = "studies_dispatcher.file_to_large" + code = "studies_dispatcher.file_to_large" # type: ignore msg_template = "File size {file_size_in_mb} MB is over allowed limit" class ServiceNotFound(StudyDispatcherError): - code = "studies_dispatcher.service_not_found" + code = "studies_dispatcher.service_not_found" # type: ignore msg_template = "Service {service_key}:{service_version} not found" class InvalidRedirectionParams(StudyDispatcherError): - code = "studies_dispatcher.invalid_redirection_params" + code = "studies_dispatcher.invalid_redirection_params" # type: ignore msg_template = ( "The link you provided is invalid because it doesn't contain any information related to data or a service." " Please check the link and make sure it is correct." diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py index 30aa1387269..c697b409c0f 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py @@ -1,6 +1,6 @@ from aiopg.sa.result import RowProxy from models_library.services import ServiceKey, ServiceVersion -from pydantic import BaseModel, Field, HttpUrl, PositiveInt, parse_obj_as +from pydantic import BaseModel, Field, HttpUrl, PositiveInt, TypeAdapter class ServiceInfo(BaseModel): @@ -10,7 +10,9 @@ class ServiceInfo(BaseModel): label: str = Field(..., description="Display name") thumbnail: HttpUrl = Field( - default=parse_obj_as(HttpUrl, "https://via.placeholder.com/170x120.png") + default=TypeAdapter(HttpUrl).validate_python( + "https://via.placeholder.com/170x120.png" + ) ) is_guest_allowed: bool = True diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py index e4b71213ee6..53f61713a43 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py @@ -13,12 +13,12 @@ from aiohttp import web from models_library.projects import DateTimeStr, Project, ProjectID -from models_library.projects_access import AccessRights +from models_library.projects_access import AccessRights, GroupIDStr from models_library.projects_nodes import Node from models_library.projects_nodes_io import DownloadLink, NodeID, PortLink from models_library.projects_ui import StudyUI from models_library.services import ServiceKey, ServiceVersion -from pydantic import AnyUrl, HttpUrl, parse_obj_as +from pydantic import AnyUrl, HttpUrl, TypeAdapter from servicelib.logging_utils import log_decorator from ..projects.db import ProjectDBAPI @@ -32,10 +32,12 @@ _logger = logging.getLogger(__name__) -_FILE_PICKER_KEY: ServiceKey = parse_obj_as( - ServiceKey, "simcore/services/frontend/file-picker" +_FILE_PICKER_KEY: ServiceKey = TypeAdapter(ServiceKey).validate_python( + "simcore/services/frontend/file-picker" +) +_FILE_PICKER_VERSION: ServiceVersion = TypeAdapter(ServiceVersion).validate_python( + "1.0.0" ) -_FILE_PICKER_VERSION: ServiceVersion = parse_obj_as(ServiceVersion, "1.0.0") def _generate_nodeids(project_id: ProjectID) -> tuple[NodeID, NodeID]: @@ -55,12 +57,12 @@ def _create_file_picker(download_link: str, output_label: str | None): # also to name the file in case it is downloaded data = {} - data["downloadLink"] = url = parse_obj_as(AnyUrl, download_link) + data["downloadLink"] = url = TypeAdapter(AnyUrl).validate_python(download_link) if output_label: - data["label"] = Path(output_label).name + data["label"] = Path(output_label).name # type: ignore[assignment] elif url.path: - data["label"] = Path(url.path).name - output = DownloadLink.parse_obj(data) + data["label"] = Path(url.path).name # type: ignore[assignment] + output = DownloadLink.model_validate(data) output_id = "outFile" node = Node( @@ -69,7 +71,7 @@ def _create_file_picker(download_link: str, output_label: str | None): label="File Picker", inputs={}, inputNodes=[], - outputs={output_id: output}, # type: ignore[dict-item] + outputs={output_id: output}, progress=0, ) return node, output_id @@ -94,12 +96,12 @@ def _create_project( uuid=project_id, name=name, description=description, - thumbnail=thumbnail, # type: ignore[arg-type] + thumbnail=thumbnail, prjOwner=owner.email, - accessRights={owner.primary_gid: access_rights}, # type: ignore[dict-item] + accessRights={GroupIDStr(owner.primary_gid): access_rights}, creationDate=DateTimeStr(now_str()), lastChangeDate=DateTimeStr(now_str()), - workbench=workbench, # type: ignore[arg-type] + workbench=workbench, ui=StudyUI(workbench=workbench_ui), # type: ignore[arg-type] ) @@ -145,7 +147,7 @@ def _create_project_with_filepicker_and_service( viewer_info: ViewerInfo, ) -> Project: file_picker, file_picker_output_id = _create_file_picker( - download_link, output_label=None + f"{download_link}", output_label=None ) viewer_service = Node( @@ -153,7 +155,7 @@ def _create_project_with_filepicker_and_service( version=viewer_info.version, label=viewer_info.label, inputs={ - viewer_info.input_port_key: PortLink( # type: ignore[dict-item] + viewer_info.input_port_key: PortLink( nodeUuid=file_picker_id, output=file_picker_output_id, ) @@ -194,7 +196,9 @@ async def _add_new_project( db: ProjectDBAPI = app[APP_PROJECT_DBAPI] # validated project is transform in dict via json to use only primitive types - project_in: dict = json.loads(project.json(exclude_none=True, by_alias=True)) + project_in: dict = json.loads( + project.model_dump_json(exclude_none=True, by_alias=True) + ) # update metadata (uuid, timestamps, ownership) and save _project_db: dict = await db.insert_project( @@ -344,7 +348,7 @@ async def get_or_create_project_with_file( ): # nodes file_picker, _ = _create_file_picker( - file_params.download_link, output_label=file_params.file_name + f"{file_params.download_link}", output_label=file_params.file_name ) # project diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py index 055f0f78fcf..9308f4e3c81 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py @@ -4,7 +4,7 @@ import sqlalchemy as sa from aiohttp import web from models_library.projects import ProjectID, ProjectIDStr -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects import ProjectType, projects @@ -58,8 +58,7 @@ def create_permalink_for_study( # create url_for = create_url_for_function(request) - permalink = parse_obj_as( - HttpUrl, + permalink = TypeAdapter(HttpUrl).validate_python( url_for(route_name="get_redirection_to_study_page", id=f"{project_uuid}"), ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py index 237aed0f7fd..647a523f23c 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py @@ -12,7 +12,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services import ServiceKey, ServiceVersion -from pydantic import BaseModel, Extra, ValidationError, validator +from pydantic import field_validator, ConfigDict, BaseModel, ValidationError from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -153,15 +153,13 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class ServiceQueryParams(ServiceParams): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class FileQueryParams(FileParams): - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") - @validator("file_type") + @field_validator("file_type") @classmethod def ensure_extension_upper_and_dotless(cls, v): # NOTE: see filetype constraint-check @@ -172,14 +170,13 @@ def ensure_extension_upper_and_dotless(cls, v): class ServiceAndFileParams(FileQueryParams, ServiceParams): - class Config: - # Optional configuration to exclude duplicates from schema - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "allOf": [ {"$ref": "#/definitions/FileParams"}, {"$ref": "#/definitions/ServiceParams"}, ] - } + }) class ViewerQueryParams(BaseModel): @@ -190,13 +187,13 @@ class ViewerQueryParams(BaseModel): @staticmethod def from_viewer(viewer: ViewerInfo) -> "ViewerQueryParams": # can safely construct w/o validation from a viewer - return ViewerQueryParams.construct( + return ViewerQueryParams.model_construct( file_type=viewer.filetype, viewer_key=viewer.key, viewer_version=viewer.version, ) - @validator("file_type") + @field_validator("file_type") @classmethod def ensure_extension_upper_and_dotless(cls, v): # NOTE: see filetype constraint-check diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py index 9f66cd460b0..b003ad55963 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py @@ -3,13 +3,19 @@ """ import logging from dataclasses import asdict -from typing import Any, ClassVar from aiohttp import web from aiohttp.web import Request from models_library.services import ServiceKey from models_library.services_types import ServiceVersion -from pydantic import BaseModel, Field, ValidationError, parse_obj_as, validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + TypeAdapter, + ValidationError, + field_validator, +) from pydantic.networks import HttpUrl from .._meta import API_VTAG @@ -32,11 +38,11 @@ def _compose_file_and_service_dispatcher_prefix_url( request: web.Request, viewer: ViewerInfo ) -> HttpUrl: """This is denoted PREFIX URL because it needs to append extra query parameters""" - params = ViewerQueryParams.from_viewer(viewer).dict() + params = ViewerQueryParams.from_viewer(viewer).model_dump() absolute_url = request.url.join( request.app.router["get_redirection_to_viewer"].url_for().with_query(**params) ) - absolute_url_: HttpUrl = parse_obj_as(HttpUrl, f"{absolute_url}") + absolute_url_: HttpUrl = TypeAdapter(HttpUrl).validate_python(f"{absolute_url}") return absolute_url_ @@ -46,11 +52,11 @@ def _compose_service_only_dispatcher_prefix_url( params = ViewerQueryParams( viewer_key=ServiceKey(service_key), viewer_version=ServiceVersion(service_version), - ).dict(exclude_none=True, exclude_unset=True) + ).model_dump(exclude_none=True, exclude_unset=True) absolute_url = request.url.join( request.app.router["get_redirection_to_viewer"].url_for().with_query(**params) ) - absolute_url_: HttpUrl = parse_obj_as(HttpUrl, f"{absolute_url}") + absolute_url_: HttpUrl = TypeAdapter(HttpUrl).validate_python(f"{absolute_url}") return absolute_url_ @@ -125,15 +131,15 @@ def create(cls, meta: ServiceMetaData, request: web.Request): **asdict(meta), ) - @validator("file_extensions") + @field_validator("file_extensions") @classmethod def remove_dot_prefix_from_extension(cls, v): if v: return [ext.removeprefix(".") for ext in v] return v - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "key": "simcore/services/dynamic/sim4life", "title": "Sim4Life Mattermost", @@ -143,6 +149,7 @@ class Config: "view_url": "https://host.com/view?viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=1.2.3", } } + ) # @@ -177,7 +184,7 @@ async def list_viewers(request: Request): file_type: str | None = request.query.get("file_type", None) viewers = [ - Viewer.create(request, viewer).dict() + Viewer.create(request, viewer).model_dump() for viewer in await list_viewers_info(request.app, file_type=file_type) ] return envelope_json_response(viewers) @@ -189,7 +196,7 @@ async def list_default_viewers(request: Request): file_type: str | None = request.query.get("file_type", None) viewers = [ - Viewer.create(request, viewer).dict() + Viewer.create(request, viewer).model_dump() for viewer in await list_viewers_info( request.app, file_type=file_type, only_default=True ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py index c9ff40adbd9..568a534832a 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py @@ -12,12 +12,12 @@ import secrets import string from contextlib import suppress -from datetime import datetime +from datetime import UTC, datetime import redis.asyncio as aioredis from aiohttp import web from models_library.emails import LowerCaseEmailStr -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, TypeAdapter from redis.exceptions import LockNotOwnedError from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.logging_utils import log_decorator @@ -80,7 +80,9 @@ async def create_temporary_guest_user(request: web.Request): random_user_name = "".join( secrets.choice(string.ascii_lowercase) for _ in range(10) ) - email = parse_obj_as(LowerCaseEmailStr, f"{random_user_name}@guest-at-osparc.io") + email = TypeAdapter(LowerCaseEmailStr).validate_python( + f"{random_user_name}@guest-at-osparc.io" + ) password = generate_password(length=12) expires_at = datetime.utcnow() + settings.STUDIES_GUEST_ACCOUNT_LIFETIME diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index a79c4865f12..00024ce2d23 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -1,10 +1,11 @@ from datetime import timedelta -from typing import Any, ClassVar +from typing import Annotated from aiohttp import web from common_library.pydantic_validators import validate_numeric_string_as_timedelta -from pydantic import ByteSize, HttpUrl, parse_obj_as, validator +from pydantic import ByteSize, HttpUrl, TypeAdapter, field_validator from pydantic.fields import Field +from pydantic_settings import SettingsConfigDict from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from settings_library.base import BaseCustomSettings @@ -22,22 +23,26 @@ class StudiesDispatcherSettings(BaseCustomSettings): ) STUDIES_DEFAULT_SERVICE_THUMBNAIL: HttpUrl = Field( - default=parse_obj_as(HttpUrl, "https://via.placeholder.com/170x120.png"), + default=TypeAdapter(HttpUrl).validate_python( + "https://via.placeholder.com/170x120.png" + ), description="Default thumbnail for services or dispatch project with a service", ) STUDIES_DEFAULT_FILE_THUMBNAIL: HttpUrl = Field( - default=parse_obj_as(HttpUrl, "https://via.placeholder.com/170x120.png"), + default=TypeAdapter(HttpUrl).validate_python( + "https://via.placeholder.com/170x120.png" + ), description="Default thumbnail for dispatch projects with only data (i.e. file-picker)", ) STUDIES_MAX_FILE_SIZE_ALLOWED: ByteSize = Field( - default=parse_obj_as(ByteSize, "50Mib"), + default=TypeAdapter(ByteSize).validate_python("50Mib"), description="Limits the size of the files that can be dispatched" "Note that the accuracy of the file size is not guaranteed and this limit might be surpassed", ) - @validator("STUDIES_GUEST_ACCOUNT_LIFETIME") + @field_validator("STUDIES_GUEST_ACCOUNT_LIFETIME") @classmethod def _is_positive_lifetime(cls, v): if v and isinstance(v, timedelta) and v.total_seconds() <= 0: @@ -55,13 +60,14 @@ def is_login_required(self): "STUDIES_GUEST_ACCOUNT_LIFETIME" ) - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = SettingsConfigDict( + json_schema_extra={ "example": { "STUDIES_GUEST_ACCOUNT_LIFETIME": "2 1:10:00", # 2 days 1h and 10 mins "STUDIES_ACCESS_ANONYMOUS_ALLOWED": "1", }, } + ) def get_plugin_settings(app: web.Application) -> StudiesDispatcherSettings: diff --git a/services/web/server/src/simcore_service_webserver/tags/_api.py b/services/web/server/src/simcore_service_webserver/tags/_api.py index 6f3a74853e7..dacedc603f7 100644 --- a/services/web/server/src/simcore_service_webserver/tags/_api.py +++ b/services/web/server/src/simcore_service_webserver/tags/_api.py @@ -22,7 +22,7 @@ async def create_tag( read=True, write=True, delete=True, - **new_tag.dict(exclude_unset=True), + **new_tag.model_dump(exclude_unset=True), ) return TagGet.from_db(tag) @@ -46,7 +46,7 @@ async def update_tag( tag = await repo.update( user_id=user_id, tag_id=tag_id, - **tag_updates.dict(exclude_unset=True), + **tag_updates.model_dump(exclude_unset=True), ) return TagGet.from_db(tag) diff --git a/services/web/server/src/simcore_service_webserver/tags/_handlers.py b/services/web/server/src/simcore_service_webserver/tags/_handlers.py index 8862f0320c1..24dff16d066 100644 --- a/services/web/server/src/simcore_service_webserver/tags/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/tags/_handlers.py @@ -1,7 +1,7 @@ import functools from aiohttp import web -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -58,7 +58,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @_handle_tags_exceptions async def create_tag(request: web.Request): assert request.app # nosec - req_ctx = TagRequestContext.parse_obj(request) + req_ctx = TagRequestContext.model_validate(request) new_tag = await parse_request_body_as(TagCreate, request) created = await _api.create_tag( @@ -73,7 +73,7 @@ async def create_tag(request: web.Request): @_handle_tags_exceptions async def list_tags(request: web.Request): - req_ctx = TagRequestContext.parse_obj(request) + req_ctx = TagRequestContext.model_validate(request) got = await _api.list_tags(request.app, user_id=req_ctx.user_id) return envelope_json_response(got) @@ -83,7 +83,7 @@ async def list_tags(request: web.Request): @permission_required("tag.crud.*") @_handle_tags_exceptions async def update_tag(request: web.Request): - req_ctx = TagRequestContext.parse_obj(request) + req_ctx = TagRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(TagPathParams, request) tag_updates = await parse_request_body_as(TagUpdate, request) @@ -101,7 +101,7 @@ async def update_tag(request: web.Request): @permission_required("tag.crud.*") @_handle_tags_exceptions async def delete_tag(request: web.Request): - req_ctx = TagRequestContext.parse_obj(request) + req_ctx = TagRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(TagPathParams, request) await _api.delete_tag( @@ -124,7 +124,7 @@ async def list_tag_groups(request: web.Request): path_params = parse_request_path_parameters_as(TagPathParams, request) assert path_params # nosec - assert envelope_json_response(parse_obj_as(list[TagGroupGet], [])) + assert envelope_json_response(TypeAdapter(list[TagGroupGet]).validate_python([])) raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/tags/schemas.py b/services/web/server/src/simcore_service_webserver/tags/schemas.py index 01663e0d337..0851c99f99a 100644 --- a/services/web/server/src/simcore_service_webserver/tags/schemas.py +++ b/services/web/server/src/simcore_service_webserver/tags/schemas.py @@ -1,9 +1,10 @@ import re from datetime import datetime +from typing import Annotated from models_library.api_schemas_webserver._base import InputSchema, OutputSchema from models_library.users import GroupID, UserID -from pydantic import ConstrainedStr, Field, PositiveInt +from pydantic import Field, PositiveInt, StringConstraints from servicelib.aiohttp.requests_validation import RequestParams, StrictRequestParams from servicelib.request_keys import RQT_USERID_KEY from simcore_postgres_database.utils_tags import TagDict @@ -17,8 +18,9 @@ class TagPathParams(StrictRequestParams): tag_id: PositiveInt -class ColorStr(ConstrainedStr): - regex = re.compile(r"^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$") +ColorStr = Annotated[ + str, StringConstraints(pattern=re.compile(r"^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$")) +] class TagUpdate(InputSchema): diff --git a/services/web/server/src/simcore_service_webserver/users/_api.py b/services/web/server/src/simcore_service_webserver/users/_api.py index a054bfe5927..458366367f5 100644 --- a/services/web/server/src/simcore_service_webserver/users/_api.py +++ b/services/web/server/src/simcore_service_webserver/users/_api.py @@ -6,7 +6,7 @@ from models_library.emails import LowerCaseEmailStr from models_library.payments import UserInvoiceAddress from models_library.users import UserBillingDetails, UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.models.users import UserStatus from ..db.plugin import get_database_engine @@ -50,7 +50,7 @@ async def get_user_credentials( ) return UserCredentialsTuple( - email=parse_obj_as(LowerCaseEmailStr, row.email), + email=TypeAdapter(LowerCaseEmailStr).validate_python(row.email), password_hash=row.password_hash, display_name=row.first_name or row.name.capitalize(), ) @@ -116,7 +116,7 @@ async def pre_register_user( if found: raise AlreadyPreRegisteredError(num_found=len(found), email=profile.email) - details = profile.dict( + details = profile.model_dump( include={ "first_name", "last_name", diff --git a/services/web/server/src/simcore_service_webserver/users/_db.py b/services/web/server/src/simcore_service_webserver/users/_db.py index f7d8769f963..2071034d2e6 100644 --- a/services/web/server/src/simcore_service_webserver/users/_db.py +++ b/services/web/server/src/simcore_service_webserver/users/_db.py @@ -212,4 +212,4 @@ async def get_user_billing_details( user_billing_details = await UsersRepo.get_billing_details(conn, user_id) if not user_billing_details: raise BillingDetailsNotFoundError(user_id=user_id) - return UserBillingDetails.from_orm(user_billing_details) + return UserBillingDetails.model_validate(user_billing_details) diff --git a/services/web/server/src/simcore_service_webserver/users/_handlers.py b/services/web/server/src/simcore_service_webserver/users/_handlers.py index 3462602f74b..4d69e9ffaab 100644 --- a/services/web/server/src/simcore_service_webserver/users/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_handlers.py @@ -68,7 +68,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @login_required @_handle_users_exceptions async def get_my_profile(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) profile: ProfileGet = await api.get_user_profile( request.app, req_ctx.user_id, req_ctx.product_name ) @@ -80,7 +80,7 @@ async def get_my_profile(request: web.Request) -> web.Response: @permission_required("user.profile.update") @_handle_users_exceptions async def update_my_profile(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) profile_update = await parse_request_body_as(ProfileUpdate, request) await api.update_user_profile( request.app, req_ctx.user_id, profile_update, as_patch=False @@ -105,7 +105,7 @@ class _SearchQueryParams(BaseModel): @permission_required("user.users.*") @_handle_users_exceptions async def search_users(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) assert req_ctx.product_name # nosec query_params: _SearchQueryParams = parse_request_query_parameters_as( @@ -117,7 +117,7 @@ async def search_users(request: web.Request) -> web.Response: ) return envelope_json_response( - [_.dict(**_RESPONSE_MODEL_MINIMAL_POLICY) for _ in found] + [_.model_dump(**_RESPONSE_MODEL_MINIMAL_POLICY) for _ in found] ) @@ -126,7 +126,7 @@ async def search_users(request: web.Request) -> web.Response: @permission_required("user.users.*") @_handle_users_exceptions async def pre_register_user(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) pre_user_profile = await parse_request_body_as(PreUserProfile, request) try: @@ -134,7 +134,7 @@ async def pre_register_user(request: web.Request) -> web.Response: request.app, profile=pre_user_profile, creator_user_id=req_ctx.user_id ) return envelope_json_response( - user_profile.dict(**_RESPONSE_MODEL_MINIMAL_POLICY) + user_profile.model_dump(**_RESPONSE_MODEL_MINIMAL_POLICY) ) except AlreadyPreRegisteredError as err: raise web.HTTPConflict(reason=f"{err}") from err diff --git a/services/web/server/src/simcore_service_webserver/users/_notifications.py b/services/web/server/src/simcore_service_webserver/users/_notifications.py index 256e521f89c..39e6fda9208 100644 --- a/services/web/server/src/simcore_service_webserver/users/_notifications.py +++ b/services/web/server/src/simcore_service_webserver/users/_notifications.py @@ -6,7 +6,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.utils.enums import StrAutoEnum -from pydantic import BaseModel, NonNegativeInt, validator +from pydantic import field_validator, ConfigDict, BaseModel, NonNegativeInt MAX_NOTIFICATIONS_FOR_USER_TO_SHOW: Final[NonNegativeInt] = 10 MAX_NOTIFICATIONS_FOR_USER_TO_KEEP: Final[NonNegativeInt] = 100 @@ -33,7 +33,7 @@ class BaseUserNotification(BaseModel): date: datetime product: Literal["UNDEFINED"] | ProductName = "UNDEFINED" - @validator("category", pre=True) + @field_validator("category", mode="before") @classmethod def category_to_upper(cls, value: str) -> str: return value.upper() @@ -58,10 +58,9 @@ class UserNotification(BaseUserNotification): def create_from_request_data( cls, request_data: UserNotificationCreate ) -> "UserNotification": - return cls.construct(id=f"{uuid4()}", read=False, **request_data.dict()) - - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + return cls.model_construct(id=f"{uuid4()}", read=False, **request_data.model_dump()) + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "id": "3fb96d89-ff5d-4d27-b5aa-d20d46e20eb8", @@ -120,3 +119,4 @@ class Config: }, ] } + ) diff --git a/services/web/server/src/simcore_service_webserver/users/_notifications_handlers.py b/services/web/server/src/simcore_service_webserver/users/_notifications_handlers.py index 3a9588d39a5..58fb1a483e5 100644 --- a/services/web/server/src/simcore_service_webserver/users/_notifications_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_notifications_handlers.py @@ -52,7 +52,7 @@ async def _get_user_notifications( # Filter by product included = [product_name, "UNDEFINED"] filtered_notifications = [n for n in notifications if n["product"] in included] - return [UserNotification.parse_obj(x) for x in filtered_notifications] + return [UserNotification.model_validate(x) for x in filtered_notifications] @routes.get(f"/{API_VTAG}/me/notifications", name="list_user_notifications") @@ -60,7 +60,7 @@ async def _get_user_notifications( @permission_required("user.notifications.read") async def list_user_notifications(request: web.Request) -> web.Response: redis_client = get_redis_user_notifications_client(request.app) - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) product_name = get_product_name(request) notifications = await _get_user_notifications( redis_client, req_ctx.user_id, product_name @@ -80,7 +80,7 @@ async def create_user_notification(request: web.Request) -> web.Response: # insert at the head of the list and discard extra notifications redis_client = get_redis_user_notifications_client(request.app) async with redis_client.pipeline(transaction=True) as pipe: - pipe.lpush(key, user_notification.json()) + pipe.lpush(key, user_notification.model_dump_json()) pipe.ltrim(key, 0, MAX_NOTIFICATIONS_FOR_USER_TO_KEEP - 1) await pipe.execute() @@ -99,21 +99,21 @@ class _NotificationPathParams(BaseModel): @permission_required("user.notifications.update") async def mark_notification_as_read(request: web.Request) -> web.Response: redis_client = get_redis_user_notifications_client(request.app) - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) req_path_params = parse_request_path_parameters_as(_NotificationPathParams, request) body = await parse_request_body_as(UserNotificationPatch, request) # NOTE: only the user's notifications can be patched key = get_notification_key(req_ctx.user_id) all_user_notifications: list[UserNotification] = [ - UserNotification.parse_raw(x) + UserNotification.model_validate_json(x) for x in await handle_redis_returns_union_types(redis_client.lrange(key, 0, -1)) ] for k, user_notification in enumerate(all_user_notifications): if req_path_params.notification_id == user_notification.id: user_notification.read = body.read await handle_redis_returns_union_types( - redis_client.lset(key, k, user_notification.json()) + redis_client.lset(key, k, user_notification.model_dump_json()) ) return web.json_response(status=status.HTTP_204_NO_CONTENT) @@ -124,13 +124,15 @@ async def mark_notification_as_read(request: web.Request) -> web.Response: @login_required @permission_required("user.permissions.read") async def list_user_permissions(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) list_permissions: list[Permission] = await _api.list_user_permissions( request.app, req_ctx.user_id, req_ctx.product_name ) return envelope_json_response( [ - PermissionGet.construct(_fields_set=p.__fields_set__, **p.dict()) + PermissionGet.model_construct( + _fields_set=p.model_fields_set, **p.model_dump() + ) for p in list_permissions ] ) diff --git a/services/web/server/src/simcore_service_webserver/users/_preferences_api.py b/services/web/server/src/simcore_service_webserver/users/_preferences_api.py index 8e17a4a25d4..fb55ac58d2f 100644 --- a/services/web/server/src/simcore_service_webserver/users/_preferences_api.py +++ b/services/web/server/src/simcore_service_webserver/users/_preferences_api.py @@ -13,7 +13,7 @@ PreferenceName, ) from models_library.users import UserID -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from servicelib.utils import logged_gather from simcore_postgres_database.utils_groups_extra_properties import ( GroupExtraPropertiesRepo, @@ -96,7 +96,7 @@ def include_preference(identifier: PreferenceIdentifier) -> bool: return True aggregated_preferences: AggregatedPreferences = { - p.preference_identifier: Preference.parse_obj( + p.preference_identifier: Preference.model_validate( {"value": p.value, "default_value": p.get_default_value()} ) for p in await _get_frontend_user_preferences(app, user_id, product_name) @@ -130,6 +130,6 @@ async def set_frontend_user_preference( await _preferences_db.set_user_preference( app, user_id=user_id, - preference=parse_obj_as(preference_class, {"value": value}), # type: ignore[arg-type] # GitHK this is suspicious + preference=TypeAdapter(preference_class).validate_python({"value": value}), # type: ignore[arg-type] # GitHK this is suspicious product_name=product_name, ) diff --git a/services/web/server/src/simcore_service_webserver/users/_preferences_db.py b/services/web/server/src/simcore_service_webserver/users/_preferences_db.py index 45903403af9..e64ce5e579b 100644 --- a/services/web/server/src/simcore_service_webserver/users/_preferences_db.py +++ b/services/web/server/src/simcore_service_webserver/users/_preferences_db.py @@ -31,7 +31,7 @@ async def get_user_preference( return ( None if preference_payload is None - else preference_class.parse_obj(preference_payload) + else preference_class.model_validate(preference_payload) ) diff --git a/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py b/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py index 0c537278f9c..9581b46b335 100644 --- a/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_preferences_handlers.py @@ -55,7 +55,7 @@ async def wrapper(request: web.Request) -> web.StreamResponse: @login_required @_handle_users_exceptions async def set_frontend_preference(request: web.Request) -> web.Response: - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) req_body = await parse_request_body_as(PatchRequestBody, request) req_path_params = parse_request_path_parameters_as(PatchPathParams, request) diff --git a/services/web/server/src/simcore_service_webserver/users/_preferences_models.py b/services/web/server/src/simcore_service_webserver/users/_preferences_models.py index 01b6b87e377..6a871bcfafe 100644 --- a/services/web/server/src/simcore_service_webserver/users/_preferences_models.py +++ b/services/web/server/src/simcore_service_webserver/users/_preferences_models.py @@ -132,7 +132,7 @@ class BillingCenterUsageColumnOrderFrontendUserPreference(FrontendUserPreference ] _PREFERENCE_NAME_TO_IDENTIFIER_MAPPING: dict[PreferenceName, PreferenceIdentifier] = { - p.get_preference_name(): p.__fields__["preference_identifier"].default + p.get_preference_name(): p.model_fields["preference_identifier"].default for p in ALL_FRONTEND_PREFERENCES } _PREFERENCE_IDENTIFIER_TO_NAME_MAPPING: dict[PreferenceIdentifier, PreferenceName] = { diff --git a/services/web/server/src/simcore_service_webserver/users/_schemas.py b/services/web/server/src/simcore_service_webserver/users/_schemas.py index 1dd4f59992f..8df6c890a8b 100644 --- a/services/web/server/src/simcore_service_webserver/users/_schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/_schemas.py @@ -12,7 +12,7 @@ from models_library.api_schemas_webserver._base import InputSchema, OutputSchema from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName -from pydantic import Field, root_validator, validator +from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator from simcore_postgres_database.models.users import UserStatus @@ -33,7 +33,7 @@ class UserProfile(OutputSchema): ) # authorization - invited_by: str | None = None + invited_by: str | None = Field(default=None) # user status registered: bool @@ -43,10 +43,10 @@ class UserProfile(OutputSchema): description="List of products this users is included or None if fields is unset", ) - @validator("status") + @field_validator("status") @classmethod - def _consistency_check(cls, v, values): - registered = values["registered"] + def _consistency_check(cls, v, info: ValidationInfo): + registered = info.data["registered"] status = v if not registered and status is not None: msg = f"{registered=} and {status=} is not allowed" @@ -61,12 +61,12 @@ class PreUserProfile(InputSchema): first_name: str last_name: str email: LowerCaseEmailStr - institution: str | None = Field(None, description="company, university, ...") + institution: str | None = Field(default=None, description="company, university, ...") phone: str | None # billing details address: str city: str - state: str | None + state: str | None = Field(default=None) postal_code: str country: str extras: dict[str, Any] = Field( @@ -74,11 +74,9 @@ class PreUserProfile(InputSchema): description="Keeps extra information provided in the request form. At most MAX_NUM_EXTRAS fields", ) - class Config(InputSchema.Config): - anystr_strip_whitespace = True - max_anystr_length = 200 + model_config = ConfigDict(str_strip_whitespace=True, str_max_length=200) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def _preprocess_aliases_and_extras(cls, values): # multiple aliases for "institution" @@ -92,8 +90,8 @@ def _preprocess_aliases_and_extras(cls, values): # collect extras extra_fields = {} field_names_and_aliases = ( - set(cls.__fields__.keys()) - | {f.alias for f in cls.__fields__.values() if f.alias} + set(cls.model_fields.keys()) + | {f.alias for f in cls.model_fields.values() if f.alias} | set(alias_by_priority) ) for key, value in values.items(): @@ -111,7 +109,7 @@ def _preprocess_aliases_and_extras(cls, values): return values - @validator("first_name", "last_name", "institution", pre=True) + @field_validator("first_name", "last_name", "institution", mode="before") @classmethod def _pre_normalize_given_names(cls, v): if v: @@ -120,7 +118,7 @@ def _pre_normalize_given_names(cls, v): return re.sub(r"\b\w+\b", lambda m: m.group(0).capitalize(), name) return v - @validator("country", pre=True) + @field_validator("country", mode="before") @classmethod def _pre_check_and_normalize_country(cls, v): if v: @@ -131,4 +129,4 @@ def _pre_check_and_normalize_country(cls, v): return v -assert set(PreUserProfile.__fields__).issubset(UserProfile.__fields__) # nosec +assert set(PreUserProfile.model_fields).issubset(UserProfile.model_fields) # nosec diff --git a/services/web/server/src/simcore_service_webserver/users/_tokens_handlers.py b/services/web/server/src/simcore_service_webserver/users/_tokens_handlers.py index 40b884c4eb9..9f5dfc941b8 100644 --- a/services/web/server/src/simcore_service_webserver/users/_tokens_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_tokens_handlers.py @@ -44,7 +44,7 @@ async def _wrapper(request: web.Request) -> web.StreamResponse: @_handle_tokens_errors @permission_required("user.tokens.*") async def list_tokens(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) all_tokens = await _tokens.list_tokens(request.app, req_ctx.user_id) return envelope_json_response(all_tokens) @@ -54,7 +54,7 @@ async def list_tokens(request: web.Request) -> web.Response: @_handle_tokens_errors @permission_required("user.tokens.*") async def create_token(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) token_create = await parse_request_body_as(TokenCreate, request) await _tokens.create_token(request.app, req_ctx.user_id, token_create) return envelope_json_response(token_create, web.HTTPCreated) @@ -69,7 +69,7 @@ class _TokenPathParams(BaseModel): @_handle_tokens_errors @permission_required("user.tokens.*") async def get_token(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) req_path_params = parse_request_path_parameters_as(_TokenPathParams, request) token = await _tokens.get_token( request.app, req_ctx.user_id, req_path_params.service @@ -82,7 +82,7 @@ async def get_token(request: web.Request) -> web.Response: @_handle_tokens_errors @permission_required("user.tokens.*") async def delete_token(request: web.Request) -> web.Response: - req_ctx = UsersRequestContext.parse_obj(request) + req_ctx = UsersRequestContext.model_validate(request) req_path_params = parse_request_path_parameters_as(_TokenPathParams, request) await _tokens.delete_token(request.app, req_ctx.user_id, req_path_params.service) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/users/api.py b/services/web/server/src/simcore_service_webserver/users/api.py index 52736a1e8d8..50dfdc4e12d 100644 --- a/services/web/server/src/simcore_service_webserver/users/api.py +++ b/services/web/server/src/simcore_service_webserver/users/api.py @@ -16,7 +16,7 @@ from models_library.basic_types import IDStr from models_library.products import ProductName from models_library.users import GroupID, UserID -from pydantic import EmailStr, ValidationError, parse_obj_as +from pydantic import EmailStr, TypeAdapter, ValidationError from simcore_postgres_database.models.users import UserRole from simcore_postgres_database.utils_groups_extra_properties import ( GroupExtraPropertiesNotFoundError, @@ -38,7 +38,7 @@ def _parse_as_user(user_id: Any) -> UserID: try: - return parse_obj_as(UserID, user_id) + return TypeAdapter(UserID).validate_python(user_id) except ValidationError as err: raise UserNotFoundError(uid=user_id) from err @@ -159,7 +159,7 @@ async def update_user_profile( user_id = _parse_as_user(user_id) async with get_database_engine(app).acquire() as conn: - to_update = update.dict( + to_update = update.model_dump( include={ "first_name", "last_name", diff --git a/services/web/server/src/simcore_service_webserver/users/exceptions.py b/services/web/server/src/simcore_service_webserver/users/exceptions.py index 51fb1cc2b19..39791ea39fe 100644 --- a/services/web/server/src/simcore_service_webserver/users/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/users/exceptions.py @@ -9,12 +9,16 @@ class UsersBaseError(WebServerBaseError): class UserNotFoundError(UsersBaseError): def __init__(self, *, uid: int | None = None, email: str | None = None, **ctx: Any): - super().__init__(**ctx) + super().__init__( + msg_template=( + "User id {uid} not found" + if uid + else f"User with email {email} not found" + ), + **ctx, + ) self.uid = uid self.email = email - self.msg_template = ( - "User id {uid} not found" if uid else f"User with email {email} not found" - ) class TokenNotFoundError(UsersBaseError): diff --git a/services/web/server/src/simcore_service_webserver/users/schemas.py b/services/web/server/src/simcore_service_webserver/users/schemas.py index cae3f5b7df5..0f2f8ee1684 100644 --- a/services/web/server/src/simcore_service_webserver/users/schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/schemas.py @@ -8,7 +8,7 @@ from models_library.api_schemas_webserver.users_preferences import AggregatedPreferences from models_library.emails import LowerCaseEmailStr from models_library.users import FirstNameStr, LastNameStr, UserID -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field from simcore_postgres_database.models.users import UserRole from ..utils import gravatar_hash @@ -28,13 +28,14 @@ class ThirdPartyToken(BaseModel): token_key: UUID = Field(..., description="basic token key") token_secret: UUID | None = None - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "service": "github-api-v1", "token_key": "5f21abf5-c596-47b7-bfd1-c0e436ef1107", } } + ) class TokenCreate(ThirdPartyToken): @@ -50,13 +51,14 @@ class ProfileUpdate(BaseModel): first_name: FirstNameStr | None = None last_name: LastNameStr | None = None - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "example": { "first_name": "Pedro", "last_name": "Crespo", } } + ) class ProfileGet(BaseModel): @@ -74,13 +76,11 @@ class ProfileGet(BaseModel): ) preferences: AggregatedPreferences - class Config: + model_config = ConfigDict( # NOTE: old models have an hybrid between snake and camel cases! # Should be unified at some point - allow_population_by_field_name = True - json_dumps = json_dumps - - schema_extra: ClassVar[dict[str, Any]] = { + populate_by_name=True, + json_schema_extra={ "examples": [ { "id": 1, @@ -92,14 +92,15 @@ class Config: { "id": 42, "login": "bla@foo.com", - "role": UserRole.ADMIN, + "role": UserRole.ADMIN.value, "expirationDate": "2022-09-14", "preferences": {}, }, ] } + ) - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def _auto_generate_gravatar(cls, values): gravatar_id = values.get("gravatar_id") @@ -108,7 +109,7 @@ def _auto_generate_gravatar(cls, values): values["gravatar_id"] = gravatar_hash(email) return values - @validator("role", pre=True) + @field_validator("role", mode="before") @classmethod def _to_upper_string(cls, v): if isinstance(v, str): diff --git a/services/web/server/src/simcore_service_webserver/utils.py b/services/web/server/src/simcore_service_webserver/utils.py index 657e347d1a6..0807cee96b1 100644 --- a/services/web/server/src/simcore_service_webserver/utils.py +++ b/services/web/server/src/simcore_service_webserver/utils.py @@ -11,11 +11,12 @@ import tracemalloc from datetime import datetime from pathlib import Path -from typing import Any, TypedDict, cast +from typing import Any, cast import orjson from common_library.error_codes import ErrorCodeStr from models_library.basic_types import SHA1Str +from typing_extensions import TypedDict _CURRENT_DIR = ( Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -190,4 +191,4 @@ def compute_sha1_on_small_dataset(d: Any) -> SHA1Str: """ # SEE options in https://github.com/ijl/orjson#option data_bytes = orjson.dumps(d, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS) - return cast(SHA1Str, hashlib.sha1(data_bytes).hexdigest()) # nosec + return SHA1Str(hashlib.sha1(data_bytes).hexdigest()) # nosec # NOSONAR diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index 4adaa5ef468..bb60b8a1b8f 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -9,7 +9,6 @@ from common_library.json_serialization import json_dumps from models_library.generics import Envelope from pydantic import BaseModel, Field -from pydantic.generics import GenericModel from servicelib.common_headers import X_FORWARDED_PROTO from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.rest_constants import RESPONSE_MODEL_POLICY @@ -71,7 +70,7 @@ def envelope_json_response( enveloped = Envelope[Any](data=obj) return web.Response( - text=json_dumps(enveloped.dict(**RESPONSE_MODEL_POLICY)), + text=json_dumps(enveloped.model_dump(**RESPONSE_MODEL_POLICY)), content_type=MIMETYPE_APPLICATION_JSON, status=status_cls.status_code, ) @@ -116,7 +115,7 @@ def create_redirect_to_page_response( PageParameters = TypeVar("PageParameters", bound=BaseModel) -class NextPage(GenericModel, Generic[PageParameters]): +class NextPage(BaseModel, Generic[PageParameters]): """ This is the body of a 2XX response to pass the front-end what kind of page shall be display next and some information about it diff --git a/services/web/server/src/simcore_service_webserver/version_control/_core.py b/services/web/server/src/simcore_service_webserver/version_control/_core.py index 53f10829b48..860d124ce48 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/_core.py +++ b/services/web/server/src/simcore_service_webserver/version_control/_core.py @@ -12,7 +12,7 @@ from uuid import UUID from aiopg.sa.result import RowProxy -from pydantic import NonNegativeInt, PositiveInt, validate_arguments +from pydantic import NonNegativeInt, PositiveInt, validate_call from .db import VersionControlRepository from .errors import CleanRequiredError @@ -136,7 +136,7 @@ async def get_workbench( # prefer actual project to snapshot content = await vc_repo.get_workbench_view(repo_id, commit_id) - return WorkbenchView.parse_obj(content) + return WorkbenchView.model_validate(content) # @@ -146,10 +146,10 @@ async def get_workbench( _CONFIG = {"arbitrary_types_allowed": True} -list_repos_safe = validate_arguments(list_repos, config=_CONFIG) # type: ignore -list_checkpoints_safe = validate_arguments(list_checkpoints, config=_CONFIG) # type: ignore -create_checkpoint_safe = validate_arguments(create_checkpoint, config=_CONFIG) # type: ignore -get_checkpoint_safe = validate_arguments(get_checkpoint, config=_CONFIG) # type: ignore -update_checkpoint_safe = validate_arguments(update_checkpoint, config=_CONFIG) # type: ignore -checkout_checkpoint_safe = validate_arguments(checkout_checkpoint, config=_CONFIG) # type: ignore -get_workbench_safe = validate_arguments(get_workbench, config=_CONFIG) # type: ignore +list_repos_safe = validate_call(list_repos, config=_CONFIG) # type: ignore +list_checkpoints_safe = validate_call(list_checkpoints, config=_CONFIG) # type: ignore +create_checkpoint_safe = validate_call(create_checkpoint, config=_CONFIG) # type: ignore +get_checkpoint_safe = validate_call(get_checkpoint, config=_CONFIG) # type: ignore +update_checkpoint_safe = validate_call(update_checkpoint, config=_CONFIG) # type: ignore +checkout_checkpoint_safe = validate_call(checkout_checkpoint, config=_CONFIG) # type: ignore +get_workbench_safe = validate_call(get_workbench, config=_CONFIG) # type: ignore diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py index 0cf849effb0..22b6270019c 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py @@ -4,7 +4,7 @@ from models_library.projects import ProjectID from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -46,7 +46,7 @@ class _CheckpointsPathParam(BaseModel): project_uuid: ProjectID ref_id: RefID - @validator("ref_id", pre=True) + @field_validator("ref_id", mode="before") @classmethod def _normalize_refid(cls, v): if v and v == "HEAD": @@ -81,7 +81,7 @@ async def _list_repos_handler(request: web.Request): # parse and validate repos_list = [ - RepoApiModel.parse_obj( + RepoApiModel.model_validate( { "url": url_for("list_repos"), **dict(row.items()), @@ -90,7 +90,7 @@ async def _list_repos_handler(request: web.Request): for row in repos_rows ] - page = Page[RepoApiModel].parse_obj( + page = Page[RepoApiModel].model_validate( paginate_data( chunk=repos_list, request_url=request.url, @@ -100,7 +100,7 @@ async def _list_repos_handler(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type="application/json", ) @@ -116,22 +116,22 @@ async def _create_checkpoint_handler(request: web.Request): vc_repo = VersionControlRepository.create_from_request(request) path_params = parse_request_path_parameters_as(_ProjectPathParam, request) - _body = CheckpointNew.parse_obj(await request.json()) + _body = CheckpointNew.model_validate(await request.json()) checkpoint: Checkpoint = await create_checkpoint( vc_repo, project_uuid=path_params.project_uuid, - **_body.dict(include={"tag", "message"}), + **_body.model_dump(include={"tag", "message"}), ) - data = CheckpointApiModel.parse_obj( + data = CheckpointApiModel.model_validate( { "url": url_for( "get_checkpoint", project_uuid=path_params.project_uuid, ref_id=checkpoint.id, ), - **checkpoint.dict(), + **checkpoint.model_dump(), } ) return envelope_json_response(data, status_cls=web.HTTPCreated) @@ -163,20 +163,20 @@ async def _list_checkpoints_handler(request: web.Request): # parse and validate checkpoints_list = [ - CheckpointApiModel.parse_obj( + CheckpointApiModel.model_validate( { "url": url_for( "get_checkpoint", project_uuid=path_params.project_uuid, ref_id=checkpoint.id, ), - **checkpoint.dict(), + **checkpoint.model_dump(), } ) for checkpoint in checkpoints ] - page = Page[CheckpointApiModel].parse_obj( + page = Page[CheckpointApiModel].model_validate( paginate_data( chunk=checkpoints_list, request_url=request.url, @@ -186,7 +186,7 @@ async def _list_checkpoints_handler(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type="application/json", ) @@ -211,14 +211,14 @@ async def _get_checkpoint_handler(request: web.Request): ref_id=path_params.ref_id, ) - data = CheckpointApiModel.parse_obj( + data = CheckpointApiModel.model_validate( { "url": url_for( "get_checkpoint", project_uuid=path_params.project_uuid, ref_id=checkpoint.id, ), - **checkpoint.dict(**RESPONSE_MODEL_POLICY), + **checkpoint.model_dump(**RESPONSE_MODEL_POLICY), } ) return envelope_json_response(data) @@ -242,17 +242,17 @@ async def _update_checkpoint_annotations_handler(request: web.Request): vc_repo, project_uuid=path_params.project_uuid, ref_id=path_params.ref_id, - **update.dict(include={"tag", "message"}, exclude_none=True), + **update.model_dump(include={"tag", "message"}, exclude_none=True), ) - data = CheckpointApiModel.parse_obj( + data = CheckpointApiModel.model_validate( { "url": url_for( "get_checkpoint", project_uuid=path_params.project_uuid, ref_id=checkpoint.id, ), - **checkpoint.dict(**RESPONSE_MODEL_POLICY), + **checkpoint.model_dump(**RESPONSE_MODEL_POLICY), } ) return envelope_json_response(data) @@ -277,14 +277,14 @@ async def _checkout_handler(request: web.Request): ref_id=path_params.ref_id, ) - data = CheckpointApiModel.parse_obj( + data = CheckpointApiModel.model_validate( { "url": url_for( "get_checkpoint", project_uuid=path_params.project_uuid, ref_id=checkpoint.id, ), - **checkpoint.dict(**RESPONSE_MODEL_POLICY), + **checkpoint.model_dump(**RESPONSE_MODEL_POLICY), } ) return envelope_json_response(data) @@ -315,7 +315,7 @@ async def _view_project_workbench_handler(request: web.Request): ref_id=checkpoint.id, ) - data = WorkbenchViewApiModel.parse_obj( + data = WorkbenchViewApiModel.model_validate( { # = request.url?? "url": url_for( @@ -328,7 +328,7 @@ async def _view_project_workbench_handler(request: web.Request): project_uuid=path_params.project_uuid, ref_id=checkpoint.id, ), - **view.dict(**RESPONSE_MODEL_POLICY), + **view.model_dump(**RESPONSE_MODEL_POLICY), } ) diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py index c40b27946f2..3424788fafa 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py +++ b/services/web/server/src/simcore_service_webserver/version_control/_handlers_base.py @@ -4,7 +4,7 @@ from aiohttp import web from common_library.json_serialization import json_dumps -from pydantic.error_wrappers import ValidationError +from pydantic import ValidationError from servicelib.aiohttp.typing_extension import Handler from ..projects.exceptions import ProjectNotFoundError diff --git a/services/web/server/src/simcore_service_webserver/version_control/models.py b/services/web/server/src/simcore_service_webserver/version_control/models.py index a562459547e..451d302d90f 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/models.py +++ b/services/web/server/src/simcore_service_webserver/version_control/models.py @@ -5,7 +5,7 @@ from models_library.basic_types import SHA1Str from models_library.projects import ProjectID from models_library.projects_nodes import Node -from pydantic import BaseModel, Field, PositiveInt, StrictBool, StrictFloat, StrictInt +from pydantic import ConfigDict, BaseModel, Field, PositiveInt, StrictBool, StrictFloat, StrictInt from pydantic.networks import HttpUrl BuiltinTypes: TypeAlias = Union[StrictBool, StrictInt, StrictFloat, str] @@ -24,7 +24,7 @@ CommitID: TypeAlias = int BranchID: TypeAlias = int -RefID: TypeAlias = Union[CommitID, str] +RefID: TypeAlias = CommitID | str CheckpointID: TypeAlias = PositiveInt @@ -35,7 +35,7 @@ class Checkpoint(BaseModel): created_at: datetime tags: tuple[str, ...] message: str | None = None - parents_ids: tuple[PositiveInt, ...] = Field(default=None) + parents_ids: tuple[PositiveInt, ...] | None = Field(default=None) @classmethod def from_commit_log(cls, commit: RowProxy, tags: list[RowProxy]) -> "Checkpoint": @@ -44,16 +44,14 @@ def from_commit_log(cls, commit: RowProxy, tags: list[RowProxy]) -> "Checkpoint" checksum=commit.snapshot_checksum, tags=tuple(tag.name for tag in tags), message=commit.message, - parents_ids=(commit.parent_commit_id,) if commit.parent_commit_id else None, # type: ignore[arg-type] + parents_ids=(commit.parent_commit_id,) if commit.parent_commit_id else None, created_at=commit.created, ) class WorkbenchView(BaseModel): """A view (i.e. read-only and visual) of the project's workbench""" - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) # NOTE: Tmp replacing UUIDS by str due to a problem serializing to json UUID keys # in the response https://github.com/samuelcolvin/pydantic/issues/2096#issuecomment-814860206 diff --git a/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py b/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py index 50d2cae1e76..cc3559c118b 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py +++ b/services/web/server/src/simcore_service_webserver/version_control/vc_changes.py @@ -31,7 +31,7 @@ def compute_workbench_checksum(workbench: dict[str, Any]) -> SHA1Str: checksum = compute_sha1_on_small_dataset( { - k: node.dict( + k: node.model_dump( exclude_unset=True, exclude_defaults=True, exclude_none=True, diff --git a/services/web/server/src/simcore_service_webserver/wallets/_api.py b/services/web/server/src/simcore_service_webserver/wallets/_api.py index 8a528fe5db2..c2af4074378 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_api.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_api.py @@ -13,7 +13,7 @@ from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import UserWalletDB, WalletDB, WalletID, WalletStatus -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..resource_usage.api import get_wallet_total_available_credits from ..users import api as users_api @@ -42,7 +42,7 @@ async def create_wallet( thumbnail=thumbnail, product_name=product_name, ) - wallet_api: WalletGet = parse_obj_as(WalletGet, wallet_db) + wallet_api: WalletGet = WalletGet.model_validate(wallet_db) return wallet_api @@ -122,7 +122,9 @@ async def get_user_default_wallet_with_available_credits( ) if user_default_wallet_preference is None: raise UserDefaultWalletNotFoundError(uid=user_id) - default_wallet_id = parse_obj_as(WalletID, user_default_wallet_preference.value) + default_wallet_id = TypeAdapter(WalletID).validate_python( + user_default_wallet_preference.value + ) return await get_wallet_with_available_credits_by_user_and_wallet( app, user_id=user_id, wallet_id=default_wallet_id, product_name=product_name ) @@ -136,7 +138,7 @@ async def list_wallets_for_user( user_wallets: list[UserWalletDB] = await db.list_wallets_for_user( app=app, user_id=user_id, product_name=product_name ) - return parse_obj_as(list[WalletGet], user_wallets) + return TypeAdapter(list[WalletGet]).validate_python(user_wallets) async def any_wallet_owned_by_user( @@ -178,7 +180,7 @@ async def update_wallet( user_id=user_id, wallet_id=wallet_id, product_name=product_name, - user_acces_rights_on_wallet=wallet.dict( + user_acces_rights_on_wallet=wallet.model_dump( include={"read", "write", "delete"} ), ) @@ -193,7 +195,7 @@ async def update_wallet( product_name=product_name, ) - wallet_api: WalletGet = parse_obj_as(WalletGet, wallet_db) + wallet_api: WalletGet = WalletGet.model_validate(wallet_db) return wallet_api @@ -212,7 +214,7 @@ async def delete_wallet( user_id=user_id, wallet_id=wallet_id, product_name=product_name, - user_acces_rights_on_wallet=wallet.dict( + user_acces_rights_on_wallet=wallet.model_dump( include={"read", "write", "delete"} ), ) @@ -235,7 +237,7 @@ async def get_wallet_by_user( user_id=user_id, wallet_id=wallet_id, product_name=product_name, - user_acces_rights_on_wallet=wallet.dict( + user_acces_rights_on_wallet=wallet.model_dump( include={"read", "write", "delete"} ), ) @@ -263,5 +265,5 @@ async def get_wallet_with_permissions_by_user( app=app, user_id=user_id, wallet_id=wallet_id, product_name=product_name ) - permissions: WalletGetPermissions = parse_obj_as(WalletGetPermissions, wallet) + permissions: WalletGetPermissions = WalletGetPermissions.model_validate(wallet) return permissions diff --git a/services/web/server/src/simcore_service_webserver/wallets/_db.py b/services/web/server/src/simcore_service_webserver/wallets/_db.py index 467bc69e437..413b68ff84f 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_db.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_db.py @@ -9,7 +9,6 @@ from models_library.products import ProductName from models_library.users import GroupID, UserID from models_library.wallets import UserWalletDB, WalletDB, WalletID, WalletStatus -from pydantic import parse_obj_as from simcore_postgres_database.models.groups import user_to_groups from simcore_postgres_database.models.wallet_to_groups import wallet_to_groups from simcore_postgres_database.models.wallets import wallets @@ -47,7 +46,7 @@ async def create_wallet( .returning(literal_column("*")) ) row = await result.first() - return parse_obj_as(WalletDB, row) + return WalletDB.model_validate(row) _SELECTION_ARGS = ( @@ -98,7 +97,7 @@ async def list_wallets_for_user( async with get_database_engine(app).acquire() as conn: result = await conn.execute(stmt) rows = await result.fetchall() or [] - output: list[UserWalletDB] = [parse_obj_as(UserWalletDB, row) for row in rows] + output: list[UserWalletDB] = [UserWalletDB.model_validate(row) for row in rows] return output @@ -160,7 +159,7 @@ async def get_wallet_for_user( wallet_id=wallet_id, product_name=product_name, ) - return parse_obj_as(UserWalletDB, row) + return UserWalletDB.model_validate(row) async def get_wallet( @@ -188,7 +187,7 @@ async def get_wallet( row = await result.first() if row is None: raise WalletNotFoundError(reason=f"Wallet {wallet_id} not found.") - return parse_obj_as(WalletDB, row) + return WalletDB.model_validate(row) async def update_wallet( @@ -219,7 +218,7 @@ async def update_wallet( row = await result.first() if row is None: raise WalletNotFoundError(reason=f"Wallet {wallet_id} not found.") - return parse_obj_as(WalletDB, row) + return WalletDB.model_validate(row) async def delete_wallet( diff --git a/services/web/server/src/simcore_service_webserver/wallets/_groups_api.py b/services/web/server/src/simcore_service_webserver/wallets/_groups_api.py index 98dcd40058b..bdace14a9de 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_groups_api.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_groups_api.py @@ -5,7 +5,7 @@ from models_library.products import ProductName from models_library.users import GroupID, UserID from models_library.wallets import UserWalletDB, WalletID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, ConfigDict from ..users import api as users_api from . import _db as wallets_db @@ -23,6 +23,10 @@ class WalletGroupGet(BaseModel): delete: bool created: datetime modified: datetime + + model_config = ConfigDict( + from_attributes=True + ) async def create_wallet_group( @@ -45,7 +49,7 @@ async def create_wallet_group( user_id=user_id, wallet_id=wallet_id, product_name=product_name, - user_acces_rights_on_wallet=wallet.dict( + user_acces_rights_on_wallet=wallet.model_dump( include={"read", "write", "delete"} ), ) @@ -58,7 +62,7 @@ async def create_wallet_group( write=write, delete=delete, ) - wallet_group_api: WalletGroupGet = WalletGroupGet(**wallet_group_db.dict()) + wallet_group_api: WalletGroupGet = WalletGroupGet(**wallet_group_db.model_dump()) return wallet_group_api @@ -79,7 +83,7 @@ async def list_wallet_groups_by_user_and_wallet( user_id=user_id, wallet_id=wallet_id, product_name=product_name, - user_acces_rights_on_wallet=wallet.dict( + user_acces_rights_on_wallet=wallet.model_dump( include={"read", "write", "delete"} ), ) @@ -89,7 +93,7 @@ async def list_wallet_groups_by_user_and_wallet( ] = await wallets_groups_db.list_wallet_groups(app=app, wallet_id=wallet_id) wallet_groups_api: list[WalletGroupGet] = [ - parse_obj_as(WalletGroupGet, group) for group in wallet_groups_db + WalletGroupGet.model_validate(group) for group in wallet_groups_db ] return wallet_groups_api @@ -105,7 +109,7 @@ async def list_wallet_groups_with_read_access_by_wallet( ] = await wallets_groups_db.list_wallet_groups(app=app, wallet_id=wallet_id) wallet_groups_api: list[WalletGroupGet] = [ - parse_obj_as(WalletGroupGet, group) + WalletGroupGet.model_validate(group) for group in wallet_groups_db if group.read is True ] @@ -140,7 +144,7 @@ async def update_wallet_group( user_id=user_id, wallet_id=wallet_id, product_name=product_name, - user_acces_rights_on_wallet=wallet.dict( + user_acces_rights_on_wallet=wallet.model_dump( include={"read", "write", "delete"} ), ) @@ -154,7 +158,7 @@ async def update_wallet_group( delete=delete, ) - wallet_api: WalletGroupGet = WalletGroupGet(**wallet_group_db.dict()) + wallet_api: WalletGroupGet = WalletGroupGet(**wallet_group_db.model_dump()) return wallet_api diff --git a/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py b/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py index f9d42cc6ddd..949978a470f 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py @@ -9,7 +9,7 @@ from aiohttp import web from models_library.users import GroupID from models_library.wallets import WalletID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, TypeAdapter from simcore_postgres_database.models.wallet_to_groups import wallet_to_groups from sqlalchemy import func, literal_column from sqlalchemy.sql import select @@ -58,7 +58,7 @@ async def create_wallet_group( .returning(literal_column("*")) ) row = await result.first() - return parse_obj_as(WalletGroupGetDB, row) + return WalletGroupGetDB.model_validate(row) async def list_wallet_groups( @@ -81,7 +81,7 @@ async def list_wallet_groups( async with get_database_engine(app).acquire() as conn: result = await conn.execute(stmt) rows = await result.fetchall() or [] - return parse_obj_as(list[WalletGroupGetDB], rows) + return TypeAdapter(list[WalletGroupGetDB]).validate_python(rows) async def get_wallet_group( @@ -112,7 +112,7 @@ async def get_wallet_group( raise WalletGroupNotFoundError( reason=f"Wallet {wallet_id} group {group_id} not found" ) - return parse_obj_as(WalletGroupGetDB, row) + return WalletGroupGetDB.model_validate(row) async def update_wallet_group( @@ -143,7 +143,7 @@ async def update_wallet_group( raise WalletGroupNotFoundError( reason=f"Wallet {wallet_id} group {group_id} not found" ) - return parse_obj_as(WalletGroupGetDB, row) + return WalletGroupGetDB.model_validate(row) async def delete_wallet_group( diff --git a/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py index 1115a239d62..b8477396e26 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_groups_handlers.py @@ -8,7 +8,7 @@ from aiohttp import web from models_library.users import GroupID, UserID from models_library.wallets import WalletID -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -60,18 +60,14 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _WalletsGroupsPathParams(BaseModel): wallet_id: WalletID group_id: GroupID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _WalletsGroupsBodyParams(BaseModel): read: bool write: bool delete: bool - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.post( @@ -81,7 +77,7 @@ class Config: @permission_required("wallets.*") @_handle_wallets_groups_exceptions async def create_wallet_group(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_WalletsGroupsPathParams, request) body_params = await parse_request_body_as(_WalletsGroupsBodyParams, request) @@ -104,7 +100,7 @@ async def create_wallet_group(request: web.Request): @permission_required("wallets.*") @_handle_wallets_groups_exceptions async def list_wallet_groups(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) wallets: list[ @@ -127,7 +123,7 @@ async def list_wallet_groups(request: web.Request): @permission_required("wallets.*") @_handle_wallets_groups_exceptions async def update_wallet_group(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_WalletsGroupsPathParams, request) body_params = await parse_request_body_as(_WalletsGroupsBodyParams, request) @@ -151,7 +147,7 @@ async def update_wallet_group(request: web.Request): @permission_required("wallets.*") @_handle_wallets_groups_exceptions async def delete_wallet_group(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_WalletsGroupsPathParams, request) await _groups_api.delete_wallet_group( diff --git a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py index 3aa26158acb..90ac4572f6f 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py @@ -128,7 +128,7 @@ class WalletsPathParams(StrictRequestParams): @permission_required("wallets.*") @handle_wallets_exceptions async def create_wallet(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) body_params = await parse_request_body_as(CreateWalletBodyParams, request) wallet: WalletGet = await _api.create_wallet( @@ -148,7 +148,7 @@ async def create_wallet(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def list_wallets(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) wallets: list[ WalletGetWithAvailableCredits @@ -164,7 +164,7 @@ async def list_wallets(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def get_default_wallet(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) wallet: WalletGetWithAvailableCredits = ( await _api.get_user_default_wallet_with_available_credits( @@ -179,7 +179,7 @@ async def get_default_wallet(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def get_wallet(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) wallet: WalletGetWithAvailableCredits = ( @@ -202,7 +202,7 @@ async def get_wallet(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def update_wallet(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) body_params = await parse_request_body_as(PutWalletBodyParams, request) diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 9a03bc2d2a5..66c73b5a293 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -65,7 +65,7 @@ @permission_required("wallets.*") @handle_wallets_exceptions async def _create_payment(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) body_params = await parse_request_body_as(CreateWalletPayment, request) @@ -113,7 +113,7 @@ async def _list_all_payments(request: web.Request): be listed here. """ - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) query_params: PageQueryParameters = parse_request_query_parameters_as( PageQueryParameters, request ) @@ -126,7 +126,7 @@ async def _list_all_payments(request: web.Request): offset=query_params.offset, ) - page = Page[PaymentTransaction].parse_obj( + page = Page[PaymentTransaction].model_validate( paginate_data( chunk=payments, request_url=request.url, @@ -148,7 +148,7 @@ async def _list_all_payments(request: web.Request): @handle_wallets_exceptions async def _get_payment_invoice_link(request: web.Request): """Get invoice for concrete payment""" - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(PaymentsPathParams, request) payment_invoice = await get_payment_invoice_url( @@ -174,7 +174,7 @@ class PaymentsPathParams(WalletsPathParams): @permission_required("wallets.*") @handle_wallets_exceptions async def _cancel_payment(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(PaymentsPathParams, request) await api.cancel_payment_to_wallet( @@ -208,7 +208,7 @@ async def _init_creation_of_payment_method(request: web.Request): """Triggers the creation of a new payment method. Note that creating a payment-method follows the init-prompt-ack flow """ - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) with log_context( @@ -241,7 +241,7 @@ async def _init_creation_of_payment_method(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def _cancel_creation_of_payment_method(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(PaymentMethodsPathParams, request) with log_context( @@ -272,7 +272,7 @@ async def _cancel_creation_of_payment_method(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def _list_payments_methods(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) payments_methods: list[PaymentMethodGet] = await list_wallet_payment_methods( @@ -292,7 +292,7 @@ async def _list_payments_methods(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def _get_payment_method(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(PaymentMethodsPathParams, request) payment_method: PaymentMethodGet = await get_wallet_payment_method( @@ -313,7 +313,7 @@ async def _get_payment_method(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def _delete_payment_method(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(PaymentMethodsPathParams, request) await delete_wallet_payment_method( @@ -337,7 +337,7 @@ async def _delete_payment_method(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def _pay_with_payment_method(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(PaymentMethodsPathParams, request) body_params = await parse_request_body_as(CreateWalletPayment, request) @@ -409,7 +409,7 @@ async def _notify_payment_completed_after_response(app, user_id, payment): @permission_required("wallets.*") @handle_wallets_exceptions async def _get_wallet_autorecharge(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) auto_recharge = await get_wallet_payment_autorecharge( @@ -426,7 +426,7 @@ async def _get_wallet_autorecharge(request: web.Request): product_name=req_ctx.product_name, ) - return envelope_json_response(GetWalletAutoRecharge.parse_obj(auto_recharge)) + return envelope_json_response(GetWalletAutoRecharge.model_validate(auto_recharge)) @routes.put( @@ -437,7 +437,7 @@ async def _get_wallet_autorecharge(request: web.Request): @permission_required("wallets.*") @handle_wallets_exceptions async def _replace_wallet_autorecharge(request: web.Request): - req_ctx = WalletsRequestContext.parse_obj(request) + req_ctx = WalletsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WalletsPathParams, request) body_params = await parse_request_body_as(ReplaceWalletAutoRecharge, request) @@ -454,4 +454,4 @@ async def _replace_wallet_autorecharge(request: web.Request): wallet_id=path_params.wallet_id, new=body_params, ) - return envelope_json_response(GetWalletAutoRecharge.parse_obj(udpated)) + return envelope_json_response(GetWalletAutoRecharge.model_validate(udpated)) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py index 0ec1e44618e..cca4da82e4e 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py @@ -5,7 +5,7 @@ from models_library.products import ProductName from models_library.users import GroupID, UserID from models_library.workspaces import UserWorkspaceAccessRightsDB, WorkspaceID -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, ConfigDict from ..users import api as users_api from . import _groups_db as workspaces_groups_db @@ -24,6 +24,10 @@ class WorkspaceGroupGet(BaseModel): delete: bool created: datetime modified: datetime + + model_config = ConfigDict( + from_attributes=True + ) async def create_workspace_group( @@ -56,7 +60,7 @@ async def create_workspace_group( ) ) workspace_group_api: WorkspaceGroupGet = WorkspaceGroupGet( - **workspace_group_db.dict() + **workspace_group_db.model_dump() ) return workspace_group_api @@ -84,7 +88,7 @@ async def list_workspace_groups_by_user_and_workspace( ) workspace_groups_api: list[WorkspaceGroupGet] = [ - parse_obj_as(WorkspaceGroupGet, group) for group in workspace_groups_db + WorkspaceGroupGet.model_validate(group) for group in workspace_groups_db ] return workspace_groups_api @@ -102,7 +106,7 @@ async def list_workspace_groups_with_read_access_by_workspace( ) workspace_groups_api: list[WorkspaceGroupGet] = [ - parse_obj_as(WorkspaceGroupGet, group) + WorkspaceGroupGet.model_validate(group) for group in workspace_groups_db if group.read is True ] @@ -147,7 +151,9 @@ async def update_workspace_group( ) ) - workspace_api: WorkspaceGroupGet = WorkspaceGroupGet(**workspace_group_db.dict()) + workspace_api: WorkspaceGroupGet = WorkspaceGroupGet( + **workspace_group_db.model_dump() + ) return workspace_api diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_db.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_db.py index daeba51ae80..c186786f603 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_db.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_db.py @@ -9,7 +9,7 @@ from aiohttp import web from models_library.users import GroupID from models_library.workspaces import WorkspaceID -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel from simcore_postgres_database.models.workspaces_access_rights import ( workspaces_access_rights, ) @@ -31,9 +31,7 @@ class WorkspaceGroupGetDB(BaseModel): delete: bool created: datetime modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) ## DB API @@ -63,7 +61,7 @@ async def create_workspace_group( .returning(literal_column("*")) ) row = await result.first() - return WorkspaceGroupGetDB.from_orm(row) + return WorkspaceGroupGetDB.model_validate(row) async def list_workspace_groups( @@ -86,7 +84,7 @@ async def list_workspace_groups( async with get_database_engine(app).acquire() as conn: result = await conn.execute(stmt) rows = await result.fetchall() or [] - return [WorkspaceGroupGetDB.from_orm(row) for row in rows] + return [WorkspaceGroupGetDB.model_validate(row) for row in rows] async def get_workspace_group( @@ -117,7 +115,7 @@ async def get_workspace_group( raise WorkspaceGroupNotFoundError( workspace_id=workspace_id, group_id=group_id ) - return WorkspaceGroupGetDB.from_orm(row) + return WorkspaceGroupGetDB.model_validate(row) async def update_workspace_group( @@ -148,7 +146,7 @@ async def update_workspace_group( raise WorkspaceGroupNotFoundError( workspace_id=workspace_id, group_id=group_id ) - return WorkspaceGroupGetDB.from_orm(row) + return WorkspaceGroupGetDB.model_validate(row) async def delete_workspace_group( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py index d4ae7c4b74f..22572cee19c 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py @@ -8,7 +8,7 @@ from aiohttp import web from models_library.users import GroupID, UserID from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -60,18 +60,14 @@ async def wrapper(request: web.Request) -> web.StreamResponse: class _WorkspacesGroupsPathParams(BaseModel): workspace_id: WorkspaceID group_id: GroupID - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class _WorkspacesGroupsBodyParams(BaseModel): read: bool write: bool delete: bool - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.post( @@ -82,7 +78,7 @@ class Config: @permission_required("workspaces.*") @_handle_workspaces_groups_exceptions async def create_workspace_group(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_WorkspacesGroupsPathParams, request) body_params = await parse_request_body_as(_WorkspacesGroupsBodyParams, request) @@ -105,7 +101,7 @@ async def create_workspace_group(request: web.Request): @permission_required("workspaces.*") @_handle_workspaces_groups_exceptions async def list_workspace_groups(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) workspaces: list[ @@ -128,7 +124,7 @@ async def list_workspace_groups(request: web.Request): @permission_required("workspaces.*") @_handle_workspaces_groups_exceptions async def replace_workspace_group(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_WorkspacesGroupsPathParams, request) body_params = await parse_request_body_as(_WorkspacesGroupsBodyParams, request) @@ -152,7 +148,7 @@ async def replace_workspace_group(request: web.Request): @permission_required("workspaces.*") @_handle_workspaces_groups_exceptions async def delete_workspace_group(request: web.Request): - req_ctx = _RequestContext.parse_obj(request) + req_ctx = _RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_WorkspacesGroupsPathParams, request) await _groups_api.delete_workspace_group( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_db.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_db.py index 23de15c3b19..4f007bc7552 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_db.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_db.py @@ -66,7 +66,7 @@ async def create_workspace( .returning(*_SELECTION_ARGS) ) row = await result.first() - return WorkspaceDB.from_orm(row) + return WorkspaceDB.model_validate(row) access_rights_subquery = ( @@ -155,7 +155,7 @@ async def list_workspaces_for_user( result = await conn.execute(list_query) rows = await result.fetchall() or [] results: list[UserWorkspaceAccessRightsDB] = [ - UserWorkspaceAccessRightsDB.from_orm(row) for row in rows + UserWorkspaceAccessRightsDB.model_validate(row) for row in rows ] return cast(int, total_count), results @@ -191,7 +191,7 @@ async def get_workspace_for_user( raise WorkspaceAccessForbiddenError( reason=f"User {user_id} does not have access to the workspace {workspace_id}. Or workspace does not exist.", ) - return UserWorkspaceAccessRightsDB.from_orm(row) + return UserWorkspaceAccessRightsDB.model_validate(row) async def update_workspace( @@ -220,7 +220,7 @@ async def update_workspace( row = await result.first() if row is None: raise WorkspaceNotFoundError(reason=f"Workspace {workspace_id} not found.") - return WorkspaceDB.from_orm(row) + return WorkspaceDB.model_validate(row) async def delete_workspace( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py index fa9a2e4aa67..3a22237865b 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py @@ -14,7 +14,7 @@ from models_library.rest_pagination_utils import paginate_data from models_library.users import UserID from models_library.workspaces import WorkspaceID -from pydantic import Extra, Field, Json, parse_obj_as, validator +from pydantic import field_validator, ConfigDict, Field, Json from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( RequestParams, @@ -75,11 +75,11 @@ class WorkspacesListWithJsonStrQueryParams(PageQueryParameters): order_by: Json[OrderBy] = Field( default=OrderBy(field=IDStr("modified"), direction=OrderDirection.DESC), description="Order by field (modified_at|name|description) and direction (asc|desc). The default sorting order is ascending.", - example='{"field": "name", "direction": "desc"}', + examples=['{"field": "name", "direction": "desc"}'], alias="order_by", ) - @validator("order_by", check_fields=False) + @field_validator("order_by", check_fields=False) @classmethod def validate_order_by_field(cls, v): if v.field not in { @@ -92,9 +92,7 @@ def validate_order_by_field(cls, v): if v.field == "modified_at": v.field = "modified" return v - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") @routes.post(f"/{VTAG}/workspaces", name="create_workspace") @@ -102,7 +100,7 @@ class Config: @permission_required("workspaces.*") @handle_workspaces_exceptions async def create_workspace(request: web.Request): - req_ctx = WorkspacesRequestContext.parse_obj(request) + req_ctx = WorkspacesRequestContext.model_validate(request) body_params = await parse_request_body_as(CreateWorkspaceBodyParams, request) workspace: WorkspaceGet = await _workspaces_api.create_workspace( @@ -122,7 +120,7 @@ async def create_workspace(request: web.Request): @permission_required("workspaces.*") @handle_workspaces_exceptions async def list_workspaces(request: web.Request): - req_ctx = WorkspacesRequestContext.parse_obj(request) + req_ctx = WorkspacesRequestContext.model_validate(request) query_params: WorkspacesListWithJsonStrQueryParams = ( parse_request_query_parameters_as(WorkspacesListWithJsonStrQueryParams, request) ) @@ -133,10 +131,10 @@ async def list_workspaces(request: web.Request): product_name=req_ctx.product_name, offset=query_params.offset, limit=query_params.limit, - order_by=parse_obj_as(OrderBy, query_params.order_by), + order_by=OrderBy.model_validate(query_params.order_by), ) - page = Page[WorkspaceGet].parse_obj( + page = Page[WorkspaceGet].model_validate( paginate_data( chunk=workspaces.items, request_url=request.url, @@ -146,7 +144,7 @@ async def list_workspaces(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -156,7 +154,7 @@ async def list_workspaces(request: web.Request): @permission_required("workspaces.*") @handle_workspaces_exceptions async def get_workspace(request: web.Request): - req_ctx = WorkspacesRequestContext.parse_obj(request) + req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) workspace: WorkspaceGet = await _workspaces_api.get_workspace( @@ -177,7 +175,7 @@ async def get_workspace(request: web.Request): @permission_required("workspaces.*") @handle_workspaces_exceptions async def replace_workspace(request: web.Request): - req_ctx = WorkspacesRequestContext.parse_obj(request) + req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) body_params = await parse_request_body_as(PutWorkspaceBodyParams, request) @@ -201,7 +199,7 @@ async def replace_workspace(request: web.Request): @permission_required("workspaces.*") @handle_workspaces_exceptions async def delete_workspace(request: web.Request): - req_ctx = WorkspacesRequestContext.parse_obj(request) + req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) await _workspaces_api.delete_workspace( diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index 8aa8ea86309..97a96fa2847 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -327,9 +327,9 @@ async def _creator( data, error = await assert_status(result, status.HTTP_200_OK) assert data assert not error - task_status = TaskStatus.parse_obj(data) + task_status = TaskStatus.model_validate(data) assert task_status - print(f"<-- status: {task_status.json(indent=2)}") + print(f"<-- status: {task_status.model_dump_json(indent=2)}") assert task_status.done, "task incomplete" print( f"-- project creation completed: {json.dumps(attempt.retry_state.retry_object.statistics, indent=2)}" @@ -403,7 +403,7 @@ async def _creator( # the access rights are set to use the logged user primary group + whatever was inside the project expected_data["accessRights"].update( { - str(primary_group["gid"]): { + f"{primary_group['gid']}": { "read": True, "write": True, "delete": True, diff --git a/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py b/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py index c15921c7d5d..f1d4aa62187 100644 --- a/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py +++ b/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py @@ -294,7 +294,7 @@ async def test_log_workflow_only_receives_messages_if_subscribed( log_message.user_id, message={ "event_type": SOCKET_IO_LOG_EVENT, - "data": log_message.dict(exclude={"user_id", "channel_name"}), + "data": log_message.model_dump(exclude={"user_id", "channel_name"}), }, ignore_queue=True, ), @@ -493,7 +493,7 @@ async def test_instrumentation_workflow( mocked_metrics_method, mock.call( client.app, - **rabbit_message.dict(include=set(included_labels)), + **rabbit_message.model_dump(include=set(included_labels)), ), ) diff --git a/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py b/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py index 014418a25fb..4ae3ca6a3e1 100644 --- a/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py +++ b/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py @@ -145,7 +145,7 @@ async def test_scicrunch_get_fields_from_invalid_rrid( async def test_scicrunch_service_autocomplete_by_name(settings: SciCrunchSettings): - expected: list[dict[str, Any]] = ListOfResourceHits.parse_obj( + expected: list[dict[str, Any]] = ListOfResourceHits.model_validate( [ { "rid": "SCR_000860", @@ -159,7 +159,7 @@ async def test_scicrunch_service_autocomplete_by_name(settings: SciCrunchSetting }, {"rid": "SCR_014398", "original_id": "SCR_014398", "name": "GNU Octave"}, ] - ).dict()["__root__"] + ).model_dump()["root"] async with ClientSession() as client: @@ -167,6 +167,6 @@ async def test_scicrunch_service_autocomplete_by_name(settings: SciCrunchSetting resource_hits = await autocomplete_by_name("octave", client, settings) - hits = resource_hits.dict()["__root__"] + hits = resource_hits.model_dump()["root"] assert expected == hits, f"for {guess_name}" diff --git a/services/web/server/tests/unit/isolated/conftest.py b/services/web/server/tests/unit/isolated/conftest.py index 9cc0948ff88..31f0aa33f98 100644 --- a/services/web/server/tests/unit/isolated/conftest.py +++ b/services/web/server/tests/unit/isolated/conftest.py @@ -96,6 +96,7 @@ def mock_env_deployer_pipeline(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: def mock_env_devel_environment( mock_env_devel_environment: EnvVarsDict, # pylint: disable=redefined-outer-name monkeypatch: pytest.MonkeyPatch, + faker: Faker ) -> EnvVarsDict: # Overrides to ensure dev-features are enabled testings return mock_env_devel_environment | setenvs_from_dict( @@ -218,8 +219,8 @@ def mock_webserver_service_environment( "STORAGE_PORT": os.environ.get("STORAGE_PORT", "8080"), "SWARM_STACK_NAME": os.environ.get("SWARM_STACK_NAME", "simcore"), "WEBSERVER_LOGLEVEL": os.environ.get("LOG_LEVEL", "WARNING"), - "SESSION_COOKIE_MAX_AGE": str(7 * 24 * 60 * 60), - }, + "SESSION_COOKIE_MAX_AGE": str(7 * 24 * 60 * 60) + } ) return ( diff --git a/services/web/server/tests/unit/isolated/notifications/test_rabbitmq_consumers.py b/services/web/server/tests/unit/isolated/notifications/test_rabbitmq_consumers.py index 62840490d68..4a9bc655df2 100644 --- a/services/web/server/tests/unit/isolated/notifications/test_rabbitmq_consumers.py +++ b/services/web/server/tests/unit/isolated/notifications/test_rabbitmq_consumers.py @@ -32,7 +32,7 @@ node_id=UUID("6925403d-5464-4d92-9ec9-72c5793ca203"), progress_type=ProgressType.SERVICE_OUTPUTS_PULLING, report=ProgressReport(actual_value=0.4, total=1), - ).json(), + ).model_dump_json(), SocketMessageDict( event_type=WebSocketNodeProgress.get_event_type(), data={ @@ -56,7 +56,7 @@ user_id=123, progress_type=ProgressType.PROJECT_CLOSING, report=ProgressReport(actual_value=0.4, total=1), - ).json(), + ).model_dump_json(), SocketMessageDict( event_type=WebSocketNodeProgress.get_event_type(), data={ diff --git a/services/web/server/tests/unit/isolated/test_application_settings.py b/services/web/server/tests/unit/isolated/test_application_settings.py index 17e3502bfa5..84c8ee46871 100644 --- a/services/web/server/tests/unit/isolated/test_application_settings.py +++ b/services/web/server/tests/unit/isolated/test_application_settings.py @@ -7,7 +7,7 @@ import pytest from aiohttp import web from common_library.json_serialization import json_dumps -from pydantic import HttpUrl, parse_obj_as +from pydantic import HttpUrl, TypeAdapter from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_webserver.application_settings import ( APP_SETTINGS_KEY, @@ -22,10 +22,11 @@ def app_settings( ) -> ApplicationSettings: app = web.Application() + print("envs\n", json.dumps(mock_webserver_service_environment, indent=1)) + # init and validation happens here settings = setup_settings(app) - print("envs\n", json.dumps(mock_webserver_service_environment, indent=1)) - print("settings:\n", settings.json(indent=1)) + print("settings:\n", settings.model_dump_json(indent=1)) assert APP_SETTINGS_KEY in app assert app[APP_SETTINGS_KEY] == settings @@ -97,7 +98,7 @@ def test_settings_to_client_statics_plugins( ) assert statics["vcsReleaseTag"] - assert parse_obj_as(HttpUrl, statics["vcsReleaseUrl"]) + assert TypeAdapter(HttpUrl).validate_python(statics["vcsReleaseUrl"]) assert set(statics["pluginsDisabled"]) == (disable_plugins | {"WEBSERVER_CLUSTERS"}) diff --git a/services/web/server/tests/unit/isolated/test_application_settings_utils.py b/services/web/server/tests/unit/isolated/test_application_settings_utils.py index f4f0f901199..2897b8bf358 100644 --- a/services/web/server/tests/unit/isolated/test_application_settings_utils.py +++ b/services/web/server/tests/unit/isolated/test_application_settings_utils.py @@ -19,7 +19,7 @@ def test_settings_infered_from_default_tests_config( settings = ApplicationSettings.create_from_envs() - print("settings=\n", settings.json(indent=1, sort_keys=True)) + print("settings=\n", settings.model_dump_json(indent=1, sort_keys=True)) infered_config = convert_to_app_config(settings) diff --git a/services/web/server/tests/unit/isolated/test_catalog_api_units.py b/services/web/server/tests/unit/isolated/test_catalog_api_units.py index 479165189d2..39d1824a775 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_api_units.py +++ b/services/web/server/tests/unit/isolated/test_catalog_api_units.py @@ -45,8 +45,8 @@ def test_can_connect_enums(unit_registry: UnitRegistry): } assert can_connect( - from_output=ServiceOutput.parse_obj(enum_port), - to_input=ServiceInput.parse_obj(enum_port), + from_output=ServiceOutput.model_validate(enum_port), + to_input=ServiceInput.model_validate(enum_port), units_registry=unit_registry, ) @@ -71,15 +71,15 @@ def test_can_connect_generic_data_types(unit_registry: UnitRegistry): # data:*/* -> data:text/plain assert can_connect( - from_output=ServiceOutput.parse_obj(file_picker_outfile), - to_input=ServiceInput.parse_obj(input_sleeper_input_1), + from_output=ServiceOutput.model_validate(file_picker_outfile), + to_input=ServiceInput.model_validate(input_sleeper_input_1), units_registry=unit_registry, ) # data:text/plain -> data:*/* assert can_connect( - from_output=ServiceOutput.parse_obj(input_sleeper_input_1), - to_input=ServiceInput.parse_obj(file_picker_outfile), + from_output=ServiceOutput.model_validate(input_sleeper_input_1), + to_input=ServiceInput.model_validate(file_picker_outfile), units_registry=unit_registry, ) @@ -127,15 +127,15 @@ def test_can_connect_no_units_with_units( ): # w/o -> w assert can_connect( - from_output=ServiceOutput.parse_obj(port_without_unit), - to_input=ServiceInput.parse_obj(port_with_unit), + from_output=ServiceOutput.model_validate(port_without_unit), + to_input=ServiceInput.model_validate(port_with_unit), units_registry=unit_registry, ) # w -> w/o assert can_connect( - from_output=ServiceOutput.parse_obj(port_with_unit), - to_input=ServiceInput.parse_obj(port_without_unit), + from_output=ServiceOutput.model_validate(port_with_unit), + to_input=ServiceInput.model_validate(port_without_unit), units_registry=unit_registry, ) @@ -178,8 +178,8 @@ def test_units_compatible( assert ( can_connect( - from_output=ServiceOutput.parse_obj(from_port), - to_input=ServiceInput.parse_obj(to_port), + from_output=ServiceOutput.model_validate(from_port), + to_input=ServiceInput.model_validate(to_port), units_registry=unit_registry, ) == are_compatible diff --git a/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py b/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py index f35d7991539..b35b2b378f4 100644 --- a/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py +++ b/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py @@ -5,9 +5,10 @@ # pylint: disable=unused-variable import asyncio +import json import logging -import time from collections.abc import Callable, Coroutine +import time import pytest import simcore_service_webserver @@ -88,11 +89,13 @@ def mock_environment( { **mock_env_devel_environment, "AIODEBUG_SLOW_DURATION_SECS": f"{SLOW_HANDLER_DELAY_SECS / 10}", - "DIAGNOSTICS_MAX_TASK_DELAY": f"{SLOW_HANDLER_DELAY_SECS}", - "DIAGNOSTICS_MAX_AVG_LATENCY": f"{2.0}", - "DIAGNOSTICS_START_SENSING_DELAY": f"{0}", + "WEBSERVER_DIAGNOSTICS": json.dumps({ + "DIAGNOSTICS_MAX_AVG_LATENCY": "2.0", + "DIAGNOSTICS_MAX_TASK_DELAY": f"{SLOW_HANDLER_DELAY_SECS}", + "DIAGNOSTICS_START_SENSING_DELAY": f"{0}", + "DIAGNOSTICS_HEALTHCHECK_ENABLED": "1", + }), "SC_HEALTHCHECK_TIMEOUT": "2m", - "DIAGNOSTICS_HEALTHCHECK_ENABLED": "1", }, ) diff --git a/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py b/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py index 6308141d254..944a958baf2 100644 --- a/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py +++ b/services/web/server/tests/unit/isolated/test_dynamic_scheduler.py @@ -47,18 +47,23 @@ def mock_rpc_client( @pytest.fixture def dynamic_service_start() -> DynamicServiceStart: - return DynamicServiceStart.parse_obj( - DynamicServiceStart.Config.schema_extra["example"] + return DynamicServiceStart.model_validate( + DynamicServiceStart.model_config["json_schema_extra"]["example"] ) @pytest.mark.parametrize( "expected_response", [ - *[NodeGet.parse_obj(x) for x in NodeGet.Config.schema_extra["examples"]], - NodeGetIdle.parse_obj(NodeGetIdle.Config.schema_extra["example"]), - DynamicServiceGet.parse_obj( - DynamicServiceGet.Config.schema_extra["examples"][0] + *[ + NodeGet.model_validate(x) + for x in NodeGet.model_config["json_schema_extra"]["examples"] + ], + NodeGetIdle.model_validate( + NodeGetIdle.model_config["json_schema_extra"]["example"] + ), + DynamicServiceGet.model_validate( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][0] ), ], ) @@ -98,9 +103,12 @@ async def test_get_service_status_raises_rpc_server_error( @pytest.mark.parametrize( "expected_response", [ - *[NodeGet.parse_obj(x) for x in NodeGet.Config.schema_extra["examples"]], - DynamicServiceGet.parse_obj( - DynamicServiceGet.Config.schema_extra["examples"][0] + *[ + NodeGet.model_validate(x) + for x in NodeGet.model_config["json_schema_extra"]["examples"] + ], + DynamicServiceGet.model_validate( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][0] ), ], ) diff --git a/services/web/server/tests/unit/isolated/test_garbage_collector_core.py b/services/web/server/tests/unit/isolated/test_garbage_collector_core.py index 0d84fbb534c..920dfb2b035 100644 --- a/services/web/server/tests/unit/isolated/test_garbage_collector_core.py +++ b/services/web/server/tests/unit/isolated/test_garbage_collector_core.py @@ -123,8 +123,8 @@ async def test_remove_orphaned_services_with_no_running_services_does_nothing( @pytest.fixture def faker_dynamic_service_get() -> Callable[[], DynamicServiceGet]: def _() -> DynamicServiceGet: - return DynamicServiceGet.parse_obj( - DynamicServiceGet.Config.schema_extra["examples"][1] + return DynamicServiceGet.model_validate( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] ) return _ diff --git a/services/web/server/tests/unit/isolated/test_groups_models.py b/services/web/server/tests/unit/isolated/test_groups_models.py index 10b49979b1c..5ad6f8863cc 100644 --- a/services/web/server/tests/unit/isolated/test_groups_models.py +++ b/services/web/server/tests/unit/isolated/test_groups_models.py @@ -14,7 +14,7 @@ def test_models_library_and_postgress_database_enums_are_equivalent(): def test_sanitize_legacy_data(): - users_group_1 = UsersGroup.parse_obj( + users_group_1 = UsersGroup.model_validate( { "gid": "27", "label": "A user", @@ -26,7 +26,7 @@ def test_sanitize_legacy_data(): assert users_group_1.thumbnail is None - users_group_2 = UsersGroup.parse_obj( + users_group_2 = UsersGroup.model_validate( { "gid": "27", "label": "A user", diff --git a/services/web/server/tests/unit/isolated/test_statics.py b/services/web/server/tests/unit/isolated/test_isolated_statics.py similarity index 100% rename from services/web/server/tests/unit/isolated/test_statics.py rename to services/web/server/tests/unit/isolated/test_isolated_statics.py diff --git a/services/web/server/tests/unit/isolated/test_login_settings.py b/services/web/server/tests/unit/isolated/test_login_settings.py index b6872fce92d..0bfff446911 100644 --- a/services/web/server/tests/unit/isolated/test_login_settings.py +++ b/services/web/server/tests/unit/isolated/test_login_settings.py @@ -121,15 +121,15 @@ def test_smtp_settings(mock_env_devel_environment: dict[str, Any]): settings = SMTPSettings.create_from_envs() - cfg = settings.dict(exclude_unset=True) + cfg = settings.model_dump(exclude_unset=True) for env_name in cfg: assert env_name in os.environ - cfg = settings.dict() + cfg = settings.model_dump() config = LoginOptions(**cfg) - print(config.json(indent=1)) + print(config.model_dump_json(indent=1)) assert not hasattr(config, "SMTP_SENDER"), "was deprecated and now we use product" @@ -137,6 +137,6 @@ def test_smtp_settings(mock_env_devel_environment: dict[str, Any]): def test_product_login_settings_in_plugin_settings(): # pylint: disable=no-member customizable_attributes = set(ProductLoginSettingsDict.__annotations__.keys()) - settings_atrributes = set(LoginSettingsForProduct.__fields__.keys()) + settings_atrributes = set(LoginSettingsForProduct.model_fields.keys()) assert customizable_attributes.issubset(settings_atrributes) diff --git a/services/web/server/tests/unit/isolated/test_products_model.py b/services/web/server/tests/unit/isolated/test_products_model.py index 31c4ac60699..147540adce6 100644 --- a/services/web/server/tests/unit/isolated/test_products_model.py +++ b/services/web/server/tests/unit/isolated/test_products_model.py @@ -34,13 +34,17 @@ def test_product_examples( def test_product_to_static(): - product = Product.parse_obj(Product.Config.schema_extra["examples"][0]) + product = Product.model_validate( + Product.model_config["json_schema_extra"]["examples"][0] + ) assert product.to_statics() == { "displayName": "o²S²PARC", "supportEmail": "support@osparc.io", } - product = Product.parse_obj(Product.Config.schema_extra["examples"][2]) + product = Product.model_validate( + Product.model_config["json_schema_extra"]["examples"][2] + ) assert product.to_statics() == { "displayName": "o²S²PARC FOO", @@ -78,7 +82,7 @@ def test_product_to_static(): def test_product_host_regex_with_spaces(): - data = Product.Config.schema_extra["examples"][2] + data = Product.model_config["json_schema_extra"]["examples"][2] # with leading and trailing spaces and uppercase (tests anystr_strip_whitespace ) data["support_email"] = " fOO@BaR.COM " @@ -88,7 +92,7 @@ def test_product_host_regex_with_spaces(): data["host_regex"] = expected + " " # parsing should strip all whitespaces and normalize email - product = Product.parse_obj(data) + product = Product.model_validate(data) assert product.host_regex.pattern == expected assert product.host_regex.search("osparc.bar.com") diff --git a/services/web/server/tests/unit/isolated/test_projects__nodes_api.py b/services/web/server/tests/unit/isolated/test_projects__nodes_api.py index ef58b4b2451..e7e4bd8a926 100644 --- a/services/web/server/tests/unit/isolated/test_projects__nodes_api.py +++ b/services/web/server/tests/unit/isolated/test_projects__nodes_api.py @@ -3,7 +3,6 @@ import pytest from models_library.api_schemas_storage import FileMetaDataGet -from pydantic import parse_obj_as from simcore_service_webserver.projects._nodes_api import ( _SUPPORTED_PREVIEW_FILE_EXTENSIONS, _FileWithThumbnail, @@ -12,13 +11,12 @@ _PROJECT_ID = uuid4() _NODE_ID = uuid4() -_UTC_NOW = datetime.datetime.now(tz=datetime.timezone.utc) +_UTC_NOW = datetime.datetime.now(tz=datetime.UTC) def _c(file_name: str) -> FileMetaDataGet: """simple converter utility""" - return parse_obj_as( - FileMetaDataGet, + return FileMetaDataGet.model_validate( { "file_uuid": f"{_PROJECT_ID}/{_NODE_ID}/{file_name}", "location_id": 0, diff --git a/services/web/server/tests/unit/isolated/test_projects__nodes_resources.py b/services/web/server/tests/unit/isolated/test_projects__nodes_resources.py index 12f6bfc23b4..70ca1ce3b9d 100644 --- a/services/web/server/tests/unit/isolated/test_projects__nodes_resources.py +++ b/services/web/server/tests/unit/isolated/test_projects__nodes_resources.py @@ -5,7 +5,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_service_webserver.projects._nodes_utils import ( validate_new_service_resources, ) @@ -17,8 +17,10 @@ @pytest.mark.parametrize( "resources", [ - parse_obj_as(ServiceResourcesDict, example) - for example in ServiceResourcesDictHelpers.Config.schema_extra["examples"] + TypeAdapter(ServiceResourcesDict).validate_python(example) + for example in ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ] ], ) def test_check_can_update_service_resources_with_same_does_not_raise( @@ -31,8 +33,10 @@ def test_check_can_update_service_resources_with_same_does_not_raise( @pytest.mark.parametrize( "resources", [ - parse_obj_as(ServiceResourcesDict, example) - for example in ServiceResourcesDictHelpers.Config.schema_extra["examples"] + TypeAdapter(ServiceResourcesDict).validate_python(example) + for example in ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ] ], ) def test_check_can_update_service_resources_with_invalid_container_name_raises( @@ -50,15 +54,19 @@ def test_check_can_update_service_resources_with_invalid_container_name_raises( @pytest.mark.parametrize( "resources", [ - parse_obj_as(ServiceResourcesDict, example) - for example in ServiceResourcesDictHelpers.Config.schema_extra["examples"] + TypeAdapter(ServiceResourcesDict).validate_python(example) + for example in ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ] ], ) def test_check_can_update_service_resources_with_invalid_image_name_raises( resources: ServiceResourcesDict, ): new_resources = { - resource_name: resource_data.copy(update={"image": "some-invalid-image-name"}) + resource_name: resource_data.model_copy( + update={"image": "some-invalid-image-name"} + ) for resource_name, resource_data in resources.items() } with pytest.raises( diff --git a/services/web/server/tests/unit/isolated/test_projects_utils.py b/services/web/server/tests/unit/isolated/test_projects_utils.py index e83e02e295f..0178882d760 100644 --- a/services/web/server/tests/unit/isolated/test_projects_utils.py +++ b/services/web/server/tests/unit/isolated/test_projects_utils.py @@ -58,7 +58,7 @@ def test_clone_project_document( # # SEE https://swagger.io/docs/specification/data-models/data-types/#Null - assert Project.parse_obj(clone) is not None + assert Project.model_validate(clone) is not None @pytest.mark.parametrize( @@ -145,4 +145,4 @@ def test_validate_project_json_schema(): with open(CURRENT_DIR / "data/project-data.json") as f: project: ProjectDict = json.load(f) - Project.parse_obj(project) + Project.model_validate(project) diff --git a/services/web/server/tests/unit/isolated/test_security_api.py b/services/web/server/tests/unit/isolated/test_security_api.py index e60cab4985b..079fa68e529 100644 --- a/services/web/server/tests/unit/isolated/test_security_api.py +++ b/services/web/server/tests/unit/isolated/test_security_api.py @@ -17,7 +17,7 @@ from aiohttp_session import get_session from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status @@ -167,7 +167,7 @@ async def _set_other_product(request: web.Request): async def _login(request: web.Request): product_name = await _get_product_name(request) body = await request.json() - email = parse_obj_as(LowerCaseEmailStr, body["email"]) + email = TypeAdapter(LowerCaseEmailStr).validate_python(body["email"]) # Permission in this product: Has user access to product? if product_name not in registered_users[email]["registered_products"]: @@ -219,7 +219,7 @@ def client( # mocks 'setup_session': patch to avoid setting up all ApplicationSettings session_settings = SessionSettings.create_from_envs() - print(session_settings.json(indent=1)) + print(session_settings.model_dump_json(indent=1)) mocker.patch( "simcore_service_webserver.session.plugin.get_plugin_settings", autospec=True, diff --git a/services/web/server/tests/unit/isolated/test_statics_settings.py b/services/web/server/tests/unit/isolated/test_statics_settings.py index 376a8330eb9..30d9035e05d 100644 --- a/services/web/server/tests/unit/isolated/test_statics_settings.py +++ b/services/web/server/tests/unit/isolated/test_statics_settings.py @@ -4,7 +4,7 @@ import json -from pydantic import AnyHttpUrl, BaseModel, parse_obj_as +from pydantic import AnyHttpUrl, BaseModel, TypeAdapter from simcore_service_webserver.statics.settings import ( _THIRD_PARTY_REFERENCES, FrontEndAppSettings, @@ -23,7 +23,7 @@ class OsparcDependency(BaseModel): def test_valid_osparc_dependencies(): - deps = parse_obj_as(list[OsparcDependency], _THIRD_PARTY_REFERENCES) + deps = TypeAdapter(list[OsparcDependency]).validate_python(_THIRD_PARTY_REFERENCES) assert deps @@ -36,7 +36,7 @@ def test_frontend_app_settings(mock_env_devel_environment: dict[str, str]): statics = settings.to_statics() assert json.dumps(statics) - parse_obj_as(list[OsparcDependency], statics["thirdPartyReferences"]) + TypeAdapter(list[OsparcDependency]).validate_python(statics["thirdPartyReferences"]) def test_static_webserver_module_settings(mock_env_devel_environment: dict[str, str]): diff --git a/services/web/server/tests/unit/isolated/test_storage_schemas.py b/services/web/server/tests/unit/isolated/test_storage_schemas.py index c11ce1f1345..31ea4260bb4 100644 --- a/services/web/server/tests/unit/isolated/test_storage_schemas.py +++ b/services/web/server/tests/unit/isolated/test_storage_schemas.py @@ -20,4 +20,4 @@ def test_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): print(example_name, ":", json.dumps(example_data)) - assert model_cls.parse_obj(example_data) + assert model_cls.model_validate(example_data) diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_core.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_core.py index 8faada91005..ef842a25a98 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_core.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_core.py @@ -12,7 +12,7 @@ import pytest from models_library.projects import Project, ProjectID from models_library.projects_nodes_io import NodeID -from pydantic import validator +from pydantic import field_validator from pydantic.main import BaseModel from pydantic.networks import HttpUrl from pytest_simcore.helpers.webserver_fake_services_data import list_fake_file_consumers @@ -46,11 +46,11 @@ async def test_create_project_with_viewer(view: dict[str, Any]): assert list(project.workbench.keys()) # converts into equivalent Dict - project_in: dict = json.loads(project.json(exclude_none=True, by_alias=True)) + project_in: dict = json.loads(project.model_dump_json(exclude_none=True, by_alias=True)) print(json.dumps(project_in, indent=2)) # This operation is done exactly before adding to the database in projects_handlers.create_projects - Project.parse_obj(project_in) + Project.model_validate(project_in) def test_url_quoting_and_validation(): @@ -63,7 +63,7 @@ def test_url_quoting_and_validation(): class M(BaseModel): url: HttpUrl - @validator("url", pre=True) + @field_validator("url", mode="before") @classmethod def unquote_url(cls, v): w = urllib.parse.unquote(v) @@ -71,14 +71,14 @@ def unquote_url(cls, v): w = w.replace(SPACE, "%20") return w - M.parse_obj( + M.model_validate( { # encoding %20 as %2520 "url": "https://raw.githubusercontent.com/pcrespov/osparc-sample-studies/master/files%2520samples/sample.ipynb" } ) - obj2 = M.parse_obj( + obj2 = M.model_validate( { # encoding space as %20 "url": "https://raw.githubusercontent.com/pcrespov/osparc-sample-studies/master/files%20samples/sample.ipynb" @@ -86,7 +86,7 @@ def unquote_url(cls, v): ) url_with_url_in_query = "http://127.0.0.1:9081/view?file_type=IPYNB&viewer_key=simcore/services/dynamic/jupyter-octave-python-math&viewer_version=1.6.9&file_size=1&download_link=https://raw.githubusercontent.com/pcrespov/osparc-sample-studies/master/files%2520samples/sample.ipynb" - obj4 = M.parse_obj({"url": URL(url_with_url_in_query).query["download_link"]}) + obj4 = M.model_validate({"url": URL(url_with_url_in_query).query["download_link"]}) assert obj2.url.path == obj4.url.path @@ -94,7 +94,7 @@ def unquote_url(cls, v): "https://raw.githubusercontent.com/pcrespov/osparc-sample-studies/master/files%20samples/sample.ipynb" ) M(url=quoted_url) - M.parse_obj({"url": url_with_url_in_query}) + M.model_validate({"url": url_with_url_in_query}) assert ( URL(url_with_url_in_query).query["download_link"] diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py index 0ab58dfd77e..24570916463 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py @@ -11,7 +11,7 @@ import pytest from aiohttp.test_utils import make_mocked_request from models_library.utils.pydantic_tools_extension import parse_obj_or_none -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from simcore_service_webserver.studies_dispatcher._models import ( FileParams, @@ -23,7 +23,7 @@ ) from yarl import URL -_SIZEBYTES = parse_obj_as(ByteSize, "3MiB") +_SIZEBYTES = TypeAdapter(ByteSize).validate_python("3MiB") # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3951#issuecomment-1489992645 # AWS download links have query arg @@ -63,25 +63,25 @@ def test_download_link_validators_1(url_in: str, expected_download_link: str): @pytest.fixture def file_and_service_params() -> dict[str, Any]: - return dict( - file_name="dataset_description.slsx", - file_size=_SIZEBYTES, - file_type="MSExcel", - viewer_key="simcore/services/dynamic/fooo", - viewer_version="1.0.0", - download_link=_DOWNLOAD_LINK, - ) + return { + "file_name": "dataset_description.slsx", + "file_size": _SIZEBYTES, + "file_type": "MSExcel", + "viewer_key": "simcore/services/dynamic/fooo", + "viewer_version": "1.0.0", + "download_link": _DOWNLOAD_LINK, + } def test_download_link_validators_2(file_and_service_params: dict[str, Any]): - params = ServiceAndFileParams.parse_obj(file_and_service_params) + params = ServiceAndFileParams.model_validate(file_and_service_params) assert params.download_link - assert params.download_link.host and params.download_link.host.endswith( + assert params.download_link.host + assert params.download_link.host.endswith( "s3.amazonaws.com" ) - assert params.download_link.host_type == "domain" query = parse_qs(params.download_link.query) assert {"AWSAccessKeyId", "Signature", "Expires", "x-amz-request-payer"} == set( @@ -105,12 +105,12 @@ def test_file_and_service_params(file_and_service_params: dict[str, Any]): def test_file_only_params(): - request_params = dict( - file_name="dataset_description.slsx", - file_size=_SIZEBYTES, - file_type="MSExcel", - download_link=_DOWNLOAD_LINK, - ) + request_params = { + "file_name": "dataset_description.slsx", + "file_size": _SIZEBYTES, + "file_type": "MSExcel", + "download_link": _DOWNLOAD_LINK, + } file_params = parse_obj_or_none(FileParams, request_params) assert file_params @@ -125,10 +125,10 @@ def test_file_only_params(): def test_service_only_params(): - request_params = dict( - viewer_key="simcore/services/dynamic/fooo", - viewer_version="1.0.0", - ) + request_params = { + "viewer_key": "simcore/services/dynamic/fooo", + "viewer_version": "1.0.0", + } file_params = parse_obj_or_none(FileParams, request_params) assert not file_params diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py index 03a0eb5920f..6858ac07ad9 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py @@ -44,13 +44,17 @@ def app_environment( "WEBSERVER_DIAGNOSTICS": "null", "WEBSERVER_DIRECTOR_V2": "null", "WEBSERVER_EXPORTER": "null", + "WEBSERVER_EMAIL": "null", "WEBSERVER_GARBAGE_COLLECTOR": "null", "WEBSERVER_GROUPS": "1", + "WEBSERVER_LOGIN": "null", "WEBSERVER_META_MODELING": "0", + "WEBSERVER_PAYMENTS": "null", "WEBSERVER_PRODUCTS": "1", "WEBSERVER_PUBLICATIONS": "0", "WEBSERVER_RABBITMQ": "null", "WEBSERVER_REMOTE_DEBUG": "0", + "WEBSERVER_SCICRUNCH": "null", "WEBSERVER_STORAGE": "null", "WEBSERVER_SOCKETIO": "0", "WEBSERVER_TAGS": "1", @@ -63,6 +67,7 @@ def app_environment( ) # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG # setup_logging(level=logging.DEBUG) + print(env_vars) return env_vars diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py index 91364e64beb..5c4377d56fd 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py @@ -21,7 +21,7 @@ def environment(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: envs = setenvs_from_dict( monkeypatch, - envs=StudiesDispatcherSettings.Config.schema_extra["example"], + envs=StudiesDispatcherSettings.model_config["json_schema_extra"]["example"], ) return envs @@ -37,8 +37,9 @@ def test_studies_dispatcher_settings(environment: EnvVarsDict): assert not settings.is_login_required() # 2 days 1h and 10 mins - assert settings.STUDIES_GUEST_ACCOUNT_LIFETIME == timedelta( - days=2, hours=1, minutes=10 + assert ( + timedelta(days=2, hours=1, minutes=10) + == settings.STUDIES_GUEST_ACCOUNT_LIFETIME ) @@ -50,10 +51,7 @@ def test_studies_dispatcher_settings_invalid_lifetime( with pytest.raises(ValidationError) as exc_info: StudiesDispatcherSettings.create_from_envs() - validation_error: ErrorDict = exc_info.value.errors()[0] + validation_error: ErrorDict = next(iter(exc_info.value.errors())) + assert validation_error["loc"] == ("STUDIES_GUEST_ACCOUNT_LIFETIME",) assert "-2" in validation_error["msg"] - assert validation_error == { - "loc": ("STUDIES_GUEST_ACCOUNT_LIFETIME",), - "type": "value_error", - "msg": validation_error["msg"], - } + assert validation_error["type"] == "value_error" diff --git a/services/web/server/tests/unit/isolated/test_user_notifications.py b/services/web/server/tests/unit/isolated/test_user_notifications.py index d606a84297f..3caca391881 100644 --- a/services/web/server/tests/unit/isolated/test_user_notifications.py +++ b/services/web/server/tests/unit/isolated/test_user_notifications.py @@ -12,9 +12,11 @@ ) -@pytest.mark.parametrize("raw_data", UserNotification.Config.schema_extra["examples"]) +@pytest.mark.parametrize( + "raw_data", UserNotification.model_config["json_schema_extra"]["examples"] +) def test_user_notification(raw_data: dict[str, Any]): - assert UserNotification.parse_obj(raw_data) + assert UserNotification.model_validate(raw_data) @pytest.mark.parametrize("user_id", [10]) @@ -26,7 +28,7 @@ def test_get_notification_key(user_id: UserID): "request_data", [ pytest.param( - UserNotificationCreate.parse_obj( + UserNotificationCreate.model_validate( { "user_id": "1", "category": NotificationCategory.NEW_ORGANIZATION, @@ -40,7 +42,7 @@ def test_get_notification_key(user_id: UserID): id="normal_usage", ), pytest.param( - UserNotificationCreate.parse_obj( + UserNotificationCreate.model_validate( { "user_id": "1", "category": NotificationCategory.NEW_ORGANIZATION, @@ -55,7 +57,7 @@ def test_get_notification_key(user_id: UserID): id="read_is_always_set_false", ), pytest.param( - UserNotificationCreate.parse_obj( + UserNotificationCreate.model_validate( { "id": "some_id", "user_id": "1", @@ -70,7 +72,7 @@ def test_get_notification_key(user_id: UserID): id="a_new_id_is_alway_recreated", ), pytest.param( - UserNotificationCreate.parse_obj( + UserNotificationCreate.model_validate( { "id": "some_id", "user_id": "1", @@ -85,7 +87,7 @@ def test_get_notification_key(user_id: UserID): id="category_from_string", ), pytest.param( - UserNotificationCreate.parse_obj( + UserNotificationCreate.model_validate( { "id": "some_id", "user_id": "1", diff --git a/services/web/server/tests/unit/isolated/test_users_models.py b/services/web/server/tests/unit/isolated/test_users_models.py index ef5ee03c7b0..a5670b5054e 100644 --- a/services/web/server/tests/unit/isolated/test_users_models.py +++ b/services/web/server/tests/unit/isolated/test_users_models.py @@ -1,5 +1,5 @@ from copy import deepcopy -from datetime import datetime +from datetime import UTC, datetime from pprint import pformat from typing import Any @@ -26,12 +26,12 @@ def test_user_models_examples( assert model_instance, f"Failed with {name}" model_enveloped = Envelope[model_cls].from_data( - model_instance.dict(by_alias=True) + model_instance.model_dump(by_alias=True) ) model_array_enveloped = Envelope[list[model_cls]].from_data( [ - model_instance.dict(by_alias=True), - model_instance.dict(by_alias=True), + model_instance.model_dump(by_alias=True), + model_instance.model_dump(by_alias=True), ] ) @@ -40,19 +40,19 @@ def test_user_models_examples( def test_profile_get_expiration_date(faker: Faker): - fake_expiration = datetime.utcnow() + fake_expiration = datetime.now(UTC) profile = ProfileGet( id=1, login=faker.email(), role=UserRole.ADMIN, - expiration_date=fake_expiration, + expiration_date=fake_expiration.date(), preferences={}, ) assert fake_expiration.date() == profile.expiration_date - body = jsonable_encoder(profile.dict(exclude_unset=True, by_alias=True)) + body = jsonable_encoder(profile.model_dump(exclude_unset=True, by_alias=True)) assert body["expirationDate"] == fake_expiration.date().isoformat() @@ -68,7 +68,7 @@ def test_auto_compute_gravatar(faker: Faker): ) envelope = Envelope[Any](data=profile) - data = envelope.dict(**RESPONSE_MODEL_POLICY)["data"] + data = envelope.model_dump(**RESPONSE_MODEL_POLICY)["data"] assert data["gravatar_id"] assert data["id"] == profile.id @@ -81,7 +81,7 @@ def test_auto_compute_gravatar(faker: Faker): @pytest.mark.parametrize("user_role", [u.name for u in UserRole]) def test_profile_get_role(user_role: str): - for example in ProfileGet.Config.schema_extra["examples"]: + for example in ProfileGet.model_config["json_schema_extra"]["examples"]: data = deepcopy(example) data["role"] = user_role m1 = ProfileGet(**data) @@ -134,5 +134,5 @@ def test_parsing_output_of_get_user_profile(): }, } - profile = ProfileGet.parse_obj(result_from_db_query_and_composition) - assert "password" not in profile.dict(exclude_unset=True) + profile = ProfileGet.model_validate(result_from_db_query_and_composition) + assert "password" not in profile.model_dump(exclude_unset=True) diff --git a/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py b/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py index 5e2b5a891b0..6568b1b7db4 100644 --- a/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py +++ b/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py @@ -10,8 +10,9 @@ from aiohttp import web from aiohttp.test_utils import TestClient from aiohttp.web_exceptions import HTTPOk, HTTPTooManyRequests -from pydantic import ValidationError, conint, parse_obj_as +from pydantic import Field, TypeAdapter, ValidationError from simcore_service_webserver.utils_rate_limiting import global_rate_limit_route +from typing_extensions import Annotated TOTAL_TEST_TIME = 1 # secs MAX_NUM_REQUESTS = 3 @@ -110,7 +111,7 @@ async def test_global_rate_limit_route(requests_per_second: float, client: TestC for t in tasks: if retry_after := t.result().headers.get("Retry-After"): try: - parse_obj_as(conint(ge=1), retry_after) + TypeAdapter(Annotated[int, Field(ge=1)]).validate_python(retry_after) except ValidationError as err: failed.append((retry_after, f"{err}")) assert not failed diff --git a/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py b/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py index c3f6b1d8570..e88d13474fe 100644 --- a/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py +++ b/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py @@ -12,6 +12,7 @@ from typing import Any import hypothesis +import hypothesis.provisional import pytest from aiohttp.test_utils import TestClient from faker import Faker @@ -21,7 +22,14 @@ ClusterPatch, ClusterPing, ) -from models_library.clusters import CLUSTER_ADMIN_RIGHTS, Cluster, SimpleAuthentication +from models_library.clusters import ( + CLUSTER_ADMIN_RIGHTS, + Cluster, + ClusterTypeInModel, + SimpleAuthentication, +) +from pydantic import HttpUrl, TypeAdapter +from pydantic_core import Url from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_parametrizations import ( # nopycln: import @@ -39,25 +47,48 @@ ) +@st.composite +def http_url_strategy(draw): + return TypeAdapter(HttpUrl).validate_python(draw(hypothesis.provisional.urls())) + + +@st.composite +def cluster_patch_strategy(draw): + return ClusterPatch( + name=draw(st.text()), + description=draw(st.text()), + owner=draw(st.integers(min_value=1)), + type=draw(st.sampled_from(ClusterTypeInModel)), + thumbnail=draw(http_url_strategy()), + endpoint=draw(http_url_strategy()), + authentication=None, + accessRights={}, + ) + + +st.register_type_strategy(ClusterPatch, cluster_patch_strategy()) +st.register_type_strategy(Url, http_url_strategy()) + + @pytest.fixture def mocked_director_v2_api(mocker: MockerFixture): mocked_director_v2_api = mocker.patch( "simcore_service_webserver.clusters._handlers.director_v2_api", autospec=True ) - mocked_director_v2_api.create_cluster.return_value = Cluster.parse_obj( - random.choice(Cluster.Config.schema_extra["examples"]) + mocked_director_v2_api.create_cluster.return_value = random.choice( + Cluster.model_config["json_schema_extra"]["examples"] ) mocked_director_v2_api.list_clusters.return_value = [] - mocked_director_v2_api.get_cluster.return_value = Cluster.parse_obj( - random.choice(Cluster.Config.schema_extra["examples"]) + mocked_director_v2_api.get_cluster.return_value = random.choice( + Cluster.model_config["json_schema_extra"]["examples"] ) mocked_director_v2_api.get_cluster_details.return_value = { "scheduler": {"status": "running"}, "dashboardLink": "https://link.to.dashboard", } - mocked_director_v2_api.update_cluster.return_value = Cluster.parse_obj( - random.choice(Cluster.Config.schema_extra["examples"]) + mocked_director_v2_api.update_cluster.return_value = random.choice( + Cluster.model_config["json_schema_extra"]["examples"] ) mocked_director_v2_api.delete_cluster.return_value = None mocked_director_v2_api.ping_cluster.return_value = None @@ -94,6 +125,7 @@ def cluster_create(faker: Faker) -> ClusterCreate: name=faker.name(), endpoint=faker.uri(), type=random.choice(list(ClusterType)), + owner=faker.pyint(), authentication=SimpleAuthentication( username=faker.user_name(), password=faker.password() ), @@ -120,7 +152,9 @@ async def test_create_cluster( url = client.app.router["create_cluster"].url_for() rsp = await client.post( f"{url}", - json=json.loads(cluster_create.json(by_alias=True, exclude_unset=True)), + json=json.loads( + cluster_create.model_dump_json(by_alias=True, exclude_unset=True) + ), ) data, error = await assert_status( rsp, @@ -132,7 +166,7 @@ async def test_create_cluster( # we are done here return - created_cluster = Cluster.parse_obj(data) + created_cluster = Cluster.model_validate(data) assert created_cluster @@ -259,7 +293,9 @@ async def test_ping_cluster( print(f"--> pinging {cluster_ping=!r}") assert client.app url = client.app.router["ping_cluster"].url_for() - rsp = await client.post(f"{url}", json=json.loads(cluster_ping.json(by_alias=True))) + rsp = await client.post( + f"{url}", json=json.loads(cluster_ping.model_dump_json(by_alias=True)) + ) data, error = await assert_status(rsp, expected.no_content) if not error: assert data is None @@ -307,7 +343,9 @@ async def test_create_cluster_with_error( url = client.app.router["create_cluster"].url_for() rsp = await client.post( f"{url}", - json=json.loads(cluster_create.json(by_alias=True, exclude_unset=True)), + json=json.loads( + cluster_create.model_dump_json(by_alias=True, exclude_unset=True) + ), ) data, error = await assert_status(rsp, expected_http_error) assert not data @@ -408,7 +446,7 @@ async def test_update_cluster_with_error( url = client.app.router["update_cluster"].url_for(cluster_id=f"{25}") rsp = await client.patch( f"{url}", - json=json.loads(ClusterPatch().json(**_PATCH_EXPORT)), + json=json.loads(ClusterPatch().model_dump_json(**_PATCH_EXPORT)), ) data, error = await assert_status(rsp, expected_http_error) assert not data diff --git a/services/web/server/tests/unit/with_dbs/01/folders/test_folders.py b/services/web/server/tests/unit/with_dbs/01/folders/test_folders.py index 345e3875628..bb0c830ef27 100644 --- a/services/web/server/tests/unit/with_dbs/01/folders/test_folders.py +++ b/services/web/server/tests/unit/with_dbs/01/folders/test_folders.py @@ -58,7 +58,7 @@ async def test_folders_full_workflow( url = client.app.router["create_folder"].url_for() resp = await client.post(url.path, json={"name": "My first folder"}) added_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) - assert FolderGet.parse_obj(added_folder) + assert FolderGet.model_validate(added_folder) # list user folders url = client.app.router["list_folders"].url_for() @@ -76,9 +76,9 @@ async def test_folders_full_workflow( url = client.app.router["get_folder"].url_for( folder_id=f"{added_folder['folderId']}" ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert FolderGet.parse_obj(data) + assert FolderGet.model_validate(data) assert data["folderId"] == added_folder["folderId"] assert data["name"] == "My first folder" @@ -93,7 +93,7 @@ async def test_folders_full_workflow( }, ) data, _ = await assert_status(resp, status.HTTP_200_OK) - assert FolderGet.parse_obj(data) + assert FolderGet.model_validate(data) # list user folders url = client.app.router["list_folders"].url_for() @@ -157,7 +157,7 @@ async def test_sub_folders_full_workflow( # list user specific folder base_url = client.app.router["list_folders"].url_for() url = base_url.with_query({"folder_id": f"{subfolder_folder['folderId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 0 @@ -175,7 +175,7 @@ async def test_sub_folders_full_workflow( # list user subfolder folders base_url = client.app.router["list_folders"].url_for() url = base_url.with_query({"folder_id": f"{subfolder_folder['folderId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["name"] == "My sub sub folder" @@ -206,12 +206,12 @@ async def test_sub_folders_full_workflow( }, ) data, _ = await assert_status(resp, status.HTTP_200_OK) - assert FolderGet.parse_obj(data) + assert FolderGet.model_validate(data) # list user root folders base_url = client.app.router["list_folders"].url_for() url = base_url.with_query({"folder_id": "null"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 @@ -298,7 +298,7 @@ async def test_project_listing_inside_of_private_folder( # list project in user private folder base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"folder_id": f"{original_user_folder['folderId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == user_project["uuid"] @@ -310,7 +310,7 @@ async def test_project_listing_inside_of_private_folder( # Try to list folder that user doesn't have access to base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"folder_id": f"{original_user_folder['folderId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") _, errors = await assert_status( resp, status.HTTP_403_FORBIDDEN, @@ -330,7 +330,7 @@ async def test_project_listing_inside_of_private_folder( # list new user root folder base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"folder_id": "null"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == user_project["uuid"] @@ -353,7 +353,7 @@ async def test_project_listing_inside_of_private_folder( # list new user specific folder base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"folder_id": f"{new_user_folder['folderId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == user_project["uuid"] @@ -396,7 +396,7 @@ async def test_folders_deletion( url = client.app.router["create_folder"].url_for() resp = await client.post(url.path, json={"name": "My first folder"}) root_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) - assert FolderGet.parse_obj(root_folder) + assert FolderGet.model_validate(root_folder) # create a subfolder folder url = client.app.router["create_folder"].url_for() @@ -448,14 +448,14 @@ async def test_folders_deletion( # list subfolder projects base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"folder_id": f"{subfolder_2['folderId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == user_project["uuid"] # list root projects base_url = client.app.router["list_projects"].url_for() - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 0 @@ -474,13 +474,13 @@ async def test_folders_deletion( await assert_status(resp, status.HTTP_204_NO_CONTENT) fire_and_forget_tasks = client.app[APP_FIRE_AND_FORGET_TASKS_KEY] - t: asyncio.Task = list(fire_and_forget_tasks)[0] + t: asyncio.Task = next(iter(fire_and_forget_tasks)) assert t.get_name().startswith("fire_and_forget_task_delete_project_task_") await t assert len(client.app[APP_FIRE_AND_FORGET_TASKS_KEY]) == 0 # list root projects (The project should have been deleted) base_url = client.app.router["list_projects"].url_for() - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 0 diff --git a/services/web/server/tests/unit/with_dbs/01/products/test_products_db.py b/services/web/server/tests/unit/with_dbs/01/products/test_products_db.py index 4cd80c74a16..bd399948c14 100644 --- a/services/web/server/tests/unit/with_dbs/01/products/test_products_db.py +++ b/services/web/server/tests/unit/with_dbs/01/products/test_products_db.py @@ -138,9 +138,9 @@ async def test_product_repository_get_product( } # check RowProxy -> pydantic's Product - product = Product.from_orm(product_row) + product = Product.model_validate(product_row) - print(product.json(indent=1)) + print(product.model_dump_json(indent=1)) # product repo assert product_repository.engine diff --git a/services/web/server/tests/unit/with_dbs/01/products/test_products_rpc.py b/services/web/server/tests/unit/with_dbs/01/products/test_products_rpc.py index 3de1f7a95c8..4505a6f4e3e 100644 --- a/services/web/server/tests/unit/with_dbs/01/products/test_products_rpc.py +++ b/services/web/server/tests/unit/with_dbs/01/products/test_products_rpc.py @@ -10,7 +10,7 @@ from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE from models_library.products import CreditResultGet, ProductName from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -70,26 +70,26 @@ async def test_get_credit_amount( ): result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_credit_amount"), + TypeAdapter(RPCMethodName).validate_python("get_credit_amount"), dollar_amount=Decimal(900), product_name="s4l", ) - credit_result = parse_obj_as(CreditResultGet, result) + credit_result = CreditResultGet.model_validate(result) assert credit_result.credit_amount == 100 result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_credit_amount"), + TypeAdapter(RPCMethodName).validate_python("get_credit_amount"), dollar_amount=Decimal(900), product_name="tis", ) - credit_result = parse_obj_as(CreditResultGet, result) + credit_result = CreditResultGet.model_validate(result) assert credit_result.credit_amount == 180 with pytest.raises(RPCServerError) as exc_info: await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_credit_amount"), + TypeAdapter(RPCMethodName).validate_python("get_credit_amount"), dollar_amount=Decimal(900), product_name="osparc", ) diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py index a074c4d77e1..a03a5713e59 100644 --- a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py +++ b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py @@ -58,7 +58,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc ) plugin_settings = StudiesDispatcherSettings.create_from_envs() - print(plugin_settings.json(indent=1)) + print(plugin_settings.model_dump_json(indent=1)) return {**app_environment, **envs_plugins, **envs_studies_dispatcher} diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py index 420adf921ac..14f673ce5da 100644 --- a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -18,7 +18,7 @@ from aioresponses import aioresponses from common_library.json_serialization import json_dumps from models_library.projects_state import ProjectLocked, ProjectStatus -from pydantic import BaseModel, ByteSize, parse_obj_as +from pydantic import BaseModel, ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict, UserRole @@ -78,7 +78,9 @@ def web_server(redis_service: RedisSettings, web_server: TestServer) -> TestServ # # Extends web_server to start redis_service # - print("Redis service started with settings: ", redis_service.json(indent=1)) + print( + "Redis service started with settings: ", redis_service.model_dump_json(indent=1) + ) return web_server @@ -241,7 +243,7 @@ def test_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): print(example_name, ":", json_dumps(example_data)) - model = model_cls.parse_obj(example_data) + model = model_cls.model_validate(example_data) assert model @@ -253,7 +255,7 @@ async def test_api_list_services(client: TestClient): data, error = await assert_status(response, status.HTTP_200_OK) - services = parse_obj_as(list[ServiceGet], data) + services = TypeAdapter(list[ServiceGet]).validate_python(data) assert services # latest versions of services with everyone + ospar-product (see stmt_create_services_access_rights) @@ -350,7 +352,7 @@ def redirect_url(redirect_type: str, client: TestClient) -> URL: if redirect_type == "service_and_file": query = { "file_name": "users.csv", - "file_size": parse_obj_as(ByteSize, "100KB"), + "file_size": TypeAdapter(ByteSize).validate_python("100KB"), "file_type": "CSV", "viewer_key": "simcore/services/dynamic/raw-graphs", "viewer_version": "2.11.1", @@ -366,7 +368,7 @@ def redirect_url(redirect_type: str, client: TestClient) -> URL: elif redirect_type == "file_only": query = { "file_name": "users.csv", - "file_size": parse_obj_as(ByteSize, "1MiB"), + "file_size": TypeAdapter(ByteSize).validate_python("1MiB"), "file_type": "CSV", "download_link": URL( "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/8987c95d0ca0090e14f3a5b52db724fa24114cf5/services/storage/tests/data/users.csv" diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_projects.py index 48aacf56c6c..cd9bc502089 100644 --- a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_projects.py +++ b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_projects.py @@ -106,7 +106,7 @@ async def test_add_new_project_from_model_instance( project_id=project_id, service_id=viewer_id, owner=user, - service_info=ServiceInfo.parse_obj(viewer_info), + service_info=ServiceInfo.model_validate(viewer_info), ) else: project = _create_project_with_filepicker_and_service( diff --git a/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py b/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py index 7ad51c739d7..51467f3c822 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py +++ b/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py @@ -12,7 +12,7 @@ from models_library.api_schemas_webserver.auth import ApiKeyCreate from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -104,7 +104,7 @@ async def test_api_key_get( for api_key_name in fake_user_api_keys: result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "api_key_get"), + TypeAdapter(RPCMethodName).validate_python("api_key_get"), product_name=osparc_product_name, user_id=logged_user["id"], name=api_key_name, @@ -124,7 +124,7 @@ async def test_api_keys_workflow( # creating a key created_api_key = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "create_api_keys"), + TypeAdapter(RPCMethodName).validate_python("create_api_keys"), product_name=osparc_product_name, user_id=logged_user["id"], new=ApiKeyCreate(display_name=key_name, expiration=None), @@ -134,7 +134,7 @@ async def test_api_keys_workflow( # query the key is still present queried_api_key = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "api_key_get"), + TypeAdapter(RPCMethodName).validate_python("api_key_get"), product_name=osparc_product_name, user_id=logged_user["id"], name=key_name, @@ -146,7 +146,7 @@ async def test_api_keys_workflow( # remove the key delete_key_result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "delete_api_keys"), + TypeAdapter(RPCMethodName).validate_python("delete_api_keys"), product_name=osparc_product_name, user_id=logged_user["id"], name=key_name, @@ -156,7 +156,7 @@ async def test_api_keys_workflow( # key no longer present query_missing_query = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "api_key_get"), + TypeAdapter(RPCMethodName).validate_python("api_key_get"), product_name=osparc_product_name, user_id=logged_user["id"], name=key_name, diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__pricing_plan.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__pricing_plan.py index b328ddc4c7d..35733d100e6 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__pricing_plan.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__pricing_plan.py @@ -13,7 +13,6 @@ PricingPlanGet, ) from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import parse_obj_as from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict @@ -30,9 +29,8 @@ def mock_rut_api_responses( assert client.app settings: ResourceUsageTrackerSettings = get_plugin_settings(client.app) - service_pricing_plan_get = parse_obj_as( - PricingPlanGet, - PricingPlanGet.Config.schema_extra["examples"][0], + service_pricing_plan_get = PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][0], ) aioresponses_mocker.get( re.compile(f"^{settings.api_base_url}/services/+.+$"), diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index 396e3a1f8a0..96ada757900 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -19,7 +19,7 @@ from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -55,8 +55,8 @@ async def _list( assert product_name assert user_id - items = parse_obj_as( - list[ServiceGetV2], ServiceGetV2.Config.schema_extra["examples"] + items = TypeAdapter(list[ServiceGetV2]).validate_python( + ServiceGetV2.model_config["json_schema_extra"]["examples"], ) total_count = len(items) @@ -79,8 +79,8 @@ async def _get( assert product_name assert user_id - got = parse_obj_as( - ServiceGetV2, ServiceGetV2.Config.schema_extra["examples"][0] + got = ServiceGetV2.model_validate( + ServiceGetV2.model_config["json_schema_extra"]["examples"][0] ) got.version = service_version got.key = service_key @@ -100,12 +100,12 @@ async def _update( assert product_name assert user_id - got = parse_obj_as( - ServiceGetV2, ServiceGetV2.Config.schema_extra["examples"][0] + got = ServiceGetV2.model_validate( + ServiceGetV2.model_config["json_schema_extra"]["examples"][0] ) got.version = service_version got.key = service_key - return got.copy(update=update.dict(exclude_unset=True)) + return got.model_copy(update=update.model_dump(exclude_unset=True)) return { "list_services_paginated": mocker.patch( @@ -146,7 +146,7 @@ async def test_list_services_latest( assert data assert error is None - model = parse_obj_as(Page[CatalogServiceGet], data) + model = Page[CatalogServiceGet].model_validate(data) assert model assert model.data assert len(model.data) == model.meta.count @@ -180,7 +180,7 @@ async def test_get_and_patch_service( assert data assert error is None - model = parse_obj_as(CatalogServiceGet, data) + model = CatalogServiceGet.model_validate(data) assert model.key == service_key assert model.version == service_version @@ -205,7 +205,7 @@ async def test_get_and_patch_service( assert data assert error is None - model = parse_obj_as(CatalogServiceGet, data) + model = CatalogServiceGet.model_validate(data) assert model.key == service_key assert model.version == service_version assert model.name == update.name diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services_resources.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services_resources.py index afffca3652a..5c65109ef0a 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services_resources.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services_resources.py @@ -13,7 +13,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict @@ -32,9 +32,8 @@ def mock_catalog_service_api_responses( url_pattern = re.compile(f"^{settings.base_url}+/.+$") - service_resources = parse_obj_as( - ServiceResourcesDict, - ServiceResourcesDictHelpers.Config.schema_extra["examples"][0], + service_resources = TypeAdapter(ServiceResourcesDict).validate_python( + ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], ) jsonable_service_resources = ServiceResourcesDictHelpers.create_jsonable( service_resources diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups.py b/services/web/server/tests/unit/with_dbs/01/test_groups.py index f6e41225ff7..95682c21f56 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups.py @@ -64,7 +64,7 @@ def client( app = create_safe_application(cfg) settings = setup_settings(app) - print(settings.json(indent=1)) + print(settings.model_dump_json(indent=1)) setup_db(app) setup_session(app) diff --git a/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py b/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py index 9e46f5a27f7..00f3ce6d576 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py +++ b/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py @@ -3,7 +3,7 @@ # pylint: disable=too-many-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable - +import json import time from collections.abc import Callable from random import randint @@ -12,10 +12,12 @@ import pytest import redis.asyncio as aioredis from aiohttp import web +from faker import Faker from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from servicelib.aiohttp.application import create_safe_application from servicelib.aiohttp.application_setup import is_setup_completed from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.payments._methods_api import Faker from simcore_service_webserver.resource_manager.plugin import setup_resource_manager from simcore_service_webserver.resource_manager.registry import ( _ALIVE_SUFFIX, @@ -33,10 +35,12 @@ @pytest.fixture def mock_env_devel_environment( - mock_env_devel_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch + mock_env_devel_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch, faker: Faker ): return mock_env_devel_environment | setenvs_from_dict( - monkeypatch, {"RESOURCE_MANAGER_RESOURCE_TTL_S": "3"} + monkeypatch, { + "RESOURCE_MANAGER_RESOURCE_TTL_S": "3", + } ) diff --git a/services/web/server/tests/unit/with_dbs/01/test_statics.py b/services/web/server/tests/unit/with_dbs/01/test_statics.py index f6cd1191577..1edb437b20a 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_statics.py +++ b/services/web/server/tests/unit/with_dbs/01/test_statics.py @@ -68,7 +68,7 @@ def client( app = create_safe_application(cfg) settings = setup_settings(app) - print(settings.json(indent=1)) + print(settings.model_dump_json(indent=1)) setup_rest(app) setup_db(app) diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/conftest.py b/services/web/server/tests/unit/with_dbs/01/wallets/conftest.py index 1686507e13d..36164328005 100644 --- a/services/web/server/tests/unit/with_dbs/01/wallets/conftest.py +++ b/services/web/server/tests/unit/with_dbs/01/wallets/conftest.py @@ -7,6 +7,7 @@ from copy import deepcopy from pathlib import Path +from faker import Faker import pytest import sqlalchemy as sa from aioresponses import aioresponses @@ -23,6 +24,7 @@ def app_environment( app_environment: EnvVarsDict, env_devel_dict: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, + faker: Faker ): new_envs = setenvs_from_dict( monkeypatch, diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/conftest.py b/services/web/server/tests/unit/with_dbs/01/wallets/payments/conftest.py index 2b59b77c3b5..5fce5fad9cb 100644 --- a/services/web/server/tests/unit/with_dbs/01/wallets/payments/conftest.py +++ b/services/web/server/tests/unit/with_dbs/01/wallets/payments/conftest.py @@ -10,6 +10,7 @@ from typing import Any, TypeAlias, cast from unittest.mock import MagicMock +import pycountry import pytest import sqlalchemy as sa from aiohttp import web @@ -79,7 +80,7 @@ async def _create(): }, ) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - return WalletGet.parse_obj(data) + return WalletGet.model_validate(data) return _create @@ -334,7 +335,7 @@ def setup_user_pre_registration_details_db( address=faker.address().replace("\n", ", "), city=faker.city(), state=faker.state(), - country=faker.country(), + country=faker.random_element([c.name for c in pycountry.countries]), postal_code=faker.postcode(), created_by=None, ) diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments.py b/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments.py index f6519735ed1..719eb7e6dc8 100644 --- a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments.py +++ b/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments.py @@ -17,7 +17,7 @@ WalletPaymentInitiated, ) from models_library.rest_pagination import Page -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import LoggedUser, NewUser, UserInfoDict @@ -105,7 +105,7 @@ async def test_one_time_payment_worfklow( data, error = await assert_status(response, expected_status) if not error: - payment = WalletPaymentInitiated.parse_obj(data) + payment = WalletPaymentInitiated.model_validate(data) assert payment.payment_id assert payment.payment_form_url @@ -134,7 +134,7 @@ async def test_one_time_payment_worfklow( response = await client.get("/v0/wallets/-/payments") data, error = await assert_status(response, status.HTTP_200_OK) - page = parse_obj_as(Page[PaymentTransaction], data) + page = Page[PaymentTransaction].model_validate(data) assert page.data assert page.meta.total == 1 @@ -200,7 +200,7 @@ async def test_multiple_payments( data, error = await assert_status(response, status.HTTP_201_CREATED) assert data assert not error - payment = WalletPaymentInitiated.parse_obj(data) + payment = WalletPaymentInitiated.model_validate(data) if n % 2: transaction = await _ack_creation_of_wallet_payment( @@ -233,7 +233,7 @@ async def test_multiple_payments( response = await client.get("/v0/wallets/-/payments") data, error = await assert_status(response, status.HTTP_200_OK) - page = parse_obj_as(Page[PaymentTransaction], data) + page = Page[PaymentTransaction].model_validate(data) assert page.meta.total == num_payments all_transactions = {t.payment_id: t for t in page.data} @@ -286,7 +286,7 @@ async def test_complete_payment_errors( assert mock_rpc_payments_service_api["init_payment"].called data, _ = await assert_status(response, status.HTTP_201_CREATED) - payment = WalletPaymentInitiated.parse_obj(data) + payment = WalletPaymentInitiated.model_validate(data) # Cannot complete as PENDING with pytest.raises(ValueError): @@ -362,9 +362,11 @@ async def test_payment_not_found( def test_payment_transaction_state_and_literals_are_in_sync(): - state_literals = PaymentTransaction.__fields__["state"].type_ + state_literals = PaymentTransaction.model_fields["state"].annotation assert ( - parse_obj_as(list[state_literals], [f"{s}" for s in PaymentTransactionState]) + TypeAdapter(list[state_literals]).validate_python( + [f"{s}" for s in PaymentTransactionState] + ) is not None ) diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_methods.py b/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_methods.py index 0980e45caa2..d10109a5295 100644 --- a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_methods.py +++ b/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_methods.py @@ -22,7 +22,7 @@ ) from models_library.rest_pagination import Page from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import status @@ -65,7 +65,7 @@ async def test_payment_method_worfklow( ) data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert error is None - inited = PaymentMethodInitiated.parse_obj(data) + inited = PaymentMethodInitiated.model_validate(data) assert inited.payment_method_id assert inited.payment_method_form_url.query @@ -103,7 +103,7 @@ async def test_payment_method_worfklow( data, _ = await assert_status(response, status.HTTP_200_OK) assert mock_rpc_payments_service_api["list_payment_methods"].called - wallet_payments_methods = parse_obj_as(list[PaymentMethodGet], data) + wallet_payments_methods = TypeAdapter(list[PaymentMethodGet]).validate_python(data) assert wallet_payments_methods == [payment_method] # Delete @@ -140,7 +140,7 @@ async def test_init_and_cancel_payment_method( ) data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert error is None - inited = PaymentMethodInitiated.parse_obj(data) + inited = PaymentMethodInitiated.model_validate(data) # cancel Create response = await client.post( @@ -165,7 +165,7 @@ async def _add_payment_method( ) data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert error is None - inited = PaymentMethodInitiated.parse_obj(data) + inited = PaymentMethodInitiated.model_validate(data) await _ack_creation_of_wallet_payment_method( client.app, payment_method_id=inited.payment_method_id, @@ -249,7 +249,7 @@ async def test_wallet_autorecharge( ) data, error = await assert_status(response, expected_status) if not error: - updated_auto_recharge = GetWalletAutoRecharge.parse_obj(data) + updated_auto_recharge = GetWalletAutoRecharge.model_validate(data) assert updated_auto_recharge == GetWalletAutoRecharge( payment_method_id=payment_method_id, min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, @@ -263,12 +263,14 @@ async def test_wallet_autorecharge( f"/v0/wallets/{wallet.wallet_id}/auto-recharge", ) data, _ = await assert_status(response, status.HTTP_200_OK) - assert updated_auto_recharge == GetWalletAutoRecharge.parse_obj(data) + assert updated_auto_recharge == GetWalletAutoRecharge.model_validate(data) # payment-methods.auto_recharge response = await client.get(f"/v0/wallets/{wallet.wallet_id}/payments-methods") data, _ = await assert_status(response, status.HTTP_200_OK) - wallet_payment_methods = parse_obj_as(list[PaymentMethodGet], data) + wallet_payment_methods = TypeAdapter(list[PaymentMethodGet]).validate_python( + data + ) for payment_method in wallet_payment_methods: assert payment_method.auto_recharge == ( @@ -305,7 +307,7 @@ async def test_delete_primary_payment_method_in_autorecharge( }, ) data, _ = await assert_status(response, status.HTTP_200_OK) - auto_recharge = GetWalletAutoRecharge.parse_obj(data) + auto_recharge = GetWalletAutoRecharge.model_validate(data) assert auto_recharge.enabled is True assert auto_recharge.payment_method_id == payment_method_id assert auto_recharge.monthly_limit_in_usd == 123 @@ -321,7 +323,7 @@ async def test_delete_primary_payment_method_in_autorecharge( f"/v0/wallets/{wallet.wallet_id}/auto-recharge", ) data, _ = await assert_status(response, status.HTTP_200_OK) - auto_recharge_after_delete = GetWalletAutoRecharge.parse_obj(data) + auto_recharge_after_delete = GetWalletAutoRecharge.model_validate(data) assert auto_recharge_after_delete.payment_method_id is None assert auto_recharge_after_delete.enabled is False @@ -334,7 +336,7 @@ async def test_delete_primary_payment_method_in_autorecharge( f"/v0/wallets/{wallet.wallet_id}/auto-recharge", ) data, _ = await assert_status(response, status.HTTP_200_OK) - auto_recharge = GetWalletAutoRecharge.parse_obj(data) + auto_recharge = GetWalletAutoRecharge.model_validate(data) assert auto_recharge.payment_method_id == new_payment_method_id assert auto_recharge.enabled is False @@ -398,7 +400,7 @@ async def test_one_time_payment_with_payment_method( ) data, error = await assert_status(response, expected_status) if not error: - payment = WalletPaymentInitiated.parse_obj(data) + payment = WalletPaymentInitiated.model_validate(data) assert mock_rpc_payments_service_api["pay_with_payment_method"].called assert payment.payment_id @@ -417,7 +419,7 @@ async def test_one_time_payment_with_payment_method( response = await client.get("/v0/wallets/-/payments") data, error = await assert_status(response, status.HTTP_200_OK) - page = parse_obj_as(Page[PaymentTransaction], data) + page = Page[PaymentTransaction].model_validate(data) assert page.data assert page.meta.total == 1 diff --git a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_rpc.py b/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_rpc.py index 756c008adba..af0f7d304ca 100644 --- a/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_rpc.py +++ b/services/web/server/tests/unit/with_dbs/01/wallets/payments/test_payments_rpc.py @@ -12,7 +12,7 @@ from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE from models_library.payments import InvoiceDataGet from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict @@ -77,11 +77,11 @@ async def test_one_time_payment_worfklow( result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_invoice_data"), + TypeAdapter(RPCMethodName).validate_python("get_invoice_data"), user_id=logged_user["id"], dollar_amount=Decimal(900), product_name="osparc", ) - invoice_data_get = parse_obj_as(InvoiceDataGet, result) + invoice_data_get = InvoiceDataGet.model_validate(result) assert invoice_data_get assert len(invoice_data_get.user_invoice_address.country) == 2 diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces.py b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces.py index e2ace9daa6a..25de918f878 100644 --- a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces.py @@ -59,7 +59,7 @@ async def test_workspaces_workflow( }, ) added_workspace, _ = await assert_status(resp, status.HTTP_201_CREATED) - assert WorkspaceGet.parse_obj(added_workspace) + assert WorkspaceGet.model_validate(added_workspace) # list user workspaces url = client.app.router["list_workspaces"].url_for() @@ -78,7 +78,7 @@ async def test_workspaces_workflow( url = client.app.router["get_workspace"].url_for( workspace_id=f"{added_workspace['workspaceId']}" ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] == added_workspace["workspaceId"] assert data["name"] == "My first workspace" @@ -96,7 +96,7 @@ async def test_workspaces_workflow( }, ) data, _ = await assert_status(resp, status.HTTP_200_OK) - assert WorkspaceGet.parse_obj(data) + assert WorkspaceGet.model_validate(data) # list user workspaces url = client.app.router["list_workspaces"].url_for() diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__folders_and_projects_crud.py index c95aebe6fdd..0ebb5518b28 100644 --- a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__folders_and_projects_crud.py @@ -79,7 +79,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( # List project in workspace base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"workspace_id": f"{added_workspace['workspaceId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] @@ -88,7 +88,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["uuid"] == project["uuid"] assert data["workspaceId"] == added_workspace["workspaceId"] @@ -110,7 +110,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( url = base_url.with_query( {"workspace_id": f"{added_workspace['workspaceId']}", "folder_id": "null"} ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["folderId"] == first_folder["folderId"] @@ -131,7 +131,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( "folder_id": f"{first_folder['folderId']}", } ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] @@ -142,7 +142,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( # Try to list folder that user doesn't have access to base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"workspace_id": f"{added_workspace['workspaceId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") _, errors = await assert_status( resp, status.HTTP_403_FORBIDDEN, @@ -164,7 +164,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( url = base_url.with_query( {"workspace_id": f"{added_workspace['workspaceId']}", "folder_id": "null"} ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 @@ -176,7 +176,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( "folder_id": "none", } ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 0 @@ -188,7 +188,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( "folder_id": f"{first_folder['folderId']}", } ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] @@ -230,7 +230,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( url = base_url.with_query( {"workspace_id": f"{added_workspace['workspaceId']}", "folder_id": "null"} ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 @@ -299,7 +299,7 @@ async def test_workspaces_delete_folders( # List project in workspace base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"workspace_id": f"{added_workspace['workspaceId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 @@ -362,6 +362,7 @@ async def test_workspaces_delete_folders( # List project in workspace (The projects should have been deleted) base_url = client.app.router["list_projects"].url_for() url = base_url.with_query({"workspace_id": f"{added_workspace['workspaceId']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 0 + diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__list_projects_full_search.py b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__list_projects_full_search.py index a7efd64b485..6c146bb5a1f 100644 --- a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__list_projects_full_search.py +++ b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__list_projects_full_search.py @@ -82,7 +82,7 @@ async def test_workspaces__list_projects_full_search( # List project with full search base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "solution"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project_1["uuid"] @@ -104,7 +104,7 @@ async def test_workspaces__list_projects_full_search( # List project with full search base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "Orion"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project_2["uuid"] @@ -137,7 +137,7 @@ async def test_workspaces__list_projects_full_search( # List project with full search base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "Skyline"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project_3["uuid"] @@ -147,7 +147,7 @@ async def test_workspaces__list_projects_full_search( # List project with full search (it should return data across all workspaces/folders) base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "solution"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) sorted_data = sorted(data, key=lambda x: x["uuid"]) assert len(sorted_data) == 3 @@ -190,7 +190,7 @@ async def test__list_projects_full_search_with_query_parameters( # Full search with text base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "Orion"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] @@ -203,7 +203,7 @@ async def test__list_projects_full_search_with_query_parameters( "order_by": json.dumps({"field": "uuid", "direction": "desc"}), } ) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] @@ -211,7 +211,7 @@ async def test__list_projects_full_search_with_query_parameters( # Full search with tag_ids base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "Orion", "tag_ids": "1,2"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 0 @@ -232,7 +232,7 @@ async def test__list_projects_full_search_with_query_parameters( # Full search with tag_ids base_url = client.app.router["list_projects_full_search"].url_for() url = base_url.with_query({"text": "Orion", "tag_ids": f"{added_tag['id']}"}) - resp = await client.get(url) + resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["uuid"] == project["uuid"] diff --git a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__moving_projects_between_workspaces.py index f583b9e3ecf..21b16ea9738 100644 --- a/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/01/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -58,7 +58,7 @@ async def test_moving_between_workspaces_user_role_permissions( base_url = client.app.router["replace_project_workspace"].url_for( project_id=fake_project["uuid"], workspace_id="null" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") await assert_status(resp, expected.no_content) @@ -98,7 +98,7 @@ async def test_moving_between_private_and_shared_workspaces( # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID @@ -106,12 +106,12 @@ async def test_moving_between_private_and_shared_workspaces( base_url = client.app.router["replace_project_workspace"].url_for( project_id=project["uuid"], workspace_id="null" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] is None # <-- Workspace ID is None @@ -119,12 +119,12 @@ async def test_moving_between_private_and_shared_workspaces( base_url = client.app.router["replace_project_workspace"].url_for( project_id=project["uuid"], workspace_id=f"{added_workspace['workspaceId']}" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID @@ -177,7 +177,7 @@ async def test_moving_between_shared_and_shared_workspaces( # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID @@ -185,12 +185,12 @@ async def test_moving_between_shared_and_shared_workspaces( base_url = client.app.router["replace_project_workspace"].url_for( project_id=project["uuid"], workspace_id=f"{second_workspace['workspaceId']}" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] == second_workspace["workspaceId"] # <-- Workspace ID @@ -257,7 +257,7 @@ async def test_moving_between_workspaces_check_removed_from_folder( # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] == added_workspace["workspaceId"] # <-- Workspace ID @@ -265,12 +265,12 @@ async def test_moving_between_workspaces_check_removed_from_folder( base_url = client.app.router["replace_project_workspace"].url_for( project_id=project["uuid"], workspace_id="none" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) # Get project in workspace base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["workspaceId"] is None # <-- Workspace ID is None diff --git a/services/web/server/tests/unit/with_dbs/02/conftest.py b/services/web/server/tests/unit/with_dbs/02/conftest.py index dbc3890f0b9..7e765694d3d 100644 --- a/services/web/server/tests/unit/with_dbs/02/conftest.py +++ b/services/web/server/tests/unit/with_dbs/02/conftest.py @@ -24,7 +24,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -39,9 +39,8 @@ @pytest.fixture def mock_service_resources() -> ServiceResourcesDict: - return parse_obj_as( - ServiceResourcesDict, - ServiceResourcesDictHelpers.Config.schema_extra["examples"][0], + return TypeAdapter(ServiceResourcesDict).validate_python( + ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], ) @@ -255,7 +254,7 @@ async def _assert_it( ) -> dict: # GET /v0/projects/{project_id} with a project owned by user url = client.app.router["get_project"].url_for(project_id=project["uuid"]) - resp = await client.get(url) + resp = await client.get(f"{url}") data, error = await assert_status(resp, expected) if not error: @@ -509,4 +508,4 @@ def workbench_db_column() -> dict[str, Any]: @pytest.fixture def workbench(workbench_db_column: dict[str, Any]) -> dict[NodeID, Node]: # convert to model - return parse_obj_as(dict[NodeID, Node], workbench_db_column) + return TypeAdapter(dict[NodeID, Node]).validate_python(workbench_db_column) diff --git a/services/web/server/tests/unit/with_dbs/02/test_announcements.py b/services/web/server/tests/unit/with_dbs/02/test_announcements.py index cd87e2526c6..19ca7413827 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_announcements.py +++ b/services/web/server/tests/unit/with_dbs/02/test_announcements.py @@ -185,7 +185,7 @@ async def test_list_announcements_filtered( def test_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): - assert model_cls.parse_obj( + assert model_cls.model_validate( example_data ), f"Failed {example_name} : {json.dumps(example_data)}" @@ -193,7 +193,7 @@ def test_model_examples( def test_invalid_announcement(faker: Faker): now = arrow.utcnow() with pytest.raises(ValidationError): - Announcement.parse_obj( + Announcement.model_validate( { "id": "Student_Competition_2023", "products": ["s4llite", "osparc"], @@ -209,7 +209,7 @@ def test_invalid_announcement(faker: Faker): def test_announcement_expired(faker: Faker): now = arrow.utcnow() - model = Announcement.parse_obj( + model = Announcement.model_validate( { "id": "Student_Competition_2023", "products": ["s4llite", "osparc"], diff --git a/services/web/server/tests/unit/with_dbs/02/test_project_lock.py b/services/web/server/tests/unit/with_dbs/02/test_project_lock.py index f70a44c7bbc..5c33586e151 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_project_lock.py +++ b/services/web/server/tests/unit/with_dbs/02/test_project_lock.py @@ -12,7 +12,7 @@ from models_library.projects_access import Owner from models_library.projects_state import ProjectLocked, ProjectStatus from models_library.users import UserID -from pydantic import parse_raw_as +from pydantic import TypeAdapter from simcore_service_webserver.projects.exceptions import ProjectLockError from simcore_service_webserver.projects.lock import ( PROJECT_REDIS_LOCK_KEY, @@ -51,7 +51,7 @@ async def test_lock_project( PROJECT_REDIS_LOCK_KEY.format(project_uuid) ) assert redis_value - lock_value = parse_raw_as(ProjectLocked, redis_value) + lock_value = TypeAdapter(ProjectLocked).validate_json(redis_value) assert lock_value == ProjectLocked( value=True, owner=Owner(user_id=user_id, **user_fullname), @@ -137,7 +137,7 @@ async def test_is_project_locked( faker: Faker, ): assert client.app - assert await is_project_locked(client.app, project_uuid) == False + assert await is_project_locked(client.app, project_uuid) is False user_name: FullNameDict = { "first_name": faker.first_name(), "last_name": faker.last_name(), @@ -149,7 +149,7 @@ async def test_is_project_locked( user_id=user_id, user_fullname=user_name, ): - assert await is_project_locked(client.app, project_uuid) == True + assert await is_project_locked(client.app, project_uuid) is True @pytest.mark.parametrize( @@ -170,9 +170,9 @@ async def test_get_project_locked_state( ): assert client.app # no lock - assert await get_project_locked_state(client.app, project_uuid) == None + assert await get_project_locked_state(client.app, project_uuid) is None - assert await is_project_locked(client.app, project_uuid) == False + assert await is_project_locked(client.app, project_uuid) is False user_name: FullNameDict = { "first_name": faker.first_name(), "last_name": faker.last_name(), diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py index 6c841fa8650..74c932ca600 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py @@ -9,7 +9,7 @@ import pytest from aiohttp.test_utils import TestClient -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -150,14 +150,14 @@ async def test_copying_large_project_and_retrieving_copy_task( create_url = create_url.with_query(from_study=user_project["uuid"]) resp = await client.post(f"{create_url}", json={}) data, error = await assert_status(resp, expected.accepted) - created_copy_task = TaskGet.parse_obj(data) + created_copy_task = TaskGet.model_validate(data) # list current tasks list_task_url = client.app.router["list_tasks"].url_for() resp = await client.get(f"{list_task_url}") data, error = await assert_status(resp, expected.ok) assert data assert not error - list_of_tasks = parse_obj_as(list[TaskGet], data) + list_of_tasks = TypeAdapter(list[TaskGet]).validate_python(data) assert len(list_of_tasks) == 1 task = list_of_tasks[0] assert task.task_name == f"POST {create_url}" @@ -290,9 +290,9 @@ async def test_copying_too_large_project_returns_422( large_project_total_size = ( app_settings.WEBSERVER_PROJECTS.PROJECTS_MAX_COPY_SIZE_BYTES + 1 ) - storage_subsystem_mock.get_project_total_size_simcore_s3.return_value = ( - parse_obj_as(ByteSize, large_project_total_size) - ) + storage_subsystem_mock.get_project_total_size_simcore_s3.return_value = TypeAdapter( + ByteSize + ).validate_python(large_project_total_size) # POST /v0/projects await request_create_project( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_comments_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_comments_handlers.py index 13fe3a7633a..604ee40308c 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_comments_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_comments_handlers.py @@ -42,7 +42,7 @@ async def test_project_comments_user_role_access( base_url = client.app.router["list_project_comments"].url_for( project_uuid=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") assert resp.status == 401 if user_role == UserRole.ANONYMOUS else 200 @@ -65,7 +65,7 @@ async def test_project_comments_full_workflow( base_url = client.app.router["list_project_comments"].url_for( project_uuid=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _, meta, links = await assert_status( resp, expected, @@ -78,7 +78,7 @@ async def test_project_comments_full_workflow( # Now we will add first comment body = {"contents": "My first comment"} - resp = await client.post(base_url, json=body) + resp = await client.post(f"{base_url}", json=body) data, _ = await assert_status( resp, status.HTTP_201_CREATED, @@ -86,7 +86,7 @@ async def test_project_comments_full_workflow( first_comment_id = data["comment_id"] # Now we will add second comment - resp = await client.post(base_url, json={"contents": "My second comment"}) + resp = await client.post(f"{base_url}", json={"contents": "My second comment"}) data, _ = await assert_status( resp, status.HTTP_201_CREATED, @@ -94,7 +94,7 @@ async def test_project_comments_full_workflow( second_comment_id = data["comment_id"] # Now we will list all comments for the project - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _, meta, links = await assert_status( resp, expected, @@ -108,7 +108,7 @@ async def test_project_comments_full_workflow( # Now we will update the second comment updated_comment = "Updated second comment" resp = await client.put( - base_url / f"{second_comment_id}", + f"{base_url}/{second_comment_id}", json={"contents": updated_comment}, ) data, _ = await assert_status( @@ -117,7 +117,7 @@ async def test_project_comments_full_workflow( ) # Now we will get the second comment - resp = await client.get(base_url / f"{second_comment_id}") + resp = await client.get(f"{base_url}/{second_comment_id}") data, _ = await assert_status( resp, expected, @@ -125,14 +125,14 @@ async def test_project_comments_full_workflow( assert data["contents"] == updated_comment # Now we will delete the second comment - resp = await client.delete(base_url / f"{second_comment_id}") + resp = await client.delete(f"{base_url}/{second_comment_id}") data, _ = await assert_status( resp, status.HTTP_204_NO_CONTENT, ) # Now we will list all comments for the project - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _, meta, links = await assert_status( resp, expected, @@ -146,14 +146,14 @@ async def test_project_comments_full_workflow( # Now we will log as a different user async with LoggedUser(client) as new_logged_user: # As this user does not have access to the project, they should get 403 - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") _, errors = await assert_status( resp, status.HTTP_403_FORBIDDEN, ) assert errors - resp = await client.get(base_url / f"{first_comment_id}") + resp = await client.get(f"{base_url}/{first_comment_id}") _, errors = await assert_status( resp, status.HTTP_403_FORBIDDEN, @@ -173,7 +173,7 @@ async def test_project_comments_full_workflow( # Now the user should have access to the project now # New user will add comment resp = await client.post( - base_url, + f"{base_url}", json={"contents": "My first comment as a new user"}, ) data, _ = await assert_status( @@ -185,7 +185,7 @@ async def test_project_comments_full_workflow( # New user will modify the comment updated_comment = "Updated My first comment as a new user" resp = await client.put( - base_url / f"{new_user_comment_id}", + f"{base_url}/{new_user_comment_id}", json={"contents": updated_comment}, ) data, _ = await assert_status( @@ -195,7 +195,7 @@ async def test_project_comments_full_workflow( assert data["contents"] == updated_comment # New user will list all comments - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _, meta, links = await assert_status( resp, expected, @@ -209,7 +209,7 @@ async def test_project_comments_full_workflow( # New user will modify comment of the previous user updated_comment = "Updated comment of previous user" resp = await client.put( - base_url / f"{first_comment_id}", + f"{base_url}/{first_comment_id}", json={"contents": updated_comment}, ) data, _ = await assert_status( @@ -219,7 +219,7 @@ async def test_project_comments_full_workflow( assert data["contents"] == updated_comment # New user will delete comment of the previous user - resp = await client.delete(base_url / f"{first_comment_id}") + resp = await client.delete(f"{base_url}/{first_comment_id}") data, _ = await assert_status( resp, status.HTTP_204_NO_CONTENT, diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index bf69984d6af..99cf4e105d4 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -18,7 +18,9 @@ from faker import Faker from models_library.products import ProductName from models_library.projects_state import ProjectState -from pydantic import parse_obj_as +from models_library.services import ServiceKey +from models_library.utils.fastapi_encoders import jsonable_encoder +from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_parametrizations import ( @@ -168,10 +170,10 @@ async def _assert_get_same_project( assert data == project if project_state: - assert parse_obj_as(ProjectState, project_state) + assert ProjectState.model_validate(project_state) if project_permalink: - assert parse_obj_as(ProjectPermalink, project_permalink) + assert ProjectPermalink.model_validate(project_permalink) assert folder_id is None @@ -209,7 +211,7 @@ async def test_list_projects( assert not ProjectState( **project_state ).locked.value, "Templates are not locked" - assert parse_obj_as(ProjectPermalink, project_permalink) + assert ProjectPermalink.model_validate(project_permalink) # standard project project_state = data[1].pop("state") @@ -252,7 +254,7 @@ async def test_list_projects( assert not ProjectState( **project_state ).locked.value, "Templates are not locked" - assert parse_obj_as(ProjectPermalink, project_permalink) + assert ProjectPermalink.model_validate(project_permalink) @pytest.fixture(scope="session") @@ -432,7 +434,7 @@ async def test_new_project_from_template( if new_project: # check uuid replacement for node_name in new_project["workbench"]: - parse_obj_as(uuidlib.UUID, node_name) + TypeAdapter(uuidlib.UUID).validate_python(node_name) @pytest.mark.parametrize(*standard_user_role_response()) @@ -461,7 +463,7 @@ async def test_new_project_from_other_study( # check uuid replacement assert new_project["name"].endswith("(Copy)") for node_name in new_project["workbench"]: - parse_obj_as(uuidlib.UUID, node_name) + TypeAdapter(uuidlib.UUID).validate_python(node_name) @pytest.mark.parametrize(*standard_user_role_response()) @@ -515,7 +517,7 @@ async def test_new_project_from_template_with_body( # check uuid replacement for node_name in project["workbench"]: - parse_obj_as(uuidlib.UUID, node_name) + TypeAdapter(uuidlib.UUID).validate_python(node_name) @pytest.mark.parametrize(*standard_user_role_response()) @@ -571,7 +573,7 @@ async def test_new_template_from_project( # check uuid replacement for node_name in template_project["workbench"]: - parse_obj_as(uuidlib.UUID, node_name) + TypeAdapter(uuidlib.UUID).validate_python(node_name) # do the same with a body predefined = { @@ -631,7 +633,7 @@ async def test_new_template_from_project( # check uuid replacement for node_name in template_project["workbench"]: - parse_obj_as(uuidlib.UUID, node_name) + TypeAdapter(uuidlib.UUID).validate_python(node_name) @pytest.fixture diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone.py index 6ca7392dd4b..657b19e20d6 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone.py @@ -14,7 +14,7 @@ from faker import Faker from models_library.api_schemas_webserver.projects import ProjectGet from models_library.projects import ProjectID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_parametrizations import ( MockedStorageSubsystem, @@ -49,7 +49,7 @@ async def _request_clone_project(client: TestClient, url: URL) -> ProjectGet: data = await long_running_task.result() assert data is not None - return ProjectGet.parse_obj(data) + return ProjectGet.model_validate(data) @pytest.mark.parametrize(*standard_role_response(), ids=str) @@ -105,9 +105,9 @@ async def test_clone_project( # check whether it's a clone assert ProjectID(project["uuid"]) != cloned_project.uuid assert project["description"] == cloned_project.description - assert parse_obj_as(datetime, project["creationDate"]) < parse_obj_as( - datetime, cloned_project.creation_date - ) + assert TypeAdapter(datetime).validate_python(project["creationDate"]) < TypeAdapter( + datetime + ).validate_python(cloned_project.creation_date) assert len(project["workbench"]) == len(cloned_project.workbench) assert set(project["workbench"].keys()) != set( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py index 051f522fcd9..18ba745eaee 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py @@ -80,7 +80,7 @@ async def _request_clone_project(client: TestClient, url: URL) -> ProjectGet: data = await long_running_task.result() assert data is not None - return ProjectGet.parse_obj(data) + return ProjectGet.model_validate(data) @pytest.mark.parametrize( @@ -114,7 +114,7 @@ async def test_clone_project( "folder_id": f"{create_workspace_and_folder[1]}", } url = base_url.with_query(**query_parameters) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 assert len(data["data"]) == 1 @@ -135,7 +135,7 @@ async def test_clone_project( "folder_id": f"{create_workspace_and_folder[1]}", } url = base_url.with_query(**query_parameters) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 assert len(data["data"]) == 2 diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py index dae689d1974..a9f111e9c4a 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py @@ -55,7 +55,7 @@ async def _list_projects( if query_parameters: url = url.with_query(**query_parameters) - resp = await client.get(url) + resp = await client.get(f"{url}") data, errors, meta, links = await assert_status( resp, expected, @@ -121,9 +121,9 @@ def standard_user_role() -> tuple[str, tuple[UserRole, ExpectedResponse]]: @pytest.mark.parametrize( "limit, offset, expected_error_msg", [ - (-7, 0, "ensure this value is greater than or equal to 1"), - (0, 0, "ensure this value is greater than or equal to 1"), - (43, -2, "ensure this value is greater than or equal to 0"), + (-7, 0, "Input should be greater than or equal to 1"), + (0, 0, "Input should be greater than or equal to 1"), + (43, -2, "Input should be greater than or equal to 0"), ], ) @pytest.mark.parametrize(*standard_user_role()) @@ -145,7 +145,7 @@ async def test_list_projects_with_invalid_pagination_parameters( status.HTTP_422_UNPROCESSABLE_ENTITY, query_parameters={"limit": limit, "offset": offset}, expected_error_msg=expected_error_msg, - expected_error_code="value_error.number.not_ge", + expected_error_code="greater_than_equal", ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py index 89b7fed1544..abb26a3f3e3 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py @@ -153,7 +153,7 @@ async def test_list_projects_with_search_parameter( base_url = client.app.router["list_projects"].url_for() assert f"{base_url}" == f"/{api_version_prefix}/projects" - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data = await resp.json() assert resp.status == 200 @@ -164,7 +164,7 @@ async def test_list_projects_with_search_parameter( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?search=" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -175,7 +175,7 @@ async def test_list_projects_with_search_parameter( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?search=nAmE+5" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -188,7 +188,7 @@ async def test_list_projects_with_search_parameter( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?search=2-fe1b-11ed-b038-cdb1" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -207,7 +207,7 @@ async def test_list_projects_with_search_parameter( == f"/{api_version_prefix}/projects?search={user_name_substring_query_parsed}" ) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -225,7 +225,7 @@ async def test_list_projects_with_search_parameter( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?search=oda" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -236,7 +236,7 @@ async def test_list_projects_with_search_parameter( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?search=does+not+exists" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -249,7 +249,7 @@ async def test_list_projects_with_search_parameter( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?search=oda&offset=0&limit=1" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -327,7 +327,7 @@ async def test_list_projects_with_order_by_parameter( f"{url}" == f"/{api_version_prefix}/projects?order_by=%7B%22field%22:+%22uuid%22,+%22direction%22:+%22asc%22%7D" ) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 assert [item["uuid"][0] for item in data["data"]] == _alphabetically_ordered_list @@ -337,7 +337,7 @@ async def test_list_projects_with_order_by_parameter( url = base_url.with_query( order_by=json.dumps({"field": "uuid", "direction": "desc"}) ) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 assert [item["uuid"][0] for item in data["data"]] == _alphabetically_ordered_list[ @@ -349,7 +349,7 @@ async def test_list_projects_with_order_by_parameter( url = base_url.with_query( order_by=json.dumps({"field": "name", "direction": "asc"}) ) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 assert [item["name"][0] for item in data["data"]] == _alphabetically_ordered_list @@ -359,7 +359,7 @@ async def test_list_projects_with_order_by_parameter( url = base_url.with_query( order_by=json.dumps({"field": "description", "direction": "asc"}) ) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 assert [ @@ -453,7 +453,7 @@ async def test_list_projects_for_specific_folder_id( base_url = client.app.router["list_projects"].url_for() assert f"{base_url}" == f"/{api_version_prefix}/projects" - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data = await resp.json() assert resp.status == 200 @@ -463,7 +463,7 @@ async def test_list_projects_for_specific_folder_id( query_parameters = {"folder_id": "null"} url = base_url.with_query(**query_parameters) - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 @@ -476,7 +476,7 @@ async def test_list_projects_for_specific_folder_id( url = base_url.with_query(**query_parameters) assert f"{url}" == f"/{api_version_prefix}/projects?folder_id={setup_folders_db}" - resp = await client.get(url) + resp = await client.get(f"{url}") data = await resp.json() assert resp.status == 200 diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 7a7056d7bbc..c960a86fa13 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -18,7 +18,7 @@ ) from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_parametrizations import ( @@ -83,12 +83,12 @@ async def test_custom_metadata_handlers( project_id=user_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).model_dump() ) data, _ = await assert_status(response, expected_status_code=expected.ok) - assert parse_obj_as(ProjectMetadataGet, data).custom == custom_metadata + assert ProjectMetadataGet.model_validate(data).custom == custom_metadata # delete project url = client.app.router["delete_project"].url_for(project_id=user_project["uuid"]) @@ -138,9 +138,9 @@ async def test_new_project_with_parent_project_node( ) assert parent_project - parent_project_uuid = parse_obj_as(ProjectID, parent_project["uuid"]) - parent_node_id = parse_obj_as( - NodeID, random.choice(list(parent_project["workbench"])) # noqa: S311 + parent_project_uuid = TypeAdapter(ProjectID).validate_python(parent_project["uuid"]) + parent_node_id = TypeAdapter(NodeID).validate_python( + random.choice(list(parent_project["workbench"])) # noqa: S311 ) child_project = await request_create_project( client, @@ -175,10 +175,10 @@ async def test_new_project_with_parent_project_node( project_id=child_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).model_dump() ) data, _ = await assert_status(response, expected_status_code=status.HTTP_200_OK) - assert parse_obj_as(ProjectMetadataGet, data).custom == custom_metadata + assert ProjectMetadataGet.model_validate(data).custom == custom_metadata # check child project has parent unchanged async with aiopg_engine.acquire() as connection: project_db_metadata = await get_db_project_metadata( @@ -216,13 +216,13 @@ async def test_new_project_with_invalid_parent_project_node( ) assert parent_project - parent_project_uuid = parse_obj_as(ProjectID, parent_project["uuid"]) - parent_node_id = parse_obj_as( - NodeID, random.choice(list(parent_project["workbench"])) # noqa: S311 + parent_project_uuid = TypeAdapter(ProjectID).validate_python(parent_project["uuid"]) + parent_node_id = TypeAdapter(NodeID).validate_python( + random.choice(list(parent_project["workbench"])) # noqa: S311 ) # creating with random project UUID should fail - random_project_uuid = parse_obj_as(ProjectID, faker.uuid4()) + random_project_uuid = TypeAdapter(ProjectID).validate_python(faker.uuid4()) child_project = await request_create_project( client, expected.accepted, @@ -235,7 +235,7 @@ async def test_new_project_with_invalid_parent_project_node( assert not child_project # creating with a random node ID should fail too - random_node_id = parse_obj_as(NodeID, faker.uuid4()) + random_node_id = TypeAdapter(NodeID).validate_python(faker.uuid4()) child_project = await request_create_project( client, expected.accepted, @@ -259,7 +259,7 @@ async def test_new_project_with_invalid_parent_project_node( assert not child_project # creating with only a parent node ID should fail too - random_node_id = parse_obj_as(NodeID, faker.uuid4()) + random_node_id = TypeAdapter(NodeID).validate_python(faker.uuid4()) child_project = await request_create_project( client, expected.unprocessable, @@ -320,10 +320,10 @@ async def test_set_project_parent_backward_compatibility( project_id=child_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).model_dump() ) data, _ = await assert_status(response, expected_status_code=status.HTTP_200_OK) - assert parse_obj_as(ProjectMetadataGet, data).custom == custom_metadata + assert ProjectMetadataGet.model_validate(data).custom == custom_metadata # check child project has parent set correctly async with aiopg_engine.acquire() as connection: project_db_metadata = await get_db_project_metadata( @@ -362,7 +362,7 @@ async def test_update_project_metadata_backward_compatibility_with_same_project_ project_id=child_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).model_dump() ) await assert_status(response, expected_status_code=expected.ok) @@ -377,7 +377,7 @@ async def test_update_project_metadata_backward_compatibility_with_same_project_ project_id=child_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).model_dump() ) await assert_status(response, expected_status_code=expected.ok) @@ -427,10 +427,10 @@ async def test_update_project_metadata_s4lacad_backward_compatibility_passing_ni project_id=child_project["uuid"] ) response = await client.patch( - f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).dict() + f"{url}", json=ProjectMetadataUpdate(custom=custom_metadata).model_dump() ) data, _ = await assert_status(response, expected_status_code=status.HTTP_200_OK) - assert parse_obj_as(ProjectMetadataGet, data).custom == custom_metadata + assert ProjectMetadataGet.model_validate(data).custom == custom_metadata # check project has no parent async with aiopg_engine.acquire() as connection: diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index 8243228681b..8945e2c9296 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -33,7 +33,7 @@ ServiceResourcesDictHelpers, ) from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import NonNegativeFloat, NonNegativeInt, parse_obj_as +from pydantic import NonNegativeFloat, NonNegativeInt, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.webserver_parametrizations import ( @@ -72,13 +72,16 @@ async def test_get_node_resources( data, error = await assert_status(response, expected) if data: assert not error - node_resources = parse_obj_as(ServiceResourcesDict, data) + node_resources = TypeAdapter(ServiceResourcesDict).validate_python(data) assert node_resources assert DEFAULT_SINGLE_SERVICE_NAME in node_resources - assert ( - node_resources - == ServiceResourcesDictHelpers.Config.schema_extra["examples"][0] - ) + assert {k: v.model_dump() for k, v in node_resources.items()} == next( + iter( + ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ] + ) + ) # type: ignore else: assert not data assert error @@ -145,18 +148,22 @@ async def test_replace_node_resources_is_forbidden_by_default( response = await client.put( f"{url}", json=ServiceResourcesDictHelpers.create_jsonable( - ServiceResourcesDictHelpers.Config.schema_extra["examples"][0] + ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][0] ), ) data, error = await assert_status(response, expected) if data: assert not error - node_resources = parse_obj_as(ServiceResourcesDict, data) + node_resources = TypeAdapter(ServiceResourcesDict).validate_python(data) assert node_resources assert DEFAULT_SINGLE_SERVICE_NAME in node_resources assert ( node_resources - == ServiceResourcesDictHelpers.Config.schema_extra["examples"][0] + == ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][0] ) @@ -183,18 +190,22 @@ async def test_replace_node_resources_is_ok_if_explicitly_authorized( response = await client.put( f"{url}", json=ServiceResourcesDictHelpers.create_jsonable( - ServiceResourcesDictHelpers.Config.schema_extra["examples"][0] + ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][0] ), ) data, error = await assert_status(response, expected) if data: assert not error - node_resources = parse_obj_as(ServiceResourcesDict, data) + node_resources = TypeAdapter(ServiceResourcesDict).validate_python(data) assert node_resources assert DEFAULT_SINGLE_SERVICE_NAME in node_resources assert ( - node_resources - == ServiceResourcesDictHelpers.Config.schema_extra["examples"][0] + {k: v.model_dump() for k, v in node_resources.items()} + == ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][0] ) @@ -218,7 +229,9 @@ async def test_replace_node_resources_raises_422_if_resource_does_not_validate( f"{url}", json=ServiceResourcesDictHelpers.create_jsonable( # NOTE: we apply a different resource set - ServiceResourcesDictHelpers.Config.schema_extra["examples"][1] + ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][1] ), ) await assert_status(response, expected) @@ -383,8 +396,8 @@ def num_services( self, *args, **kwargs ) -> list[DynamicServiceGet]: # noqa: ARG002 return [ - DynamicServiceGet.parse_obj( - DynamicServiceGet.Config.schema_extra["examples"][1] + DynamicServiceGet.model_validate( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][1] | {"service_uuid": service_uuid, "project_id": user_project["uuid"]} ) for service_uuid in self.running_services_uuids @@ -928,8 +941,7 @@ def mock_storage_calls(aioresponses_mocker: aioresponses, faker: Faker) -> None: payload=jsonable_encoder( Envelope[list[FileMetaDataGet]]( data=[ - parse_obj_as( - FileMetaDataGet, + FileMetaDataGet.model_validate( { "file_uuid": file_uuid, "location_id": 0, @@ -979,7 +991,7 @@ async def test_read_project_nodes_previews( assert not error assert len(data) == 3 - nodes_previews = parse_obj_as(list[_ProjectNodePreview], data) + nodes_previews = TypeAdapter(list[_ProjectNodePreview]).validate_python(data) # GET node's preview for node_preview in nodes_previews: @@ -995,4 +1007,4 @@ async def test_read_project_nodes_previews( status.HTTP_200_OK, ) - assert parse_obj_as(_ProjectNodePreview, data) == node_preview + assert _ProjectNodePreview.model_validate(data) == node_preview diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py index 06957402de2..a867d2e3c40 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py @@ -18,7 +18,6 @@ PricingUnitGet, ) from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import parse_obj_as from pytest_mock.plugin import MockerFixture from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status @@ -53,7 +52,7 @@ async def test_project_node_pricing_unit_user_role_access( base_url = client.app.router["get_project_node_pricing_unit"].url_for( project_id=user_project["uuid"], node_id=node_id ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") assert ( resp.status == status.HTTP_401_UNAUTHORIZED if user_role == UserRole.ANONYMOUS @@ -72,7 +71,7 @@ async def test_project_node_pricing_unit_user_project_access( base_url = client.app.router["get_project_node_pricing_unit"].url_for( project_id=user_project["uuid"], node_id=node_id ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) assert data == None @@ -81,7 +80,7 @@ async def test_project_node_pricing_unit_user_project_access( base_url = client.app.router["get_project_node_pricing_unit"].url_for( project_id=user_project["uuid"], node_id=node_id ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") _, errors = await assert_status(resp, status.HTTP_403_FORBIDDEN) assert errors @@ -98,8 +97,8 @@ def mock_rut_api_responses( assert client.app settings: ResourceUsageTrackerSettings = get_plugin_settings(client.app) - pricing_unit_get_base = parse_obj_as( - PricingUnitGet, PricingUnitGet.Config.schema_extra["examples"][0] + pricing_unit_get_base = PricingUnitGet.model_validate( + PricingUnitGet.model_config["json_schema_extra"]["examples"][0] ) pricing_unit_get_1 = pricing_unit_get_base.copy() pricing_unit_get_1.pricing_unit_id = _PRICING_UNIT_ID_1 diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py index ae1b62e0558..8a82df500b6 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py @@ -15,7 +15,7 @@ from models_library.api_schemas_directorv2.comp_tasks import TasksOutputs from models_library.api_schemas_webserver.projects import ProjectGet from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_fake_ports_data import ( PROJECTS_METADATA_PORTS_RESPONSE_BODY_DATA, @@ -276,13 +276,13 @@ async def test_clone_project_and_set_inputs( data = await long_running_task.result() assert data is not None - cloned_project = ProjectGet.parse_obj(data) + cloned_project = ProjectGet.model_validate(data) assert parent_project_id != cloned_project.uuid assert user_project["description"] == cloned_project.description - assert parse_obj_as(datetime, user_project["creationDate"]) < parse_obj_as( - datetime, cloned_project.creation_date - ) + assert TypeAdapter(datetime).validate_python( + user_project["creationDate"] + ) < TypeAdapter(datetime).validate_python(cloned_project.creation_date) # - set_inputs project_clone_id ---------------------------------------------- job_inputs_values = {"X": 42} # like JobInputs.values diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index b5550400006..693372aab27 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -9,7 +9,7 @@ import time from collections.abc import Awaitable, Callable, Iterator from copy import deepcopy -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from http import HTTPStatus from typing import Any from unittest import mock @@ -228,7 +228,7 @@ async def _assert_project_state_updated( jsonable_encoder( { "project_uuid": shared_project["uuid"], - "data": p_state.dict(by_alias=True, exclude_unset=True), + "data": p_state.model_dump(by_alias=True, exclude_unset=True), } ) ) @@ -285,8 +285,8 @@ async def test_share_project( ) if new_project: assert new_project["accessRights"] == { - str(primary_group["gid"]): {"read": True, "write": True, "delete": True}, - str(all_group["gid"]): share_rights, + f'{primary_group["gid"]}': {"read": True, "write": True, "delete": True}, + f'{(all_group["gid"])}': share_rights, } # user 1 can always get to his project @@ -716,7 +716,7 @@ async def test_open_project_with_deprecated_services_ok_but_does_not_start_dynam mocked_notifications_plugin: dict[str, mock.Mock], ): mock_catalog_api["get_service"].return_value["deprecated"] = ( - datetime.utcnow() - timedelta(days=1) + datetime.now(UTC) - timedelta(days=1) ).isoformat() url = client.app.router["open_project"].url_for(project_id=user_project["uuid"]) resp = await client.post(url, json=client_session_id_factory()) @@ -1043,10 +1043,10 @@ async def test_project_node_lifetime( # noqa: PLR0915 project_id=user_project["uuid"], node_id=dynamic_node_id ) - node_sample = deepcopy(NodeGet.Config.schema_extra["examples"][1]) + node_sample = deepcopy(NodeGet.model_config["json_schema_extra"]["examples"][1]) mocked_director_v2_api[ "dynamic_scheduler.api.get_dynamic_service" - ].return_value = NodeGet.parse_obj( + ].return_value = NodeGet.model_validate( { **node_sample, "service_state": "running", @@ -1065,7 +1065,7 @@ async def test_project_node_lifetime( # noqa: PLR0915 ) mocked_director_v2_api[ "dynamic_scheduler.api.get_dynamic_service" - ].return_value = NodeGetIdle.parse_obj( + ].return_value = NodeGetIdle.model_validate( { "service_uuid": node_sample["service_uuid"], "service_state": "idle", diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py index 9443f773c03..30aaa89abbc 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py @@ -12,7 +12,6 @@ import sqlalchemy as sa from aiohttp.test_utils import TestClient from models_library.api_schemas_webserver.wallets import WalletGet -from pydantic import parse_obj_as from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import LoggedUser, UserInfoDict from servicelib.aiohttp import status @@ -43,7 +42,7 @@ async def test_project_wallets_user_role_access( base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") assert ( resp.status == status.HTTP_401_UNAUTHORIZED if user_role == UserRole.ANONYMOUS @@ -62,7 +61,7 @@ async def test_project_wallets_user_project_access( base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) assert data == None @@ -71,7 +70,7 @@ async def test_project_wallets_user_project_access( base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") _, errors = await assert_status(resp, status.HTTP_403_FORBIDDEN) assert errors @@ -93,7 +92,7 @@ def setup_wallets_db( ) .returning(sa.literal_column("*")) ) - output.append(parse_obj_as(WalletGet, result.fetchone())) + output.append(WalletGet.model_validate(result.fetchone())) yield output con.execute(wallets.delete()) @@ -111,7 +110,7 @@ async def test_project_wallets_full_workflow( base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) assert data == None @@ -119,14 +118,14 @@ async def test_project_wallets_full_workflow( base_url = client.app.router["connect_wallet_to_project"].url_for( project_id=user_project["uuid"], wallet_id=f"{setup_wallets_db[0].wallet_id}" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") data, _ = await assert_status(resp, expected) assert data["walletId"] == setup_wallets_db[0].wallet_id base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) assert data["walletId"] == setup_wallets_db[0].wallet_id @@ -134,13 +133,13 @@ async def test_project_wallets_full_workflow( base_url = client.app.router["connect_wallet_to_project"].url_for( project_id=user_project["uuid"], wallet_id=f"{setup_wallets_db[1].wallet_id}" ) - resp = await client.put(base_url) + resp = await client.put(f"{base_url}") data, _ = await assert_status(resp, expected) assert data["walletId"] == setup_wallets_db[1].wallet_id base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) - resp = await client.get(base_url) + resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) assert data["walletId"] == setup_wallets_db[1].wallet_id diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py b/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py index ea792b8f726..014ed5db536 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py @@ -66,7 +66,7 @@ def fake_osparc_invitation( Emulates an invitation for osparc product """ oas = deepcopy(invitations_service_openapi_specs) - content = ApiInvitationContent.parse_obj( + content = ApiInvitationContent.model_validate( oas["components"]["schemas"]["ApiInvitationContent"]["example"] ) content.product = "osparc" @@ -117,11 +117,11 @@ def mock_invitations_service_http_api( assert "/v1/invitations:extract" in oas["paths"] def _extract(url, **kwargs): - fake_code = URL(URL(kwargs["json"]["invitation_url"]).fragment).query[ + fake_code = URL(URL(f'{kwargs["json"]["invitation_url"]}').fragment).query[ "invitation" ] # if nothing is encoded in fake_code, just return fake_osparc_invitation - body = fake_osparc_invitation.dict() + body = fake_osparc_invitation.model_dump() with suppress(Exception): decoded = json.loads(binascii.unhexlify(fake_code).decode()) body.update(decoded) @@ -150,7 +150,7 @@ def _generate(url, **kwargs): return CallbackResult( status=status.HTTP_200_OK, payload=jsonable_encoder( - ApiInvitationContentAndLink.parse_obj( + ApiInvitationContentAndLink.model_validate( { **example, **body, @@ -212,5 +212,5 @@ def app_environment( ) # tests envs - print(ApplicationSettings.create_from_envs().json(indent=2)) + print(ApplicationSettings.create_from_envs().model_dump_json(indent=2)) return envs diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/test_login_handlers_registration_invitations.py b/services/web/server/tests/unit/with_dbs/03/invitations/test_login_handlers_registration_invitations.py index 7fa3ee144a7..8c4daca29df 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/test_login_handlers_registration_invitations.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/test_login_handlers_registration_invitations.py @@ -48,11 +48,11 @@ async def test_check_registration_invitation_when_not_required( response = await client.post( "/v0/auth/register/invitations:check", - json=InvitationCheck(invitation="*" * 100).dict(), + json=InvitationCheck(invitation="*" * 100).model_dump(), ) data, _ = await assert_status(response, status.HTTP_200_OK) - invitation = InvitationInfo.parse_obj(data) + invitation = InvitationInfo.model_validate(data) assert invitation.email is None @@ -70,11 +70,11 @@ async def test_check_registration_invitations_with_old_code( response = await client.post( "/v0/auth/register/invitations:check", - json=InvitationCheck(invitation="short-code").dict(), + json=InvitationCheck(invitation="short-code").model_dump(), ) data, _ = await assert_status(response, status.HTTP_200_OK) - invitation = InvitationInfo.parse_obj(data) + invitation = InvitationInfo.model_validate(data) assert invitation.email is None @@ -96,11 +96,11 @@ async def test_check_registration_invitation_and_get_email( response = await client.post( "/v0/auth/register/invitations:check", - json=InvitationCheck(invitation="*" * 105).dict(), + json=InvitationCheck(invitation="*" * 105).model_dump(), ) data, _ = await assert_status(response, status.HTTP_200_OK) - invitation = InvitationInfo.parse_obj(data) + invitation = InvitationInfo.model_validate(data) assert invitation.email == fake_osparc_invitation.guest diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py b/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py index 0f8a85544f4..71da6536363 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py @@ -56,7 +56,7 @@ async def test_role_access_to_generate_invitation( ) data, error = await assert_status(response, expected_status) if not error: - got = InvitationGenerated.parse_obj(data) + got = InvitationGenerated.model_validate(data) assert got.guest == guest_email else: assert error @@ -92,22 +92,22 @@ async def test_product_owner_generates_invitation( # request response = await client.post( "/v0/invitation:generate", - json=request_model.dict(exclude_none=True), + json=request_model.model_dump(exclude_none=True), ) # checks data, error = await assert_status(response, expected_status) assert not error - got = InvitationGenerated.parse_obj(data) + got = InvitationGenerated.model_validate(data) expected = { "issuer": logged_user["email"][:_MAX_LEN], - **request_model.dict(exclude_none=True), + **request_model.model_dump(exclude_none=True), } - assert got.dict(include=set(expected), by_alias=False) == expected + assert got.model_dump(include=set(expected), by_alias=False) == expected product_base_url = f"{client.make_url('/')}" - assert got.invitation_link.startswith(product_base_url) + assert f"{got.invitation_link}".startswith(product_base_url) assert before_dt < got.created assert got.created < datetime.now(tz=timezone.utc) @@ -150,7 +150,7 @@ async def test_pre_registration_and_invitation_workflow( guest=guest_email, trial_account_days=None, extra_credits_in_usd=10, - ).dict() + ).model_dump() # Search user -> nothing response = await client.get("/v0/users:search", params={"email": guest_email}) @@ -186,7 +186,7 @@ async def test_pre_registration_and_invitation_workflow( response = await client.post("/v0/invitation:generate", json=invitation) data, _ = await assert_status(response, status.HTTP_200_OK) assert data["guest"] == guest_email - got_invitation = InvitationGenerated.parse_obj(data) + got_invitation = InvitationGenerated.model_validate(data) # register user assert got_invitation.invitation_link.fragment diff --git a/services/web/server/tests/unit/with_dbs/03/login/conftest.py b/services/web/server/tests/unit/with_dbs/03/login/conftest.py index 167315facb4..b9a458add16 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/login/conftest.py @@ -20,7 +20,7 @@ @pytest.fixture -def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): +def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker): envs_plugins = setenvs_from_dict( monkeypatch, { @@ -41,6 +41,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc "WEBSERVER_TRACING": "null", "WEBSERVER_VERSION_CONTROL": "0", "WEBSERVER_WALLETS": "1", + "WEBSERVER_TRACING": "null" }, ) diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa.py index d6dc34bcdfe..29324b2af23 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa.py @@ -50,7 +50,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc "LOGIN_2FA_CODE_EXPIRATION_SEC": "60", }, ) - print(ApplicationSettings.create_from_envs().json(indent=1)) + print(ApplicationSettings.create_from_envs().model_dump_json(indent=1)) return {**app_environment, **envs_login} @@ -148,7 +148,7 @@ def _get_confirmation_link_from_email(): url = _get_confirmation_link_from_email() # 2. confirmation - response = await client.get(url) + response = await client.get(f"{url}") assert response.status == status.HTTP_200_OK # check email+password registered diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py index 7139811a6b1..2cf5b63eb24 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py @@ -6,7 +6,6 @@ import pytest import sqlalchemy as sa from aiohttp.test_utils import TestClient -from pydantic import parse_obj_as from pytest_mock import MockFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict @@ -29,7 +28,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc }, ) - print(ApplicationSettings.create_from_envs().json(indent=2)) + print(ApplicationSettings.create_from_envs().model_dump_json(indent=2)) return {**app_environment, **envs_login} @@ -106,7 +105,7 @@ async def test_resend_2fa_workflow( }, ) data, _ = await assert_status(response, status.HTTP_202_ACCEPTED) - next_page = parse_obj_as(NextPage[CodePageParams], data) + next_page = NextPage[CodePageParams].model_validate(data) assert next_page.name == CODE_2FA_SMS_CODE_REQUIRED assert next_page.parameters.expiration_2fa > 0 diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_auth.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_auth.py index c73020d0638..7d16e912414 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_auth.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_auth.py @@ -30,7 +30,7 @@ def test_login_plugin_setup_succeeded(client: TestClient): assert client.app - print(client.app[APP_SETTINGS_KEY].json(indent=1, sort_keys=True)) + print(client.app[APP_SETTINGS_KEY].model_dump_json(indent=1)) # this should raise AssertionError if not succeedd settings = get_plugin_settings(client.app) diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_change_password.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_change_password.py index 41f90807925..a171ec63ae2 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_change_password.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_change_password.py @@ -85,7 +85,7 @@ async def test_wrong_confirm_pass(client: TestClient, new_password: str): "errors": [ { "code": "value_error", - "message": MSG_PASSWORD_MISMATCH, + "message": f"Value error, {MSG_PASSWORD_MISMATCH}", "resource": "/v0/auth/change-password", "field": "confirm", } diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py index 61ade5ec24b..d99f8f1f297 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py @@ -95,14 +95,14 @@ async def test_register_body_validation( "status": 422, "errors": [ { - "code": "value_error.email", - "message": "value is not a valid email address", + "code": "value_error", + "message": "value is not a valid email address: An email address must have an @-sign.", "resource": "/v0/auth/register", "field": "email", }, { "code": "value_error", - "message": MSG_PASSWORD_MISMATCH, + "message": f"Value error, {MSG_PASSWORD_MISMATCH}", "resource": "/v0/auth/register", "field": "confirm", }, @@ -494,7 +494,7 @@ async def test_registraton_with_invitation_for_trial_account( url = client.app.router["get_my_profile"].url_for() response = await client.get(url.path) data, _ = await assert_status(response, status.HTTP_200_OK) - profile = ProfileGet.parse_obj(data) + profile = ProfileGet.model_validate(data) expected = invitation.user["created_at"] + timedelta(days=TRIAL_DAYS) assert profile.expiration_date diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_registration_handlers.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_registration_handlers.py index 782d6bba93d..655afde42b7 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_registration_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_registration_handlers.py @@ -184,7 +184,7 @@ async def test_request_an_account( assert client.app # A form similar to the one in https://github.com/ITISFoundation/osparc-simcore/pull/5378 user_data = { - **AccountRequestInfo.Config.schema_extra["example"]["form"], + **AccountRequestInfo.model_config["json_schema_extra"]["example"]["form"], # fields required in the form "firstName": faker.first_name(), "lastName": faker.last_name(), @@ -197,7 +197,7 @@ async def test_request_an_account( response = await client.post( "/v0/auth/request-account", - json={"form": user_data, "captcha": 123456}, + json={"form": user_data, "captcha": "123456"}, ) await assert_status(response, status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py index 043176ae5ee..a9ea81e32fb 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py @@ -132,7 +132,7 @@ async def test_iterators_workflow( project_id=project_data["uuid"] ) for node_id, node_data in modifications["workbench"].items(): - node = Node.parse_obj(node_data) + node = Node.model_validate(node_data) response = await client.post( f"{create_node_url}", json={ @@ -190,7 +190,7 @@ async def _mock_start(project_id, user_id, product_name, **options): f"/v0/projects/{project_uuid}/checkpoint/{head_ref_id}/iterations?offset=0" ) body = await response.json() - first_iterlist = Page[ProjectIterationItem].parse_obj(body).data + first_iterlist = Page[ProjectIterationItem].model_validate(body).data assert len(first_iterlist) == 3 @@ -233,7 +233,7 @@ async def _mock_catalog_get(*args, **kwarg): assert response.status == status.HTTP_200_OK, await response.text() body = await response.json() - assert Page[ProjectIterationResultItem].parse_obj(body).data is not None + assert Page[ProjectIterationResultItem].model_validate(body).data is not None # GET project and MODIFY iterator values---------------------------------------------- # - Change iterations from 0:4 -> HEAD+1 @@ -247,7 +247,7 @@ async def _mock_catalog_get(*args, **kwarg): # Dict keys are usually some sort of identifier, typically a UUID or # and index but nothing prevents a dict from using any other type of key types # - project = Project.parse_obj(body["data"]) + project = Project.model_validate(body["data"]) new_project = project.copy( update={ # TODO: HACK to overcome export from None -> string @@ -295,7 +295,7 @@ async def _mock_catalog_get(*args, **kwarg): ) body = await response.json() assert response.status == status.HTTP_200_OK, f"{body=}" # nosec - second_iterlist = Page[ProjectIterationItem].parse_obj(body).data + second_iterlist = Page[ProjectIterationItem].model_validate(body).data assert len(second_iterlist) == 4 assert len({it.workcopy_project_id for it in second_iterlist}) == len( diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/conftest.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/conftest.py index 4dc9da94974..a4691fcc3a2 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/conftest.py @@ -17,7 +17,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): - # print( ApplicationSettings.create_from_envs().json(indent=1 ) + # print( ApplicationSettings.create_from_envs().model_dump_json((indent=1 ) return app_environment | setenvs_from_dict( monkeypatch, diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py index 6e67883e357..ad508f523e4 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py @@ -17,7 +17,6 @@ PricingUnitGet, ) from models_library.resource_tracker import PricingPlanClassification -from pydantic import parse_obj_as from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict @@ -35,52 +34,52 @@ def mock_rpc_resource_usage_tracker_service_api( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_plans.list_pricing_plans", autospec=True, return_value=[ - parse_obj_as( - PricingPlanGet, PricingPlanGet.Config.schema_extra["examples"][0] + PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][0], ) ], ), "get_pricing_plan": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_plans.get_pricing_plan", autospec=True, - return_value=parse_obj_as( - PricingPlanGet, PricingPlanGet.Config.schema_extra["examples"][0] + return_value=PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][0], ), ), "create_pricing_plan": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_plans.create_pricing_plan", autospec=True, - return_value=parse_obj_as( - PricingPlanGet, PricingPlanGet.Config.schema_extra["examples"][0] + return_value=PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][0], ), ), "update_pricing_plan": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_plans.update_pricing_plan", autospec=True, - return_value=parse_obj_as( - PricingPlanGet, PricingPlanGet.Config.schema_extra["examples"][0] + return_value=PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][0], ), ), ## Pricing units "get_pricing_unit": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_units.get_pricing_unit", autospec=True, - return_value=parse_obj_as( - PricingUnitGet, PricingUnitGet.Config.schema_extra["examples"][0] + return_value=PricingUnitGet.model_validate( + PricingUnitGet.model_config["json_schema_extra"]["examples"][0], ), ), "create_pricing_unit": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_units.create_pricing_unit", autospec=True, - return_value=parse_obj_as( - PricingUnitGet, PricingUnitGet.Config.schema_extra["examples"][0] + return_value=PricingUnitGet.model_validate( + PricingUnitGet.model_config["json_schema_extra"]["examples"][0], ), ), "update_pricing_unit": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_units.update_pricing_unit", autospec=True, - return_value=parse_obj_as( - PricingUnitGet, PricingUnitGet.Config.schema_extra["examples"][0] + return_value=PricingUnitGet.model_validate( + PricingUnitGet.model_config["json_schema_extra"]["examples"][0], ), ), ## Pricing plan to service @@ -88,18 +87,20 @@ def mock_rpc_resource_usage_tracker_service_api( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_plans.list_connected_services_to_pricing_plan_by_pricing_plan", autospec=True, return_value=[ - parse_obj_as( - PricingPlanToServiceGet, - PricingPlanToServiceGet.Config.schema_extra["examples"][0], + PricingPlanToServiceGet.model_validate( + PricingPlanToServiceGet.model_config["json_schema_extra"][ + "examples" + ][0], ) ], ), "connect_service_to_pricing_plan": mocker.patch( "simcore_service_webserver.resource_usage._pricing_plans_admin_api.pricing_plans.connect_service_to_pricing_plan", autospec=True, - return_value=parse_obj_as( - PricingPlanToServiceGet, - PricingPlanToServiceGet.Config.schema_extra["examples"][0], + return_value=PricingPlanToServiceGet.model_validate( + PricingPlanToServiceGet.model_config["json_schema_extra"]["examples"][ + 0 + ], ), ), } diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_pricing_plans.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_pricing_plans.py index 7b25e33a799..70114820036 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_pricing_plans.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_pricing_plans.py @@ -15,7 +15,6 @@ PricingUnitGet, ) from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import parse_obj_as from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict @@ -32,13 +31,12 @@ def mock_rut_api_responses( assert client.app settings: ResourceUsageTrackerSettings = get_plugin_settings(client.app) - pricing_unit_get = parse_obj_as( - PricingUnitGet, PricingUnitGet.Config.schema_extra["examples"][0] + pricing_unit_get = PricingUnitGet.model_validate( + PricingUnitGet.model_config["json_schema_extra"]["examples"][0] ) - service_pricing_plan_get = parse_obj_as( - PricingPlanGet, - PricingPlanGet.Config.schema_extra["examples"][0], + service_pricing_plan_get = PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][0], ) aioresponses_mocker.get( diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py index 6a80bccca0d..3ac7ffaa665 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py @@ -16,7 +16,7 @@ from aiohttp.test_utils import TestClient from models_library.resource_tracker import ServiceResourceUsagesFilters from models_library.rest_ordering import OrderBy -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status @@ -29,7 +29,7 @@ def mock_export_usage_services(mocker: MockerFixture) -> MagicMock: return mocker.patch( "simcore_service_webserver.resource_usage._service_runs_api.service_runs.export_service_runs", spec=True, - return_value=parse_obj_as(AnyUrl, "https://www.google.com/"), + return_value=TypeAdapter(AnyUrl).validate_python("https://www.google.com/"), ) @@ -115,5 +115,5 @@ async def test_list_service_usage( assert mock_export_usage_services.called args = mock_export_usage_services.call_args[1] - assert args["order_by"] == parse_obj_as(OrderBy, _order_by) - assert args["filters"] == parse_obj_as(ServiceResourceUsagesFilters, _filter) + assert args["order_by"] == OrderBy.model_validate(_order_by) + assert args["filters"] == ServiceResourceUsagesFilters.model_validate(_filter) diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py index 33b9d9146f5..6e3a12bfe84 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py @@ -45,6 +45,8 @@ "started_at": "2023-08-26T14:18:17.600493+00:00", "stopped_at": "2023-08-26T14:18:19.358355+00:00", "service_run_status": "SUCCESS", + "credit_cost": None, + "transaction_status": None } ) ], @@ -199,7 +201,7 @@ async def test_list_service_usage_with_order_by_query_param( assert mock_list_usage_services.called assert error["status"] == status.HTTP_422_UNPROCESSABLE_ENTITY assert error["errors"][0]["message"].startswith( - "We do not support ordering by provided field" + "Value error, We do not support ordering by provided field" ) # with non-parsable field in order by query parameter @@ -237,7 +239,7 @@ async def test_list_service_usage_with_order_by_query_param( assert mock_list_usage_services.called assert error["status"] == status.HTTP_422_UNPROCESSABLE_ENTITY assert error["errors"][0]["message"].startswith( - "value is not a valid enumeration member" + "Input should be 'asc' or 'desc'" ) # without field @@ -251,7 +253,7 @@ async def test_list_service_usage_with_order_by_query_param( _, error = await assert_status(resp, status.HTTP_422_UNPROCESSABLE_ENTITY) assert mock_list_usage_services.called assert error["status"] == status.HTTP_422_UNPROCESSABLE_ENTITY - assert error["errors"][0]["message"].startswith("field required") + assert error["errors"][0]["message"].startswith("Field required") @pytest.mark.parametrize("user_role", [(UserRole.USER)]) diff --git a/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py b/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py index ebd268074ab..eafb6ed29b9 100644 --- a/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py +++ b/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py @@ -56,7 +56,7 @@ def app(app_environment: EnvVarsDict) -> web.Application: # - all plugins are setup but app is NOT started (i.e events are not triggered) # app_ = create_application() - print(get_application_settings(app_).json(indent=1)) + print(get_application_settings(app_).model_dump_json(indent=1)) return app_ diff --git a/services/web/server/tests/unit/with_dbs/03/test_email.py b/services/web/server/tests/unit/with_dbs/03/test_email.py index e2164071c16..c208162d318 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_email.py +++ b/services/web/server/tests/unit/with_dbs/03/test_email.py @@ -136,10 +136,10 @@ async def test_email_handlers( assert error is None with pytest.raises(ValidationError): - EmailTestFailed.parse_obj(data) + EmailTestFailed.model_validate(data) - passed = EmailTestPassed.parse_obj(data) - print(passed.json(indent=1)) + passed = EmailTestPassed.model_validate(data) + print(passed.model_dump_json(indent=1)) class IndexParser(HTMLParser): diff --git a/services/web/server/tests/unit/with_dbs/03/test_socketio.py b/services/web/server/tests/unit/with_dbs/03/test_socketio.py index 05be09f7749..699ff0ccef9 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_socketio.py +++ b/services/web/server/tests/unit/with_dbs/03/test_socketio.py @@ -46,7 +46,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc }, ) - print(ApplicationSettings.create_from_envs().json(indent=1)) + print(ApplicationSettings.create_from_envs().model_dump_json(indent=1)) return app_environment | overrides diff --git a/services/web/server/tests/unit/with_dbs/03/test_storage_handlers.py b/services/web/server/tests/unit/with_dbs/03/test_storage_handlers.py index 28e7d0590c9..694d49a998b 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_storage_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/test_storage_handlers.py @@ -17,7 +17,7 @@ FileUploadLinks, FileUploadSchema, ) -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -59,7 +59,7 @@ async def _resp(*args, **kwargs) -> tuple[Any, int]: ) def _resolve(*args, **kwargs) -> AnyUrl: - return parse_obj_as(AnyUrl, "http://private-url") + return TypeAdapter(AnyUrl).validate_python("http://private-url") mocker.patch( "simcore_service_webserver.storage._handlers._from_storage_url", @@ -69,16 +69,19 @@ def _resolve(*args, **kwargs) -> AnyUrl: MOCK_FILE_UPLOAD_SCHEMA = FileUploadSchema( - chunk_size=parse_obj_as(ByteSize, "5GiB"), - urls=[parse_obj_as(AnyUrl, "s3://file_id")], + chunk_size=TypeAdapter(ByteSize).validate_python("5GiB"), + urls=[TypeAdapter(AnyUrl).validate_python("s3://file_id")], links=FileUploadLinks( - abort_upload=parse_obj_as(AnyUrl, "http://private-url/operation:abort"), - complete_upload=parse_obj_as(AnyUrl, "http://private-url/operation:complete"), + abort_upload=TypeAdapter(AnyUrl).validate_python( + "http://private-url/operation:abort" + ), + complete_upload=TypeAdapter(AnyUrl).validate_python( + "http://private-url/operation:complete" + ), ), ) -MOCK_FILE_UPLOAD_SCHEMA = parse_obj_as( - FileUploadSchema, +MOCK_FILE_UPLOAD_SCHEMA = FileUploadSchema.model_validate( { "chunk_size": "5", "urls": ["s3://file_id"], @@ -90,8 +93,8 @@ def _resolve(*args, **kwargs) -> AnyUrl: ) -MOCK_FILE_UPLOAD_COMPLETE_RESPONSE = parse_obj_as( - FileUploadCompleteResponse, {"links": {"state": "http://private-url"}} +MOCK_FILE_UPLOAD_COMPLETE_RESPONSE = FileUploadCompleteResponse.model_validate( + {"links": {"state": "http://private-url"}} ) @@ -124,7 +127,7 @@ def _resolve(*args, **kwargs) -> AnyUrl: "PUT", "/v0/storage/locations/0/files/{file_id}", None, - json.loads(MOCK_FILE_UPLOAD_SCHEMA.json()), + json.loads(MOCK_FILE_UPLOAD_SCHEMA.model_dump_json()), id="upload_file", ), pytest.param( @@ -145,14 +148,14 @@ def _resolve(*args, **kwargs) -> AnyUrl: "POST", "/v0/storage/locations/0/files/{file_id}:complete", {"parts": []}, - json.loads(MOCK_FILE_UPLOAD_COMPLETE_RESPONSE.json()), + json.loads(MOCK_FILE_UPLOAD_COMPLETE_RESPONSE.model_dump_json()), id="complete_upload_file", ), pytest.param( "POST", "/v0/storage/locations/0/files/{file_id}:complete/futures/RANDOM_FUTURE_ID", None, - json.loads(MOCK_FILE_UPLOAD_SCHEMA.json()), + json.loads(MOCK_FILE_UPLOAD_SCHEMA.model_dump_json()), id="is_completed_upload_file", ), ], @@ -208,7 +211,7 @@ def test_url_storage_resolver_helpers(faker: Faker, app_environment: EnvVarsDict # storage -> web web_url: AnyUrl = _from_storage_url( - web_request, parse_obj_as(AnyUrl, str(storage_url)) + web_request, TypeAdapter(AnyUrl).validate_python(f"{storage_url}") ) assert storage_url.host != web_url.host @@ -216,4 +219,4 @@ def test_url_storage_resolver_helpers(faker: Faker, app_environment: EnvVarsDict assert isinstance(storage_url, URL) # this is a bit inconvenient assert isinstance(web_url, AnyUrl) - assert str(web_url) == str(web_request.url) + assert f"{web_url}" == f"{web_request.url}" diff --git a/services/web/server/tests/unit/with_dbs/03/test_users.py b/services/web/server/tests/unit/with_dbs/03/test_users.py index 80c0c7912af..33d22b4e815 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users.py @@ -82,12 +82,14 @@ async def test_get_profile( data, error = await assert_status(resp, expected) # check enveloped - e = Envelope[ProfileGet].parse_obj(await resp.json()) + e = Envelope[ProfileGet].model_validate(await resp.json()) assert e.error == error - assert e.data.dict(**RESPONSE_MODEL_POLICY) == data if e.data else e.data == data + assert ( + e.data.model_dump(**RESPONSE_MODEL_POLICY, mode="json") == data if e.data else e.data == data + ) if not error: - profile = ProfileGet.parse_obj(data) + profile = ProfileGet.model_validate(data) product_group = { "accessRights": {"delete": False, "read": False, "write": False}, @@ -105,7 +107,7 @@ async def test_get_profile( assert profile.role == user_role.name assert profile.groups - got_profile_groups = profile.groups.dict(**RESPONSE_MODEL_POLICY) + got_profile_groups = profile.groups.model_dump(**RESPONSE_MODEL_POLICY, mode="json") assert got_profile_groups["me"] == primary_group assert got_profile_groups["all"] == all_group @@ -147,7 +149,7 @@ async def test_update_profile( data, _ = await assert_status(resp, status.HTTP_200_OK) # This is a PUT! i.e. full replace of profile variable fields! - assert data["first_name"] == ProfileUpdate.__fields__["first_name"].default + assert data["first_name"] == ProfileUpdate.model_fields["first_name"].default assert data["last_name"] == "Foo" assert data["role"] == user_role.name @@ -250,7 +252,9 @@ def account_request_form(faker: Faker) -> dict[str, Any]: } # keeps in sync fields from example and this fixture - assert set(form) == set(AccountRequestInfo.Config.schema_extra["example"]["form"]) + assert set(form) == set( + AccountRequestInfo.model_config["json_schema_extra"]["example"]["form"] + ) return form @@ -292,7 +296,7 @@ async def test_search_and_pre_registration( "registered": True, "status": UserStatus.ACTIVE, } - assert got.dict(include=set(expected)) == expected + assert got.model_dump(include=set(expected)) == expected # NOT in `users` and ONLY `users_pre_registration_details` @@ -307,7 +311,7 @@ async def test_search_and_pre_registration( assert len(found) == 1 got = UserProfile(**found[0]) - assert got.dict(include={"registered", "status"}) == { + assert got.model_dump(include={"registered", "status"}) == { "registered": False, "status": None, } @@ -329,7 +333,7 @@ async def test_search_and_pre_registration( found, _ = await assert_status(resp, status.HTTP_200_OK) assert len(found) == 1 got = UserProfile(**found[0]) - assert got.dict(include={"registered", "status"}) == { + assert got.model_dump(include={"registered", "status"}) == { "registered": True, "status": new_user["status"].name, } @@ -356,7 +360,7 @@ def test_preuserprofile_parse_model_from_request_form_data( # pre-processors pre_user_profile = PreUserProfile(**data) - print(pre_user_profile.json(indent=1)) + print(pre_user_profile.model_dump_json(indent=1)) # institution aliases assert pre_user_profile.institution == account_request_form["company"] @@ -377,7 +381,7 @@ def test_preuserprofile_parse_model_without_extras( account_request_form: dict[str, Any] ): required = { - f.alias or f.name for f in PreUserProfile.__fields__.values() if f.required + f.alias or f_name for f_name, f in PreUserProfile.model_fields.items() if f.is_required() } data = {k: account_request_form[k] for k in required} assert not PreUserProfile(**data).extras @@ -401,6 +405,6 @@ def test_preuserprofile_pre_given_names( account_request_form["lastName"] = given_name pre_user_profile = PreUserProfile(**account_request_form) - print(pre_user_profile.json(indent=1)) + print(pre_user_profile.model_dump_json(indent=1)) assert pre_user_profile.first_name in ["Pedro-Luis", "Pedro Luis"] assert pre_user_profile.first_name == pre_user_profile.last_name diff --git a/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py b/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py index 77aaccade51..048dc779c51 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py @@ -18,7 +18,7 @@ import redis.asyncio as aioredis from aiohttp.test_utils import TestClient from models_library.products import ProductName -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.webserver_login import UserInfoDict @@ -71,7 +71,7 @@ def _create_notification( notification_categories = tuple(NotificationCategory) notification: UserNotification = UserNotification.create_from_request_data( - UserNotificationCreate.parse_obj( + UserNotificationCreate.model_validate( { "user_id": user_id, "category": random.choice(notification_categories), @@ -104,7 +104,7 @@ async def _create_notifications( redis_key = get_notification_key(user_id) if user_notifications: for notification in user_notifications: - await redis_client.lpush(redis_key, notification.json()) + await redis_client.lpush(redis_key, notification.model_dump_json()) yield user_notifications @@ -154,7 +154,9 @@ async def test_list_user_notifications( response = await client.get(url.path) json_response = await response.json() - result = parse_obj_as(list[UserNotification], json_response["data"]) + result = TypeAdapter(list[UserNotification]).validate_python( + json_response["data"] + ) # noqa: F821 assert len(result) <= MAX_NOTIFICATIONS_FOR_USER_TO_SHOW assert result == list( reversed(created_notifications[:MAX_NOTIFICATIONS_FOR_USER_TO_SHOW]) @@ -377,7 +379,7 @@ async def test_update_user_notification_at_correct_index( async def _get_stored_notifications() -> list[UserNotification]: return [ - UserNotification.parse_raw(x) + UserNotification.model_validate_json(x) for x in await notification_redis_client.lrange( get_notification_key(user_id), 0, -1 ) @@ -444,7 +446,7 @@ async def test_list_permissions( data, error = await assert_status(resp, expected_response) if data: assert not error - list_of_permissions = parse_obj_as(list[PermissionGet], data) + list_of_permissions = TypeAdapter(list[PermissionGet]).validate_python(data) assert ( len(list_of_permissions) == 1 ), "for now there is only 1 permission, but when we sync frontend/backend permissions there will be more" @@ -475,7 +477,7 @@ async def test_list_permissions_with_overriden_extra_properties( data, error = await assert_status(resp, expected_response) assert data assert not error - list_of_permissions = parse_obj_as(list[PermissionGet], data) + list_of_permissions = TypeAdapter(list[PermissionGet]).validate_python(data) filtered_permissions = list( filter( lambda x: x.name == "override_services_specifications", list_of_permissions diff --git a/services/web/server/tests/unit/with_dbs/03/test_users__preferences_api.py b/services/web/server/tests/unit/with_dbs/03/test_users__preferences_api.py index 3835883af8b..8db8935616d 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users__preferences_api.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users__preferences_api.py @@ -11,12 +11,13 @@ from aiohttp import web from aiohttp.test_utils import TestClient from faker import Faker +from common_library.pydantic_fields_extension import get_type from models_library.api_schemas_webserver.users_preferences import Preference from models_library.products import ProductName from models_library.user_preferences import FrontendUserPreference from models_library.users import UserID from pydantic import BaseModel -from pydantic.fields import ModelField +from pydantic.fields import FieldInfo from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.webserver_login import NewUser from simcore_postgres_database.models.groups_extra_properties import ( @@ -64,8 +65,8 @@ def product_name() -> ProductName: return "osparc" -def _get_model_field(model_class: type[BaseModel], field_name: str) -> ModelField: - return model_class.__dict__["__fields__"][field_name] +def _get_model_field(model_class: type[BaseModel], field_name: str) -> FieldInfo: + return model_class.model_fields[field_name] def _get_default_field_value(model_class: type[BaseModel]) -> Any: @@ -83,7 +84,7 @@ def _get_non_default_value( """given a default value transforms into something that is different""" model_field = _get_model_field(model_class, "value") - value_type = model_field.type_ + value_type = get_type(model_field) value = _get_default_field_value(model_class) if isinstance(value, bool): diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index ed8a2c2979f..679091f6e85 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -210,7 +210,7 @@ async def _go(client: TestClient, project_uuid: UUID) -> None: # add a node node_id = faker.uuid4() - node = Node.parse_obj( + node = Node.model_validate( { "key": f"simcore/services/comp/test_{__name__}", "version": "1.0.0", diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control.py index ed04b3728e2..ae95f95f9f9 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control.py @@ -4,16 +4,13 @@ from models_library.projects import NodesDict -from pydantic import BaseModel +from pydantic import ConfigDict, RootModel from simcore_service_webserver.projects.models import ProjectDict from simcore_service_webserver.version_control.db import compute_workbench_checksum -class WorkbenchModel(BaseModel): - __root__: NodesDict - - class Config: - allow_population_by_field_name = True +class WorkbenchModel(RootModel[NodesDict]): + model_config = ConfigDict(populate_by_name=True) def test_compute_workbench_checksum(fake_project: ProjectDict): @@ -21,12 +18,12 @@ def test_compute_workbench_checksum(fake_project: ProjectDict): # as a dict sha1_w_dict = compute_workbench_checksum(fake_project["workbench"]) - workbench = WorkbenchModel.parse_obj(fake_project["workbench"]) + workbench = WorkbenchModel.model_validate(fake_project["workbench"]) # with pydantic models, i.e. Nodes # # e.g. order after parse maps order in BaseModel but not in dict # - sha1_w_model = compute_workbench_checksum(workbench.__root__) + sha1_w_model = compute_workbench_checksum(workbench.root) assert sha1_w_model == sha1_w_dict diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py index 5b660286cea..705b0458188 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py @@ -71,9 +71,9 @@ async def test_workflow( vc_repo, project_uuid, HEAD, message="updated message" ) - assert checkpoint2_updated.dict(exclude={"message"}) == checkpoint2.dict( + assert checkpoint2_updated.model_dump( exclude={"message"} - ) + ) == checkpoint2.model_dump(exclude={"message"}) # ------------------------------------- # checking out to v1 diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index 05ab31ccdf8..5270a17c04c 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -32,7 +32,7 @@ async def assert_resp_page( assert resp.status == status.HTTP_200_OK, f"Got {await resp.text()}" body = await resp.json() - page = expected_page_cls.parse_obj(body) + page = expected_page_cls.model_validate(body) assert page.meta.total == expected_total assert page.meta.count == expected_count return page @@ -42,7 +42,7 @@ async def assert_status_and_body( resp, expected_cls: HTTPStatus, expected_model: type[BaseModel] ) -> BaseModel: data, _ = await assert_status(resp, expected_cls) - return expected_model.parse_obj(data) + return expected_model.model_validate(data) @pytest.mark.acceptance_test() @@ -59,7 +59,7 @@ async def test_workflow( # get existing project resp = await client.get(f"/{VX}/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - project = Project.parse_obj(data) + project = Project.model_validate(data) assert project.uuid == UUID(project_uuid) # @@ -78,7 +78,7 @@ async def test_workflow( data, _ = await assert_status(resp, status.HTTP_201_CREATED) assert data - checkpoint1 = CheckpointApiModel.parse_obj(data) # NOTE: this is NOT API model + checkpoint1 = CheckpointApiModel.model_validate(data) # NOTE: this is NOT API model # # this project now has a repo @@ -87,20 +87,20 @@ async def test_workflow( resp, expected_page_cls=Page[ProjectDict], expected_total=1, expected_count=1 ) - repo = RepoApiModel.parse_obj(page.data[0]) + repo = RepoApiModel.model_validate(page.data[0]) assert repo.project_uuid == UUID(project_uuid) # GET checkpoint with HEAD resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/HEAD") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert CheckpointApiModel.parse_obj(data) == checkpoint1 + assert CheckpointApiModel.model_validate(data) == checkpoint1 # TODO: GET checkpoint with tag with pytest.raises(aiohttp.ClientResponseError) as excinfo: resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/v1") resp.raise_for_status() - assert CheckpointApiModel.parse_obj(data) == checkpoint1 + assert CheckpointApiModel.model_validate(data) == checkpoint1 assert excinfo.value.status == status.HTTP_501_NOT_IMPLEMENTED @@ -108,8 +108,8 @@ async def test_workflow( resp = await client.get( f"/{VX}/repos/projects/{project_uuid}/checkpoints/{checkpoint1.id}" ) - assert str(resp.url) == checkpoint1.url - assert CheckpointApiModel.parse_obj(data) == checkpoint1 + assert f"{resp.url}" == f"{checkpoint1.url}" + assert CheckpointApiModel.model_validate(data) == checkpoint1 # LIST checkpoints resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") @@ -120,7 +120,7 @@ async def test_workflow( expected_count=1, ) - assert CheckpointApiModel.parse_obj(page.data[0]) == checkpoint1 + assert CheckpointApiModel.model_validate(page.data[0]) == checkpoint1 # UPDATE checkpoint annotations resp = await client.patch( @@ -128,7 +128,7 @@ async def test_workflow( json={"message": "updated message", "tag": "Version 1"}, ) data, _ = await assert_status(resp, status.HTTP_200_OK) - checkpoint1_updated = CheckpointApiModel.parse_obj(data) + checkpoint1_updated = CheckpointApiModel.model_validate(data) assert checkpoint1.id == checkpoint1_updated.id assert checkpoint1.checksum == checkpoint1_updated.checksum @@ -142,7 +142,7 @@ async def test_workflow( data, _ = await assert_status(resp, status.HTTP_200_OK) assert ( data["workbench"] - == project.dict(exclude_none=True, exclude_unset=True)["workbench"] + == project.model_dump(exclude_none=True, exclude_unset=True)["workbench"] ) # do some changes in project @@ -154,30 +154,30 @@ async def test_workflow( json={"tag": "v2", "message": "new commit"}, ) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - checkpoint2 = CheckpointApiModel.parse_obj(data) + checkpoint2 = CheckpointApiModel.model_validate(data) assert checkpoint2.tags == ("v2",) # GET checkpoint with HEAD resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/HEAD") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert CheckpointApiModel.parse_obj(data) == checkpoint2 + assert CheckpointApiModel.model_validate(data) == checkpoint2 # CHECKOUT resp = await client.post( f"/{VX}/repos/projects/{project_uuid}/checkpoints/{checkpoint1.id}:checkout" ) data, _ = await assert_status(resp, status.HTTP_200_OK) - assert CheckpointApiModel.parse_obj(data) == checkpoint1_updated + assert CheckpointApiModel.model_validate(data) == checkpoint1_updated # GET checkpoint with HEAD resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/HEAD") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert CheckpointApiModel.parse_obj(data) == checkpoint1_updated + assert CheckpointApiModel.model_validate(data) == checkpoint1_updated # get working copy resp = await client.get(f"/{VX}/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - project_wc = Project.parse_obj(data) + project_wc = Project.model_validate(data) assert project_wc.uuid == UUID(project_uuid) assert project_wc != project @@ -193,7 +193,7 @@ async def test_create_checkpoint_without_changes( data, _ = await assert_status(resp, status.HTTP_201_CREATED) assert data - checkpoint1 = CheckpointApiModel.parse_obj(data) # NOTE: this is NOT API model + checkpoint1 = CheckpointApiModel.model_validate(data) # NOTE: this is NOT API model # CREATE checkpoint WITHOUT changes resp = await client.post( @@ -203,7 +203,7 @@ async def test_create_checkpoint_without_changes( data, _ = await assert_status(resp, status.HTTP_201_CREATED) assert data - checkpoint2 = CheckpointApiModel.parse_obj(data) # NOTE: this is NOT API model + checkpoint2 = CheckpointApiModel.model_validate(data) # NOTE: this is NOT API model assert ( checkpoint1 == checkpoint2 diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 84ffd71830f..c943be8c76c 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -41,7 +41,7 @@ from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.products import ProductName from models_library.services_enums import ServiceState -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.dict_tools import ConfigDict from pytest_simcore.helpers.faker_factories import random_product @@ -382,7 +382,7 @@ async def _mock_result(): mock3 = mocker.patch( "simcore_service_webserver.projects._crud_api_create.get_project_total_size_simcore_s3", autospec=True, - return_value=parse_obj_as(ByteSize, "1Gib"), + return_value=TypeAdapter(ByteSize).validate_python("1Gib"), ) return MockedStorageSubsystem(mock, mock1, mock2, mock3) From 0527b1e121e39634af1f80d76db529af50816eda Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Wed, 6 Nov 2024 12:40:08 +0100 Subject: [PATCH 30/53] =?UTF-8?q?=F0=9F=90=9B=20pydantic=20v2=20fixed=20br?= =?UTF-8?q?oken=20unit=20tests=20(#6649)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- services/api-server/requirements/_base.txt | 2 +- services/api-server/requirements/_test.txt | 2 +- services/autoscaling/requirements/_base.txt | 2 +- services/autoscaling/requirements/_test.txt | 2 +- services/catalog/requirements/_base.txt | 2 +- services/catalog/requirements/_test.txt | 2 +- .../clusters-keeper/requirements/_base.txt | 2 +- .../clusters-keeper/requirements/_test.txt | 2 +- services/dask-sidecar/requirements/_base.txt | 2 +- .../requirements/_dask-distributed.txt | 2 +- services/dask-sidecar/requirements/_test.txt | 2 +- .../datcore-adapter/requirements/_base.txt | 2 +- .../datcore-adapter/requirements/_test.txt | 2 +- services/director-v2/requirements/_base.txt | 2 +- services/director-v2/requirements/_test.txt | 2 +- .../core/settings.py | 19 +++++++++++++++---- .../services/service_tracker/_models.py | 4 +++- .../tests/unit/service_tracker/test__api.py | 4 ++++ .../unit/service_tracker/test__models.py | 5 +++-- .../dynamic-sidecar/requirements/_base.txt | 2 +- .../dynamic-sidecar/requirements/_test.txt | 2 +- .../api/rest/health.py | 2 +- .../tests/unit/test_api_rest_containers.py | 4 ++-- ..._api_rest_containers_long_running_tasks.py | 7 ++++--- .../tests/unit/test_api_rest_health.py | 6 ++++-- .../unit/test_api_rest_prometheus_metrics.py | 5 +++-- .../test_api_rest_workflow_service_metrics.py | 7 ++++--- services/invitations/requirements/_base.txt | 2 +- .../requirements/_test.txt | 2 +- .../tests/system/requirements/_test.txt | 2 +- services/payments/requirements/_base.txt | 2 +- services/payments/requirements/_test.txt | 2 +- .../requirements/_base.txt | 2 +- .../requirements/_test.txt | 2 +- services/storage/requirements/_base.txt | 2 +- services/storage/requirements/_test.txt | 2 +- services/web/server/requirements/_base.txt | 2 +- services/web/server/requirements/_test.txt | 2 +- 38 files changed, 71 insertions(+), 48 deletions(-) diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 712567f425e..c190f06e6f7 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -826,7 +826,7 @@ ujson==5.9.0 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt index 19d8d350199..26f93fde738 100644 --- a/services/api-server/requirements/_test.txt +++ b/services/api-server/requirements/_test.txt @@ -327,7 +327,7 @@ typing-extensions==4.10.0 # boto3-stubs # mypy # sqlalchemy2-stubs -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index 29aa4ae7a5e..95ee384b190 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -693,7 +693,7 @@ typing-extensions==4.11.0 # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/autoscaling/requirements/_test.txt b/services/autoscaling/requirements/_test.txt index 7784681d8b9..e019e4f118b 100644 --- a/services/autoscaling/requirements/_test.txt +++ b/services/autoscaling/requirements/_test.txt @@ -331,7 +331,7 @@ typing-extensions==4.11.0 # types-aiobotocore-iam # types-aiobotocore-s3 # types-aiobotocore-ssm -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 9004709650e..e48ea8af4ca 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -520,7 +520,7 @@ ujson==5.9.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/catalog/requirements/_test.txt b/services/catalog/requirements/_test.txt index c824bb0f6c3..a379e35f4d5 100644 --- a/services/catalog/requirements/_test.txt +++ b/services/catalog/requirements/_test.txt @@ -185,7 +185,7 @@ typing-extensions==4.10.0 # alembic # mypy # sqlalchemy2-stubs -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 806dca3beb0..61e4c8295d0 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -691,7 +691,7 @@ typing-extensions==4.11.0 # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index a43369668cc..4e297870fd4 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -317,7 +317,7 @@ typing-extensions==4.11.0 # cfn-lint # pydantic # pydantic-core -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 01fd1d54565..a75d6cff0d5 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -496,7 +496,7 @@ typing-extensions==4.11.0 # typer tzdata==2024.1 # via pandas -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/dask-sidecar/requirements/_dask-distributed.txt b/services/dask-sidecar/requirements/_dask-distributed.txt index e1b822b67bb..78a222ea415 100644 --- a/services/dask-sidecar/requirements/_dask-distributed.txt +++ b/services/dask-sidecar/requirements/_dask-distributed.txt @@ -90,7 +90,7 @@ tornado==6.4 # via # -c requirements/./_base.txt # distributed -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/./_base.txt # distributed diff --git a/services/dask-sidecar/requirements/_test.txt b/services/dask-sidecar/requirements/_test.txt index 92130c87b77..db273aba95a 100644 --- a/services/dask-sidecar/requirements/_test.txt +++ b/services/dask-sidecar/requirements/_test.txt @@ -253,7 +253,7 @@ typing-extensions==4.11.0 # cfn-lint # pydantic # pydantic-core -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index 04e4eed01d9..a007dbea811 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -420,7 +420,7 @@ typing-extensions==4.10.0 # pydantic # pydantic-core # typer -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/datcore-adapter/requirements/_test.txt b/services/datcore-adapter/requirements/_test.txt index b09942fe970..3ab05285f93 100644 --- a/services/datcore-adapter/requirements/_test.txt +++ b/services/datcore-adapter/requirements/_test.txt @@ -122,7 +122,7 @@ typing-extensions==4.10.0 # via # -c requirements/_base.txt # boto3-stubs -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 1ca7e9a0e9a..151f1a87e1d 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -978,7 +978,7 @@ ujson==5.10.0 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index ccfb429b50f..ee97fe23500 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -324,7 +324,7 @@ typing-extensions==4.11.0 # sqlalchemy2-stubs tzdata==2024.2 # via pandas -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index d1471052a9f..8fd79446760 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,6 +1,7 @@ import datetime from pydantic import AliasChoices, Field, TypeAdapter, field_validator +from pydantic_settings import SettingsConfigDict from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -32,19 +33,27 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, validation_alias=AliasChoices( - "DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", + "DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED", + ), + description=( + "Enables local development log format. WARNING: make sure it " + "is disabled if you want to have structured logs!" ), - description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING: dict[ LoggerName, list[MessageSubstring] ] = Field( default_factory=dict, validation_alias=AliasChoices( - "DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + "LOG_FILTER_MAPPING", + "DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", + ), + description=( + "is a dictionary that maps specific loggers " + "(such as 'uvicorn.access' or 'gunicorn.access') to a list " + "of log message patterns that should be filtered out." ), - description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: datetime.timedelta = Field( @@ -60,6 +69,8 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def _validate_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + model_config = SettingsConfigDict(extra="allow") + class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py index 70e063462bc..55a30cf2e8a 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_models.py @@ -131,7 +131,9 @@ def set_last_status_notification_to_now(self) -> None: ##################### def to_bytes(self) -> bytes: - result: bytes = umsgpack.packb(self.dict(), ext_handlers=_PACKB_EXTENSION_TYPES) + result: bytes = umsgpack.packb( + self.model_dump(), ext_handlers=_PACKB_EXTENSION_TYPES + ) return result @classmethod diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index 489dd14f2b8..f8b4b442a8e 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -45,6 +45,9 @@ pytest_simcore_core_services_selection = [ "redis", ] +pytest_simcore_ops_services_selection = [ + # "redis-commander", +] @pytest.fixture @@ -52,6 +55,7 @@ def app_environment( disable_rabbitmq_setup: None, disable_deferred_manager_setup: None, disable_notifier_setup: None, + disable_status_monitor_setup: None, app_environment: EnvVarsDict, redis_service: RedisSettings, remove_redis_data: None, diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py index 077da84dcc7..b07a41ed3fe 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__models.py @@ -12,6 +12,7 @@ DynamicServiceStart, ) from models_library.projects import ProjectID +from pydantic import TypeAdapter from servicelib.deferred_tasks import TaskUID from simcore_service_dynamic_scheduler.services.service_tracker._models import ( SchedulerServiceState, @@ -51,8 +52,8 @@ def test_serialization( "dynamic_service_start", [ None, - DynamicServiceStart.parse_obj( - DynamicServiceStart.Config.schema_extra["example"] + TypeAdapter(DynamicServiceStart).validate_python( + DynamicServiceStart.model_config["json_schema_extra"]["example"] ), ], ) diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index a1cfdbe68da..b2c2dbbd1b2 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -719,7 +719,7 @@ typing-extensions==4.11.0 # typer u-msgpack-python==2.8.0 # via -r requirements/_base.in -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/dynamic-sidecar/requirements/_test.txt b/services/dynamic-sidecar/requirements/_test.txt index 3a87310dcdb..6ad99539f08 100644 --- a/services/dynamic-sidecar/requirements/_test.txt +++ b/services/dynamic-sidecar/requirements/_test.txt @@ -144,7 +144,7 @@ typing-extensions==4.11.0 # mypy # sqlalchemy2-stubs # types-aiobotocore-s3 -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/health.py index f55c8dad9ef..848821715e9 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/health.py @@ -29,7 +29,7 @@ async def health_endpoint( ) -> ApplicationHealth: if not application_health.is_healthy: raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, detail=application_health.dict() + status.HTTP_503_SERVICE_UNAVAILABLE, detail=application_health.model_dump() ) if not rabbitmq_client.healthy or not rabbitmq_rpc_server.healthy: diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index 0877e7ab0fe..a5179cd87e7 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -104,7 +104,7 @@ async def _start_containers( response = await test_client.post( f"/{API_VTAG}/containers", - json={"metrics_params": mock_metrics_params.dict()}, + json={"metrics_params": mock_metrics_params.model_dump()}, ) assert response.status_code == status.HTTP_202_ACCEPTED, response.text task_id: TaskId = response.json() @@ -776,7 +776,7 @@ def mock_inactive_since_command_response( activity_response: ActivityInfo, ) -> None: mocker.patch( - "simcore_service_dynamic_sidecar.api.containers.run_command_in_container", + "simcore_service_dynamic_sidecar.api.rest.containers.run_command_in_container", return_value=activity_response.model_dump_json(), ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 66a31f1c13c..98122863397 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -290,15 +290,16 @@ async def _get_task_id_create_service_containers( *args, **kwargs, ) -> TaskId: - ctontainers_compose_spec = ContainersComposeSpec( + containers_compose_spec = ContainersComposeSpec( docker_compose_yaml=compose_spec, ) await httpx_async_client.post( - f"/{API_VTAG}/containers/compose-spec", json=ctontainers_compose_spec.dict() + f"/{API_VTAG}/containers/compose-spec", + json=containers_compose_spec.model_dump(), ) containers_create = ContainersCreate(metrics_params=mock_metrics_params) response = await httpx_async_client.post( - f"/{API_VTAG}/containers", json=containers_create.dict() + f"/{API_VTAG}/containers", json=containers_create.model_dump() ) task_id: TaskId = response.json() assert isinstance(task_id, str) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_health.py b/services/dynamic-sidecar/tests/unit/test_api_rest_health.py index f9b18e3d795..b7dca61cf63 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_health.py @@ -18,14 +18,16 @@ async def test_is_healthy(test_client: TestClient) -> None: test_client.application.state.application_health.is_healthy = True response = await test_client.get("/health") assert response.status_code == status.HTTP_200_OK, response - assert response.json() == ApplicationHealth(is_healthy=True).dict() + assert response.json() == ApplicationHealth(is_healthy=True).model_dump() async def test_is_unhealthy(test_client: TestClient) -> None: test_client.application.state.application_health.is_healthy = False response = await test_client.get("/health") assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE, response - assert response.json() == {"detail": ApplicationHealth(is_healthy=False).dict()} + assert response.json() == { + "detail": ApplicationHealth(is_healthy=False).model_dump() + } async def test_is_unhealthy_via_rabbitmq(test_client: TestClient) -> None: diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index e8206dc47a9..1767e403991 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -111,11 +111,12 @@ async def _get_task_id_create_service_containers( docker_compose_yaml=compose_spec, ) await httpx_async_client.post( - f"/{API_VTAG}/containers/compose-spec", json=ctontainers_compose_spec.dict() + f"/{API_VTAG}/containers/compose-spec", + json=ctontainers_compose_spec.model_dump(), ) containers_create = ContainersCreate(metrics_params=mock_metrics_params) response = await httpx_async_client.post( - f"/{API_VTAG}/containers", json=containers_create.dict() + f"/{API_VTAG}/containers", json=containers_create.model_dump() ) task_id: TaskId = response.json() assert isinstance(task_id, str) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index ddb5b18df1a..1b2bb2a16fa 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -149,15 +149,16 @@ async def _get_task_id_create_service_containers( compose_spec: str, mock_metrics_params: CreateServiceMetricsAdditionalParams, ) -> TaskId: - ctontainers_compose_spec = ContainersComposeSpec( + containers_compose_spec = ContainersComposeSpec( docker_compose_yaml=compose_spec, ) await httpx_async_client.post( - f"/{API_VTAG}/containers/compose-spec", json=ctontainers_compose_spec.dict() + f"/{API_VTAG}/containers/compose-spec", + json=containers_compose_spec.model_dump(), ) containers_create = ContainersCreate(metrics_params=mock_metrics_params) response = await httpx_async_client.post( - f"/{API_VTAG}/containers", json=containers_create.dict() + f"/{API_VTAG}/containers", json=containers_create.model_dump() ) task_id: TaskId = response.json() assert isinstance(task_id, str) diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index f1a478c30be..4849f95a92f 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -410,7 +410,7 @@ typing-extensions==4.10.0 # pydantic # pydantic-core # typer -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/osparc-gateway-server/requirements/_test.txt b/services/osparc-gateway-server/requirements/_test.txt index a9fff835004..1fc9e930b69 100644 --- a/services/osparc-gateway-server/requirements/_test.txt +++ b/services/osparc-gateway-server/requirements/_test.txt @@ -192,7 +192,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # mypy # sqlalchemy2-stubs -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/../../dask-sidecar/requirements/_dask-distributed.txt diff --git a/services/osparc-gateway-server/tests/system/requirements/_test.txt b/services/osparc-gateway-server/tests/system/requirements/_test.txt index 0977f99f778..29d4e7666d4 100644 --- a/services/osparc-gateway-server/tests/system/requirements/_test.txt +++ b/services/osparc-gateway-server/tests/system/requirements/_test.txt @@ -175,7 +175,7 @@ tornado==6.4 # -c requirements/../../../../dask-sidecar/requirements/_dask-distributed.txt # dask-gateway # distributed -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../../../requirements/constraints.txt # -c requirements/../../../../dask-sidecar/requirements/_dask-distributed.txt diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index e5c6a6edb48..cb5f1245458 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -541,7 +541,7 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # typer -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/payments/requirements/_test.txt b/services/payments/requirements/_test.txt index 8a2b7c3d174..f6d7b7e540a 100644 --- a/services/payments/requirements/_test.txt +++ b/services/payments/requirements/_test.txt @@ -178,7 +178,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # mypy # sqlalchemy2-stubs -urllib3==2.2.2 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 3587be89a74..9221cb3577d 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -745,7 +745,7 @@ tzdata==2024.1 # via pandas tzlocal==5.2 # via dateparser -urllib3==2.0.7 +urllib3==2.2.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/resource-usage-tracker/requirements/_test.txt b/services/resource-usage-tracker/requirements/_test.txt index 2a0809dfa17..484ce4158da 100644 --- a/services/resource-usage-tracker/requirements/_test.txt +++ b/services/resource-usage-tracker/requirements/_test.txt @@ -318,7 +318,7 @@ typing-extensions==4.10.0 # pydantic # pydantic-core # sqlalchemy2-stubs -urllib3==2.0.7 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index 701d9577d62..385ab816619 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -733,7 +733,7 @@ ujson==5.9.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiohttp-swagger -urllib3==2.0.7 +urllib3==2.2.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 2915015ad89..6157f084c4d 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -333,7 +333,7 @@ typing-extensions==4.10.0 # sqlalchemy2-stubs tzdata==2024.2 # via pandas -urllib3==2.0.7 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index 9f0548262a8..59c96ab1fd4 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -813,7 +813,7 @@ ujson==5.5.0 # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt # aiohttp-swagger -urllib3==1.26.11 +urllib3==2.2.3 # via # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index d787c281d34..54ecb02f5e4 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -234,7 +234,7 @@ typing-extensions==4.12.0 # asyncpg-stubs # mypy # sqlalchemy2-stubs -urllib3==1.26.11 +urllib3==2.2.3 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt From 60fe257f7e834b78478e9e00d21733f799056b5c Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Fri, 8 Nov 2024 13:44:26 +0100 Subject: [PATCH 31/53] =?UTF-8?q?=F0=9F=8E=A8=20Add=20examples=20to=20pyda?= =?UTF-8?q?ntic=20response=20models=20(#6658)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_schemas_webserver/product.py | 15 +- .../api_schemas_webserver/resource_usage.py | 7 +- .../api_schemas_webserver/wallets.py | 16 ++- .../src/pytest_simcore/pydantic_models.py | 4 + .../simcore_webserver_projects_rest_api.py | 3 + scripts/openapi-diff.bash | 2 +- services/api-server/Makefile | 19 ++- services/api-server/openapi.json | 136 ++++++++++++------ .../api/routes/solvers_jobs_getters.py | 8 +- .../models/schemas/files.py | 10 +- .../models/schemas/jobs.py | 28 ++++ .../models/schemas/meta.py | 8 +- .../models/schemas/studies.py | 22 ++- .../services/webserver.py | 4 +- .../tests/mocks/create_study_job.json | 3 + .../mocks/for_test_api_routes_studies.json | 6 + .../tests/mocks/get_job_outputs.json | 3 + .../get_job_pricing_unit_invalid_solver.json | 3 + .../mocks/get_job_pricing_unit_success.json | 9 +- .../tests/mocks/get_solver_outputs.json | 3 + .../get_solver_pricing_plan_success.json | 6 +- .../api-server/tests/mocks/on_list_jobs.json | 6 + .../tests/mocks/run_study_workflow.json | 3 + .../mocks/start_job_not_enough_credit.json | 3 + .../tests/mocks/start_job_with_payment.json | 3 + ...est_get_and_update_study_job_metadata.json | 3 + .../test_api_routers_solvers_jobs.py | 3 +- 27 files changed, 262 insertions(+), 74 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/product.py b/packages/models-library/src/models_library/api_schemas_webserver/product.py index f967e15d548..1e747c554fb 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/product.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/product.py @@ -1,7 +1,15 @@ from datetime import datetime from typing import Annotated, TypeAlias -from pydantic import ConfigDict, Field, HttpUrl, NonNegativeInt, PositiveInt +from pydantic import ( + ConfigDict, + Field, + HttpUrl, + NonNegativeFloat, + NonNegativeInt, + PlainSerializer, + PositiveInt, +) from ..basic_types import IDStr, NonNegativeDecimal from ..emails import LowerCaseEmailStr @@ -11,7 +19,10 @@ class GetCreditPrice(OutputSchema): product_name: str - usd_per_credit: NonNegativeDecimal | None = Field( + usd_per_credit: Annotated[ + NonNegativeDecimal, + PlainSerializer(float, return_type=NonNegativeFloat, when_used="json"), + ] | None = Field( ..., description="Price of a credit in USD. " "If None, then this product's price is UNDEFINED", diff --git a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py index 3eea55c6d67..bdf0992910e 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py @@ -1,7 +1,8 @@ from datetime import datetime from decimal import Decimal +from typing import Annotated -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, PlainSerializer from ..projects import ProjectID from ..projects_nodes_io import NodeID @@ -49,7 +50,9 @@ class PricingUnitGet(OutputSchema): pricing_unit_id: PricingUnitId unit_name: str unit_extra_info: UnitExtraInfo - current_cost_per_unit: Decimal + current_cost_per_unit: Annotated[ + Decimal, PlainSerializer(float, return_type=float, when_used="json") + ] default: bool diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index ae20bee1e62..21fe990b88b 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -2,7 +2,14 @@ from decimal import Decimal from typing import Annotated, Literal, TypeAlias -from pydantic import ConfigDict, Field, HttpUrl, PlainSerializer, ValidationInfo, field_validator +from pydantic import ( + ConfigDict, + Field, + HttpUrl, + PlainSerializer, + ValidationInfo, + field_validator, +) from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID @@ -20,10 +27,7 @@ class WalletGet(OutputSchema): created: datetime modified: datetime - model_config = ConfigDict( - from_attributes=True, - frozen=False - ) + model_config = ConfigDict(from_attributes=True, frozen=False) class WalletGetWithAvailableCredits(WalletGet): @@ -163,7 +167,7 @@ class PaymentMethodGet(OutputSchema): "autoRecharge": "False", }, ], - } + }, ) diff --git a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py index 05e85361eba..1c2809fa52e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py @@ -70,10 +70,14 @@ def test_model_examples( def _is_model_cls(obj) -> bool: with suppress(TypeError): # NOTE: issubclass( dict[models_library.services.ConstrainedStrValue, models_library.services.ServiceInput] ) raises TypeError + is_parametrized = False + if hasattr(obj, "__parameters__"): + is_parametrized = len(obj.__parameters__) == 0 return ( obj is not BaseModel and inspect.isclass(obj) and issubclass(obj, BaseModel) + and not is_parametrized ) return False diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py index 99452834be1..ca386f1914c 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py @@ -96,6 +96,9 @@ def request_desc(self) -> str: "creationDate": "2021-12-06T10:13:03.100Z", "lastChangeDate": "2021-12-06T10:13:03.100Z", "workbench": {}, + "workspaceId": 123, + "folderId": 2, + "trashedAt": "2021-12-06T10:13:18.100Z", "accessRights": {"2": {"read": True, "write": True, "delete": True}}, "dev": {}, "classifiers": [], diff --git a/scripts/openapi-diff.bash b/scripts/openapi-diff.bash index c611422f14e..519bcb6fe51 100755 --- a/scripts/openapi-diff.bash +++ b/scripts/openapi-diff.bash @@ -16,5 +16,5 @@ exec docker run \ --volume="/etc/passwd:/etc/passwd:ro" \ --user="$(id --user "$USER")":"$(id --group "$USER")" \ --volume "$(pwd):/specs" \ - openapitools/openapi-diff:latest \ + tufin/oasdiff:latest \ "$@" diff --git a/services/api-server/Makefile b/services/api-server/Makefile index 5b675b22ce4..39672c9764e 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -4,6 +4,18 @@ include ../../scripts/common.Makefile include ../../scripts/common-service.Makefile +# Check that given variables are set and all have non-empty values, +# die with an error otherwise. +# +# Params: +# 1. Variable name(s) to test. +# 2. (optional) Error message to print. +guard-%: + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable $* not set"; \ + exit 1; \ + fi + .PHONY: reqs reqs: ## compiles pip requirements (.in -> .txt) @$(MAKE_C) requirements reqs @@ -57,7 +69,8 @@ openapi-client-master.json: define _openapi_diff_inputs - $(SCRIPTS_DIR)/openapi-diff.bash $(foreach f,$^,/specs/$f) --fail-on-incompatible --json=/specs/$@ + $(SCRIPTS_DIR)/openapi-diff.bash diff $(foreach f,$^,/specs/$f) --format json > $@ + $(SCRIPTS_DIR)/openapi-diff.bash breaking $(foreach f,$^,/specs/$f) --fail-on ERR > /dev/null endef # Examples: @@ -67,7 +80,9 @@ endef openapi-%-diff.json: openapi.json openapi-%.json ## Diffs against newer or older openapi-%.json and checks backwards compatibility $(call _openapi_diff_inputs) - +openapi-diff.md: guard-OPENAPI_JSON_BASE_URL openapi.json ## Diffs against a remote openapi.json. E.g. OPENAPI_JSON_BASE_URL=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/api-server/openapi.json + $(SCRIPTS_DIR)/openapi-diff.bash diff $(OPENAPI_JSON_BASE_URL) /specs/openapi.json --format markup --flatten-allof --exclude-elements title,description > $@ + $(SCRIPTS_DIR)/openapi-diff.bash breaking $(OPENAPI_JSON_BASE_URL) /specs/openapi.json --flatten-allof --fail-on ERR # SEE https://schemathesis.readthedocs.io/en/stable/index.html APP_URL:=http://$(get_my_ip).nip.io:8006 diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 51ba054e5af..5b23b44603d 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -3788,15 +3788,7 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/WalletGetWithAvailableCredits" - }, - { - "type": "null" - } - ], - "title": "Response Get Job Wallet V0 Solvers Solver Key Releases Version Jobs Job Id Wallet Get" + "$ref": "#/components/schemas/WalletGetWithAvailableCredits" } } } @@ -3935,15 +3927,7 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/PricingUnitGet" - }, - { - "type": "null" - } - ], - "title": "Response Get Job Pricing Unit V0 Solvers Solver Key Releases Version Jobs Job Id Pricing Unit Get" + "$ref": "#/components/schemas/PricingUnitGet" } } } @@ -5414,16 +5398,8 @@ "description": "File name" }, "filesize": { - "anyOf": [ - { - "type": "string", - "pattern": "^\\s*(\\d*\\.?\\d+)\\s*(\\w+)?" - }, - { - "type": "integer", - "minimum": 0 - } - ], + "type": "integer", + "minimum": 0, "title": "Filesize", "description": "File size in bytes" }, @@ -5568,6 +5544,7 @@ "urls": { "items": { "type": "string", + "maxLength": 65536, "minLength": 1, "format": "uri" }, @@ -5595,7 +5572,8 @@ "usdPerCredit": { "anyOf": [ { - "type": "string" + "type": "number", + "minimum": 0.0 }, { "type": "null" @@ -5934,7 +5912,17 @@ "metadata", "url" ], - "title": "JobMetadata" + "title": "JobMetadata", + "example": { + "job_id": "3497e4de-0e69-41fb-b08f-7f3875a1ac4b", + "metadata": { + "bool": "true", + "float": "3.14", + "int": "42", + "str": "hej med dig" + }, + "url": "https://f02b2452-1dd8-4882-b673-af06373b41b3.fake" + } }, "JobMetadataUpdate": { "properties": { @@ -5961,7 +5949,15 @@ } }, "type": "object", - "title": "JobMetadataUpdate" + "title": "JobMetadataUpdate", + "example": { + "metadata": { + "bool": "true", + "float": "3.14", + "int": "42", + "str": "hej med dig" + } + } }, "JobOutputs": { "properties": { @@ -6163,6 +6159,7 @@ }, "download_link": { "type": "string", + "maxLength": 65536, "minLength": 1, "format": "uri", "title": "Download Link" @@ -6204,12 +6201,14 @@ }, "docs_url": { "type": "string", + "maxLength": 65536, "minLength": 1, "format": "uri", "title": "Docs Url" }, "docs_dev_url": { "type": "string", + "maxLength": 65536, "minLength": 1, "format": "uri", "title": "Docs Dev Url" @@ -6244,8 +6243,15 @@ "title": "Items" }, "total": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Total" } }, @@ -6265,8 +6271,15 @@ "title": "Items" }, "total": { - "type": "integer", - "minimum": 0, + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], "title": "Total" } }, @@ -6474,11 +6487,10 @@ "title": "Unitname" }, "unitExtraInfo": { - "type": "object", - "title": "Unitextrainfo" + "$ref": "#/components/schemas/UnitExtraInfo" }, "currentCostPerUnit": { - "type": "string", + "type": "number", "title": "Currentcostperunit" }, "default": { @@ -6808,11 +6820,25 @@ "title": "Uid" }, "title": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Title" }, "description": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Description" } }, @@ -6869,6 +6895,34 @@ "kind": "input" } }, + "UnitExtraInfo": { + "properties": { + "CPU": { + "type": "integer", + "minimum": 0, + "title": "Cpu" + }, + "RAM": { + "type": "integer", + "minimum": 0, + "title": "Ram" + }, + "VRAM": { + "type": "integer", + "minimum": 0, + "title": "Vram" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "CPU", + "RAM", + "VRAM" + ], + "title": "UnitExtraInfo", + "description": "Custom information that is propagated to the frontend. Defined fields are mandatory." + }, "UploadLinks": { "properties": { "abort_upload": { @@ -7045,9 +7099,7 @@ "required": [ "walletId", "name", - "description", "owner", - "thumbnail", "status", "created", "modified", diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index 4e36afe7ccd..708b9871a68 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -376,7 +376,7 @@ async def get_job_custom_metadata( @router.get( "/{solver_key:path}/releases/{version}/jobs/{job_id:uuid}/wallet", - response_model=WalletGetWithAvailableCredits | None, + response_model=WalletGetWithAvailableCredits, responses=WALLET_STATUS_CODES, description=("Get job wallet\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7")), ) @@ -385,18 +385,18 @@ async def get_job_wallet( version: VersionStr, job_id: JobID, webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -) -> WalletGetWithAvailableCredits | None: +) -> WalletGetWithAvailableCredits: job_name = _compose_job_resource_name(solver_key, version, job_id) _logger.debug("Getting wallet for job '%s'", job_name) if project_wallet := await webserver_api.get_project_wallet(project_id=job_id): return await webserver_api.get_wallet(wallet_id=project_wallet.wallet_id) - return None + raise MissingWalletError(job_id=job_id) @router.get( "/{solver_key:path}/releases/{version}/jobs/{job_id:uuid}/pricing_unit", - response_model=PricingUnitGet | None, + response_model=PricingUnitGet, responses=_PRICING_UNITS_STATUS_CODES, description=( "Get job pricing unit\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7") diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/files.py b/services/api-server/src/simcore_service_api_server/models/schemas/files.py index 42686f0876e..f56fa0da669 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/files.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/files.py @@ -13,9 +13,9 @@ from pydantic import ( AnyUrl, BaseModel, - ByteSize, ConfigDict, Field, + NonNegativeInt, StringConstraints, TypeAdapter, ValidationInfo, @@ -33,7 +33,7 @@ class ClientFile(BaseModel): """Represents a file stored on the client side""" filename: FileName = Field(..., description="File name") - filesize: ByteSize = Field(..., description="File size in bytes") + filesize: NonNegativeInt = Field(..., description="File size in bytes") sha256_checksum: SHA256Str = Field(..., description="SHA256 checksum") @@ -166,8 +166,10 @@ class UploadLinks(BaseModel): class FileUploadData(BaseModel): - chunk_size: ByteSize - urls: list[AnyUrl] + chunk_size: NonNegativeInt + urls: list[ + Annotated[AnyUrl, StringConstraints(max_length=65536)] + ] # maxlength added for backwards compatibility links: UploadLinks diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index 485d0c920c1..2a7dc8c4f32 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -133,6 +133,19 @@ class JobMetadataUpdate(BaseModel): default_factory=dict, description="Custom key-value map" ) + model_config = ConfigDict( + json_schema_extra={ + "example": { + "metadata": { + "bool": "true", + "int": "42", + "float": "3.14", + "str": "hej med dig", + } + } + } + ) + class JobMetadata(BaseModel): job_id: JobID = Field(..., description="Parent Job") @@ -141,6 +154,21 @@ class JobMetadata(BaseModel): # Links url: HttpUrl | None = Field(..., description="Link to get this resource (self)") + model_config = ConfigDict( + json_schema_extra={ + "example": { + "job_id": "3497e4de-0e69-41fb-b08f-7f3875a1ac4b", + "metadata": { + "bool": "true", + "int": "42", + "float": "3.14", + "str": "hej med dig", + }, + "url": "https://f02b2452-1dd8-4882-b673-af06373b41b3.fake", + } + } + ) + # JOBS ---------- # - A job can be create on a specific solver or other type of future runner (e.g. a pipeline) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/meta.py b/services/api-server/src/simcore_service_api_server/models/schemas/meta.py index 64f80a45b9b..6332d5ae1d5 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/meta.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/meta.py @@ -1,10 +1,12 @@ +from typing import Annotated + from models_library.api_schemas__common.meta import BaseMeta -from pydantic import AnyHttpUrl, ConfigDict +from pydantic import AnyHttpUrl, ConfigDict, StringConstraints class Meta(BaseMeta): - docs_url: AnyHttpUrl - docs_dev_url: AnyHttpUrl + docs_url: Annotated[AnyHttpUrl, StringConstraints(max_length=65536)] + docs_dev_url: Annotated[AnyHttpUrl, StringConstraints(max_length=65536)] model_config = ConfigDict( json_schema_extra={ "example": { diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/studies.py b/services/api-server/src/simcore_service_api_server/models/schemas/studies.py index eb93027ebe6..6815fcc5216 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/studies.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/studies.py @@ -1,7 +1,7 @@ -from typing import TypeAlias +from typing import Annotated, TypeAlias from models_library import projects, projects_nodes_io -from pydantic import AnyUrl, BaseModel, Field +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, StringConstraints from .. import api_resources from . import solvers @@ -28,11 +28,27 @@ class StudyPort(solvers.SolverPort): "Correponds to the UUID of the parameter/probe node in the study", title="Key name", ) + model_config = ConfigDict( + extra="ignore", + json_schema_extra={ + "example": { + "key": "f763658f-a89a-4a90-ace4-c44631290f12", + "kind": "input", + "content_schema": { + "title": "Sleep interval", + "type": "integer", + "x_unit": "second", + "minimum": 0, + "maximum": 5, + }, + } + }, + ) class LogLink(BaseModel): node_name: NodeName - download_link: DownloadLink + download_link: Annotated[DownloadLink, StringConstraints(max_length=65536)] class JobLogsMap(BaseModel): diff --git a/services/api-server/src/simcore_service_api_server/services/webserver.py b/services/api-server/src/simcore_service_api_server/services/webserver.py index 5a2326258af..0c5428bf0e6 100644 --- a/services/api-server/src/simcore_service_api_server/services/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services/webserver.py @@ -408,7 +408,7 @@ async def update_project_metadata( @_exception_mapper({status.HTTP_404_NOT_FOUND: PricingUnitNotFoundError}) async def get_project_node_pricing_unit( self, *, project_id: UUID, node_id: UUID - ) -> PricingUnitGet | None: + ) -> PricingUnitGet: response = await self.client.get( f"/projects/{project_id}/nodes/{node_id}/pricing-unit", cookies=self.session_cookies, @@ -559,7 +559,7 @@ async def get_wallet(self, *, wallet_id: int) -> WalletGetWithAvailableCredits: return data @_exception_mapper(_WALLET_STATUS_MAP) - async def get_project_wallet(self, *, project_id: ProjectID) -> WalletGet | None: + async def get_project_wallet(self, *, project_id: ProjectID) -> WalletGet: response = await self.client.get( f"/projects/{project_id}/wallet", cookies=self.session_cookies, diff --git a/services/api-server/tests/mocks/create_study_job.json b/services/api-server/tests/mocks/create_study_job.json index c47c19e6972..e2b522e66db 100644 --- a/services/api-server/tests/mocks/create_study_job.json +++ b/services/api-server/tests/mocks/create_study_job.json @@ -82,6 +82,9 @@ "lastChangeDate": "2024-05-14T09:43:20.099Z", "workbench": {}, "prjOwner": "frubio@example.net", + "workspaceId": 23, + "folderId": 4, + "trashedAt": "2024-05-14T09:55:20.099Z", "accessRights": { "3": { "read": true, diff --git a/services/api-server/tests/mocks/for_test_api_routes_studies.json b/services/api-server/tests/mocks/for_test_api_routes_studies.json index 4543853948d..9e5b0e679ae 100644 --- a/services/api-server/tests/mocks/for_test_api_routes_studies.json +++ b/services/api-server/tests/mocks/for_test_api_routes_studies.json @@ -89,6 +89,9 @@ "thumbnail": "", "creationDate": "2023-07-18T12:18:04.314Z", "lastChangeDate": "2023-07-20T20:02:42.535Z", + "workspaceId": 278, + "folderId": 123, + "trashedAt": "2023-07-20T20:02:55.535Z", "workbench": { "deea006c-a223-4103-b46e-7b677428de9f": { "key": "simcore/services/frontend/file-picker", @@ -308,6 +311,9 @@ "thumbnail": "", "creationDate": "2023-07-18T12:18:04.314Z", "lastChangeDate": "2023-07-20T20:04:05.607Z", + "workspaceId": 278, + "folderId": 123, + "trashedAt": "2023-07-20T20:04:10.607Z", "workbench": { "deea006c-a223-4103-b46e-7b677428de9f": { "key": "simcore/services/frontend/file-picker", diff --git a/services/api-server/tests/mocks/get_job_outputs.json b/services/api-server/tests/mocks/get_job_outputs.json index e610a86e6ec..cc49e55fe27 100644 --- a/services/api-server/tests/mocks/get_job_outputs.json +++ b/services/api-server/tests/mocks/get_job_outputs.json @@ -144,6 +144,9 @@ "thumbnail": "", "creationDate": "2024-07-16T12:56:51.922Z", "lastChangeDate": "2024-07-16T12:56:51.922Z", + "workspaceId": 5, + "folderId": 2, + "trashedAt": null, "workbench": { "dd875b4f-7663-529f-bd7f-3716b19e28af": { "key": "simcore/services/comp/itis/sleeper", diff --git a/services/api-server/tests/mocks/get_job_pricing_unit_invalid_solver.json b/services/api-server/tests/mocks/get_job_pricing_unit_invalid_solver.json index 43071193260..a99eca3ed8f 100644 --- a/services/api-server/tests/mocks/get_job_pricing_unit_invalid_solver.json +++ b/services/api-server/tests/mocks/get_job_pricing_unit_invalid_solver.json @@ -36,6 +36,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2023-10-10T20:15:22.096Z", "lastChangeDate": "2023-10-10T20:15:22.096Z", + "workspaceId": 3, + "folderId": 31, + "trashedAt": null, "workbench": { "4b03863d-107a-5c77-a3ca-c5ba1d7048c0": { "key": "simcore/services/comp/isolve", diff --git a/services/api-server/tests/mocks/get_job_pricing_unit_success.json b/services/api-server/tests/mocks/get_job_pricing_unit_success.json index f1e166bebae..3c10f684e46 100644 --- a/services/api-server/tests/mocks/get_job_pricing_unit_success.json +++ b/services/api-server/tests/mocks/get_job_pricing_unit_success.json @@ -36,6 +36,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2023-10-10T20:15:22.096Z", "lastChangeDate": "2023-10-10T20:15:22.096Z", + "workspaceId": 3, + "folderId": 1, + "trashedAt": null, "workbench": { "4b03863d-107a-5c77-a3ca-c5ba1d7048c0": { "key": "simcore/services/comp/isolve", @@ -152,7 +155,11 @@ "data": { "pricingUnitId": 1, "unitName": "small", - "unitExtraInfo": {}, + "unitExtraInfo": { + "CPU": 500, + "RAM": 26598, + "VRAM": 456789123456 + }, "currentCostPerUnit": 50, "default": true } diff --git a/services/api-server/tests/mocks/get_solver_outputs.json b/services/api-server/tests/mocks/get_solver_outputs.json index 2e9f02a235c..adda3db505e 100644 --- a/services/api-server/tests/mocks/get_solver_outputs.json +++ b/services/api-server/tests/mocks/get_solver_outputs.json @@ -36,6 +36,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2024-01-18T12:33:56.952Z", "lastChangeDate": "2024-01-18T12:34:13.002Z", + "workspaceId": 2, + "folderId": 2, + "trashedAt": null, "workbench": { "df42d273-b6f0-509c-bfb5-4abbc5bb0581": { "key": "simcore/services/comp/itis/sleeper", diff --git a/services/api-server/tests/mocks/get_solver_pricing_plan_success.json b/services/api-server/tests/mocks/get_solver_pricing_plan_success.json index e12c2db1dc2..1cb46e85b8f 100644 --- a/services/api-server/tests/mocks/get_solver_pricing_plan_success.json +++ b/services/api-server/tests/mocks/get_solver_pricing_plan_success.json @@ -57,7 +57,11 @@ { "pricingUnitId": 1, "unitName": "small", - "unitExtraInfo": {}, + "unitExtraInfo": { + "CPU": 2, + "RAM": 1239876234, + "VRAM": 23454676789 + }, "currentCostPerUnit": 50, "default": true } diff --git a/services/api-server/tests/mocks/on_list_jobs.json b/services/api-server/tests/mocks/on_list_jobs.json index 48eadf3612b..d954da588fa 100644 --- a/services/api-server/tests/mocks/on_list_jobs.json +++ b/services/api-server/tests/mocks/on_list_jobs.json @@ -101,6 +101,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2023-06-22T18:42:35.506Z", "lastChangeDate": "2023-06-22T18:42:35.506Z", + "workspaceId": 7, + "folderId": 1, + "trashedAt": "2023-06-22T18:42:36.506Z", "workbench": { "05c7ed3b-0be1-5077-8065-fb55f5e59ff3": { "key": "simcore/services/comp/itis/sleeper", @@ -171,6 +174,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2023-06-22T18:42:32.201Z", "lastChangeDate": "2023-06-22T18:42:32.201Z", + "workspaceId": 4, + "folderId": 8, + "trashedAt": "2023-06-22T18:42:33.201Z", "workbench": { "34805d7e-c2d0-561f-831f-c74a28fc9bd1": { "key": "simcore/services/comp/itis/sleeper", diff --git a/services/api-server/tests/mocks/run_study_workflow.json b/services/api-server/tests/mocks/run_study_workflow.json index 7c77a69ade0..56b92873e50 100644 --- a/services/api-server/tests/mocks/run_study_workflow.json +++ b/services/api-server/tests/mocks/run_study_workflow.json @@ -301,6 +301,9 @@ "thumbnail": "", "creationDate": "2024-04-15T15:50:28.196Z", "lastChangeDate": "2024-04-15T15:50:28.196Z", + "workspaceId": 3, + "folderId": 3, + "trashedAt": null, "workbench": { "ab014072-a95f-5775-bb34-5582a13245a6": { "key": "simcore/services/frontend/iterator-consumer/probe/file", diff --git a/services/api-server/tests/mocks/start_job_not_enough_credit.json b/services/api-server/tests/mocks/start_job_not_enough_credit.json index 6f433ebeee4..2167313c683 100644 --- a/services/api-server/tests/mocks/start_job_not_enough_credit.json +++ b/services/api-server/tests/mocks/start_job_not_enough_credit.json @@ -36,6 +36,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2023-10-26T14:13:08.013Z", "lastChangeDate": "2023-10-26T14:13:08.013Z", + "workspaceId": 3, + "folderId": 2, + "trashedAt": null, "workbench": { "3b0b20e0-c860-51d9-9f82-d6b4bc5c2f24": { "key": "simcore/services/comp/itis/sleeper", diff --git a/services/api-server/tests/mocks/start_job_with_payment.json b/services/api-server/tests/mocks/start_job_with_payment.json index 8e52a63a964..1a7a829cf11 100644 --- a/services/api-server/tests/mocks/start_job_with_payment.json +++ b/services/api-server/tests/mocks/start_job_with_payment.json @@ -36,6 +36,9 @@ "thumbnail": "https://via.placeholder.com/170x120.png", "creationDate": "2023-10-26T14:10:11.118Z", "lastChangeDate": "2023-10-26T14:10:11.118Z", + "workspaceId": 12, + "folderId": 2, + "trashedAt": null, "workbench": { "657b124c-0697-5166-b820-a2ea2704ae84": { "key": "simcore/services/comp/itis/sleeper", diff --git a/services/api-server/tests/mocks/test_get_and_update_study_job_metadata.json b/services/api-server/tests/mocks/test_get_and_update_study_job_metadata.json index 05c81c5cfc5..a8f690f0827 100644 --- a/services/api-server/tests/mocks/test_get_and_update_study_job_metadata.json +++ b/services/api-server/tests/mocks/test_get_and_update_study_job_metadata.json @@ -208,6 +208,9 @@ "thumbnail": "", "creationDate": "2024-05-30T10:29:54.150Z", "lastChangeDate": "2024-05-30T10:29:54.150Z", + "workspaceId": 3, + "folderId": 12, + "trashedAt": "2024-05-30T10:30:54.137359", "workbench": { "45043872-d6d3-530b-bf40-67bfde79191c": { "key": "simcore/services/dynamic/jupyter-math", diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index 7846dbe1e84..237b846abaf 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -3,7 +3,6 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments -from collections.abc import Iterator from pathlib import Path from pprint import pprint from typing import Any @@ -310,6 +309,8 @@ async def test_run_solver_job( "contact", "inputs", "outputs", + "classifiers", + "owner", } == set(oas["components"]["schemas"]["ServiceGet"]["required"]) example = next( From f07afbf19271d7853dc94934c0a6493996a11f19 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Mon, 11 Nov 2024 08:53:54 +0100 Subject: [PATCH 32/53] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20packages=20mypy,?= =?UTF-8?q?=20pylint=20and=20tests=20(#6678)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../tests/test_pydantic_validators.py | 4 +- .../api_schemas_directorv2/clusters.py | 2 +- .../api_schemas_directorv2/comp_tasks.py | 13 ++++- .../api_schemas_dynamic_sidecar/telemetry.py | 23 +------- .../src/models_library/api_schemas_storage.py | 8 +-- .../api_schemas_webserver/projects.py | 10 +++- .../src/models_library/progress_bar.py | 4 +- .../src/models_library/rest_pagination.py | 32 +---------- .../src/models_library/rpc_pagination.py | 32 +---------- .../models_library/service_settings_labels.py | 17 +++--- .../src/models_library/services_base.py | 12 ++-- .../src/models_library/services_types.py | 4 +- .../models_library/utils/common_validators.py | 3 +- .../tests/test__models_examples.py | 17 +++++- ...t_api_schemas_dynamic_sidecar_telemetry.py | 8 +-- .../tests/test_project_nodes.py | 5 +- .../models-library/tests/test_rest_filters.py | 12 ++-- .../tests/test_rest_pagination.py | 19 +++++-- .../tests/test_rpc_pagination.py | 5 +- .../src/notifications_library/errors.py | 5 +- .../utils_projects.py | 4 +- .../src/pytest_simcore/examples/__init__.py | 0 .../pytest_simcore/examples/models_library.py | 55 +++++++++++++++++++ .../src/pytest_simcore/pydantic_models.py | 9 +++ .../simcore_webserver_projects_rest_api.py | 22 ++++++++ .../src/servicelib/aiohttp/tracing.py | 4 +- .../src/servicelib/fastapi/tracing.py | 4 +- .../src/servicelib/rabbitmq/_models.py | 1 - .../tests/fastapi/test_http_client_thin.py | 6 +- .../src/settings_library/base.py | 8 ++- .../src/settings_library/node_ports.py | 14 ++--- .../src/settings_library/twilio.py | 3 +- .../tests/test_base_w_postgres.py | 1 - .../node_ports_common/storage_endpoint.py | 1 + .../tests/unit/test_node_ports_v2_links.py | 9 ++- .../unit/test_node_ports_v2_nodeports_v2.py | 7 +-- .../_logging_event_handler.py | 6 +- .../modules/outputs/_event_handler.py | 4 +- 38 files changed, 219 insertions(+), 174 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/examples/__init__.py create mode 100644 packages/pytest-simcore/src/pytest_simcore/examples/models_library.py diff --git a/packages/common-library/tests/test_pydantic_validators.py b/packages/common-library/tests/test_pydantic_validators.py index 89e57bc0999..c1cfea84c67 100644 --- a/packages/common-library/tests/test_pydantic_validators.py +++ b/packages/common-library/tests/test_pydantic_validators.py @@ -3,7 +3,7 @@ import pytest from common_library.pydantic_validators import ( - validate_legacy_timedelta_str, + _validate_legacy_timedelta_str, validate_numeric_string_as_timedelta, ) from faker import Faker @@ -16,7 +16,7 @@ def test_validate_legacy_timedelta(monkeypatch: pytest.MonkeyPatch, faker: Faker class Settings(BaseSettings): APP_NAME: str REQUEST_TIMEOUT: Annotated[ - timedelta, BeforeValidator(validate_legacy_timedelta_str) + timedelta, BeforeValidator(_validate_legacy_timedelta_str) ] = Field(default=timedelta(hours=1)) model_config = SettingsConfigDict() diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py index 9c4698a5dc3..158242195e1 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py @@ -113,7 +113,7 @@ class ClusterDetailsGet(ClusterDetails): class ClusterCreate(BaseCluster): - owner: GroupID | None + owner: GroupID | None = None # type: ignore[assignment] authentication: ExternalClusterAuthentication access_rights: dict[GroupID, ClusterAccessRights] = Field( alias="accessRights", default_factory=dict diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py index e383d45f20e..2204186c3ed 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py @@ -1,7 +1,7 @@ from typing import Any, TypeAlias from models_library.basic_types import IDStr -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, field_validator +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field, field_validator from ..clusters import ClusterID from ..projects import ProjectID @@ -19,6 +19,17 @@ class ComputationGet(ComputationTask): None, description="the link where to stop the task" ) + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + x | {"url": "http://url.local"} # type:ignore[operator] + for x in ComputationTask.model_config[ # type:ignore[index,union-attr] + "json_schema_extra" + ]["examples"] + ] + } + ) + class ComputationCreate(BaseModel): user_id: UserID diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py index b7d315a8aa6..25b71df9e0f 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py @@ -8,8 +8,7 @@ Field, NonNegativeFloat, NonNegativeInt, - root_validator, - validator, + model_validator, ) from ..projects_nodes_io import NodeID @@ -56,29 +55,13 @@ class DiskUsage(BaseModel): free: ByteSize = Field(description="remaining space") total: ByteSize = Field(description="total space = free + used") - used_percent: NonNegativeFloat = Field( + used_percent: float = Field( ge=0.00, le=100.00, description="Percent of used space relative to the total space", ) - @validator("free") - @classmethod - def _free_positive(cls, v: float) -> float: - if v < 0: - msg = f"free={v} cannot be a negative value" - raise ValueError(msg) - return v - - @validator("used") - @classmethod - def _used_positive(cls, v: float) -> float: - if v < 0: - msg = f"used={v} cannot be a negative value" - raise ValueError(msg) - return v - - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def _check_total(cls, values: dict[str, Any]) -> dict[str, Any]: total = values["total"] diff --git a/packages/models-library/src/models_library/api_schemas_storage.py b/packages/models-library/src/models_library/api_schemas_storage.py index 54c5e28d43d..cdbbeebffcd 100644 --- a/packages/models-library/src/models_library/api_schemas_storage.py +++ b/packages/models-library/src/models_library/api_schemas_storage.py @@ -311,12 +311,12 @@ class FileUploadCompleteFutureResponse(BaseModel): class FoldersBody(BaseModel): - source: dict[str, Any] = Field(default_factory=dict) - destination: dict[str, Any] = Field(default_factory=dict) - nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict) + source: Annotated[dict[str, Any], Field(default_factory=dict)] + destination: Annotated[dict[str, Any], Field(default_factory=dict)] + nodes_map: Annotated[dict[NodeID, NodeID], Field(default_factory=dict)] @model_validator(mode="after") - def ensure_consistent_entries(self) -> Self: + def ensure_consistent_entries(self: Self) -> Self: source_node_keys = (NodeID(n) for n in self.source.get("workbench", {})) if set(source_node_keys) != set(self.nodes_map.keys()): msg = "source project nodes do not fit with nodes_map entries" diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 0ae2bdee311..a7d63904350 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -107,7 +107,9 @@ class ProjectReplace(InputSchema): uuid: ProjectID name: ShortTruncatedStr description: LongTruncatedStr - thumbnail: Annotated[HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)] = Field(default=None) + thumbnail: Annotated[ + HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator) + ] = Field(default=None) creation_date: DateTimeStr last_change_date: DateTimeStr workbench: NodesDict @@ -127,13 +129,15 @@ class ProjectReplace(InputSchema): class ProjectPatch(InputSchema): name: ShortTruncatedStr | None = Field(default=None) description: LongTruncatedStr | None = Field(default=None) - thumbnail: Annotated[HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)] = Field(default=None) + thumbnail: Annotated[ + HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator) + ] = Field(default=None) access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None) classifiers: list[ClassifierID] | None = Field(default=None) dev: dict | None = Field(default=None) ui: StudyUI | None = Field(default=None) quality: dict[str, Any] | None = Field(default=None) - + __all__: tuple[str, ...] = ( "EmptyModel", diff --git a/packages/models-library/src/models_library/progress_bar.py b/packages/models-library/src/models_library/progress_bar.py index da2829b0c94..90232847bbc 100644 --- a/packages/models-library/src/models_library/progress_bar.py +++ b/packages/models-library/src/models_library/progress_bar.py @@ -12,8 +12,8 @@ class ProgressStructuredMessage(BaseModel): description: IDStr current: float total: int - unit: str | None - sub: "ProgressStructuredMessage | None" + unit: str | None = None + sub: "ProgressStructuredMessage | None" = None model_config = ConfigDict( json_schema_extra={ diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index 28556d50303..e3e5e04e6ef 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -136,34 +136,4 @@ def _check_data_compatible_with_meta(cls, v, info: ValidationInfo): raise ValueError(msg) return v - model_config = ConfigDict( - extra="forbid", - json_schema_extra={ - "examples": [ - # first page Page[str] - { - "_meta": {"total": 7, "count": 4, "limit": 4, "offset": 0}, - "_links": { - "self": "https://osparc.io/v2/listing?offset=0&limit=4", - "first": "https://osparc.io/v2/listing?offset=0&limit=4", - "prev": None, - "next": "https://osparc.io/v2/listing?offset=1&limit=4", - "last": "https://osparc.io/v2/listing?offset=1&limit=4", - }, - "data": ["data 1", "data 2", "data 3", "data 4"], - }, - # second and last page - { - "_meta": {"total": 7, "count": 3, "limit": 4, "offset": 1}, - "_links": { - "self": "https://osparc.io/v2/listing?offset=1&limit=4", - "first": "https://osparc.io/v2/listing?offset=0&limit=4", - "prev": "https://osparc.io/v2/listing?offset=0&limit=4", - "next": None, - "last": "https://osparc.io/v2/listing?offset=1&limit=4", - }, - "data": ["data 5", "data 6", "data 7"], - }, - ] - }, - ) + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/rpc_pagination.py b/packages/models-library/src/models_library/rpc_pagination.py index 92470b30d67..96d6308f66c 100644 --- a/packages/models-library/src/models_library/rpc_pagination.py +++ b/packages/models-library/src/models_library/rpc_pagination.py @@ -74,34 +74,4 @@ def create( data=chunk, ) - model_config = ConfigDict( - extra="forbid", - json_schema_extra={ - "examples": [ - # first page Page[str] - { - "_meta": {"total": 7, "count": 4, "limit": 4, "offset": 0}, - "_links": { - "self": {"offset": 0, "limit": 4}, - "first": {"offset": 0, "limit": 4}, - "prev": None, - "next": {"offset": 1, "limit": 4}, - "last": {"offset": 1, "limit": 4}, - }, - "data": ["data 1", "data 2", "data 3", "data 4"], - }, - # second and last page - { - "_meta": {"total": 7, "count": 3, "limit": 4, "offset": 1}, - "_links": { - "self": {"offset": 1, "limit": 4}, - "first": {"offset": 0, "limit": 4}, - "prev": {"offset": 0, "limit": 4}, - "next": None, - "last": {"offset": 1, "limit": 4}, - }, - "data": ["data 5", "data 6", "data 7"], - }, - ] - }, - ) + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/service_settings_labels.py b/packages/models-library/src/models_library/service_settings_labels.py index 0a632a7b70d..0a4e35717bb 100644 --- a/packages/models-library/src/models_library/service_settings_labels.py +++ b/packages/models-library/src/models_library/service_settings_labels.py @@ -3,7 +3,7 @@ from enum import Enum from functools import cached_property from pathlib import Path -from typing import Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias from common_library.json_serialization import json_dumps from pydantic import ( @@ -317,13 +317,14 @@ class DynamicSidecarServiceLabels(BaseModel): ), ) - containers_allowed_outgoing_permit_list: None | ( - Json[dict[str, list[NATRule]]] - ) = Field( - None, - alias="simcore.service.containers-allowed-outgoing-permit-list", - description="allow internet access to certain domain names and ports per container", - ) + containers_allowed_outgoing_permit_list: Annotated[ + None | (Json[dict[str, list[NATRule]]]), + Field( + None, + alias="simcore.service.containers-allowed-outgoing-permit-list", + description="allow internet access to certain domain names and ports per container", + ), + ] containers_allowed_outgoing_internet: Json[set[str]] | None = Field( None, diff --git a/packages/models-library/src/models_library/services_base.py b/packages/models-library/src/models_library/services_base.py index 5f92d6e46b6..48afb0b6c04 100644 --- a/packages/models-library/src/models_library/services_base.py +++ b/packages/models-library/src/models_library/services_base.py @@ -1,4 +1,5 @@ from typing import Annotated + from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator from .services_types import ServiceKey, ServiceVersion @@ -8,10 +9,13 @@ class ServiceKeyVersion(BaseModel): """Service `key-version` pair uniquely identifies a service""" - key: ServiceKey = Field( - ..., - description="distinctive name for the node based on the docker registry path", - ) + key: Annotated[ + ServiceKey, + Field( + ..., + description="distinctive name for the node based on the docker registry path", + ), + ] version: ServiceVersion = Field( ..., description="service version number", diff --git a/packages/models-library/src/models_library/services_types.py b/packages/models-library/src/models_library/services_types.py index e882c8dae09..03c0bb4bf5d 100644 --- a/packages/models-library/src/models_library/services_types.py +++ b/packages/models-library/src/models_library/services_types.py @@ -61,7 +61,9 @@ def create(cls) -> "RunID": @classmethod def __get_pydantic_core_schema__( - cls, source_type: Any, handler: GetCoreSchemaHandler + cls, + source_type: Any, # pylint:disable=unused-argument + handler: GetCoreSchemaHandler, ) -> CoreSchema: return core_schema.no_info_after_validator_function(cls, handler(str)) diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index d488c9d4acc..79c07249e03 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -93,7 +93,8 @@ def _validator(cls, values): assert set(alternative_field_names).issubset(cls.__fields__) # nosec got = { - field_name: values.get(field_name) for field_name in alternative_field_names + field_name: getattr(values, field_name) + for field_name in alternative_field_names } if not functools.reduce(operator.xor, (v is not None for v in got.values())): diff --git a/packages/models-library/tests/test__models_examples.py b/packages/models-library/tests/test__models_examples.py index 2345b5451f1..482f586df7c 100644 --- a/packages/models-library/tests/test__models_examples.py +++ b/packages/models-library/tests/test__models_examples.py @@ -1,15 +1,28 @@ import json +from itertools import chain from typing import Any import models_library import pytest +from models_library.rest_pagination import Page +from models_library.rpc_pagination import PageRpc from pydantic import BaseModel -from pytest_simcore.pydantic_models import walk_model_examples_in_package +from pytest_simcore.examples.models_library import PAGE_EXAMPLES, RPC_PAGE_EXAMPLES +from pytest_simcore.pydantic_models import ( + ModelExample, + iter_examples, + walk_model_examples_in_package, +) + +GENERIC_EXAMPLES: list[ModelExample] = [ + *iter_examples(model_cls=Page[str], examples=PAGE_EXAMPLES), + *iter_examples(model_cls=PageRpc[str], examples=RPC_PAGE_EXAMPLES), +] @pytest.mark.parametrize( "model_cls, example_name, example_data", - walk_model_examples_in_package(models_library), + chain(GENERIC_EXAMPLES, walk_model_examples_in_package(models_library)), ) def test_all_models_library_models_config_examples( model_cls: type[BaseModel], example_name: int, example_data: Any diff --git a/packages/models-library/tests/test_api_schemas_dynamic_sidecar_telemetry.py b/packages/models-library/tests/test_api_schemas_dynamic_sidecar_telemetry.py index 8de5c01dc83..d5ffc459397 100644 --- a/packages/models-library/tests/test_api_schemas_dynamic_sidecar_telemetry.py +++ b/packages/models-library/tests/test_api_schemas_dynamic_sidecar_telemetry.py @@ -42,8 +42,8 @@ def test_failing_validation(): with pytest.raises(ValidationError) as exc: assert DiskUsage.from_efs_guardian(100, 10) - assert "free=" in f"{exc.value}" - assert "negative value" in f"{exc.value}" + assert "free" in f"{exc.value}" + assert "input_value=-90" in f"{exc.value}" with pytest.raises(ValidationError) as exc: assert DiskUsage( @@ -52,8 +52,8 @@ def test_failing_validation(): total=ByteSize(0), used_percent=-10, ) - assert "used=" in f"{exc.value}" - assert "negative value" in f"{exc.value}" + assert "used" in f"{exc.value}" + assert "input_value=-10" in f"{exc.value}" with pytest.raises(ValidationError) as exc: DiskUsage( diff --git a/packages/models-library/tests/test_project_nodes.py b/packages/models-library/tests/test_project_nodes.py index 96f427a19cb..09b5511e2bf 100644 --- a/packages/models-library/tests/test_project_nodes.py +++ b/packages/models-library/tests/test_project_nodes.py @@ -1,6 +1,7 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument +# pylint:disable=no-member # pylint:disable=redefined-outer-name +# pylint:disable=unused-argument +# pylint:disable=unused-variable from typing import Any diff --git a/packages/models-library/tests/test_rest_filters.py b/packages/models-library/tests/test_rest_filters.py index 0a46bd3a25b..1b470fc1767 100644 --- a/packages/models-library/tests/test_rest_filters.py +++ b/packages/models-library/tests/test_rest_filters.py @@ -2,7 +2,7 @@ import pytest from models_library.rest_filters import Filters, FiltersQueryParameters -from pydantic import Extra, ValidationError +from pydantic import ConfigDict, ValidationError # 1. create filter model @@ -12,8 +12,7 @@ class CustomFilter(Filters): class CustomFilterStrict(CustomFilter): - class Config(CustomFilter.Config): - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") def test_custom_filter_query_parameters(): @@ -21,12 +20,12 @@ def test_custom_filter_query_parameters(): # 2. use generic as query parameters logging.info( "json schema is for the query \n %s", - FiltersQueryParameters[CustomFilter].schema_json(indent=1), + FiltersQueryParameters[CustomFilter].model_json_schema(), ) # lets filter only is_trashed and unset is_hidden custom_filter = CustomFilter(is_trashed=True) - assert custom_filter.json() == '{"is_trashed": true, "is_hidden": null}' + assert custom_filter.model_dump_json() == '{"is_trashed":true,"is_hidden":null}' # default to None (optional) query_param = FiltersQueryParameters[CustomFilter]() @@ -56,9 +55,8 @@ def test_invalid_filter_query_is_ignored(): assert query_param.filters == CustomFilter(is_hidden=True) -@pytest.mark.xfail def test_invalid_filter_query_fails(): - # NOTE: this should fail according to pydantic manual but it does not + # with pydantic1 this used to not pass but now passes url_query_value = '{"undefined_filter": true, "is_hidden": true}' with pytest.raises(ValidationError): diff --git a/packages/models-library/tests/test_rest_pagination.py b/packages/models-library/tests/test_rest_pagination.py index d0f5f9f7d92..a32bec673bb 100644 --- a/packages/models-library/tests/test_rest_pagination.py +++ b/packages/models-library/tests/test_rest_pagination.py @@ -3,11 +3,20 @@ import pytest from models_library.rest_pagination import Page, PageMetaInfoLimitOffset from pydantic.main import BaseModel +from pytest_simcore.examples.models_library import PAGE_EXAMPLES -@pytest.mark.parametrize("cls_model", [Page[str], PageMetaInfoLimitOffset]) -def test_page_response_limit_offset_models(cls_model: BaseModel): - examples = cls_model.model_config["json_schema_extra"]["examples"] +@pytest.mark.parametrize( + "cls_model, examples", + [ + (Page[str], PAGE_EXAMPLES), + ( + PageMetaInfoLimitOffset, + PageMetaInfoLimitOffset.model_config["json_schema_extra"]["examples"], + ), + ], +) +def test_page_response_limit_offset_models(cls_model: BaseModel, examples: list[dict]): for index, example in enumerate(examples): print(f"{index:-^10}:\n", example) @@ -35,14 +44,14 @@ def test_invalid_count(count: int, offset: int): def test_data_size_does_not_fit_count(): - example = deepcopy(Page[str].model_config["json_schema_extra"]["examples"][0]) + example = deepcopy(PAGE_EXAMPLES[0]) example["_meta"]["count"] = len(example["data"]) - 1 with pytest.raises(ValueError): Page[str](**example) def test_empty_data_is_converted_to_list(): - example = deepcopy(Page[str].model_config["json_schema_extra"]["examples"][0]) + example = deepcopy(PAGE_EXAMPLES[0]) example["data"] = None example["_meta"]["count"] = 0 model_instance = Page[str](**example) diff --git a/packages/models-library/tests/test_rpc_pagination.py b/packages/models-library/tests/test_rpc_pagination.py index 26931b9032e..b8f78c737e5 100644 --- a/packages/models-library/tests/test_rpc_pagination.py +++ b/packages/models-library/tests/test_rpc_pagination.py @@ -2,11 +2,10 @@ import pytest from models_library.rpc_pagination import PageRpc +from pytest_simcore.examples.models_library import RPC_PAGE_EXAMPLES -@pytest.mark.parametrize( - "example", PageRpc.model_config["json_schema_extra"]["examples"] -) +@pytest.mark.parametrize("example", RPC_PAGE_EXAMPLES) def test_create_page_rpc(example: dict[str, Any]): expected = PageRpc.model_validate(example) diff --git a/packages/notifications-library/src/notifications_library/errors.py b/packages/notifications-library/src/notifications_library/errors.py index 21edbbb0dc7..9c250909f21 100644 --- a/packages/notifications-library/src/notifications_library/errors.py +++ b/packages/notifications-library/src/notifications_library/errors.py @@ -1,11 +1,8 @@ -from typing import Any - from common_library.errors_classes import OsparcErrorMixin class NotifierError(OsparcErrorMixin, Exception): - def __init__(self, **ctx: Any) -> None: - super().__init__(**ctx) + pass class TemplatesNotFoundError(NotifierError): diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py index 8dfd97b759b..577f9441004 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects.py @@ -2,15 +2,15 @@ from datetime import UTC, datetime import sqlalchemy as sa +from common_library.errors_classes import OsparcErrorMixin from pydantic import TypeAdapter -from pydantic.errors import PydanticErrorMixin from sqlalchemy.ext.asyncio import AsyncConnection from .models.projects import projects from .utils_repos import transaction_context -class DBBaseProjectError(PydanticErrorMixin, Exception): +class DBBaseProjectError(OsparcErrorMixin, Exception): msg_template: str = "Project utils unexpected error" diff --git a/packages/pytest-simcore/src/pytest_simcore/examples/__init__.py b/packages/pytest-simcore/src/pytest_simcore/examples/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/pytest-simcore/src/pytest_simcore/examples/models_library.py b/packages/pytest-simcore/src/pytest_simcore/examples/models_library.py new file mode 100644 index 00000000000..8af09913f71 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/examples/models_library.py @@ -0,0 +1,55 @@ +from typing import Final + +PAGE_EXAMPLES: Final[list[dict]] = [ + # first page Page[str] + { + "_meta": {"total": 7, "count": 4, "limit": 4, "offset": 0}, + "_links": { + "self": "https://osparc.io/v2/listing?offset=0&limit=4", + "first": "https://osparc.io/v2/listing?offset=0&limit=4", + "prev": None, + "next": "https://osparc.io/v2/listing?offset=1&limit=4", + "last": "https://osparc.io/v2/listing?offset=1&limit=4", + }, + "data": ["data 1", "data 2", "data 3", "data 4"], + }, + # second and last page + { + "_meta": {"total": 7, "count": 3, "limit": 4, "offset": 1}, + "_links": { + "self": "https://osparc.io/v2/listing?offset=1&limit=4", + "first": "https://osparc.io/v2/listing?offset=0&limit=4", + "prev": "https://osparc.io/v2/listing?offset=0&limit=4", + "next": None, + "last": "https://osparc.io/v2/listing?offset=1&limit=4", + }, + "data": ["data 5", "data 6", "data 7"], + }, +] + +RPC_PAGE_EXAMPLES: Final[list[dict]] = [ + # first page Page[str] + { + "_meta": {"total": 7, "count": 4, "limit": 4, "offset": 0}, + "_links": { + "self": {"offset": 0, "limit": 4}, + "first": {"offset": 0, "limit": 4}, + "prev": None, + "next": {"offset": 1, "limit": 4}, + "last": {"offset": 1, "limit": 4}, + }, + "data": ["data 1", "data 2", "data 3", "data 4"], + }, + # second and last page + { + "_meta": {"total": 7, "count": 3, "limit": 4, "offset": 1}, + "_links": { + "self": {"offset": 1, "limit": 4}, + "first": {"offset": 0, "limit": 4}, + "prev": {"offset": 0, "limit": 4}, + "next": None, + "last": {"offset": 1, "limit": 4}, + }, + "data": ["data 5", "data 6", "data 7"], + }, +] diff --git a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py index 1c2809fa52e..e9a7c318a9c 100644 --- a/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py +++ b/packages/pytest-simcore/src/pytest_simcore/pydantic_models.py @@ -35,6 +35,15 @@ class ModelExample(NamedTuple): example_data: Any +def iter_examples( + *, model_cls: type[BaseModel], examples: list[Any] +) -> Iterator[ModelExample]: + for k, data in enumerate(examples): + yield ModelExample( + model_cls=model_cls, example_name=f"example_{k}", example_data=data + ) + + def walk_model_examples_in_package(package: ModuleType) -> Iterator[ModelExample]: """Walks recursively all sub-modules and collects BaseModel.Config examples""" assert inspect.ismodule(package) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py index ca386f1914c..fd6dd234720 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_projects_rest_api.py @@ -73,6 +73,10 @@ def request_desc(self) -> str: "locked": {"value": False, "status": "CLOSED"}, "state": {"value": "NOT_STARTED"}, }, + "dev": None, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, }, "error": None, }, @@ -109,6 +113,9 @@ def request_desc(self) -> str: "locked": {"value": False, "status": "CLOSED"}, "state": {"value": "NOT_STARTED"}, }, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, } }, ) @@ -148,6 +155,9 @@ def request_desc(self) -> str: }, "state": {"value": "NOT_STARTED"}, }, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, } }, ) @@ -275,6 +285,9 @@ def request_desc(self) -> str: }, "state": {"value": "NOT_STARTED"}, }, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, } }, ) @@ -465,6 +478,9 @@ def request_desc(self) -> str: }, "accessRights": {"2": {"read": True, "write": True, "delete": True}}, "dev": {}, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, "classifiers": [], "ui": { "mode": "workbench", @@ -663,6 +679,9 @@ def request_desc(self) -> str: }, "classifiers": [], "dev": {}, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, "quality": { "enabled": True, "tsr_target": { @@ -910,6 +929,9 @@ def request_desc(self) -> str: }, "classifiers": [], "dev": {}, + "workspace_id": None, + "folder_id": None, + "trashed_at": None, "quality": { "enabled": True, "tsr_target": { diff --git a/packages/service-library/src/servicelib/aiohttp/tracing.py b/packages/service-library/src/servicelib/aiohttp/tracing.py index 3da3b28e3b3..fa9cec661a4 100644 --- a/packages/service-library/src/servicelib/aiohttp/tracing.py +++ b/packages/service-library/src/servicelib/aiohttp/tracing.py @@ -31,7 +31,9 @@ except ImportError: HAS_BOTOCORE = False try: - from opentelemetry.instrumentation.aiopg import AiopgInstrumentor + from opentelemetry.instrumentation.aiopg import ( # type: ignore[import-not-found] + AiopgInstrumentor, + ) HAS_AIOPG = True except ImportError: diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index b5179a8a5f6..bdd371fae1a 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -28,7 +28,9 @@ HAS_ASYNCPG = False try: - from opentelemetry.instrumentation.aiopg import AiopgInstrumentor + from opentelemetry.instrumentation.aiopg import ( # type: ignore[import-not-found] + AiopgInstrumentor, + ) HAS_AIOPG = True except ImportError: diff --git a/packages/service-library/src/servicelib/rabbitmq/_models.py b/packages/service-library/src/servicelib/rabbitmq/_models.py index e48e4bb13aa..c76800a4d8a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_models.py +++ b/packages/service-library/src/servicelib/rabbitmq/_models.py @@ -1,4 +1,3 @@ -import re from collections.abc import Awaitable, Callable from typing import Any, Protocol diff --git a/packages/service-library/tests/fastapi/test_http_client_thin.py b/packages/service-library/tests/fastapi/test_http_client_thin.py index 7bd96b25eee..103584f3f77 100644 --- a/packages/service-library/tests/fastapi/test_http_client_thin.py +++ b/packages/service-library/tests/fastapi/test_http_client_thin.py @@ -3,10 +3,10 @@ import logging from collections.abc import AsyncIterable, Iterable from typing import Final -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy import arrow import pytest +from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from httpx import ( HTTPError, PoolTimeout, @@ -16,7 +16,7 @@ TransportError, codes, ) -from pydantic import AnyHttpUrl, TypeAdapter +from pydantic import TypeAdapter from respx import MockRouter from servicelib.fastapi.http_client_thin import ( BaseThinClient, @@ -78,7 +78,7 @@ async def thick_client(request_timeout: int) -> AsyncIterable[FakeThickClient]: @pytest.fixture def test_url() -> str: - url =TypeAdapter(AnyHttpUrlLegacy).validate_python("http://missing-host:1111") + url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://missing-host:1111") return f"{url}" diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index 74d2a3fa1b8..6d436ea93a9 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -54,7 +54,7 @@ def _default_factory(): def _is_auto_default_from_env_enabled(field: FieldInfo) -> bool: return bool( field.json_schema_extra is not None - and field.json_schema_extra.get("auto_default_from_env", False) + and field.json_schema_extra.get("auto_default_from_env", False) # type: ignore[union-attr] ) @@ -171,7 +171,7 @@ def create_from_envs(cls, **overrides): @classmethod def settings_customise_sources( cls, - settings_cls: BaseSettings, + settings_cls: type[BaseSettings], init_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, @@ -180,7 +180,9 @@ def settings_customise_sources( assert env_settings # nosec return ( init_settings, - EnvSettingsWithAutoDefaultSource(settings_cls, env_settings=env_settings), + EnvSettingsWithAutoDefaultSource( + settings_cls, env_settings=env_settings # type:ignore[arg-type] + ), dotenv_settings, file_secret_settings, ) diff --git a/packages/settings-library/src/settings_library/node_ports.py b/packages/settings-library/src/settings_library/node_ports.py index 8498c882e4d..562e71e038a 100644 --- a/packages/settings-library/src/settings_library/node_ports.py +++ b/packages/settings-library/src/settings_library/node_ports.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import Final, Self +from typing import Annotated, Final, Self from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator @@ -32,13 +32,13 @@ def _validate_auth_fields(self) -> Self: class NodePortsSettings(BaseCustomSettings): - NODE_PORTS_STORAGE_AUTH: StorageAuthSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) + NODE_PORTS_STORAGE_AUTH: Annotated[ + StorageAuthSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] - POSTGRES_SETTINGS: PostgresSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) + POSTGRES_SETTINGS: Annotated[ + PostgresSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] NODE_PORTS_MULTIPART_UPLOAD_COMPLETION_TIMEOUT_S: NonNegativeInt = int( timedelta(minutes=5).total_seconds() diff --git a/packages/settings-library/src/settings_library/twilio.py b/packages/settings-library/src/settings_library/twilio.py index b63e35caf61..343cbda4732 100644 --- a/packages/settings-library/src/settings_library/twilio.py +++ b/packages/settings-library/src/settings_library/twilio.py @@ -5,7 +5,6 @@ SEE https://support.twilio.com/hc/en-us/articles/223136027-Auth-Tokens-and-How-to-Change-Them """ - from typing import Annotated, TypeAlias from pydantic import BeforeValidator, Field, StringConstraints, TypeAdapter @@ -42,5 +41,5 @@ def is_alphanumeric_supported(self, phone_number: str) -> bool: ) return any( phone_number_wo_international_code.startswith(code) - for code in self.TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT + for code in self.TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT # pylint:disable=not-an-iterable ) diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index c2c877ca703..b1d4958378f 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -345,7 +345,6 @@ def test_parse_from_mixed_envs( POSTGRES_USER=test POSTGRES_PASSWORD=ssh POSTGRES_DB=db - POSTGRES_CLIENT_NAME=client-name """ with monkeypatch.context(): diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_endpoint.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_endpoint.py index 8e5e38cc0a3..7efbf45af37 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_endpoint.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_endpoint.py @@ -15,6 +15,7 @@ def is_storage_secure() -> bool: @lru_cache def get_base_url() -> str: settings = NodePortsSettings.create_from_envs() + # pylint:disable=no-member base_url: str = settings.NODE_PORTS_STORAGE_AUTH.api_base_url return base_url diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py index 95b114ae563..72ba5e76570 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_links.py @@ -1,8 +1,7 @@ -from typing import Dict from uuid import uuid4 import pytest -from pydantic import TypeAdapter, ValidationError +from pydantic import ValidationError from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink @@ -23,7 +22,7 @@ def test_valid_port_link(): {"nodeUuid": f"{uuid4()}", "output": "some:key"}, ], ) -def test_invalid_port_link(port_link: Dict[str, str]): +def test_invalid_port_link(port_link: dict[str, str]): with pytest.raises(ValidationError): PortLink(**port_link) @@ -36,7 +35,7 @@ def test_invalid_port_link(port_link: Dict[str, str]): {"label": "some stuff"}, ], ) -def test_invalid_download_link(download_link: Dict[str, str]): +def test_invalid_download_link(download_link: dict[str, str]): with pytest.raises(ValidationError): DownloadLink(**download_link) @@ -49,6 +48,6 @@ def test_invalid_download_link(download_link: Dict[str, str]): {"path": "/somefile/blahblah:"}, ], ) -def test_invalid_file_link(file_link: Dict[str, str]): +def test_invalid_file_link(file_link: dict[str, str]): with pytest.raises(ValidationError): FileLink(**file_link) diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py index 848b5d60d98..250f9d2599d 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py @@ -3,7 +3,6 @@ # pylint:disable=redefined-outer-name # pylint:disable=protected-access -import asyncio from pathlib import Path from typing import Any, Callable from unittest.mock import AsyncMock @@ -230,10 +229,7 @@ async def test_node_ports_v2_packages( @pytest.fixture def mock_port_set(mocker: MockFixture) -> None: async def _always_raise_error(*args, **kwargs): - async def _i_raise_errors(): - raise ValidationError("invalid") - - return asyncio.create_task(_i_raise_errors()) + raise ValidationError.from_exception_data(title="Just a test", line_errors=[]) mocker.patch( "simcore_sdk.node_ports_v2.port.Port._set", side_effect=_always_raise_error @@ -278,4 +274,5 @@ async def _mock_callback(*args, **kwargs): + list(original_outputs.values()) }, progress_bar=progress_bar, + outputs_callbacks=None, ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py index 527fa32f3ea..f1537a87389 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py @@ -1,5 +1,3 @@ -# pylint:disable=no-member - import logging import stat from asyncio import CancelledError, Task, create_task, get_event_loop @@ -70,7 +68,7 @@ def start_process(self) -> None: self._process = aioprocessing.AioProcess( target=self._process_worker, daemon=True ) - self._process.start() + self._process.start() # pylint:disable=no-member def _stop_process(self) -> None: with log_context( @@ -175,7 +173,7 @@ async def _health_worker(self) -> None: try: self._health_check_queue.get_nowait() heart_beat_count += 1 - except Empty: # noqa: PERF203 + except Empty: break if heart_beat_count == 0: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py index 784db423ed6..9b256a3d037 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py @@ -1,5 +1,3 @@ -# pylint:disable=no-member - import logging from asyncio import CancelledError, Task, create_task, get_event_loop from asyncio import sleep as async_sleep @@ -97,7 +95,7 @@ def start_process(self) -> None: self._process = aioprocessing.AioProcess( target=self._process_worker, daemon=True ) - self._process.start() + self._process.start() # pylint:disable=no-member def stop_process(self) -> None: # NOTE: runs in asyncio thread From a00d55ff84b6953caf85b1d8630f5b09d7ac383d Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:00:13 +0100 Subject: [PATCH 33/53] pydantic2 making ported services green (#6701) Co-authored-by: Andrei Neagu --- .../src/common_library/serialization.py | 12 ++--- .../api_schemas_clusters_keeper/clusters.py | 4 +- .../api_schemas_directorv2/clusters.py | 20 +++---- .../src/models_library/clusters.py | 2 +- .../src/models_library/projects.py | 4 +- .../src/models_library/projects_state.py | 3 +- .../helpers/httpx_calls_capture_parameters.py | 6 +-- .../src/servicelib/aiohttp/tracing.py | 4 +- .../src/servicelib/fastapi/tracing.py | 4 +- .../core/settings.py | 8 +-- .../models/schemas/solvers.py | 4 +- .../tests/unit/test_core_settings.py | 9 ++++ services/catalog/tests/unit/test_cli.py | 3 +- .../tests/unit/test_core_settings.py | 9 ++++ .../simcore_service_dask_sidecar/settings.py | 15 +++--- services/dask-sidecar/tests/unit/test_cli.py | 5 +- .../tests/unit/test_modules_dask_client.py | 4 +- .../core/settings.py | 18 ++++--- .../simcore_service_dynamic_scheduler/main.py | 1 - .../dynamic-scheduler/tests/unit/test_cli.py | 2 +- .../_logging_event_handler.py | 10 ++-- .../modules/outputs/_event_handler.py | 28 ++++++---- .../requirements/_base.in | 1 + .../requirements/_base.txt | 15 ++++-- .../osparc_gateway_server/backend/settings.py | 9 ++-- .../simcore_service_payments/core/settings.py | 5 +- services/payments/tests/unit/test_cli.py | 2 +- .../exceptions/errors.py | 2 +- .../exceptions/handlers/_http_error.py | 10 ++-- ...i_resource_tracker_service_runs__export.py | 17 +++--- .../datcore_adapter_settings.py | 9 ++-- services/storage/tests/unit/test_cli.py | 4 +- .../storage/tests/unit/test_dsm_dsmcleaner.py | 2 +- .../application_settings.py | 52 ++++++++++++------- .../director_v2/_models.py | 6 +-- .../dynamic_scheduler/api.py | 4 +- .../groups/_handlers.py | 2 +- .../login/settings.py | 26 ++++++---- .../meta_modeling/_iterations.py | 2 +- .../products/_model.py | 5 +- .../projects/_crud_api_read.py | 3 +- .../projects/_crud_handlers_models.py | 30 +++++------ .../scicrunch/_rest.py | 7 +-- .../session/settings.py | 39 +++++++------- .../studies_dispatcher/_models.py | 4 +- .../studies_dispatcher/_users.py | 2 +- .../studies_dispatcher/settings.py | 1 - .../users/_schemas.py | 18 ++++--- .../src/simcore_service_webserver/utils.py | 2 +- .../version_control/_handlers.py | 2 + .../version_control/models.py | 15 ++++-- .../server/tests/unit/isolated/conftest.py | 5 +- .../test_application_settings_utils.py | 2 +- .../01/clusters/test_clusters_handlers.py | 14 ++--- .../01/test_resource_manager_user_sessions.py | 11 ++-- .../02/test_projects_crud_handlers.py | 2 - .../02/test_projects_nodes_handler.py | 19 ++++--- ...st_projects_nodes_pricing_unit_handlers.py | 4 +- .../02/test_projects_states_handlers.py | 9 +++- .../tests/unit/with_dbs/03/login/conftest.py | 7 +-- .../test_meta_modeling_iterations.py | 2 +- .../tests/unit/with_dbs/03/test_session.py | 2 +- .../tests/unit/with_dbs/03/test_users.py | 26 +++++++--- .../test_version_control_handlers.py | 1 - 64 files changed, 334 insertions(+), 241 deletions(-) diff --git a/packages/common-library/src/common_library/serialization.py b/packages/common-library/src/common_library/serialization.py index 964dfc01ef8..4394fa8cc45 100644 --- a/packages/common-library/src/common_library/serialization.py +++ b/packages/common-library/src/common_library/serialization.py @@ -1,11 +1,10 @@ +import contextlib from datetime import timedelta from typing import Any -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel, SecretStr, TypeAdapter, ValidationError from pydantic_core import Url -from .pydantic_fields_extension import get_type - def model_dump_with_secrets( settings_obj: BaseModel, *, show_secrets: bool, **pydantic_export_options @@ -31,10 +30,11 @@ def model_dump_with_secrets( data[field_name] = str(field_data) elif isinstance(field_data, dict): - field_type = get_type(settings_obj.model_fields[field_name]) - if issubclass(field_type, BaseModel): + possible_pydantic_model = settings_obj.model_fields[field_name].annotation + # NOTE: data could be a dict which does not represent a pydantic model or a union of models + with contextlib.suppress(AttributeError, ValidationError): data[field_name] = model_dump_with_secrets( - field_type.model_validate(field_data), + TypeAdapter(possible_pydantic_model).validate_python(field_data), show_secrets=show_secrets, **pydantic_export_options, ) diff --git a/packages/models-library/src/models_library/api_schemas_clusters_keeper/clusters.py b/packages/models-library/src/models_library/api_schemas_clusters_keeper/clusters.py index 587220d5720..135b42188b8 100644 --- a/packages/models-library/src/models_library/api_schemas_clusters_keeper/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_clusters_keeper/clusters.py @@ -1,7 +1,7 @@ import datetime from enum import auto -from pydantic import AnyUrl, BaseModel +from pydantic import AnyUrl, BaseModel, Field from ..clusters import ClusterAuthentication from ..users import UserID @@ -17,7 +17,7 @@ class ClusterState(StrAutoEnum): class OnDemandCluster(BaseModel): endpoint: AnyUrl - authentication: ClusterAuthentication + authentication: ClusterAuthentication = Field(discriminator="type") state: ClusterState user_id: UserID wallet_id: WalletID | None diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py index 158242195e1..a16ba29289b 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py @@ -114,7 +114,7 @@ class ClusterDetailsGet(ClusterDetails): class ClusterCreate(BaseCluster): owner: GroupID | None = None # type: ignore[assignment] - authentication: ExternalClusterAuthentication + authentication: ExternalClusterAuthentication = Field(discriminator="type") access_rights: dict[GroupID, ClusterAccessRights] = Field( alias="accessRights", default_factory=dict ) @@ -144,9 +144,9 @@ class ClusterCreate(BaseCluster): "password": "somepassword", }, "accessRights": { - 154: CLUSTER_ADMIN_RIGHTS, # type: ignore[dict-item] - 12: CLUSTER_MANAGER_RIGHTS, # type: ignore[dict-item] - 7899: CLUSTER_USER_RIGHTS, # type: ignore[dict-item] + 154: CLUSTER_ADMIN_RIGHTS.model_dump(), # type:ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS.model_dump(), # type:ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS.model_dump(), # type:ignore[dict-item] }, }, ] @@ -174,7 +174,7 @@ class ClusterPatch(BaseCluster): owner: GroupID | None = None # type: ignore[assignment] thumbnail: HttpUrl | None = None endpoint: AnyUrl | None = None # type: ignore[assignment] - authentication: ExternalClusterAuthentication | None = None # type: ignore[assignment] + authentication: ExternalClusterAuthentication | None = Field(None, discriminator="type") # type: ignore[assignment] access_rights: dict[GroupID, ClusterAccessRights] | None = Field( # type: ignore[assignment] default=None, alias="accessRights" ) @@ -190,9 +190,9 @@ class ClusterPatch(BaseCluster): }, { "accessRights": { - 154: CLUSTER_ADMIN_RIGHTS, # type: ignore[dict-item] - 12: CLUSTER_MANAGER_RIGHTS, # type: ignore[dict-item] - 7899: CLUSTER_USER_RIGHTS, # type: ignore[dict-item] + 154: CLUSTER_ADMIN_RIGHTS.model_dump(), # type:ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS.model_dump(), # type:ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS.model_dump(), # type:ignore[dict-item] }, }, ] @@ -203,5 +203,7 @@ class ClusterPatch(BaseCluster): class ClusterPing(BaseModel): endpoint: AnyHttpUrl authentication: ClusterAuthentication = Field( - ..., description="Dask gateway authentication" + ..., + description="Dask gateway authentication", + discriminator="type", ) diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index 09540b8c16e..243144600e9 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -140,7 +140,7 @@ class BaseCluster(BaseModel): ) endpoint: AnyUrl authentication: ClusterAuthentication = Field( - ..., description="Dask gateway authentication" + ..., description="Dask gateway authentication", discriminator="type" ) access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 634b2004998..5a1d20d16f2 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -4,7 +4,7 @@ from datetime import datetime from enum import Enum -from typing import Any, Final, TypeAlias +from typing import Annotated, Any, Final, TypeAlias from uuid import UUID from models_library.basic_types import ConstrainedStr @@ -77,7 +77,7 @@ class BaseProjectModel(BaseModel): last_change_date: datetime = Field(...) # Pipeline of nodes (SEE projects_nodes.py) - workbench: NodesDict = Field(..., description="Project's pipeline") + workbench: Annotated[NodesDict, Field(..., description="Project's pipeline")] # validators _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")( diff --git a/packages/models-library/src/models_library/projects_state.py b/packages/models-library/src/models_library/projects_state.py index 44d40152760..ca5698ed6b2 100644 --- a/packages/models-library/src/models_library/projects_state.py +++ b/packages/models-library/src/models_library/projects_state.py @@ -3,6 +3,7 @@ """ from enum import Enum, unique +from typing import Annotated from pydantic import ( BaseModel, @@ -126,7 +127,7 @@ class ProjectRunningState(BaseModel): class ProjectState(BaseModel): - locked: ProjectLocked = Field(..., description="The project lock state") + locked: Annotated[ProjectLocked, Field(..., description="The project lock state")] state: ProjectRunningState = Field(..., description="The project running state") model_config = ConfigDict(extra="forbid") diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py index 5ebde2fb9d5..6c1a00c2302 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py @@ -1,6 +1,6 @@ -from typing import Literal +from typing import Annotated, Literal -from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from .httpx_calls_capture_errors import OpenApiSpecError @@ -94,7 +94,7 @@ class CapturedParameter(BaseModel): in_: Literal["path", "header", "query"] = Field(..., alias="in") name: str required: bool - schema_: CapturedParameterSchema = Field(..., alias="schema") + schema_: Annotated[CapturedParameterSchema, Field(..., alias="schema")] response_value: str | None = ( None # attribute for storing the params value in a concrete response ) diff --git a/packages/service-library/src/servicelib/aiohttp/tracing.py b/packages/service-library/src/servicelib/aiohttp/tracing.py index fa9cec661a4..3da3b28e3b3 100644 --- a/packages/service-library/src/servicelib/aiohttp/tracing.py +++ b/packages/service-library/src/servicelib/aiohttp/tracing.py @@ -31,9 +31,7 @@ except ImportError: HAS_BOTOCORE = False try: - from opentelemetry.instrumentation.aiopg import ( # type: ignore[import-not-found] - AiopgInstrumentor, - ) + from opentelemetry.instrumentation.aiopg import AiopgInstrumentor HAS_AIOPG = True except ImportError: diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index bdd371fae1a..b5179a8a5f6 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -28,9 +28,7 @@ HAS_ASYNCPG = False try: - from opentelemetry.instrumentation.aiopg import ( # type: ignore[import-not-found] - AiopgInstrumentor, - ) + from opentelemetry.instrumentation.aiopg import AiopgInstrumentor HAS_AIOPG = True except ImportError: diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index b492ec79eff..8c804df22be 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -1,4 +1,5 @@ from functools import cached_property +from typing import Annotated from models_library.basic_types import BootModeEnum, LogLevel from pydantic import ( @@ -87,9 +88,10 @@ class ApplicationSettings(BasicSettings): # DOCKER BOOT SC_BOOT_MODE: BootModeEnum | None = None - API_SERVER_POSTGRES: PostgresSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) + API_SERVER_POSTGRES: Annotated[ + PostgresSettings | None, + Field(json_schema_extra={"auto_default_from_env": True}), + ] API_SERVER_RABBITMQ: RabbitSettings | None = Field( json_schema_extra={"auto_default_from_env": True}, diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index 0e32f5343b5..8462efba68c 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -38,7 +38,7 @@ class Solver(BaseModel): """A released solver with a specific version""" - id: SolverKeyId = Field(..., description="Solver identifier") # noqa: A003 + id: SolverKeyId = Field(..., description="Solver identifier") version: VersionStr = Field( ..., description="semantic version number of the node", @@ -52,7 +52,7 @@ class Solver(BaseModel): # TODO: consider version_aliases: list[str] = [] # remaining tags # Get links to other resources - url: HttpUrl | None = Field(..., description="Link to get this resource") + url: Annotated[HttpUrl | None, Field(..., description="Link to get this resource")] model_config = ConfigDict( extra="ignore", json_schema_extra={ diff --git a/services/autoscaling/tests/unit/test_core_settings.py b/services/autoscaling/tests/unit/test_core_settings.py index 10050a56594..8ad55ec40f0 100644 --- a/services/autoscaling/tests/unit/test_core_settings.py +++ b/services/autoscaling/tests/unit/test_core_settings.py @@ -140,6 +140,9 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_valid( # noqa: N802 assert settings.AUTOSCALING_EC2_INSTANCES +@pytest.mark.xfail( + reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" +) def test_EC2_INSTANCES_ALLOWED_TYPES_passing_invalid_image_tags( # noqa: N802 app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker ): @@ -195,6 +198,9 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_passing_valid_image_tags( # noqa: N802 ] +@pytest.mark.xfail( + reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" +) def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed( # noqa: N802 app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch ): @@ -204,6 +210,9 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed( # noqa: N802 ApplicationSettings.create_from_envs() +@pytest.mark.xfail( + reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" +) def test_invalid_instance_names( app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker ): diff --git a/services/catalog/tests/unit/test_cli.py b/services/catalog/tests/unit/test_cli.py index 044cc6c5884..95d4794306d 100644 --- a/services/catalog/tests/unit/test_cli.py +++ b/services/catalog/tests/unit/test_cli.py @@ -24,7 +24,8 @@ def test_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): result = cli_runner.invoke(main, ["settings", "--show-secrets", "--as-json"]) assert result.exit_code == os.EX_OK - settings = ApplicationSettings.model_validate_json(result.output) + print(result.output) + settings = ApplicationSettings(result.output) assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() diff --git a/services/clusters-keeper/tests/unit/test_core_settings.py b/services/clusters-keeper/tests/unit/test_core_settings.py index d734bf32cff..021d7f4f107 100644 --- a/services/clusters-keeper/tests/unit/test_core_settings.py +++ b/services/clusters-keeper/tests/unit/test_core_settings.py @@ -23,6 +23,9 @@ def test_settings(app_environment: EnvVarsDict): assert settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES +@pytest.mark.xfail( + reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" +) def test_empty_primary_ec2_instances_raises( app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, @@ -34,6 +37,9 @@ def test_empty_primary_ec2_instances_raises( ApplicationSettings.create_from_envs() +@pytest.mark.xfail( + reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" +) def test_multiple_primary_ec2_instances_raises( app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, @@ -58,6 +64,9 @@ def test_multiple_primary_ec2_instances_raises( ApplicationSettings.create_from_envs() +@pytest.mark.xfail( + reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" +) @pytest.mark.parametrize( "invalid_tag", [ diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py index 75ac1fb6cb8..2c3d49ee685 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any +from typing import Annotated, Any from models_library.basic_types import LogLevel from pydantic import AliasChoices, Field, field_validator @@ -13,12 +13,15 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): SC_BUILD_TARGET: str | None = None SC_BOOT_MODE: str | None = None - LOG_LEVEL: LogLevel = Field( - LogLevel.INFO.value, - validation_alias=AliasChoices( - "DASK_SIDECAR_LOGLEVEL", "SIDECAR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + LOG_LEVEL: Annotated[ + LogLevel, + Field( + LogLevel.INFO.value, + validation_alias=AliasChoices( + "DASK_SIDECAR_LOGLEVEL", "SIDECAR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ), - ) + ] # sidecar config --- diff --git a/services/dask-sidecar/tests/unit/test_cli.py b/services/dask-sidecar/tests/unit/test_cli.py index 101b0e4bcdc..7a359d44cc0 100644 --- a/services/dask-sidecar/tests/unit/test_cli.py +++ b/services/dask-sidecar/tests/unit/test_cli.py @@ -28,6 +28,5 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): result = cli_runner.invoke(main, ["settings", "--show-secrets", "--as-json"]) assert result.exit_code == os.EX_OK, result.output - print(result.output) - settings = Settings.model_validate_json(result.output) - assert settings == Settings.create_from_envs() + settings = Settings(result.output) + assert settings.model_dump() == Settings.create_from_envs().model_dump() diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index f63381c538b..68f8464b829 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -54,7 +54,7 @@ from models_library.projects_nodes_io import NodeID from models_library.resource_tracker import HardwareInfo from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, SecretStr +from pydantic import AnyUrl, ByteSize, SecretStr, TypeAdapter from pydantic.tools import parse_obj_as from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -375,7 +375,7 @@ def _mocked_node_ports(mocker: MockerFixture) -> None: ) mocker.patch( "simcore_service_director_v2.modules.dask_client.dask_utils.compute_service_log_file_upload_link", - return_value=parse_obj_as(AnyUrl, "file://undefined"), + return_value=TypeAdapter(AnyUrl).validate_python("file://undefined"), ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 8fd79446760..e577a806712 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,7 +1,7 @@ import datetime +from typing import Annotated from pydantic import AliasChoices, Field, TypeAdapter, field_validator -from pydantic_settings import SettingsConfigDict from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -24,12 +24,16 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): # RUNTIME ----------------------------------------------------------- - DYNAMIC_SCHEDULER_LOGLEVEL: LogLevel = Field( - default=LogLevel.INFO, - validation_alias=AliasChoices( - "DYNAMIC_SCHEDULER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + DYNAMIC_SCHEDULER_LOGLEVEL: Annotated[ + LogLevel, + Field( + default=LogLevel.INFO, + validation_alias=AliasChoices( + "DYNAMIC_SCHEDULER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ), - ) + ] + DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, validation_alias=AliasChoices( @@ -69,8 +73,6 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): def _validate_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - model_config = SettingsConfigDict(extra="allow") - class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py index 55b8513d7e9..4431038df10 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py @@ -10,7 +10,6 @@ from simcore_service_dynamic_scheduler.core.settings import ApplicationSettings _the_settings = ApplicationSettings.create_from_envs() - logging.basicConfig(level=_the_settings.DYNAMIC_SCHEDULER_LOGLEVEL.value) logging.root.setLevel(_the_settings.DYNAMIC_SCHEDULER_LOGLEVEL.value) config_all_loggers( diff --git a/services/dynamic-scheduler/tests/unit/test_cli.py b/services/dynamic-scheduler/tests/unit/test_cli.py index 85b2a5e2dcd..e94c51a9e15 100644 --- a/services/dynamic-scheduler/tests/unit/test_cli.py +++ b/services/dynamic-scheduler/tests/unit/test_cli.py @@ -39,7 +39,7 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, result.output print(result.output) - settings = ApplicationSettings.model_validate_json(result.output) + settings = ApplicationSettings(result.output) assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py index f1537a87389..e8746eef08d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/attribute_monitor/_logging_event_handler.py @@ -76,12 +76,12 @@ def _stop_process(self) -> None: logging.DEBUG, f"{_LoggingEventHandlerProcess.__name__} stop_process", ): - self._stop_queue.put(None) + self._stop_queue.put(None) # pylint:disable=no-member if self._process: # force stop the process - self._process.kill() - self._process.join() + self._process.kill() # pylint:disable=no-member + self._process.join() # pylint:disable=no-member self._process = None # cleanup whatever remains @@ -109,7 +109,7 @@ def _process_worker(self) -> None: ) observer.start() - while self._stop_queue.qsize() == 0: + while self._stop_queue.qsize() == 0: # pylint:disable=no-member # NOTE: watchdog handles events internally every 1 second. # While doing so it will block this thread briefly. # Health check delivery may be delayed. @@ -171,7 +171,7 @@ async def _health_worker(self) -> None: heart_beat_count = 0 while True: try: - self._health_check_queue.get_nowait() + self._health_check_queue.get_nowait() # pylint:disable=no-member heart_beat_count += 1 except Empty: break diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py index 9b256a3d037..dbd35a2b24e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/outputs/_event_handler.py @@ -103,12 +103,12 @@ def stop_process(self) -> None: with log_context( _logger, logging.DEBUG, f"{_EventHandlerProcess.__name__} stop_process" ): - self._stop_queue.put(None) + self._stop_queue.put(None) # pylint:disable=no-member if self._process: # force stop the process - self._process.kill() - self._process.join() + self._process.kill() # pylint:disable=no-member + self._process.join() # pylint:disable=no-member self._process = None # cleanup whatever remains @@ -123,8 +123,10 @@ def shutdown(self) -> None: self.stop_process() # signal queue observers to finish - self.outputs_context.port_key_events_queue.put(None) - self.health_check_queue.put(None) + self.outputs_context.port_key_events_queue.put( + None + ) # pylint:disable=no-member + self.health_check_queue.put(None) # pylint:disable=no-member def _thread_worker_update_outputs_port_keys(self) -> None: # NOTE: runs as a thread in the created process @@ -133,7 +135,9 @@ def _thread_worker_update_outputs_port_keys(self) -> None: while True: message: dict[ str, Any - ] | None = self.outputs_context.file_system_event_handler_queue.get() + ] | None = ( + self.outputs_context.file_system_event_handler_queue.get() # pylint:disable=no-member + ) _logger.debug("received message %s", message) # no more messages quitting @@ -173,7 +177,7 @@ def _process_worker(self) -> None: ) observer.start() - while self._stop_queue.qsize() == 0: + while self._stop_queue.qsize() == 0: # pylint:disable=no-member # watchdog internally uses 1 sec interval to detect events # sleeping for less is useless. # If this value is bigger then the DEFAULT_OBSERVER_TIMEOUT @@ -183,7 +187,9 @@ def _process_worker(self) -> None: # time while handling inotify events # the health_check sending could be delayed - self.health_check_queue.put(_HEART_BEAT_MARK) + self.health_check_queue.put( # pylint:disable=no-member + _HEART_BEAT_MARK + ) blocking_sleep(self.heart_beat_interval_s) except Exception: # pylint: disable=broad-except @@ -196,7 +202,9 @@ def _process_worker(self) -> None: observer.stop() # stop created thread - self.outputs_context.file_system_event_handler_queue.put(None) + self.outputs_context.file_system_event_handler_queue.put( # pylint:disable=no-member + None + ) thread_update_outputs_port_keys.join() _logger.warning("%s exited", _EventHandlerProcess.__name__) @@ -246,7 +254,7 @@ async def _health_worker(self) -> None: heart_beat_count = 0 while True: try: - self._health_check_queue.get_nowait() + self._health_check_queue.get_nowait() # pylint:disable=no-member heart_beat_count += 1 except Empty: break diff --git a/services/osparc-gateway-server/requirements/_base.in b/services/osparc-gateway-server/requirements/_base.in index e41303cf13a..605373b2ef8 100644 --- a/services/osparc-gateway-server/requirements/_base.in +++ b/services/osparc-gateway-server/requirements/_base.in @@ -7,4 +7,5 @@ aiodocker async-timeout dask-gateway-server[local] +pydantic-settings pydantic[email,dotenv] diff --git a/services/osparc-gateway-server/requirements/_base.txt b/services/osparc-gateway-server/requirements/_base.txt index 8a734704a81..c6689413bb4 100644 --- a/services/osparc-gateway-server/requirements/_base.txt +++ b/services/osparc-gateway-server/requirements/_base.txt @@ -7,6 +7,8 @@ aiohttp==3.9.5 # dask-gateway-server aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic async-timeout==4.0.3 # via -r requirements/_base.in attrs==23.2.0 @@ -41,12 +43,17 @@ multidict==6.0.5 # yarl pycparser==2.22 # via cffi -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in -python-dotenv==1.0.1 + # pydantic-settings +pydantic-core==2.23.4 # via pydantic +pydantic-settings==2.6.1 + # via -r requirements/_base.in +python-dotenv==1.0.1 + # via pydantic-settings sqlalchemy==1.4.52 # via # -c requirements/../../../requirements/constraints.txt @@ -54,6 +61,8 @@ sqlalchemy==1.4.52 traitlets==5.14.3 # via dask-gateway-server typing-extensions==4.12.2 - # via pydantic + # via + # pydantic + # pydantic-core yarl==1.9.4 # via aiohttp diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py index 1b967564262..6df9845bbaf 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py @@ -1,6 +1,7 @@ from enum import Enum -from pydantic import BaseSettings, Field, NonNegativeInt, PositiveInt +from pydantic import AliasChoices, Field, NonNegativeInt, PositiveInt +from pydantic_settings import BaseSettings class BootModeEnum(str, Enum): @@ -23,13 +24,13 @@ class AppSettings(BaseSettings): COMPUTATIONAL_SIDECAR_LOG_LEVEL: str | None = Field( default="WARNING", description="The computational sidecar log level", - env=[ + validation_alias=AliasChoices( "COMPUTATIONAL_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL", "SIDECAR_LOG_LEVEL", "SIDECAR_LOGLEVEL", - ], + ), ) COMPUTATIONAL_SIDECAR_VOLUME_NAME: str = Field( ..., description="Named volume for the computational sidecars" @@ -58,7 +59,7 @@ class AppSettings(BaseSettings): description="The hostname of the gateway server in the GATEWAY_WORKERS_NETWORK network", ) - SC_BOOT_MODE: BootModeEnum | None + SC_BOOT_MODE: BootModeEnum | None = None GATEWAY_SERVER_ONE_WORKER_PER_NODE: bool = Field( default=True, diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index 7d33beff83f..a5075e8e4ca 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -4,7 +4,6 @@ from models_library.basic_types import NonNegativeDecimal from pydantic import ( AliasChoices, - ConfigDict, EmailStr, Field, PositiveFloat, @@ -49,7 +48,7 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): PAYMENTS_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, validation_alias=AliasChoices( - "PAYMENTS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + "LOG_FILTER_MAPPING", "PAYMENTS_LOG_FILTER_MAPPING" ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) @@ -63,8 +62,6 @@ def LOG_LEVEL(self): # noqa: N802 def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) - model_config = ConfigDict(extra="allow") # type:ignore[assignment] - class ApplicationSettings(_BaseApplicationSettings): """Web app's environment variables diff --git a/services/payments/tests/unit/test_cli.py b/services/payments/tests/unit/test_cli.py index 84c654ee159..1fb1db4eded 100644 --- a/services/payments/tests/unit/test_cli.py +++ b/services/payments/tests/unit/test_cli.py @@ -50,7 +50,7 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): assert result.exit_code == os.EX_OK, _format_cli_error(result) print(result.output) - settings = ApplicationSettings.model_validate_json(result.output) + settings = ApplicationSettings(result.output) assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py index 533bec1b114..fe620d99c62 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/errors.py @@ -1,4 +1,4 @@ -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class ResourceUsageTrackerBaseError(OsparcErrorMixin, Exception): diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py index 7879d27ae6f..3ab692a70dc 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/exceptions/handlers/_http_error.py @@ -2,10 +2,11 @@ from collections.abc import Callable from typing import Awaitable -from fastapi import HTTPException, Request, status +from fastapi import HTTPException, status from fastapi.encoders import jsonable_encoder from servicelib.logging_errors import create_troubleshotting_log_kwargs from servicelib.status_codes_utils import is_5xx_server_error +from starlette.requests import Request from starlette.responses import JSONResponse from ...exceptions.errors import RutNotFoundError @@ -34,8 +35,9 @@ async def http_error_handler(request: Request, exc: Exception) -> JSONResponse: def http404_error_handler( _: Request, # pylint: disable=unused-argument - exc: RutNotFoundError, + exc: Exception, ) -> JSONResponse: + assert isinstance(exc, RutNotFoundError) # nose return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={"message": f"{exc.msg_template}"}, @@ -44,7 +46,7 @@ def http404_error_handler( def make_http_error_handler_for_exception( status_code: int, exception_cls: type[BaseException] -) -> Callable[[Request, type[BaseException]], Awaitable[JSONResponse]]: +) -> Callable[[Request, Exception], Awaitable[JSONResponse]]: """ Produces a handler for BaseException-type exceptions which converts them into an error JSON response with a given status code @@ -52,7 +54,7 @@ def make_http_error_handler_for_exception( SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions """ - async def _http_error_handler(_: Request, exc: type[BaseException]) -> JSONResponse: + async def _http_error_handler(_: Request, exc: Exception) -> JSONResponse: assert isinstance(exc, exception_cls) # nosec return JSONResponse( content=jsonable_encoder({"errors": [str(exc)]}), status_code=status_code diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py index e0ebd28981b..56c9c102df6 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py @@ -4,12 +4,12 @@ # pylint:disable=too-many-arguments import os -from unittest.mock import Mock +from unittest.mock import AsyncMock, Mock import pytest import sqlalchemy as sa from moto.server import ThreadedMotoServer -from pydantic import AnyUrl +from pydantic import AnyUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient @@ -29,21 +29,18 @@ @pytest.fixture -async def mocked_export(mocker: MockerFixture): - mock_export = mocker.patch( +async def mocked_export(mocker: MockerFixture) -> AsyncMock: + return mocker.patch( "simcore_service_resource_usage_tracker.services.service_runs.ResourceTrackerRepository.export_service_runs_table_to_s3", autospec=True, ) @pytest.fixture -async def mocked_presigned_link(mocker: MockerFixture): - mock_presigned_link = mocker.patch( +async def mocked_presigned_link(mocker: MockerFixture) -> AsyncMock: + return mocker.patch( "simcore_service_resource_usage_tracker.services.service_runs.SimcoreS3API.create_single_presigned_download_link", - return_value=parse_obj_as( - AnyUrl, - "https://www.testing.com/", - ), + return_value=TypeAdapter(AnyUrl).validate_python("https://www.testing.com/"), ) diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py index 71905e42381..3e258d9bb1a 100644 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py +++ b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py @@ -15,10 +15,7 @@ class DatcoreAdapterSettings(BaseCustomSettings): @cached_property def endpoint(self) -> str: - endpoint = AnyHttpUrl.build( - scheme="http", - host=self.DATCORE_ADAPTER_HOST, - port=self.DATCORE_ADAPTER_PORT, - path=f"/{self.DATCORE_ADAPTER_VTAG}", + url = TypeAdapter(AnyHttpUrl).validate_python( + f"http://{self.DATCORE_ADAPTER_HOST}:{self.DATCORE_ADAPTER_PORT}/{self.DATCORE_ADAPTER_VTAG}" ) - return f"{endpoint}" + return f"{url}" diff --git a/services/storage/tests/unit/test_cli.py b/services/storage/tests/unit/test_cli.py index 7c86b85fcd7..ad31a85e31f 100644 --- a/services/storage/tests/unit/test_cli.py +++ b/services/storage/tests/unit/test_cli.py @@ -29,7 +29,7 @@ def test_cli_settings_as_json( assert result.exit_code == os.EX_OK, result # reuse resulting json to build settings settings: dict = json.loads(result.stdout) - assert Settings.model_validate(settings) + assert Settings(settings) def test_cli_settings_env_file( @@ -46,4 +46,4 @@ def test_cli_settings_env_file( with contextlib.suppress(json.decoder.JSONDecodeError): settings[key] = json.loads(str(value)) - assert Settings.model_validate(settings) + assert Settings(settings) diff --git a/services/storage/tests/unit/test_dsm_dsmcleaner.py b/services/storage/tests/unit/test_dsm_dsmcleaner.py index 7862d4d7166..1683a9d0a0d 100644 --- a/services/storage/tests/unit/test_dsm_dsmcleaner.py +++ b/services/storage/tests/unit/test_dsm_dsmcleaner.py @@ -70,7 +70,7 @@ def simcore_directory_id(simcore_file_id: SimcoreS3FileID) -> SimcoreS3FileID: ], ) @pytest.mark.parametrize("checksum", [None, _faker.sha256()]) -async def test_regression_collaborator_creates_file_upload_links( +async def test_regression_collaborator_creates_file_upload_links( # pylint:disable=too-many-positional-arguments disabled_dsm_cleaner_task, aiopg_engine: Engine, simcore_s3_dsm: SimcoreS3DataManager, diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 94d1be5358d..67b1b32696d 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -1,6 +1,6 @@ import logging from functools import cached_property -from typing import Any, Final +from typing import Annotated, Any, Final from aiohttp import web from common_library.pydantic_fields_extension import is_nullable @@ -22,7 +22,6 @@ ) from pydantic.fields import Field from pydantic.types import PositiveInt -from pydantic_settings import SettingsConfigDict from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.email import SMTPSettings @@ -115,11 +114,16 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_CREDIT_COMPUTATION_ENABLED: bool = Field( default=False, description="Enables credit computation features." ) - WEBSERVER_LOGLEVEL: LogLevel = Field( - default=LogLevel.WARNING.value, - validation_alias=AliasChoices("WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), - # NOTE: suffix '_LOGLEVEL' is used overall - ) + WEBSERVER_LOGLEVEL: Annotated[ + LogLevel, + Field( + default=LogLevel.WARNING.value, + validation_alias=AliasChoices( + "WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), + # NOTE: suffix '_LOGLEVEL' is used overall + ), + ] WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, validation_alias=AliasChoices( @@ -188,9 +192,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="invitations plugin", ) - WEBSERVER_LOGIN: LoginSettings | None = Field( - json_schema_extra={"auto_default_from_env": True}, description="login plugin" - ) + WEBSERVER_LOGIN: Annotated[ + LoginSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + description="login plugin", + ), + ] WEBSERVER_PAYMENTS: PaymentsSettings | None = Field( json_schema_extra={"auto_default_from_env": True}, @@ -222,9 +230,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="scicrunch plugin", json_schema_extra={"auto_default_from_env": True}, ) - WEBSERVER_SESSION: SessionSettings = Field( - description="session plugin", json_schema_extra={"auto_default_from_env": True} - ) + WEBSERVER_SESSION: Annotated[ + SessionSettings, + Field( + description="session plugin", + json_schema_extra={"auto_default_from_env": True}, + ), + ] WEBSERVER_STATICWEB: StaticWebserverModuleSettings | None = Field( description="static-webserver service plugin", @@ -279,21 +291,23 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): "Currently this is a system plugin and cannot be disabled", ) - @model_validator(mode="after") + @model_validator(mode="before") @classmethod - def build_vcs_release_url_if_unset(cls, v): - release_url = v.SIMCORE_VCS_RELEASE_URL + def build_vcs_release_url_if_unset(cls, values): + release_url = values.get("SIMCORE_VCS_RELEASE_URL") - if release_url is None and (vsc_release_tag := v.SIMCORE_VCS_RELEASE_TAG): + if release_url is None and ( + vsc_release_tag := values.get("SIMCORE_VCS_RELEASE_TAG") + ): if vsc_release_tag == "latest": release_url = ( "https://github.com/ITISFoundation/osparc-simcore/commits/master/" ) else: release_url = f"https://github.com/ITISFoundation/osparc-simcore/releases/tag/{vsc_release_tag}" - v.SIMCORE_VCS_RELEASE_URL = release_url + values["SIMCORE_VCS_RELEASE_URL"] = release_url - return v + return values @field_validator( # List of plugins under-development (keep up-to-date) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_models.py b/services/web/server/src/simcore_service_webserver/director_v2/_models.py index 6d14d4b709d..966229c4221 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_models.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_models.py @@ -65,9 +65,9 @@ def set_default_thumbnail_if_empty(cls, v, values): "password": "somepassword", }, "access_rights": { - 154: CLUSTER_ADMIN_RIGHTS, - 12: CLUSTER_MANAGER_RIGHTS, - 7899: CLUSTER_USER_RIGHTS, + 154: CLUSTER_ADMIN_RIGHTS.model_dump(), # type:ignore[dict-item] + 12: CLUSTER_MANAGER_RIGHTS.model_dump(), # type:ignore[dict-item] + 7899: CLUSTER_USER_RIGHTS.model_dump(), # type:ignore[dict-item] }, }, ] diff --git a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py index 3c5509e449e..be02b28bf73 100644 --- a/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py +++ b/services/web/server/src/simcore_service_webserver/dynamic_scheduler/api.py @@ -66,7 +66,9 @@ async def stop_dynamic_service( await services.stop_dynamic_service( get_rabbitmq_rpc_client(app), dynamic_service_stop=dynamic_service_stop, - timeout_s=settings.DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT, + timeout_s=int( + settings.DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT.total_seconds() + ), ) diff --git a/services/web/server/src/simcore_service_webserver/groups/_handlers.py b/services/web/server/src/simcore_service_webserver/groups/_handlers.py index 748458f4a44..428769b725d 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_handlers.py @@ -150,7 +150,7 @@ async def create_group(request: web.Request): """Creates organization groups""" req_ctx = _GroupsRequestContext.model_validate(request) create = await parse_request_body_as(GroupCreate, request) - new_group = create.model_dump(exclude_unset=True) + new_group = create.model_dump(mode="json", exclude_unset=True) created_group = await api.create_user_group(request.app, req_ctx.user_id, new_group) assert GroupGet.model_validate(created_group) is not None # nosec diff --git a/services/web/server/src/simcore_service_webserver/login/settings.py b/services/web/server/src/simcore_service_webserver/login/settings.py index 91ee1041889..909b3a64eb6 100644 --- a/services/web/server/src/simcore_service_webserver/login/settings.py +++ b/services/web/server/src/simcore_service_webserver/login/settings.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import Final, Literal +from typing import Annotated, Final, Literal from aiohttp import web from pydantic import BaseModel, ValidationInfo, field_validator @@ -21,11 +21,14 @@ class LoginSettings(BaseCustomSettings): - LOGIN_ACCOUNT_DELETION_RETENTION_DAYS: PositiveInt = Field( - default=30, - description="Retention time (in days) of all the data after a user has requested the deletion of their account" - "NOTE: exposed to the front-end as `to_client_statics`", - ) + LOGIN_ACCOUNT_DELETION_RETENTION_DAYS: Annotated[ + PositiveInt, + Field( + default=30, + description="Retention time (in days) of all the data after a user has requested the deletion of their account" + "NOTE: exposed to the front-end as `to_client_statics`", + ), + ] LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: bool = Field( default=True, @@ -44,10 +47,13 @@ class LoginSettings(BaseCustomSettings): default=120, description="Expiration time for code [sec]" ) - LOGIN_2FA_REQUIRED: bool = Field( - default=False, - description="If true, it enables two-factor authentication (2FA)", - ) + LOGIN_2FA_REQUIRED: Annotated[ + bool, + Field( + default=False, + description="If true, it enables two-factor authentication (2FA)", + ), + ] LOGIN_PASSWORD_MIN_LENGTH: PositiveInt = Field( default=12, diff --git a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py index b8adef9bcc4..4d271a5c9f7 100644 --- a/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py +++ b/services/web/server/src/simcore_service_webserver/meta_modeling/_iterations.py @@ -280,7 +280,7 @@ async def get_or_create_runnable_projects( project["workbench"].update( { # converts model in dict patching first thumbnail - nid: n.copy(update={"thumbnail": n.thumbnail or ""}).model_dump( + nid: n.model_copy(update={"thumbnail": n.thumbnail or ""}).model_dump( by_alias=True, exclude_unset=True ) for nid, n in updated_nodes.items() diff --git a/services/web/server/src/simcore_service_webserver/products/_model.py b/services/web/server/src/simcore_service_webserver/products/_model.py index 48f9f20fa01..5a00687b1a7 100644 --- a/services/web/server/src/simcore_service_webserver/products/_model.py +++ b/services/web/server/src/simcore_service_webserver/products/_model.py @@ -50,7 +50,7 @@ class Product(BaseModel): name: ProductName = Field(pattern=PUBLIC_VARIABLE_NAME_RE, validate_default=True) - display_name: str = Field(..., description="Long display name") + display_name: Annotated[str, Field(..., description="Long display name")] short_name: str | None = Field( None, pattern=re.compile(TWILIO_ALPHANUMERIC_SENDER_ID_RE), @@ -135,7 +135,8 @@ def _validate_name(cls, v): return v @field_serializer("issues", "vendor") - def _preserve_snake_case(self, v: Any) -> Any: + @staticmethod + def _preserve_snake_case(v: Any) -> Any: return v @property diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 793dd19b083..81a615730d0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -53,7 +53,8 @@ async def _append_fields( # replace project access rights (if project is in workspace) if workspace_access_rights: project["accessRights"] = { - gid: access.model_dump() for gid, access in workspace_access_rights.items() + f"{gid}": access.model_dump() + for gid, access in workspace_access_rights.items() } # validate diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py index 1f67baeabc0..52e3234876f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers_models.py @@ -4,7 +4,7 @@ """ -from typing import Any +from typing import Annotated, Self from models_library.basic_types import IDStr from models_library.folders import FolderID @@ -57,19 +57,14 @@ class ProjectCreateHeaders(BaseModel): alias=X_SIMCORE_PARENT_NODE_ID, ) - @model_validator(mode="before") - @classmethod - def check_parent_valid(cls, values: dict[str, Any]) -> dict[str, Any]: - if ( - values.get("parent_project_uuid") is None - and values.get("parent_node_id") is not None - ) or ( - values.get("parent_project_uuid") is not None - and values.get("parent_node_id") is None + @model_validator(mode="after") + def check_parent_valid(self) -> Self: + if (self.parent_project_uuid is None and self.parent_node_id is not None) or ( + self.parent_project_uuid is not None and self.parent_node_id is None ): msg = "Both parent_project_uuid and parent_node_id must be set or both null or both unset" raise ValueError(msg) - return values + return self model_config = ConfigDict(populate_by_name=False) @@ -181,11 +176,14 @@ class ProjectListFullSearchParams(PageQueryParameters): max_length=100, examples=["My Project"], ) - tag_ids: str | None = Field( - default=None, - description="Search by tag ID (multiple tag IDs may be provided separated by column)", - examples=["1,3"], - ) + tag_ids: Annotated[ + str | None, + Field( + default=None, + description="Search by tag ID (multiple tag IDs may be provided separated by column)", + examples=["1,3"], + ), + ] _empty_is_none = field_validator("text", mode="before")( empty_str_to_none_pre_validator diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py b/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py index fd2f3d243a1..4a546a589fe 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_rest.py @@ -15,7 +15,7 @@ """ import logging -from typing import Any +from typing import Annotated, Any from aiohttp import ClientSession from pydantic import BaseModel, Field, RootModel @@ -41,7 +41,7 @@ class FieldItem(BaseModel): class ResourceView(BaseModel): - resource_fields: list[FieldItem] = Field([], alias="fields") + resource_fields: Annotated[list[FieldItem], Field([], alias="fields")] version: int curation_status: str last_curated_version: int @@ -60,7 +60,8 @@ def _get_field(self, fieldname: str): for field in self.resource_fields: if field.field_name == fieldname: return field.value - raise ValueError(f"Cannot file expected field {fieldname}") + msg = f"Cannot file expected field {fieldname}" + raise ValueError(msg) def get_name(self): return str(self._get_field("Resource Name")) diff --git a/services/web/server/src/simcore_service_webserver/session/settings.py b/services/web/server/src/simcore_service_webserver/session/settings.py index f6fec9a878e..74a7f18f2e9 100644 --- a/services/web/server/src/simcore_service_webserver/session/settings.py +++ b/services/web/server/src/simcore_service_webserver/session/settings.py @@ -1,31 +1,31 @@ -from typing import Final +from typing import Annotated, Final from aiohttp import web from pydantic import AliasChoices, PositiveInt, field_validator from pydantic.fields import Field from pydantic.types import SecretStr -from pydantic_settings import SettingsConfigDict from settings_library.base import BaseCustomSettings from settings_library.utils_session import MixinSessionSettings from .._constants import APP_SETTINGS_KEY _MINUTE: Final[int] = 60 # secs -_HOUR: Final[int] = 60 * _MINUTE -_DAY: Final[int] = 24 * _HOUR class SessionSettings(BaseCustomSettings, MixinSessionSettings): - SESSION_SECRET_KEY: SecretStr = Field( - ..., - description="Secret key to encrypt cookies. " - 'TIP: python3 -c "from cryptography.fernet import *; print(Fernet.generate_key())"', - min_length=44, - validation_alias=AliasChoices( - "SESSION_SECRET_KEY", "WEBSERVER_SESSION_SECRET_KEY" + SESSION_SECRET_KEY: Annotated[ + SecretStr, + Field( + ..., + description="Secret key to encrypt cookies. " + 'TIP: python3 -c "from cryptography.fernet import *; print(Fernet.generate_key())"', + min_length=44, + validation_alias=AliasChoices( + "SESSION_SECRET_KEY", "WEBSERVER_SESSION_SECRET_KEY" + ), ), - ) + ] SESSION_ACCESS_TOKENS_EXPIRATION_INTERVAL_SECS: int = Field( 30 * _MINUTE, @@ -37,10 +37,13 @@ class SessionSettings(BaseCustomSettings, MixinSessionSettings): # - Defaults taken from https://github.com/aio-libs/aiohttp-session/blob/master/aiohttp_session/cookie_storage.py#L20-L26 # - SESSION_COOKIE_MAX_AGE: PositiveInt | None = Field( - default=None, - description="Max-Age attribute. Maximum age for session data, int seconds or None for “session cookie” which last until you close your browser.", - ) + SESSION_COOKIE_MAX_AGE: Annotated[ + PositiveInt | None, + Field( + default=None, + description="Max-Age attribute. Maximum age for session data, int seconds or None for “session cookie” which last until you close your browser.", + ), + ] SESSION_COOKIE_SAMESITE: str | None = Field( default=None, description="SameSite attribute lets servers specify whether/when cookies are sent with cross-site requests", @@ -54,10 +57,6 @@ class SessionSettings(BaseCustomSettings, MixinSessionSettings): default=True, description="This prevents JavaScript from accessing the session cookie", ) - - model_config = SettingsConfigDict( - extra="allow" - ) @field_validator("SESSION_SECRET_KEY") @classmethod diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py index c697b409c0f..a9a1cc23661 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py @@ -1,3 +1,5 @@ +from typing import Annotated + from aiopg.sa.result import RowProxy from models_library.services import ServiceKey, ServiceVersion from pydantic import BaseModel, Field, HttpUrl, PositiveInt, TypeAdapter @@ -7,7 +9,7 @@ class ServiceInfo(BaseModel): key: ServiceKey version: ServiceVersion - label: str = Field(..., description="Display name") + label: Annotated[str, Field(..., description="Display name")] thumbnail: HttpUrl = Field( default=TypeAdapter(HttpUrl).validate_python( diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py index 568a534832a..b76d8a4b3f9 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py @@ -12,7 +12,7 @@ import secrets import string from contextlib import suppress -from datetime import UTC, datetime +from datetime import datetime import redis.asyncio as aioredis from aiohttp import web diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index 00024ce2d23..4d61119d0b7 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -1,5 +1,4 @@ from datetime import timedelta -from typing import Annotated from aiohttp import web from common_library.pydantic_validators import validate_numeric_string_as_timedelta diff --git a/services/web/server/src/simcore_service_webserver/users/_schemas.py b/services/web/server/src/simcore_service_webserver/users/_schemas.py index 8df6c890a8b..4b9aa7acf63 100644 --- a/services/web/server/src/simcore_service_webserver/users/_schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/_schemas.py @@ -2,11 +2,10 @@ """ - import re import sys from contextlib import suppress -from typing import Any, Final +from typing import Annotated, Any, Final import pycountry from models_library.api_schemas_webserver._base import InputSchema, OutputSchema @@ -61,7 +60,9 @@ class PreUserProfile(InputSchema): first_name: str last_name: str email: LowerCaseEmailStr - institution: str | None = Field(default=None, description="company, university, ...") + institution: str | None = Field( + default=None, description="company, university, ..." + ) phone: str | None # billing details address: str @@ -69,10 +70,13 @@ class PreUserProfile(InputSchema): state: str | None = Field(default=None) postal_code: str country: str - extras: dict[str, Any] = Field( - default_factory=dict, - description="Keeps extra information provided in the request form. At most MAX_NUM_EXTRAS fields", - ) + extras: Annotated[ + dict[str, Any], + Field( + default_factory=dict, + description="Keeps extra information provided in the request form. At most MAX_NUM_EXTRAS fields", + ), + ] model_config = ConfigDict(str_strip_whitespace=True, str_max_length=200) diff --git a/services/web/server/src/simcore_service_webserver/utils.py b/services/web/server/src/simcore_service_webserver/utils.py index 0807cee96b1..0ffb59a493b 100644 --- a/services/web/server/src/simcore_service_webserver/utils.py +++ b/services/web/server/src/simcore_service_webserver/utils.py @@ -11,7 +11,7 @@ import tracemalloc from datetime import datetime from pathlib import Path -from typing import Any, cast +from typing import Any import orjson from common_library.error_codes import ErrorCodeStr diff --git a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py index 22b6270019c..a0847ea34ea 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/version_control/_handlers.py @@ -238,6 +238,8 @@ async def _update_checkpoint_annotations_handler(request: web.Request): path_params = parse_request_path_parameters_as(_CheckpointsPathParam, request) update = await parse_request_body_as(CheckpointAnnotations, request) + assert isinstance(path_params.ref_id, int) + checkpoint: Checkpoint = await update_checkpoint( vc_repo, project_uuid=path_params.project_uuid, diff --git a/services/web/server/src/simcore_service_webserver/version_control/models.py b/services/web/server/src/simcore_service_webserver/version_control/models.py index 451d302d90f..505758d53d2 100644 --- a/services/web/server/src/simcore_service_webserver/version_control/models.py +++ b/services/web/server/src/simcore_service_webserver/version_control/models.py @@ -1,11 +1,19 @@ from datetime import datetime -from typing import Any, TypeAlias, Union +from typing import Annotated, Any, TypeAlias, Union from aiopg.sa.result import RowProxy from models_library.basic_types import SHA1Str from models_library.projects import ProjectID from models_library.projects_nodes import Node -from pydantic import ConfigDict, BaseModel, Field, PositiveInt, StrictBool, StrictFloat, StrictInt +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PositiveInt, + StrictBool, + StrictFloat, + StrictInt, +) from pydantic.networks import HttpUrl BuiltinTypes: TypeAlias = Union[StrictBool, StrictInt, StrictFloat, str] @@ -24,7 +32,7 @@ CommitID: TypeAlias = int BranchID: TypeAlias = int -RefID: TypeAlias = CommitID | str +RefID: TypeAlias = Annotated[CommitID | str, Field(union_mode="left_to_right")] CheckpointID: TypeAlias = PositiveInt @@ -51,6 +59,7 @@ def from_commit_log(cls, commit: RowProxy, tags: list[RowProxy]) -> "Checkpoint" class WorkbenchView(BaseModel): """A view (i.e. read-only and visual) of the project's workbench""" + model_config = ConfigDict(from_attributes=True) # NOTE: Tmp replacing UUIDS by str due to a problem serializing to json UUID keys diff --git a/services/web/server/tests/unit/isolated/conftest.py b/services/web/server/tests/unit/isolated/conftest.py index 31f0aa33f98..9cc0948ff88 100644 --- a/services/web/server/tests/unit/isolated/conftest.py +++ b/services/web/server/tests/unit/isolated/conftest.py @@ -96,7 +96,6 @@ def mock_env_deployer_pipeline(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: def mock_env_devel_environment( mock_env_devel_environment: EnvVarsDict, # pylint: disable=redefined-outer-name monkeypatch: pytest.MonkeyPatch, - faker: Faker ) -> EnvVarsDict: # Overrides to ensure dev-features are enabled testings return mock_env_devel_environment | setenvs_from_dict( @@ -219,8 +218,8 @@ def mock_webserver_service_environment( "STORAGE_PORT": os.environ.get("STORAGE_PORT", "8080"), "SWARM_STACK_NAME": os.environ.get("SWARM_STACK_NAME", "simcore"), "WEBSERVER_LOGLEVEL": os.environ.get("LOG_LEVEL", "WARNING"), - "SESSION_COOKIE_MAX_AGE": str(7 * 24 * 60 * 60) - } + "SESSION_COOKIE_MAX_AGE": str(7 * 24 * 60 * 60), + }, ) return ( diff --git a/services/web/server/tests/unit/isolated/test_application_settings_utils.py b/services/web/server/tests/unit/isolated/test_application_settings_utils.py index 2897b8bf358..a8e97785754 100644 --- a/services/web/server/tests/unit/isolated/test_application_settings_utils.py +++ b/services/web/server/tests/unit/isolated/test_application_settings_utils.py @@ -19,7 +19,7 @@ def test_settings_infered_from_default_tests_config( settings = ApplicationSettings.create_from_envs() - print("settings=\n", settings.model_dump_json(indent=1, sort_keys=True)) + print("settings=\n", settings.model_dump_json(indent=1)) infered_config = convert_to_app_config(settings) diff --git a/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py b/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py index e88d13474fe..edef380dc86 100644 --- a/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py +++ b/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py @@ -152,15 +152,13 @@ async def test_create_cluster( url = client.app.router["create_cluster"].url_for() rsp = await client.post( f"{url}", - json=json.loads( - cluster_create.model_dump_json(by_alias=True, exclude_unset=True) - ), + json=json.loads(cluster_create.model_dump_json(by_alias=True)), ) data, error = await assert_status( rsp, - expected.forbidden - if user_role == UserRole.USER - else expected.created, # only accessible for TESTER + ( + expected.forbidden if user_role == UserRole.USER else expected.created + ), # only accessible for TESTER ) if error: # we are done here @@ -343,9 +341,7 @@ async def test_create_cluster_with_error( url = client.app.router["create_cluster"].url_for() rsp = await client.post( f"{url}", - json=json.loads( - cluster_create.model_dump_json(by_alias=True, exclude_unset=True) - ), + json=json.loads(cluster_create.model_dump_json(by_alias=True)), ) data, error = await assert_status(rsp, expected_http_error) assert not data diff --git a/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py b/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py index 00f3ce6d576..a82b2e5fd3f 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py +++ b/services/web/server/tests/unit/with_dbs/01/test_resource_manager_user_sessions.py @@ -3,7 +3,6 @@ # pylint: disable=too-many-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable -import json import time from collections.abc import Callable from random import randint @@ -17,7 +16,6 @@ from servicelib.aiohttp.application import create_safe_application from servicelib.aiohttp.application_setup import is_setup_completed from simcore_service_webserver.application_settings import setup_settings -from simcore_service_webserver.payments._methods_api import Faker from simcore_service_webserver.resource_manager.plugin import setup_resource_manager from simcore_service_webserver.resource_manager.registry import ( _ALIVE_SUFFIX, @@ -35,12 +33,15 @@ @pytest.fixture def mock_env_devel_environment( - mock_env_devel_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch, faker: Faker + mock_env_devel_environment: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + faker: Faker, ): return mock_env_devel_environment | setenvs_from_dict( - monkeypatch, { + monkeypatch, + { "RESOURCE_MANAGER_RESOURCE_TTL_S": "3", - } + }, ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 99cf4e105d4..a606d2c61c2 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -18,8 +18,6 @@ from faker import Faker from models_library.products import ProductName from models_library.projects_state import ProjectState -from models_library.services import ServiceKey -from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index 8945e2c9296..5496cb46458 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -201,12 +201,13 @@ async def test_replace_node_resources_is_ok_if_explicitly_authorized( node_resources = TypeAdapter(ServiceResourcesDict).validate_python(data) assert node_resources assert DEFAULT_SINGLE_SERVICE_NAME in node_resources - assert ( - {k: v.model_dump() for k, v in node_resources.items()} - == ServiceResourcesDictHelpers.model_config["json_schema_extra"][ - "examples" - ][0] - ) + assert { + k: v.model_dump() for k, v in node_resources.items() + } == ServiceResourcesDictHelpers.model_config["json_schema_extra"][ + "examples" + ][ + 0 + ] @pytest.mark.parametrize( @@ -333,6 +334,7 @@ async def test_create_node( body = { "service_key": f"simcore/services/{node_class}/{faker.pystr().lower()}", "service_version": faker.numerify("%.#.#"), + "service_id": None, } response = await client.post(url.path, json=body) data, error = await assert_status(response, expected.created) @@ -424,6 +426,7 @@ def inc_running_services(self, *args, **kwargs): # noqa: ARG002 body = { "service_key": f"simcore/services/dynamic/{faker.pystr().lower()}", "service_version": faker.numerify("%.#.#"), + "service_id": None, } NUM_DY_SERVICES = 150 responses = await asyncio.gather( @@ -485,6 +488,7 @@ async def test_create_node_does_not_start_dynamic_node_if_there_are_already_too_ body = { "service_key": f"simcore/services/dynamic/{faker.pystr().lower()}", "service_version": faker.numerify("%.#.#"), + "service_id": None, } response = await client.post(f"{ url}", json=body) await assert_status(response, expected.created) @@ -546,6 +550,7 @@ async def inc_running_services(self, *args, **kwargs): # noqa: ARG002 body = { "service_key": f"simcore/services/dynamic/{faker.pystr().lower()}", "service_version": faker.numerify("%.#.#"), + "service_id": None, } NUM_DY_SERVICES: Final[NonNegativeInt] = 150 responses = await asyncio.gather( @@ -597,6 +602,7 @@ async def test_create_node_does_start_dynamic_node_if_max_num_set_to_0( body = { "service_key": f"simcore/services/dynamic/{faker.pystr().lower()}", "service_version": faker.numerify("%.#.#"), + "service_id": None, } response = await client.post(f"{ url}", json=body) await assert_status(response, expected.created) @@ -629,6 +635,7 @@ async def test_creating_deprecated_node_returns_406_not_acceptable( body = { "service_key": f"simcore/services/{node_class}/{faker.pystr().lower()}", "service_version": f"{faker.random_int()}.{faker.random_int()}.{faker.random_int()}", + "service_id": None, } response = await client.post(url.path, json=body) data, error = await assert_status(response, expected.not_acceptable) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py index a867d2e3c40..5812190c354 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py @@ -100,9 +100,9 @@ def mock_rut_api_responses( pricing_unit_get_base = PricingUnitGet.model_validate( PricingUnitGet.model_config["json_schema_extra"]["examples"][0] ) - pricing_unit_get_1 = pricing_unit_get_base.copy() + pricing_unit_get_1 = pricing_unit_get_base.model_copy() pricing_unit_get_1.pricing_unit_id = _PRICING_UNIT_ID_1 - pricing_unit_get_2 = pricing_unit_get_base.copy() + pricing_unit_get_2 = pricing_unit_get_base.model_copy() pricing_unit_get_2.pricing_unit_id = _PRICING_UNIT_ID_2 aioresponses_mocker.get( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 693372aab27..77950f4c0be 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -995,7 +995,11 @@ async def test_project_node_lifetime( # noqa: PLR0915 # create a new dynamic node... url = client.app.router["create_node"].url_for(project_id=user_project["uuid"]) - body = {"service_key": "simcore/services/dynamic/key", "service_version": "1.3.4"} + body = { + "service_key": "simcore/services/dynamic/key", + "service_version": "1.3.4", + "service_id": None, + } resp = await client.post(url.path, json=body) data, errors = await assert_status(resp, expected_response_on_create) dynamic_node_id = None @@ -1016,6 +1020,7 @@ async def test_project_node_lifetime( # noqa: PLR0915 body = { "service_key": "simcore/services/comp/key", "service_version": "1.3.4", + "service_id": None, } resp = await client.post(f"{url}", json=body) data, errors = await assert_status(resp, expected_response_on_create) @@ -1277,7 +1282,7 @@ async def test_open_shared_project_2_users_locked( mock_project_state_updated_handler, shared_project, [ - expected_project_state_client_1.copy( + expected_project_state_client_1.model_copy( update={ "locked": ProjectLocked( value=True, status=ProjectStatus.CLOSING, owner=owner1 diff --git a/services/web/server/tests/unit/with_dbs/03/login/conftest.py b/services/web/server/tests/unit/with_dbs/03/login/conftest.py index b9a458add16..b3f8049ff51 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/login/conftest.py @@ -20,7 +20,9 @@ @pytest.fixture -def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker): +def app_environment( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker +): envs_plugins = setenvs_from_dict( monkeypatch, { @@ -38,10 +40,9 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc "WEBSERVER_SOCKETIO": "1", # for login notifications "WEBSERVER_STUDIES_DISPATCHER": "null", "WEBSERVER_TAGS": "1", - "WEBSERVER_TRACING": "null", "WEBSERVER_VERSION_CONTROL": "0", "WEBSERVER_WALLETS": "1", - "WEBSERVER_TRACING": "null" + "WEBSERVER_TRACING": "null", }, ) diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py index a9ea81e32fb..1572f862421 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py @@ -248,7 +248,7 @@ async def _mock_catalog_get(*args, **kwarg): # and index but nothing prevents a dict from using any other type of key types # project = Project.model_validate(body["data"]) - new_project = project.copy( + new_project = project.model_copy( update={ # TODO: HACK to overcome export from None -> string # SOLUTION 1: thumbnail should not be required (check with team!) diff --git a/services/web/server/tests/unit/with_dbs/03/test_session.py b/services/web/server/tests/unit/with_dbs/03/test_session.py index 127089dc802..f9f709c8e3f 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_session.py +++ b/services/web/server/tests/unit/with_dbs/03/test_session.py @@ -118,7 +118,7 @@ def test_session_settings( ): if session_key is not None: - settings = SessionSettings(SESSION_SECRET_KEY=session_key) + settings = SessionSettings(WEBSERVER_SESSION_SECRET_KEY=session_key) else: settings = SessionSettings() diff --git a/services/web/server/tests/unit/with_dbs/03/test_users.py b/services/web/server/tests/unit/with_dbs/03/test_users.py index 33d22b4e815..391053b40ac 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users.py @@ -85,7 +85,9 @@ async def test_get_profile( e = Envelope[ProfileGet].model_validate(await resp.json()) assert e.error == error assert ( - e.data.model_dump(**RESPONSE_MODEL_POLICY, mode="json") == data if e.data else e.data == data + e.data.model_dump(**RESPONSE_MODEL_POLICY, mode="json") == data + if e.data + else e.data == data ) if not error: @@ -107,7 +109,9 @@ async def test_get_profile( assert profile.role == user_role.name assert profile.groups - got_profile_groups = profile.groups.model_dump(**RESPONSE_MODEL_POLICY, mode="json") + got_profile_groups = profile.groups.model_dump( + **RESPONSE_MODEL_POLICY, mode="json" + ) assert got_profile_groups["me"] == primary_group assert got_profile_groups["all"] == all_group @@ -280,7 +284,15 @@ async def test_search_and_pre_registration( found, _ = await assert_status(resp, status.HTTP_200_OK) assert len(found) == 1 - got = UserProfile(**found[0]) + got = UserProfile( + **found[0], + institution=None, + address=None, + city=None, + state=None, + postal_code=None, + country=None, + ) expected = { "first_name": logged_user.get("first_name"), "last_name": logged_user.get("last_name"), @@ -309,7 +321,7 @@ async def test_search_and_pre_registration( ) found, _ = await assert_status(resp, status.HTTP_200_OK) assert len(found) == 1 - got = UserProfile(**found[0]) + got = UserProfile(**found[0], state=None, status=None) assert got.model_dump(include={"registered", "status"}) == { "registered": False, @@ -332,7 +344,7 @@ async def test_search_and_pre_registration( ) found, _ = await assert_status(resp, status.HTTP_200_OK) assert len(found) == 1 - got = UserProfile(**found[0]) + got = UserProfile(**found[0], state=None) assert got.model_dump(include={"registered", "status"}) == { "registered": True, "status": new_user["status"].name, @@ -381,7 +393,9 @@ def test_preuserprofile_parse_model_without_extras( account_request_form: dict[str, Any] ): required = { - f.alias or f_name for f_name, f in PreUserProfile.model_fields.items() if f.is_required() + f.alias or f_name + for f_name, f in PreUserProfile.model_fields.items() + if f.is_required() } data = {k: account_request_form[k] for k in required} assert not PreUserProfile(**data).extras diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index 5270a17c04c..ab84b68a3e8 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -121,7 +121,6 @@ async def test_workflow( ) assert CheckpointApiModel.model_validate(page.data[0]) == checkpoint1 - # UPDATE checkpoint annotations resp = await client.patch( f"/{VX}/repos/projects/{project_uuid}/checkpoints/{checkpoint1.id}", From f335bca583859091805a3c854bdd4fddd0fce862 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 14 Nov 2024 13:24:38 +0100 Subject: [PATCH 34/53] =?UTF-8?q?WIP:=20=E2=AC=86=EF=B8=8F=20Upgrade=20Dir?= =?UTF-8?q?ector=20v2=20service=20(Pydantic=20v2)=20(#6619)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu Co-authored-by: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Co-authored-by: Sylvain <35365065+sanderegg@users.noreply.github.com> --- .../aws-library/src/aws_library/s3/_client.py | 2 +- .../src/common_library/pydantic_validators.py | 13 +- .../src/common_library/serialization.py | 7 +- .../tests/test_serialization.py | 19 +- .../src/models_library/aiodocker_api.py | 20 +- .../api_schemas_directorv2/clusters.py | 11 +- .../api_schemas_directorv2/comp_tasks.py | 20 +- .../src/models_library/clusters.py | 11 +- .../src/models_library/docker.py | 8 +- .../src/models_library/errors.py | 6 +- .../generated_models/docker_rest_api.py | 499 +++++++++++++++++- .../models_library/service_settings_labels.py | 6 +- .../src/pytest_simcore/aws_s3_service.py | 4 +- .../src/pytest_simcore/minio_service.py | 9 +- .../long_running_tasks/_context_manager.py | 2 +- .../src/settings_library/basic_types.py | 8 +- .../src/settings_library/postgres.py | 36 +- .../src/settings_library/s3.py | 12 +- .../test_node_ports_common_aws_s3_cli.py | 4 +- .../test_node_ports_common_r_clone.py | 4 +- .../openapi-pydantic-models-generator.bash | 2 +- .../file_utils.py | 2 +- .../api/errors/http_error.py | 8 +- .../api/errors/validation_error.py | 4 +- .../api/routes/computations.py | 14 +- .../api/routes/dynamic_services.py | 6 +- .../cli/_close_and_save_service.py | 4 +- .../simcore_service_director_v2/cli/_core.py | 6 +- .../simcore_service_director_v2/constants.py | 3 +- .../core/application.py | 5 +- .../dynamic_services_settings/__init__.py | 12 +- .../dynamic_services_settings/scheduler.py | 4 +- .../core/dynamic_services_settings/sidecar.py | 64 ++- .../core/errors.py | 56 +- .../core/settings.py | 83 +-- .../models/comp_pipelines.py | 18 +- .../models/comp_runs.py | 29 +- .../models/comp_tasks.py | 107 ++-- .../models/dynamic_services_scheduler.py | 77 ++- .../models/pricing.py | 10 +- .../modules/catalog.py | 6 +- .../modules/comp_scheduler/_dask_scheduler.py | 4 +- .../modules/dask_client.py | 4 +- .../modules/db/repositories/clusters.py | 8 +- .../modules/db/repositories/comp_pipelines.py | 6 +- .../modules/db/repositories/comp_runs.py | 8 +- .../db/repositories/comp_tasks/_core.py | 10 +- .../db/repositories/comp_tasks/_utils.py | 28 +- .../modules/db/repositories/projects.py | 2 +- .../db/repositories/projects_networks.py | 6 +- .../repositories/user_preferences_frontend.py | 2 +- .../modules/db/repositories/users.py | 4 +- .../modules/director_v0.py | 6 +- .../dynamic_sidecar/api_client/_public.py | 16 +- .../dynamic_sidecar/api_client/_thin.py | 48 +- .../dynamic_sidecar/docker_compose_specs.py | 2 +- .../docker_service_specs/settings.py | 10 +- .../docker_service_specs/sidecar.py | 42 +- .../modules/dynamic_sidecar/docker_states.py | 6 +- .../modules/dynamic_sidecar/errors.py | 6 +- .../scheduler/_core/_event_create_sidecars.py | 8 +- .../scheduler/_core/_events_user_services.py | 4 +- .../scheduler/_core/_events_utils.py | 6 +- .../scheduler/_core/_scheduler.py | 4 +- .../scheduler/_core/_scheduler_utils.py | 2 +- .../modules/instrumentation/_models.py | 4 +- .../modules/osparc_variables/_api_auth_rpc.py | 6 +- .../modules/projects_networks.py | 14 +- .../modules/rabbitmq.py | 2 +- .../modules/resource_usage_tracker_client.py | 7 +- .../modules/storage.py | 2 +- .../utils/clients.py | 4 +- .../utils/comp_scheduler.py | 2 +- .../utils/computations.py | 14 +- .../simcore_service_director_v2/utils/dask.py | 27 +- .../utils/dask_client_utils.py | 2 +- .../simcore_service_director_v2/utils/db.py | 18 +- .../utils/osparc_variables.py | 4 +- .../utils/rabbitmq.py | 10 +- services/director-v2/tests/conftest.py | 10 +- .../tests/helpers/shared_comp_utils.py | 11 +- .../integration/01/test_computation_api.py | 6 +- .../tests/integration/02/conftest.py | 13 +- ...t_dynamic_sidecar_nodeports_integration.py | 16 +- .../director-v2/tests/integration/02/utils.py | 10 +- .../director-v2/tests/integration/conftest.py | 2 +- .../director-v2/tests/mocks/fake_task.json | 1 + services/director-v2/tests/unit/conftest.py | 25 +- .../tests/unit/test_core_settings.py | 11 +- .../tests/unit/test_models_clusters.py | 13 +- .../unit/test_models_dynamic_services.py | 4 +- ...dels_schemas_dynamic_services_scheduler.py | 12 +- .../tests/unit/test_modules_dask_client.py | 34 +- .../unit/test_modules_dask_clients_pool.py | 24 +- ...s_db_repositories_services_environments.py | 4 +- ...dules_dynamic_sidecar_client_api_public.py | 15 +- ...modules_dynamic_sidecar_client_api_thin.py | 26 +- ...es_dynamic_sidecar_docker_compose_specs.py | 12 +- .../test_modules_dynamic_sidecar_scheduler.py | 10 +- .../tests/unit/test_modules_notifier.py | 4 +- .../unit/test_modules_osparc_variables.py | 8 +- .../unit/test_modules_project_networks.py | 6 +- .../tests/unit/test_modules_rabbitmq.py | 2 +- ...test_schemas_dynamic_services_scheduler.py | 4 +- .../unit/test_utils_client_decorators.py | 6 +- .../tests/unit/test_utils_comp_scheduler.py | 4 +- .../tests/unit/test_utils_computation.py | 2 +- .../director-v2/tests/unit/test_utils_dags.py | 15 +- .../unit/test_utils_distributed_identifier.py | 10 +- .../tests/unit/with_dbs/conftest.py | 23 +- .../unit/with_dbs/test_api_route_clusters.py | 171 +++--- .../test_api_route_clusters_details.py | 10 +- .../with_dbs/test_api_route_computations.py | 139 +++-- .../test_api_route_computations_tasks.py | 10 +- .../test_api_route_dynamic_services.py | 139 +++-- .../tests/unit/with_dbs/test_cli.py | 6 +- ...t_modules_comp_scheduler_dask_scheduler.py | 54 +- ...test_modules_dynamic_sidecar_docker_api.py | 14 +- ...es_dynamic_sidecar_docker_service_specs.py | 119 +++-- .../tests/unit/with_dbs/test_utils_dask.py | 26 +- .../unit/with_dbs/test_utils_rabbitmq.py | 28 +- .../test_modules_long_running_tasks.py | 2 +- 122 files changed, 1649 insertions(+), 991 deletions(-) diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py index 38e7c0f9636..5f89a2cee7c 100644 --- a/packages/aws-library/src/aws_library/s3/_client.py +++ b/packages/aws-library/src/aws_library/s3/_client.py @@ -73,7 +73,7 @@ async def create( session = aioboto3.Session() session_client = session.client( "s3", - endpoint_url=settings.S3_ENDPOINT, + endpoint_url=f"{settings.S3_ENDPOINT}", aws_access_key_id=settings.S3_ACCESS_KEY, aws_secret_access_key=settings.S3_SECRET_KEY, region_name=settings.S3_REGION, diff --git a/packages/common-library/src/common_library/pydantic_validators.py b/packages/common-library/src/common_library/pydantic_validators.py index 471ba1a4bf3..a0122fccbe8 100644 --- a/packages/common-library/src/common_library/pydantic_validators.py +++ b/packages/common-library/src/common_library/pydantic_validators.py @@ -1,12 +1,11 @@ -import datetime +import datetime as dt import re import warnings -from datetime import timedelta from pydantic import TypeAdapter, field_validator -def _validate_legacy_timedelta_str(time_str: str | timedelta) -> str | timedelta: +def _validate_legacy_timedelta_str(time_str: str | dt.timedelta) -> str | dt.timedelta: if not isinstance(time_str, str): return time_str @@ -34,14 +33,14 @@ def validate_numeric_string_as_timedelta(field: str): """Transforms a float/int number into a valid datetime as it used to work in the past""" def _numeric_string_as_timedelta( - v: datetime.timedelta | str | float, - ) -> datetime.timedelta | str | float: + v: dt.timedelta | str | float, + ) -> dt.timedelta | str | float: if isinstance(v, str): try: converted_value = float(v) - iso8601_format = TypeAdapter(timedelta).dump_python( - timedelta(seconds=converted_value), mode="json" + iso8601_format = TypeAdapter(dt.timedelta).dump_python( + dt.timedelta(seconds=converted_value), mode="json" ) warnings.warn( f"{field}='{v}' -should be set to-> {field}='{iso8601_format}' (ISO8601 datetime format). " diff --git a/packages/common-library/src/common_library/serialization.py b/packages/common-library/src/common_library/serialization.py index 4394fa8cc45..70dd53e13c4 100644 --- a/packages/common-library/src/common_library/serialization.py +++ b/packages/common-library/src/common_library/serialization.py @@ -21,10 +21,9 @@ def model_dump_with_secrets( data[field_name] = field_data.total_seconds() elif isinstance(field_data, SecretStr): - if show_secrets: - data[field_name] = field_data.get_secret_value() - else: - data[field_name] = str(field_data) + data[field_name] = ( + field_data.get_secret_value() if show_secrets else str(field_data) + ) elif isinstance(field_data, Url): data[field_name] = str(field_data) diff --git a/packages/common-library/tests/test_serialization.py b/packages/common-library/tests/test_serialization.py index d897ff5ec5d..d5dea70ec22 100644 --- a/packages/common-library/tests/test_serialization.py +++ b/packages/common-library/tests/test_serialization.py @@ -4,22 +4,31 @@ class Credentials(BaseModel): - USERNAME: str | None = None - PASSWORD: SecretStr | None = None + username: str + password: SecretStr + + +class Access(BaseModel): + credentials: Credentials @pytest.mark.parametrize( "expected,show_secrets", [ ( - {"USERNAME": "DeepThought", "PASSWORD": "42"}, + {"credentials": {"username": "DeepThought", "password": "42"}}, True, ), ( - {"USERNAME": "DeepThought", "PASSWORD": "**********"}, + {"credentials": {"username": "DeepThought", "password": "**********"}}, False, # hide secrets ), ], ) def test_model_dump_with_secrets(expected: dict, show_secrets: bool): - assert expected == model_dump_with_secrets(Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42")), show_secrets=show_secrets) + assert expected == model_dump_with_secrets( + Access( + credentials=Credentials(username="DeepThought", password=SecretStr("42")) + ), + show_secrets=show_secrets, + ) diff --git a/packages/models-library/src/models_library/aiodocker_api.py b/packages/models-library/src/models_library/aiodocker_api.py index 865570fbe36..4f1c86b8f4d 100644 --- a/packages/models-library/src/models_library/aiodocker_api.py +++ b/packages/models-library/src/models_library/aiodocker_api.py @@ -11,12 +11,13 @@ class AioDockerContainerSpec(ContainerSpec): - Env: dict[str, str | None] | None = Field( + env: dict[str, str | None] | None = Field( # type: ignore[assignment] default=None, - description="aiodocker expects here a dictionary and re-convert it back internally`.\n", + alias="Env", + description="aiodocker expects here a dictionary and re-convert it back internally", ) - @field_validator("Env", mode="before") + @field_validator("env", mode="before") @classmethod def convert_list_to_dict(cls, v): if v is not None and isinstance(v, list): @@ -33,7 +34,7 @@ def convert_list_to_dict(cls, v): class AioDockerResources1(Resources1): # NOTE: The Docker REST API documentation is wrong!!! # Do not set that back to singular Reservation. - Reservation: ResourceObject | None = Field( + reservation: ResourceObject | None = Field( None, description="Define resources reservation.", alias="Reservations" ) @@ -41,17 +42,14 @@ class AioDockerResources1(Resources1): class AioDockerTaskSpec(TaskSpec): - ContainerSpec: AioDockerContainerSpec | None = Field( - None, + container_spec: AioDockerContainerSpec | None = Field( + default=None, alias="ContainerSpec" ) - Resources: AioDockerResources1 | None = Field( - None, - description="Resource requirements which apply to each individual container created\nas part of the service.\n", - ) + resources: AioDockerResources1 | None = Field(default=None, alias="Resources") class AioDockerServiceSpec(ServiceSpec): - TaskTemplate: AioDockerTaskSpec | None = None + task_template: AioDockerTaskSpec | None = Field(default=None, alias="TaskTemplate") model_config = ConfigDict(populate_by_name=True, alias_generator=camel_to_snake) diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py index a16ba29289b..0539ec5a3eb 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py @@ -1,4 +1,4 @@ -from typing import TypeAlias +from typing import Any, TypeAlias from pydantic import ( AnyHttpUrl, @@ -48,13 +48,12 @@ class WorkerMetrics(BaseModel): class UsedResources(DictModel[str, NonNegativeFloat]): @model_validator(mode="before") @classmethod - def ensure_negative_value_is_zero(cls, values): + def ensure_negative_value_is_zero(cls, values: dict[str, Any]): # dasks adds/remove resource values and sometimes # they end up being negative instead of 0 - if v := values.get("__root__", {}): - for res_key, res_value in v.items(): - if res_value < 0: - v[res_key] = 0 + for res_key, res_value in values.items(): + if res_value < 0: + values[res_key] = 0 return values diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py index 2204186c3ed..0663cc37f78 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py @@ -1,7 +1,15 @@ from typing import Any, TypeAlias from models_library.basic_types import IDStr -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field, field_validator +from pydantic import ( + AnyHttpUrl, + AnyUrl, + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, +) from ..clusters import ClusterID from ..projects import ProjectID @@ -63,16 +71,18 @@ class ComputationCreate(BaseModel): @field_validator("product_name") @classmethod - def ensure_product_name_defined_if_computation_starts(cls, v, values): - if "start_pipeline" in values and values["start_pipeline"] and v is None: + def _ensure_product_name_defined_if_computation_starts( + cls, v, info: ValidationInfo + ): + if info.data.get("start_pipeline") and v is None: msg = "product_name must be set if computation shall start!" raise ValueError(msg) return v @field_validator("use_on_demand_clusters") @classmethod - def ensure_expected_options(cls, v, values): - if v is True and ("cluster_id" in values and values["cluster_id"] is not None): + def _ensure_expected_options(cls, v, info: ValidationInfo): + if v and info.data.get("cluster_id") is not None: msg = "cluster_id cannot be set if use_on_demand_clusters is set" raise ValueError(msg) return v diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index 243144600e9..8b38913c246 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import Final, Literal, TypeAlias +from models_library.utils._original_fastapi_encoders import jsonable_encoder from pydantic import ( AnyUrl, BaseModel, @@ -224,6 +225,8 @@ class Cluster(BaseCluster): @model_validator(mode="before") @classmethod def check_owner_has_access_rights(cls, values): + values = jsonable_encoder(values) + is_default_cluster = bool(values["id"] == DEFAULT_CLUSTER_ID) owner_gid = values["owner"] @@ -231,11 +234,15 @@ def check_owner_has_access_rights(cls, values): access_rights = values.get("access_rights", values.get("accessRights", {})) if owner_gid not in access_rights: access_rights[owner_gid] = ( - CLUSTER_USER_RIGHTS if is_default_cluster else CLUSTER_ADMIN_RIGHTS + CLUSTER_USER_RIGHTS.model_dump() + if is_default_cluster + else CLUSTER_ADMIN_RIGHTS.model_dump() ) # check owner has the expected access if access_rights[owner_gid] != ( - CLUSTER_USER_RIGHTS if is_default_cluster else CLUSTER_ADMIN_RIGHTS + CLUSTER_USER_RIGHTS.model_dump() + if is_default_cluster + else CLUSTER_ADMIN_RIGHTS.model_dump() ): msg = f"the cluster owner access rights are incorrectly set: {access_rights[owner_gid]}" raise ValueError(msg) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index b8134b3ec73..6e87f06b62e 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -37,13 +37,7 @@ def from_key(cls, key: str) -> "DockerLabelKey": str, StringConstraints(pattern=DOCKER_GENERIC_TAG_KEY_RE) ] - -class DockerPlacementConstraint(ConstrainedStr): - strip_whitespace = True - regex = re.compile( - r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(? typing.AsyncIterator[S3Client]: exit_stack = contextlib.AsyncExitStack() session_client = session.client( "s3", - endpoint_url=s3_settings.S3_ENDPOINT, + endpoint_url=f"{s3_settings.S3_ENDPOINT}", aws_access_key_id=s3_settings.S3_ACCESS_KEY, aws_secret_access_key=s3_settings.S3_SECRET_KEY, region_name=s3_settings.S3_REGION, config=Config(signature_version="s3v4"), ) assert isinstance(session_client, ClientCreatorContext) - client = typing.cast(S3Client, await exit_stack.enter_async_context(session_client)) # type: ignore[arg-type] + client = typing.cast(S3Client, await exit_stack.enter_async_context(session_client)) # type: ignore[arg-type] yield client diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index 38b9d2bdf8d..f8adf8cda9f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -4,6 +4,7 @@ import pytest from faker import Faker +from pydantic import AnyHttpUrl, TypeAdapter from pytest_simcore.helpers.docker import get_service_published_port from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -20,7 +21,9 @@ def minio_s3_settings( return S3Settings( S3_ACCESS_KEY=testing_environ_vars["S3_ACCESS_KEY"], S3_SECRET_KEY=testing_environ_vars["S3_SECRET_KEY"], - S3_ENDPOINT=f"http://{get_localhost_ip()}:{get_service_published_port('minio')}", + S3_ENDPOINT=TypeAdapter(AnyHttpUrl).validate_python( + f"http://{get_localhost_ip()}:{get_service_published_port('minio')}" + ), S3_BUCKET_NAME=testing_environ_vars["S3_BUCKET_NAME"], S3_REGION="us-east-1", ) @@ -31,5 +34,7 @@ def minio_s3_settings_envs( minio_s3_settings: S3Settings, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - changed_envs: EnvVarsDict = minio_s3_settings.model_dump(exclude_unset=True) + changed_envs: EnvVarsDict = minio_s3_settings.model_dump( + mode="json", exclude_unset=True + ) return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py index 2c001525173..35b734ac055 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py @@ -118,7 +118,7 @@ async def _wait_task_completion() -> None: logger.debug("%s, %s", f"{task_id=}", f"{result=}") yield result - except asyncio.TimeoutError as e: + except TimeoutError as e: await client.cancel_and_delete_task(task_id) raise TaskClientTimeoutError( task_id=task_id, diff --git a/packages/settings-library/src/settings_library/basic_types.py b/packages/settings-library/src/settings_library/basic_types.py index f4c745b2284..9d1d440d5ae 100644 --- a/packages/settings-library/src/settings_library/basic_types.py +++ b/packages/settings-library/src/settings_library/basic_types.py @@ -3,7 +3,7 @@ # This is a minor evil to avoid the maintenance burden that creates # an extra dependency to a larger models_library (intra-repo library) -from enum import Enum +from enum import StrEnum from typing import Annotated, TypeAlias from pydantic import Field, StringConstraints @@ -16,14 +16,14 @@ VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")] -class LogLevel(str, Enum): +class LogLevel(StrEnum): DEBUG = "DEBUG" INFO = "INFO" WARNING = "WARNING" ERROR = "ERROR" -class BootMode(str, Enum): +class BootMode(StrEnum): """ Values taken by SC_BOOT_MODE environment variable set in Dockerfile and used during docker/boot.sh @@ -36,7 +36,7 @@ class BootMode(str, Enum): DEVELOPMENT = "development" -class BuildTargetEnum(str, Enum): +class BuildTargetEnum(StrEnum): """ Values taken by SC_BUILD_TARGET environment variable set in Dockerfile that defines the stage targeted in the diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index 883d14c3bb4..e65f02e6edc 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -56,31 +56,27 @@ def _check_size(cls, v, info: ValidationInfo): @cached_property def dsn(self) -> str: - dsn: str = str( - PostgresDsn.build( # pylint: disable=no-member - scheme="postgresql", - username=self.POSTGRES_USER, - password=self.POSTGRES_PASSWORD.get_secret_value(), - host=self.POSTGRES_HOST, - port=self.POSTGRES_PORT, - path=f"{self.POSTGRES_DB}", - ) + url = PostgresDsn.build( # pylint: disable=no-member + scheme="postgresql", + username=self.POSTGRES_USER, + password=self.POSTGRES_PASSWORD.get_secret_value(), + host=self.POSTGRES_HOST, + port=self.POSTGRES_PORT, + path=f"{self.POSTGRES_DB}", ) - return dsn + return f"{url}" @cached_property def dsn_with_async_sqlalchemy(self) -> str: - dsn: str = str( - PostgresDsn.build( # pylint: disable=no-member - scheme="postgresql+asyncpg", - username=self.POSTGRES_USER, - password=self.POSTGRES_PASSWORD.get_secret_value(), - host=self.POSTGRES_HOST, - port=self.POSTGRES_PORT, - path=f"{self.POSTGRES_DB}", - ) + url = PostgresDsn.build( # pylint: disable=no-member + scheme="postgresql+asyncpg", + username=self.POSTGRES_USER, + password=self.POSTGRES_PASSWORD.get_secret_value(), + host=self.POSTGRES_HOST, + port=self.POSTGRES_PORT, + path=f"{self.POSTGRES_DB}", ) - return dsn + return f"{url}" @cached_property def dsn_with_query(self) -> str: diff --git a/packages/settings-library/src/settings_library/s3.py b/packages/settings-library/src/settings_library/s3.py index 95268b41920..18f23860658 100644 --- a/packages/settings-library/src/settings_library/s3.py +++ b/packages/settings-library/src/settings_library/s3.py @@ -1,20 +1,16 @@ -from typing import Annotated - -from pydantic import AnyHttpUrl, BeforeValidator, Field, TypeAdapter +from pydantic import AnyHttpUrl, Field from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings from .basic_types import IDStr -ANY_HTTP_URL_ADAPTER: TypeAdapter = TypeAdapter(AnyHttpUrl) - class S3Settings(BaseCustomSettings): S3_ACCESS_KEY: IDStr S3_BUCKET_NAME: IDStr - S3_ENDPOINT: Annotated[ - str, BeforeValidator(lambda x: str(ANY_HTTP_URL_ADAPTER.validate_python(x))) - ] | None = Field(default=None, description="do not define if using standard AWS") + S3_ENDPOINT: AnyHttpUrl | None = Field( + default=None, description="do not define if using standard AWS" + ) S3_REGION: IDStr S3_SECRET_KEY: IDStr diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py index 0c0c03b0363..717a428a1ed 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_aws_s3_cli.py @@ -50,7 +50,9 @@ async def cleanup_bucket_after_test( yield - async with session.client("s3", endpoint_url=aws_s3_cli_settings.AWS_S3_CLI_S3.S3_ENDPOINT) as s3_client: # type: ignore + async with session.client( + "s3", endpoint_url=f"{aws_s3_cli_settings.AWS_S3_CLI_S3.S3_ENDPOINT}" + ) as s3_client: # List all object versions paginator = s3_client.get_paginator("list_object_versions") async for page in paginator.paginate( diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py index 5d728aad51d..c94fc524bec 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_r_clone.py @@ -51,7 +51,9 @@ async def cleanup_bucket_after_test( yield - async with session.client("s3", endpoint_url=r_clone_settings.R_CLONE_S3.S3_ENDPOINT) as s3_client: # type: ignore + async with session.client( + "s3", endpoint_url=f"{r_clone_settings.R_CLONE_S3.S3_ENDPOINT}" + ) as s3_client: # List all object versions paginator = s3_client.get_paginator("list_object_versions") async for page in paginator.paginate( diff --git a/scripts/openapi-pydantic-models-generator.bash b/scripts/openapi-pydantic-models-generator.bash index 788cb90e792..88e071a5273 100755 --- a/scripts/openapi-pydantic-models-generator.bash +++ b/scripts/openapi-pydantic-models-generator.bash @@ -1,5 +1,4 @@ #!/bin/bash -#!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ set -o errexit set -o nounset @@ -27,6 +26,7 @@ ENTRYPOINT ["datamodel-codegen", \ "--use-standard-collections", \ "--use-union-operator", \ "--use-schema-description", \ + "--allow-population-by-field-name", \ "--use-subclass-enum", \ "--use-double-quotes", \ "--field-constraints", \ diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py index e5b78bd286a..1016cfd5c5c 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py @@ -75,7 +75,7 @@ def _s3fs_settings_from_s3_settings(s3_settings: S3Settings) -> S3FsSettingsDict # setting it for the us-east-1 creates issue when creating buckets (which we do in tests) s3fs_settings["client_kwargs"]["region_name"] = s3_settings.S3_REGION if s3_settings.S3_ENDPOINT is not None: - s3fs_settings["client_kwargs"]["endpoint_url"] = s3_settings.S3_ENDPOINT + s3fs_settings["client_kwargs"]["endpoint_url"] = f"{s3_settings.S3_ENDPOINT}" return s3fs_settings diff --git a/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py b/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py index da51e6f9e26..5edfb25aa20 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py +++ b/services/director-v2/src/simcore_service_director_v2/api/errors/http_error.py @@ -6,7 +6,9 @@ from starlette.responses import JSONResponse -async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse: +async def http_error_handler(_: Request, exc: Exception) -> JSONResponse: + assert isinstance(exc, HTTPException) + return JSONResponse( content=jsonable_encoder({"errors": [exc.detail]}), status_code=exc.status_code ) @@ -14,7 +16,7 @@ async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse: def make_http_error_handler_for_exception( status_code: int, exception_cls: type[BaseException] -) -> Callable[[Request, type[BaseException]], Awaitable[JSONResponse]]: +) -> Callable[[Request, Exception], Awaitable[JSONResponse]]: """ Produces a handler for BaseException-type exceptions which converts them into an error JSON response with a given status code @@ -22,7 +24,7 @@ def make_http_error_handler_for_exception( SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions """ - async def _http_error_handler(_: Request, exc: type[BaseException]) -> JSONResponse: + async def _http_error_handler(_: Request, exc: Exception) -> JSONResponse: assert isinstance(exc, exception_cls) # nosec return JSONResponse( content=jsonable_encoder({"errors": [str(exc)]}), status_code=status_code diff --git a/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py b/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py index fb70f6791ac..b3509cbbec9 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py +++ b/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py @@ -12,8 +12,10 @@ async def http422_error_handler( _: Request, - exc: Union[RequestValidationError, ValidationError], + exc: Exception, ) -> JSONResponse: + assert isinstance(exc, RequestValidationError | ValidationError) + return JSONResponse( content=jsonable_encoder({"errors": exc.errors()}), status_code=HTTP_422_UNPROCESSABLE_ENTITY, diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index b7f47b186e7..e624ed0785c 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -35,7 +35,7 @@ from models_library.services import ServiceKeyVersion from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from servicelib.async_utils import run_sequentially_in_context from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RabbitMQRPCClient @@ -399,13 +399,11 @@ async def create_computation( # noqa: PLR0913 # pylint: disable=too-many-positi pipeline_details=await compute_pipeline_details( complete_dag, minimal_computational_dag, comp_tasks ), - url=parse_obj_as( - AnyHttpUrl, + url=TypeAdapter(AnyHttpUrl).validate_python( f"{request.url}/{computation.project_id}?user_id={computation.user_id}", ), stop_url=( - parse_obj_as( - AnyHttpUrl, + TypeAdapter(AnyHttpUrl).validate_python( f"{request.url}/{computation.project_id}:stop?user_id={computation.user_id}", ) if computation.start_pipeline @@ -510,9 +508,9 @@ async def get_computation( id=project_id, state=pipeline_state, pipeline_details=pipeline_details, - url=parse_obj_as(AnyHttpUrl, f"{request.url}"), + url=TypeAdapter(AnyHttpUrl).validate_python(f"{request.url}"), stop_url=( - parse_obj_as(AnyHttpUrl, f"{self_url}:stop?user_id={user_id}") + TypeAdapter(AnyHttpUrl).validate_python(f"{self_url}:stop?user_id={user_id}") if pipeline_state.is_running() else None ), @@ -588,7 +586,7 @@ async def stop_computation( pipeline_details=await compute_pipeline_details( complete_dag, pipeline_dag, tasks ), - url=parse_obj_as(AnyHttpUrl, f"{request.url}"), + url=TypeAdapter(AnyHttpUrl).validate_python(f"{request.url}"), stop_url=None, iteration=last_run.iteration if last_run else None, cluster_id=last_run.cluster_id if last_run else None, diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py index 743d7ae2a66..24db21cbd23 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py @@ -273,7 +273,7 @@ async def service_retrieve_data_on_ports( dynamic_services_settings.DYNAMIC_SCHEDULER ) timeout = httpx.Timeout( - dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), connect=dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT, ) @@ -281,12 +281,12 @@ async def service_retrieve_data_on_ports( response = await services_client.request( "POST", f"{service_base_url}/retrieve", - content=retrieve_settings.json(by_alias=True), + content=retrieve_settings.model_dump_json(by_alias=True), timeout=timeout, ) # validate and return - return RetrieveDataOutEnveloped.parse_obj(response.json()) + return RetrieveDataOutEnveloped.model_validate(response.json()) @router.post( diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py b/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py index be75694f55c..fb8f70bf62f 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_close_and_save_service.py @@ -5,7 +5,7 @@ import rich from fastapi import FastAPI from models_library.projects_nodes_io import NodeID -from pydantic import AnyHttpUrl, PositiveFloat, parse_obj_as +from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from rich.progress import ( BarColumn, Progress, @@ -106,7 +106,7 @@ async def async_close_and_save_service( client = Client( app=app, async_client=thin_dv2_localhost_client.client, - base_url=parse_obj_as(AnyHttpUrl, thin_dv2_localhost_client.BASE_ADDRESS), + base_url=f"{TypeAdapter(AnyHttpUrl).validate_python(thin_dv2_localhost_client.BASE_ADDRESS)}", ) if not skip_container_removal: diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_core.py b/services/director-v2/src/simcore_service_director_v2/cli/_core.py index 893aed2504e..028a882f268 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_core.py @@ -12,7 +12,7 @@ from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceType from models_library.services_enums import ServiceBootType, ServiceState -from pydantic import AnyHttpUrl, BaseModel, PositiveInt, parse_obj_as +from pydantic import AnyHttpUrl, BaseModel, PositiveInt, TypeAdapter from rich.live import Live from rich.table import Table from servicelib.services_utils import get_service_from_key @@ -58,7 +58,9 @@ def _get_dynamic_sidecar_endpoint( dynamic_sidecar_names = DynamicSidecarNamesHelper.make(NodeID(node_id)) hostname = dynamic_sidecar_names.service_name_dynamic_sidecar port = settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_PORT - url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, f"http://{hostname}:{port}") # NOSONAR + url: AnyHttpUrl = TypeAdapter(AnyHttpUrl).validate_python( + f"http://{hostname}:{port}" + ) return url diff --git a/services/director-v2/src/simcore_service_director_v2/constants.py b/services/director-v2/src/simcore_service_director_v2/constants.py index fc700254ed0..b84865745df 100644 --- a/services/director-v2/src/simcore_service_director_v2/constants.py +++ b/services/director-v2/src/simcore_service_director_v2/constants.py @@ -1,5 +1,4 @@ from typing import Final - # dynamic services DYNAMIC_SIDECAR_SERVICE_PREFIX: Final[str] = "dy-sidecar" @@ -14,7 +13,7 @@ # - itisfoundation # - 10.0.0.0:8473 (IP & Port) DYNAMIC_SIDECAR_DOCKER_IMAGE_RE = ( - r"(^([_a-zA-Z0-9:.-]+)/)?(dynamic-sidecar):([_a-zA-Z0-9.-]+$)" + r"^(([_a-zA-Z0-9:.-]+)/)?(dynamic-sidecar):([_a-zA-Z0-9.-]+)$" ) REGEX_DY_SERVICE_SIDECAR = rf"^{DYNAMIC_SIDECAR_SERVICE_PREFIX}_[a-zA-Z0-9-_]*" diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 416e5c85367..d611351f782 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -9,6 +9,7 @@ from servicelib.fastapi.profiler_middleware import ProfilerMiddleware from servicelib.fastapi.tracing import setup_tracing from servicelib.logging_utils import config_all_loggers +from settings_library.basic_types import BootMode from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY from ..api.entrypoints import api_router @@ -127,12 +128,12 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: logging.getLogger(name).setLevel(quiet_level) app = FastAPI( - debug=settings.SC_BOOT_MODE.is_devel_mode(), + debug=settings.SC_BOOT_MODE is BootMode.DEVELOPMENT, title=PROJECT_NAME, description=SUMMARY, version=API_VERSION, openapi_url=f"/api/{API_VTAG}/openapi.json", - **get_common_oas_options(settings.SC_BOOT_MODE.is_devel_mode()), + **get_common_oas_options(settings.SC_BOOT_MODE == BootMode.DEVELOPMENT), ) override_fastapi_openapi_method(app) app.state.settings = settings diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py index b4abd4f5b6e..c3ed002edd6 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py @@ -13,22 +13,22 @@ class DynamicServicesSettings(BaseCustomSettings): default=True, description="Enables/Disables the dynamic_sidecar submodule" ) - DYNAMIC_SIDECAR: DynamicSidecarSettings = Field(auto_default_from_env=True) + DYNAMIC_SIDECAR: DynamicSidecarSettings = Field(json_schema_extra={"auto_default_from_env": True}) DYNAMIC_SCHEDULER: DynamicServicesSchedulerSettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) DYNAMIC_SIDECAR_PROXY_SETTINGS: DynamicSidecarProxySettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) DYNAMIC_SIDECAR_EGRESS_PROXY_SETTINGS: EgressProxySettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) - WEBSERVER_SETTINGS: WebServerSettings = Field(auto_default_from_env=True) + WEBSERVER_SETTINGS: WebServerSettings = Field(json_schema_extra={"auto_default_from_env": True}) diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py index 74810cdd101..5072a365af6 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/scheduler.py @@ -103,8 +103,8 @@ class DynamicServicesSchedulerSettings(BaseCustomSettings): ), ) - DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: PositiveFloat = Field( - 60.0 * _MINUTE, + DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: timedelta = Field( + timedelta(hours=1), description=( "When saving and restoring the state of a dynamic service, depending on the payload " "some services take longer or shorter to save and restore. Across the " diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index 98ce21fc6a4..434c3e0941f 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -9,7 +9,7 @@ ensure_unique_dict_values_validator, ensure_unique_list_values_validator, ) -from pydantic import Field, PositiveInt, validator +from pydantic import AliasChoices, Field, PositiveInt, ValidationInfo, field_validator from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.base import BaseCustomSettings from settings_library.efs import AwsEfsSettings @@ -45,10 +45,10 @@ class RCloneSettings(SettingsLibraryRCloneSettings): description="VFS operation mode, defines how and when the disk cache is synced", ) - @validator("R_CLONE_POLL_INTERVAL_SECONDS") + @field_validator("R_CLONE_POLL_INTERVAL_SECONDS") @classmethod - def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt: - dir_cache_time = values["R_CLONE_DIR_CACHE_TIME_SECONDS"] + def enforce_r_clone_requirement(cls, v: int, info: ValidationInfo) -> PositiveInt: + dir_cache_time = info.data["R_CLONE_DIR_CACHE_TIME_SECONDS"] if v >= dir_cache_time: msg = f"R_CLONE_POLL_INTERVAL_SECONDS={v} must be lower than R_CLONE_DIR_CACHE_TIME_SECONDS={dir_cache_time}" raise ValueError(msg) @@ -60,7 +60,7 @@ class PlacementSettings(BaseCustomSettings): # https://docs.docker.com/engine/swarm/services/#control-service-placement. DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: list[DockerPlacementConstraint] = Field( default_factory=list, - example='["node.labels.region==east", "one!=yes"]', + examples=['["node.labels.region==east", "one!=yes"]'], ) DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: dict[ @@ -72,20 +72,18 @@ class PlacementSettings(BaseCustomSettings): "see https://github.com/ITISFoundation/osparc-simcore/issues/5250 " "When `None` (default), uses generic resources" ), - example='{"AIRAM": "node.labels.custom==true"}', + examples=['{"AIRAM": "node.labels.custom==true"}'], ) - _unique_custom_constraints = validator( + _unique_custom_constraints = field_validator( "DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS", - allow_reuse=True, )(ensure_unique_list_values_validator) - _unique_resource_placement_constraints_substitutions = validator( + _unique_resource_placement_constraints_substitutions = field_validator( "DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS", - allow_reuse=True, )(ensure_unique_dict_values_validator) - @validator("DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS") + @field_validator("DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS") @classmethod def warn_if_any_values_provided(cls, value: dict) -> dict: if len(value) > 0: @@ -101,40 +99,46 @@ def warn_if_any_values_provided(cls, value: dict) -> dict: class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED: bool = Field( # doc: https://docs.docker.com/engine/swarm/networking/#configure-service-discovery default=False, - env=["DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED"], + validation_alias=AliasChoices( + "DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED" + ), description="dynamic-sidecar's service 'endpoint_spec' with {'Mode': 'dnsrr'}", ) DYNAMIC_SIDECAR_SC_BOOT_MODE: BootModeEnum = Field( ..., description="Boot mode used for the dynamic-sidecar services" "By defaults, it uses the same boot mode set for the director-v2", - env=["DYNAMIC_SIDECAR_SC_BOOT_MODE", "SC_BOOT_MODE"], + validation_alias=AliasChoices("DYNAMIC_SIDECAR_SC_BOOT_MODE", "SC_BOOT_MODE"), ) DYNAMIC_SIDECAR_LOG_LEVEL: str = Field( "WARNING", description="log level of the dynamic sidecar" "If defined, it captures global env vars LOG_LEVEL and LOGLEVEL from the director-v2 service", - env=["DYNAMIC_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "DYNAMIC_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) DYNAMIC_SIDECAR_IMAGE: str = Field( ..., - regex=DYNAMIC_SIDECAR_DOCKER_IMAGE_RE, + pattern=DYNAMIC_SIDECAR_DOCKER_IMAGE_RE, description="used by the director to start a specific version of the dynamic-sidecar", ) - DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) + DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) DYNAMIC_SIDECAR_AWS_S3_CLI_SETTINGS: AwsS3CliSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) DYNAMIC_SIDECAR_EFS_SETTINGS: AwsEfsSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) # @@ -144,7 +148,7 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_MOUNT_PATH_DEV: Path | None = Field( None, description="Host path to the dynamic-sidecar project. Used as source path to mount to the dynamic-sidecar [DEVELOPMENT ONLY]", - example="osparc-simcore/services/dynamic-sidecar", + examples=["osparc-simcore/services/dynamic-sidecar"], ) DYNAMIC_SIDECAR_PORT: PortInt = Field( @@ -157,12 +161,16 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): description="Publishes the service on localhost for debuging and testing [DEVELOPMENT ONLY]" "Can be used to access swagger doc from the host as http://127.0.0.1:30023/dev/doc " "where 30023 is the host published port", + validate_default=True, ) - @validator("DYNAMIC_SIDECAR_MOUNT_PATH_DEV", pre=True) + @field_validator("DYNAMIC_SIDECAR_MOUNT_PATH_DEV", mode="before") @classmethod - def auto_disable_if_production(cls, v, values): - if v and values.get("DYNAMIC_SIDECAR_SC_BOOT_MODE") == BootModeEnum.PRODUCTION: + def auto_disable_if_production(cls, v, info: ValidationInfo): + if ( + v + and info.data.get("DYNAMIC_SIDECAR_SC_BOOT_MODE") == BootModeEnum.PRODUCTION + ): _logger.warning( "In production DYNAMIC_SIDECAR_MOUNT_PATH_DEV cannot be set to %s, enforcing None", v, @@ -170,22 +178,22 @@ def auto_disable_if_production(cls, v, values): return None return v - @validator("DYNAMIC_SIDECAR_EXPOSE_PORT", pre=True, always=True) + @field_validator("DYNAMIC_SIDECAR_EXPOSE_PORT", mode="before") @classmethod - def auto_enable_if_development(cls, v, values): + def auto_enable_if_development(cls, v, info: ValidationInfo): if ( - boot_mode := values.get("DYNAMIC_SIDECAR_SC_BOOT_MODE") + boot_mode := info.data.get("DYNAMIC_SIDECAR_SC_BOOT_MODE") ) and boot_mode.is_devel_mode(): # Can be used to access swagger doc from the host as http://127.0.0.1:30023/dev/doc return True return v - @validator("DYNAMIC_SIDECAR_IMAGE", pre=True) + @field_validator("DYNAMIC_SIDECAR_IMAGE", mode="before") @classmethod def strip_leading_slashes(cls, v: str) -> str: return v.lstrip("/") - @validator("DYNAMIC_SIDECAR_LOG_LEVEL") + @field_validator("DYNAMIC_SIDECAR_LOG_LEVEL") @classmethod def _validate_log_level(cls, value) -> str: log_level: str = cls.validate_log_level(value) diff --git a/services/director-v2/src/simcore_service_director_v2/core/errors.py b/services/director-v2/src/simcore_service_director_v2/core/errors.py index e8c47a934d3..213cd4744bf 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/core/errors.py @@ -19,18 +19,15 @@ } """ +from common_library.errors_classes import OsparcErrorMixin from models_library.errors import ErrorDict from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from pydantic.errors import PydanticErrorMixin class DirectorError(Exception): """Basic exception""" - def message(self) -> str: - return f"{self.args[0]}" - class ConfigurationError(DirectorError): """An error in the director-v2 configuration""" @@ -114,15 +111,15 @@ def __init__(self, pipeline_id: str): super().__init__(f"pipeline {pipeline_id} not found") -class ComputationalRunNotFoundError(PydanticErrorMixin, DirectorError): +class ComputationalRunNotFoundError(OsparcErrorMixin, DirectorError): msg_template = "Computational run not found" -class ComputationalTaskNotFoundError(PydanticErrorMixin, DirectorError): +class ComputationalTaskNotFoundError(OsparcErrorMixin, DirectorError): msg_template = "Computational task {node_id} not found" -class WalletNotEnoughCreditsError(PydanticErrorMixin, DirectorError): +class WalletNotEnoughCreditsError(OsparcErrorMixin, DirectorError): msg_template = "Wallet '{wallet_name}' has {wallet_credit_amount} credits." @@ -132,8 +129,6 @@ class WalletNotEnoughCreditsError(PydanticErrorMixin, DirectorError): class SchedulerError(DirectorError): - code = "scheduler_error" - def __init__(self, msg: str | None = None): super().__init__(msg or "Unexpected error in the scheduler") @@ -148,6 +143,8 @@ def __init__(self, pipeline_id: str, msg: str | None = None): class TaskSchedulingError(SchedulerError): """A task cannot be scheduled""" + code: str = "task scheduler error" + def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = None): super().__init__(msg=msg) self.project_id = project_id @@ -161,7 +158,7 @@ def get_errors(self) -> list[ErrorDict]: f"{self.project_id}", f"{self.node_id}", ), - "msg": self.message(), + "msg": f"{self.args[0]}", "type": self.code, }, ] @@ -227,39 +224,33 @@ def get_errors(self) -> list[ErrorDict]: return value_errors -class ComputationalSchedulerChangedError(PydanticErrorMixin, SchedulerError): - code = "computational_backend.scheduler_changed" +class ComputationalSchedulerChangedError(OsparcErrorMixin, SchedulerError): msg_template = "The dask scheduler ID changed from '{original_scheduler_id}' to '{current_scheduler_id}'" -class ComputationalBackendNotConnectedError(PydanticErrorMixin, SchedulerError): - code = "computational_backend.not_connected" +class ComputationalBackendNotConnectedError(OsparcErrorMixin, SchedulerError): msg_template = "The dask computational backend is not connected" -class ComputationalBackendNoS3AccessError(PydanticErrorMixin, SchedulerError): +class ComputationalBackendNoS3AccessError(OsparcErrorMixin, SchedulerError): msg_template = "The S3 backend is not ready, please try again later" -class ComputationalBackendTaskNotFoundError(PydanticErrorMixin, SchedulerError): - code = "computational_backend.task_not_found" +class ComputationalBackendTaskNotFoundError(OsparcErrorMixin, SchedulerError): msg_template = ( "The dask computational backend does not know about the task '{job_id}'" ) -class ComputationalBackendTaskResultsNotReadyError(PydanticErrorMixin, SchedulerError): - code = "computational_backend.task_result_not_ready" +class ComputationalBackendTaskResultsNotReadyError(OsparcErrorMixin, SchedulerError): msg_template = "The task result is not ready yet for job '{job_id}'" -class ClustersKeeperNotAvailableError(PydanticErrorMixin, SchedulerError): - code = "computational_backend.clusters_keeper_not_available" +class ClustersKeeperNotAvailableError(OsparcErrorMixin, SchedulerError): msg_template = "clusters-keeper service is not available!" -class ComputationalBackendOnDemandNotReadyError(PydanticErrorMixin, SchedulerError): - code = "computational_backend.on_demand_cluster.not_ready" +class ComputationalBackendOnDemandNotReadyError(OsparcErrorMixin, SchedulerError): msg_template = ( "The on demand computational cluster is not ready 'est. remaining time: {eta}'" ) @@ -268,16 +259,15 @@ class ComputationalBackendOnDemandNotReadyError(PydanticErrorMixin, SchedulerErr # # SCHEDULER/CLUSTER ERRORS # -class ClusterNotFoundError(PydanticErrorMixin, SchedulerError): - code = "cluster.not_found" +class ClusterNotFoundError(OsparcErrorMixin, SchedulerError): msg_template = "The cluster '{cluster_id}' not found" -class ClusterAccessForbiddenError(PydanticErrorMixin, SchedulerError): +class ClusterAccessForbiddenError(OsparcErrorMixin, SchedulerError): msg_template = "Insufficient rights to access cluster '{cluster_id}'" -class ClusterInvalidOperationError(PydanticErrorMixin, SchedulerError): +class ClusterInvalidOperationError(OsparcErrorMixin, SchedulerError): msg_template = "Invalid operation on cluster '{cluster_id}'" @@ -286,25 +276,21 @@ class ClusterInvalidOperationError(PydanticErrorMixin, SchedulerError): # -class DaskClientRequestError(PydanticErrorMixin, SchedulerError): - code = "dask_client.request.error" +class DaskClientRequestError(OsparcErrorMixin, SchedulerError): msg_template = ( "The dask client to cluster on '{endpoint}' did an invalid request '{error}'" ) -class DaskClusterError(PydanticErrorMixin, SchedulerError): - code = "cluster.error" +class DaskClusterError(OsparcErrorMixin, SchedulerError): msg_template = "The dask cluster on '{endpoint}' encountered an error: '{error}'" -class DaskGatewayServerError(PydanticErrorMixin, SchedulerError): - code = "gateway.error" +class DaskGatewayServerError(OsparcErrorMixin, SchedulerError): msg_template = "The dask gateway on '{endpoint}' encountered an error: '{error}'" -class DaskClientAcquisisitonError(PydanticErrorMixin, SchedulerError): - code = "dask_client.acquisition.error" +class DaskClientAcquisisitonError(OsparcErrorMixin, SchedulerError): msg_template = ( "The dask client to cluster '{cluster}' encountered an error '{error}'" ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 717694f7f25..062e1ec4d5f 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -6,13 +6,7 @@ from functools import cached_property from common_library.pydantic_validators import validate_numeric_string_as_timedelta -from models_library.basic_types import ( - BootModeEnum, - BuildTargetEnum, - LogLevel, - PortInt, - VersionTag, -) +from models_library.basic_types import LogLevel, PortInt, VersionTag from models_library.clusters import ( DEFAULT_CLUSTER_ID, Cluster, @@ -20,8 +14,16 @@ ClusterTypeInModel, NoAuthentication, ) -from pydantic import AliasChoices, AnyHttpUrl, AnyUrl, Field, NonNegativeInt, validator +from pydantic import ( + AliasChoices, + AnyHttpUrl, + AnyUrl, + Field, + NonNegativeInt, + field_validator, +) from servicelib.logging_utils_filtering import LoggerName, MessageSubstring +from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings from settings_library.docker_registry import RegistrySettings @@ -56,13 +58,13 @@ class DirectorV0Settings(BaseCustomSettings): @cached_property def endpoint(self) -> str: - url: str = AnyHttpUrl.build( + url = AnyHttpUrl.build( # pylint: disable=no-member scheme="http", host=self.DIRECTOR_HOST, - port=f"{self.DIRECTOR_PORT}", - path=f"/{self.DIRECTOR_V0_VTAG}", + port=self.DIRECTOR_PORT, + path=f"{self.DIRECTOR_V0_VTAG}", ) - return url + return f"{url}" class ComputationalBackendSettings(BaseCustomSettings): @@ -108,7 +110,7 @@ def default_cluster(self) -> Cluster: type=ClusterTypeInModel.ON_PREMISE, ) - @validator("COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH", pre=True) + @field_validator("COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH", mode="before") @classmethod def _empty_auth_is_none(cls, v): if not v: @@ -116,11 +118,7 @@ def _empty_auth_is_none(cls, v): return v -class AppSettings(BaseCustomSettings, MixinLoggingSettings): - # docker environs - SC_BOOT_MODE: BootModeEnum - SC_BOOT_TARGET: BuildTargetEnum | None - +class AppSettings(BaseApplicationSettings, MixinLoggingSettings): LOG_LEVEL: LogLevel = Field( LogLevel.INFO.value, validation_alias=AliasChoices("DIRECTOR_V2_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), @@ -166,10 +164,10 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True DIRECTOR_V2_PROFILING: bool = False - DIRECTOR_V2_REMOTE_DEBUGGING_PORT: PortInt | None + DIRECTOR_V2_REMOTE_DEBUGGING_PORT: PortInt | None = Field(default=None) # extras - SWARM_STACK_NAME: str = Field("undefined-please-check", env="SWARM_STACK_NAME") + SWARM_STACK_NAME: str = Field(default="undefined-please-check") SERVICE_TRACKING_HEARTBEAT: datetime.timedelta = Field( default=DEFAULT_RESOURCE_USAGE_HEARTBEAT_INTERVAL, description="Service scheduler heartbeat (everytime a heartbeat is sent into RabbitMQ)" @@ -191,42 +189,56 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): ) # debug settings - CLIENT_REQUEST: ClientRequestSettings = Field(auto_default_from_env=True) + CLIENT_REQUEST: ClientRequestSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) # App modules settings --------------------- - DIRECTOR_V2_STORAGE: StorageSettings = Field(auto_default_from_env=True) + DIRECTOR_V2_STORAGE: StorageSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH: StorageAuthSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) - DIRECTOR_V2_CATALOG: CatalogSettings | None = Field(auto_default_from_env=True) + DIRECTOR_V2_CATALOG: CatalogSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - DIRECTOR_V0: DirectorV0Settings = Field(auto_default_from_env=True) + DIRECTOR_V0: DirectorV0Settings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - DYNAMIC_SERVICES: DynamicServicesSettings = Field(auto_default_from_env=True) + DYNAMIC_SERVICES: DynamicServicesSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - POSTGRES: PostgresSettings = Field(auto_default_from_env=True) + POSTGRES: PostgresSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - REDIS: RedisSettings = Field(auto_default_from_env=True) + REDIS: RedisSettings = Field(json_schema_extra={"auto_default_from_env": True}) - DIRECTOR_V2_RABBITMQ: RabbitSettings = Field(auto_default_from_env=True) + DIRECTOR_V2_RABBITMQ: RabbitSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) TRAEFIK_SIMCORE_ZONE: str = Field("internal_simcore_stack") DIRECTOR_V2_COMPUTATIONAL_BACKEND: ComputationalBackendSettings = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) DIRECTOR_V2_DOCKER_REGISTRY: RegistrySettings = Field( - auto_default_from_env=True, + json_schema_extra={"auto_default_from_env": True}, description="settings for the private registry deployed with the platform", ) DIRECTOR_V2_DOCKER_HUB_REGISTRY: RegistrySettings | None = Field( - description="public DockerHub registry settings" + default=None, description="public DockerHub registry settings" ) DIRECTOR_V2_RESOURCE_USAGE_TRACKER: ResourceUsageTrackerSettings = Field( - auto_default_from_env=True, + json_schema_extra={"auto_default_from_env": True}, description="resource usage tracker service client's plugin", ) @@ -235,10 +247,11 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): description="Base URL used to access the public api e.g. http://127.0.0.1:6000 for development or https://api.osparc.io", ) DIRECTOR_V2_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) - @validator("LOG_LEVEL", pre=True) + @field_validator("LOG_LEVEL", mode="before") @classmethod def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py b/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py index 6e156607ae6..5de823d826b 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_pipelines.py @@ -1,10 +1,10 @@ from contextlib import suppress -from typing import Any, ClassVar, cast +from typing import cast import networkx as nx from models_library.projects import ProjectID from models_library.projects_state import RunningState -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, field_validator from simcore_postgres_database.models.comp_pipeline import StateType from ..utils.db import DB_TO_RUNNING_STATE @@ -15,7 +15,7 @@ class CompPipelineAtDB(BaseModel): dag_adjacency_list: dict[str, list[str]] # json serialization issue if using NodeID state: RunningState - @validator("state", pre=True) + @field_validator("state", mode="before") @classmethod def convert_state_from_state_type_enum_if_needed(cls, v): if isinstance(v, str): @@ -27,7 +27,7 @@ def convert_state_from_state_type_enum_if_needed(cls, v): return RunningState(DB_TO_RUNNING_STATE[StateType(v)]) return v - @validator("dag_adjacency_list", pre=True) + @field_validator("dag_adjacency_list", mode="before") @classmethod def auto_convert_dag(cls, v): # this enforcement is here because the serialization using json is not happy with non str Dict keys, also comparison gets funny if the lists are having sometimes UUIDs or str. @@ -42,10 +42,9 @@ def get_graph(self) -> nx.DiGraph: ), ) - class Config: - orm_mode = True - - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "examples": [ # DB model { @@ -60,4 +59,5 @@ class Config: "state": "NOT_STARTED", } ] - } + }, + ) diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index 1d7800b9788..9b466bbc49f 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -1,14 +1,14 @@ import datetime from contextlib import suppress -from typing import Any, ClassVar, TypedDict from models_library.clusters import DEFAULT_CLUSTER_ID, ClusterID from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.projects_state import RunningState from models_library.users import UserID -from pydantic import BaseModel, PositiveInt, validator +from pydantic import BaseModel, ConfigDict, PositiveInt, field_validator from simcore_postgres_database.models.comp_pipeline import StateType +from typing_extensions import TypedDict from ..utils.db import DB_TO_RUNNING_STATE @@ -44,12 +44,12 @@ class CompRunsAtDB(BaseModel): result: RunningState created: datetime.datetime modified: datetime.datetime - started: datetime.datetime | None - ended: datetime.datetime | None + started: datetime.datetime | None = None + ended: datetime.datetime | None = None metadata: RunMetadataDict = RunMetadataDict() use_on_demand_clusters: bool - @validator("result", pre=True) + @field_validator("result", mode="before") @classmethod def convert_result_from_state_type_enum_if_needed(cls, v): if isinstance(v, str): @@ -61,30 +61,30 @@ def convert_result_from_state_type_enum_if_needed(cls, v): return RunningState(DB_TO_RUNNING_STATE[StateType(v)]) return v - @validator("cluster_id", pre=True) + @field_validator("cluster_id", mode="before") @classmethod def convert_null_to_default_cluster_id(cls, v): if v is None: v = DEFAULT_CLUSTER_ID return v - @validator("created", "modified", "started", "ended") + @field_validator("created", "modified", "started", "ended") @classmethod def ensure_utc(cls, v: datetime.datetime | None) -> datetime.datetime | None: if v is not None and v.tzinfo is None: - v = v.replace(tzinfo=datetime.timezone.utc) + v = v.replace(tzinfo=datetime.UTC) return v - @validator("metadata", pre=True) + @field_validator("metadata", mode="before") @classmethod def convert_null_to_empty_metadata(cls, v): if v is None: v = RunMetadataDict() return v - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ "examples": [ # DB model { @@ -107,7 +107,7 @@ class Config: "result": "SUCCESS", "created": "2021-03-01 13:07:34.19161", "modified": "2021-03-01 13:07:34.19161", - "started": "2021-03-01 8:07:34.19161", + "started": "2021-03-01 08:07:34.19161", "ended": "2021-03-01 13:07:34.10", "metadata": { "node_id_names_map": {}, @@ -119,4 +119,5 @@ class Config: "use_on_demand_clusters": False, }, ] - } + }, + ) diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py b/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py index 5895411b2d0..e45e7aea896 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_tasks.py @@ -1,6 +1,6 @@ -import datetime +import datetime as dt from contextlib import suppress -from typing import Any, ClassVar +from typing import Any from dask_task_models_library.container_tasks.protocol import ContainerEnvsDict from models_library.api_schemas_directorv2.services import NodeRequirements @@ -17,11 +17,12 @@ from pydantic import ( BaseModel, ByteSize, - Extra, + ConfigDict, Field, PositiveInt, - parse_obj_as, - validator, + TypeAdapter, + ValidationInfo, + field_validator, ) from simcore_postgres_database.models.comp_pipeline import StateType from simcore_postgres_database.models.comp_tasks import NodeClass @@ -30,8 +31,8 @@ class Image(BaseModel): - name: str = Field(..., regex=SERVICE_KEY_RE.pattern) - tag: str = Field(..., regex=SIMPLE_VERSION_RE) + name: str = Field(..., pattern=SERVICE_KEY_RE.pattern) + tag: str = Field(..., pattern=SIMPLE_VERSION_RE) requires_gpu: bool | None = Field( default=None, deprecated=True, description="Use instead node_requirements" @@ -40,7 +41,9 @@ class Image(BaseModel): default=None, deprecated=True, description="Use instead node_requirements" ) node_requirements: NodeRequirements | None = Field( - default=None, description="the requirements for the service to run on a node" + default=None, + description="the requirements for the service to run on a node", + validate_default=True, ) boot_mode: BootMode = BootMode.CPU command: list[str] = Field( @@ -53,9 +56,9 @@ class Image(BaseModel): default_factory=dict, description="The environment to use to run the service" ) - @validator("node_requirements", pre=True, always=True) + @field_validator("node_requirements", mode="before") @classmethod - def migrate_from_requirements(cls, v, values): + def _migrate_from_requirements(cls, v, info: ValidationInfo): if v is None: # NOTE: 'node_requirements' field's default=None although is NOT declared as nullable. # Then this validator with `pre=True, always=True` is used to create a default @@ -63,21 +66,23 @@ def migrate_from_requirements(cls, v, values): # This strategy guarantees backwards compatibility v = NodeRequirements( CPU=1.0, - GPU=1 if values.get("requires_gpu") else 0, - RAM=parse_obj_as(ByteSize, "128 MiB"), + GPU=1 if info.data.get("requires_gpu") else 0, + RAM=TypeAdapter(ByteSize).validate_python("128 MiB"), ) return v - class Config: - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { - "examples": [ + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ + "examples": [ # type: ignore { "name": "simcore/services/dynamic/jupyter-octave-python-math", "tag": "1.3.1", "node_requirements": node_req_example, } - for node_req_example in NodeRequirements.Config.schema_extra["examples"] + for node_req_example in NodeRequirements.model_config[ # type: ignore + "json_schema_extra" + ]["examples"] ] + # old version @@ -89,14 +94,14 @@ class Config: "requires_mpi": False, } ] - } + }, + ) -# NOTE: for a long time defaultValue field was added to ServiceOutput wrongly in the DB. -# this flags allows parsing of the outputs without error. This MUST not leave the director-v2! class _ServiceOutputOverride(ServiceOutput): - class Config(ServiceOutput.Config): - extra = Extra.ignore + # NOTE: for a long time defaultValue field was added to ServiceOutput wrongly in the DB. + # this flags allows parsing of the outputs without error. This MUST not leave the director-v2! + model_config = ConfigDict(extra="ignore") _ServiceOutputsOverride = dict[ServicePortKey, _ServiceOutputOverride] @@ -105,10 +110,7 @@ class Config(ServiceOutput.Config): class NodeSchema(BaseModel): inputs: ServiceInputsDict = Field(..., description="the inputs scheam") outputs: _ServiceOutputsOverride = Field(..., description="the outputs schema") - - class Config: - extra = Extra.forbid - orm_mode = True + model_config = ConfigDict(extra="forbid", from_attributes=True) class CompTaskAtDB(BaseModel): @@ -125,32 +127,32 @@ class CompTaskAtDB(BaseModel): description="the hex digest of the resolved inputs +outputs hash at the time when the last outputs were generated", ) image: Image - submit: datetime.datetime - start: datetime.datetime | None = Field(default=None) - end: datetime.datetime | None = Field(default=None) + submit: dt.datetime + start: dt.datetime | None = None + end: dt.datetime | None = None state: RunningState - task_id: PositiveInt | None = Field(default=None) + task_id: PositiveInt | None = None internal_id: PositiveInt node_class: NodeClass - errors: list[ErrorDict] | None = Field(default=None) + errors: list[ErrorDict] | None = None progress: float | None = Field( default=None, ge=0.0, le=1.0, description="current progress of the task if available", ) - last_heartbeat: datetime.datetime | None = Field( + last_heartbeat: dt.datetime | None = Field( ..., description="Last time the running task was checked by the backend" ) - created: datetime.datetime - modified: datetime.datetime + created: dt.datetime + modified: dt.datetime # Additional information about price and hardware (ex. AWS EC2 instance type) pricing_info: dict | None hardware_info: HardwareInfo - @validator("state", pre=True) + @field_validator("state", mode="before") @classmethod - def convert_state_from_state_type_enum_if_needed(cls, v): + def _convert_state_from_state_type_enum_if_needed(cls, v): if isinstance(v, str): # try to convert to a StateType, if it fails the validations will continue # and pydantic will try to convert it to a RunninState later on @@ -160,30 +162,32 @@ def convert_state_from_state_type_enum_if_needed(cls, v): return RunningState(DB_TO_RUNNING_STATE[StateType(v)]) return v - @validator("start", "end", "submit") + @field_validator("start", "end", "submit") @classmethod - def ensure_utc(cls, v: datetime.datetime | None) -> datetime.datetime | None: + def _ensure_utc(cls, v: dt.datetime | None) -> dt.datetime | None: if v is not None and v.tzinfo is None: - v = v.replace(tzinfo=datetime.timezone.utc) + v = v.replace(tzinfo=dt.UTC) return v - @validator("hardware_info", pre=True) + @field_validator("hardware_info", mode="before") @classmethod - def backward_compatible_null_value(cls, v: HardwareInfo | None) -> HardwareInfo: + def _backward_compatible_null_value(cls, v: HardwareInfo | None) -> HardwareInfo: if v is None: return HardwareInfo(aws_ec2_instances=[]) return v def to_db_model(self, **exclusion_rules) -> dict[str, Any]: - comp_task_dict = self.dict(by_alias=True, exclude_unset=True, **exclusion_rules) + comp_task_dict = self.model_dump( + mode="json", by_alias=True, exclude_unset=True, **exclusion_rules + ) if "state" in comp_task_dict: comp_task_dict["state"] = RUNNING_STATE_TO_DB[comp_task_dict["state"]].value return comp_task_dict - class Config: - extra = Extra.forbid - orm_mode = True - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + extra="forbid", + from_attributes=True, + json_schema_extra={ "examples": [ # DB model { @@ -228,15 +232,16 @@ class Config: "state": "NOT_STARTED", "progress": 0.44, "last_heartbeat": None, - "created": "2022-05-20 13:28:31.139+00", - "modified": "2023-06-23 15:58:32.833081+00", + "created": "2022-05-20 13:28:31.139", + "modified": "2023-06-23 15:58:32.833081", "pricing_info": { "pricing_plan_id": 1, "pricing_unit_id": 1, "pricing_unit_cost_id": 1, }, - "hardware_info": HardwareInfo.Config.schema_extra["examples"][0], + "hardware_info": next(iter(HardwareInfo.model_config["json_schema_extra"]["examples"])), # type: ignore } - for image_example in Image.Config.schema_extra["examples"] + for image_example in Image.model_config["json_schema_extra"]["examples"] # type: ignore ] - } + }, + ) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 17408a202ec..0a2322c11c1 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -1,12 +1,11 @@ import json import logging -import re from collections.abc import Mapping from datetime import datetime from enum import Enum from functools import cached_property from pathlib import Path -from typing import Any, TypeAlias +from typing import Annotated, Any, TypeAlias from uuid import UUID import arrow @@ -31,11 +30,11 @@ from pydantic import ( AnyHttpUrl, BaseModel, - ConstrainedStr, - Extra, + ConfigDict, Field, - parse_obj_as, - validator, + StringConstraints, + TypeAdapter, + field_validator, ) from servicelib.exception_utils import DelayedExceptionHandler @@ -55,18 +54,17 @@ DockerStatus: TypeAlias = Status2 -class DockerId(ConstrainedStr): - max_length = 25 - regex = re.compile(r"[A-Za-z0-9]{25}") - +DockerId: TypeAlias = Annotated[ + str, StringConstraints(max_length=25, pattern=r"[A-Za-z0-9]{25}") +] ServiceId: TypeAlias = DockerId NetworkId: TypeAlias = DockerId -class ServiceName(ConstrainedStr): - strip_whitespace = True - min_length = 2 +ServiceName: TypeAlias = Annotated[ + str, StringConstraints(min_length=2, strip_whitespace=True) +] logger = logging.getLogger() @@ -129,8 +127,8 @@ class DockerContainerInspect(BaseModel): @cached_property def status(self) -> DockerStatus: - assert self.container_state.Status # nosec - result: DockerStatus = self.container_state.Status + assert self.container_state.status # nosec + result: DockerStatus = self.container_state.status return result @classmethod @@ -141,9 +139,7 @@ def from_container(cls, container: dict[str, Any]) -> "DockerContainerInspect": id=container["Id"], ) - class Config: - keep_untouched = (cached_property,) - allow_mutation = False + model_config = ConfigDict(ignored_types=(cached_property,), frozen=True) class ServiceRemovalState(BaseModel): @@ -202,7 +198,7 @@ class DynamicSidecar(BaseModel): is_ready: bool = Field( default=False, - scription=( + description=( "is True while the health check on the dynamic-sidecar is responding. " "Meaning that the dynamic-sidecar is reachable and can accept requests" ), @@ -224,7 +220,7 @@ def compose_spec_submitted(self) -> bool: containers_inspect: list[DockerContainerInspect] = Field( [], - scription="docker inspect results from all the container ran at regular intervals", + description="docker inspect results from all the container ran at regular intervals", ) was_dynamic_sidecar_started: bool = False @@ -279,7 +275,7 @@ def compose_spec_submitted(self) -> bool: ) instrumentation: ServicesInstrumentation = Field( - default_factory=lambda: ServicesInstrumentation.parse_obj({}), + default_factory=lambda: ServicesInstrumentation.model_validate({}), description="keeps track times for various operations", ) @@ -317,9 +313,7 @@ def compose_spec_submitted(self) -> bool: "this value will be set to None." ), ) - - class Config: - validate_assignment = True + model_config = ConfigDict(validate_assignment=True) class DynamicSidecarNamesHelper(BaseModel): @@ -337,25 +331,25 @@ class DynamicSidecarNamesHelper(BaseModel): service_name_dynamic_sidecar: str = Field( ..., - regex=REGEX_DY_SERVICE_SIDECAR, + pattern=REGEX_DY_SERVICE_SIDECAR, max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, description="unique name of the dynamic-sidecar service", ) proxy_service_name: str = Field( ..., - regex=REGEX_DY_SERVICE_PROXY, + pattern=REGEX_DY_SERVICE_PROXY, max_length=MAX_ALLOWED_SERVICE_NAME_LENGTH, description="name of the proxy for the dynamic-sidecar", ) simcore_traefik_zone: str = Field( ..., - regex=REGEX_DY_SERVICE_SIDECAR, + pattern=REGEX_DY_SERVICE_SIDECAR, description="unique name for the traefik constraints", ) dynamic_sidecar_network_name: str = Field( ..., - regex=REGEX_DY_SERVICE_SIDECAR, + pattern=REGEX_DY_SERVICE_SIDECAR, description="based on the node_id and project_id", ) @@ -392,15 +386,13 @@ class SchedulerData(CommonServiceDetails, DynamicSidecarServiceLabels): hostname: str = Field( ..., description="dy-sidecar's service hostname (provided by docker-swarm)" ) - port: PortInt = Field( - default=parse_obj_as(PortInt, 8000), description="dynamic-sidecar port" - ) + port: PortInt = Field(default=8000, description="dynamic-sidecar port") @property def endpoint(self) -> AnyHttpUrl: """endpoint where all the services are exposed""" - url: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, f"http://{self.hostname}:{self.port}" # NOSONAR + url = AnyHttpUrl.build( # pylint: disable=no-member + scheme="http", host=self.hostname, port=self.port ) return url @@ -425,7 +417,7 @@ def endpoint(self) -> AnyHttpUrl: ) service_port: PortInt = Field( - default=parse_obj_as(PortInt, TEMPORARY_PORT_NUMBER), + default=TEMPORARY_PORT_NUMBER, description=( "port where the service is exposed defined by the service; " "NOTE: temporary default because it will be changed once the service " @@ -470,8 +462,7 @@ def endpoint(self) -> AnyHttpUrl: def get_proxy_endpoint(self) -> AnyHttpUrl: """get the endpoint where the proxy's admin API is exposed""" assert self.proxy_admin_api_port # nosec - url: AnyHttpUrl = parse_obj_as( - AnyHttpUrl, + url: AnyHttpUrl = TypeAdapter(AnyHttpUrl).validate_python( f"http://{self.proxy_service_name}:{self.proxy_admin_api_port}", # nosec # NOSONAR ) return url @@ -528,9 +519,9 @@ def from_http_request( } if run_id: obj_dict["run_id"] = run_id - return cls.parse_obj(obj_dict) + return cls.model_validate(obj_dict) - @validator("user_preferences_path", pre=True) + @field_validator("user_preferences_path", mode="before") @classmethod def strip_path_serialization_to_none(cls, v): if v == "None": @@ -542,15 +533,13 @@ def from_service_inspect( cls, service_inspect: Mapping[str, Any] ) -> "SchedulerData": labels = service_inspect["Spec"]["Labels"] - return cls.parse_raw(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) + return cls.model_validate_json(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) def as_label_data(self) -> str: # compose_spec needs to be json encoded before encoding it to json # and storing it in the label - return self.copy( + return self.model_copy( update={"compose_spec": json.dumps(self.compose_spec)}, deep=True - ).json() + ).model_dump_json() - class Config: - extra = Extra.allow - allow_population_by_field_name = True + model_config = ConfigDict(extra="allow", populate_by_name=True) diff --git a/services/director-v2/src/simcore_service_director_v2/models/pricing.py b/services/director-v2/src/simcore_service_director_v2/models/pricing.py index 4aabef7cd10..43ade424954 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/pricing.py +++ b/services/director-v2/src/simcore_service_director_v2/models/pricing.py @@ -1,12 +1,11 @@ from decimal import Decimal -from typing import Any, ClassVar from models_library.resource_tracker import ( PricingPlanId, PricingUnitCostId, PricingUnitId, ) -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class PricingInfo(BaseModel): @@ -15,14 +14,15 @@ class PricingInfo(BaseModel): pricing_unit_cost_id: PricingUnitCostId pricing_unit_cost: Decimal - class Config: - schema_extra: ClassVar[dict[str, Any]] = { + model_config = ConfigDict( + json_schema_extra={ "examples": [ { "pricing_plan_id": 1, "pricing_unit_id": 1, "pricing_unit_cost_id": 1, - "pricing_unit_cost": Decimal(10), + "pricing_unit_cost": Decimal(10), # type: ignore[dict-item] } ] } + ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/catalog.py b/services/director-v2/src/simcore_service_director_v2/modules/catalog.py index f5e378afa43..2a064d14642 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/catalog.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/catalog.py @@ -8,7 +8,7 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.services_resources import ServiceResourcesDict from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from settings_library.catalog import CatalogSettings from ..utils.client_decorators import handle_errors, handle_retry @@ -90,8 +90,8 @@ async def get_service_resources( ) resp.raise_for_status() if resp.status_code == status.HTTP_200_OK: - json_response: ServiceResourcesDict = parse_obj_as( - ServiceResourcesDict, resp.json() + json_response: ServiceResourcesDict = TypeAdapter(ServiceResourcesDict).validate_python( + resp.json() ) return json_response raise HTTPException(status_code=resp.status_code, detail=resp.content) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py index 51fb3b1a3fb..abc97638608 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_dask_scheduler.py @@ -324,7 +324,7 @@ async def _process_task_result( async def _task_progress_change_handler(self, event: str) -> None: with log_catch(_logger, reraise=False): - task_progress_event = TaskProgressEvent.parse_raw(event) + task_progress_event = TaskProgressEvent.model_validate_json(event) _logger.debug("received task progress update: %s", task_progress_event) user_id = task_progress_event.task_owner.user_id project_id = task_progress_event.task_owner.project_id @@ -355,7 +355,7 @@ async def _task_progress_change_handler(self, event: str) -> None: async def _task_log_change_handler(self, event: str) -> None: with log_catch(_logger, reraise=False): - task_log_event = TaskLogEvent.parse_raw(event) + task_log_event = TaskLogEvent.model_validate_json(event) _logger.debug("received task log update: %s", task_log_event) await publish_service_log( self.rabbitmq_client, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index fcba4ad1fd3..e28e48f82f7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -48,7 +48,7 @@ from models_library.projects_nodes_io import NodeID from models_library.resource_tracker import HardwareInfo from models_library.users import UserID -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError from pydantic.networks import AnyUrl from servicelib.logging_utils import log_catch from settings_library.s3 import S3Settings @@ -583,5 +583,5 @@ def _get_worker_used_resources( assert dashboard_link # nosec return ClusterDetails( scheduler=Scheduler(status=scheduler_status, **scheduler_info), - dashboard_link=parse_obj_as(AnyUrl, dashboard_link), + dashboard_link=TypeAdapter(AnyUrl).validate_python(dashboard_link), ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py index 214c03b9dca..30381110173 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/clusters.py @@ -108,9 +108,9 @@ async def _compute_user_access_rights( ) and (primary_grp_rights := cluster.access_rights.get(primary_group_row.gid)): return primary_grp_rights - solved_rights = CLUSTER_NO_RIGHTS.dict() + solved_rights = CLUSTER_NO_RIGHTS.model_dump() for group_row in filter(lambda ugrp: ugrp[1] != GroupType.PRIMARY, user_groups): - grp_access = cluster.access_rights.get(group_row.gid, CLUSTER_NO_RIGHTS).dict() + grp_access = cluster.access_rights.get(group_row.gid, CLUSTER_NO_RIGHTS).model_dump() for operation in ["read", "write", "delete"]: solved_rights[operation] |= grp_access[operation] return ClusterAccessRights(**solved_rights) @@ -250,14 +250,14 @@ async def update_cluster( # pylint: disable=too-many-branches if updated_cluster.access_rights: for grp, rights in resolved_access_rights.items(): insert_stmt = pg_insert(cluster_to_groups).values( - **rights.dict(by_alias=True), gid=grp, cluster_id=the_cluster.id + **rights.model_dump(by_alias=True), gid=grp, cluster_id=the_cluster.id ) on_update_stmt = insert_stmt.on_conflict_do_update( index_elements=[ cluster_to_groups.c.cluster_id, cluster_to_groups.c.gid, ], - set_=rights.dict(by_alias=True), + set_=rights.model_dump(by_alias=True), ) await conn.execute(on_update_stmt) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py index 3c24694c2fd..37129141f6d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py @@ -26,7 +26,7 @@ async def get_pipeline(self, project_id: ProjectID) -> CompPipelineAtDB: row: RowProxy | None = await result.fetchone() if not row: raise PipelineNotFoundError(str(project_id)) - return CompPipelineAtDB.from_orm(row) + return CompPipelineAtDB.model_validate(row) async def upsert_pipeline( self, @@ -39,14 +39,14 @@ async def upsert_pipeline( dag_adjacency_list=nx.to_dict_of_lists(dag_graph), state=RunningState.PUBLISHED if publish else RunningState.NOT_STARTED, ) - insert_stmt = insert(comp_pipeline).values(**pipeline_at_db.dict(by_alias=True)) + insert_stmt = insert(comp_pipeline).values(**pipeline_at_db.model_dump(by_alias=True)) # FIXME: This is not a nice thing. this part of the information should be kept in comp_runs. update_exclusion_policy = set() if not dag_graph.nodes(): update_exclusion_policy.add("dag_adjacency_list") on_update_stmt = insert_stmt.on_conflict_do_update( index_elements=[comp_pipeline.c.project_id], - set_=pipeline_at_db.dict( + set_=pipeline_at_db.model_dump( by_alias=True, exclude_unset=True, exclude=update_exclusion_policy ), ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py index 4f9a8e42b53..289a0063649 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py @@ -51,7 +51,7 @@ async def get( row: RowProxy | None = await result.first() if not row: raise ComputationalRunNotFoundError - return CompRunsAtDB.from_orm(row) + return CompRunsAtDB.model_validate(row) async def list( self, filter_by_state: set[RunningState] | None = None @@ -70,7 +70,7 @@ async def list( ) ) ): - runs_in_db.append(CompRunsAtDB.from_orm(row)) + runs_in_db.append(CompRunsAtDB.model_validate(row)) return list(runs_in_db) async def create( @@ -114,7 +114,7 @@ async def create( .returning(literal_column("*")) ) row = await result.first() - return CompRunsAtDB.from_orm(row) + return CompRunsAtDB.model_validate(row) except ForeignKeyViolation as exc: raise ClusterNotFoundError(cluster_id=cluster_id) from exc @@ -133,7 +133,7 @@ async def update( .returning(literal_column("*")) ) row = await result.first() - return CompRunsAtDB.from_orm(row) if row else None + return CompRunsAtDB.model_validate(row) if row else None async def set_run_result( self, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py index dabb45dfb0f..aa72e996d7f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py @@ -44,7 +44,7 @@ async def get_task(self, project_id: ProjectID, node_id: NodeID) -> CompTaskAtDB row = await result.fetchone() if not row: raise ComputationalTaskNotFoundError(node_id=node_id) - return CompTaskAtDB.from_orm(row) + return CompTaskAtDB.model_validate(row) async def list_tasks( self, @@ -55,7 +55,7 @@ async def list_tasks( async for row in conn.execute( sa.select(comp_tasks).where(comp_tasks.c.project_id == f"{project_id}") ): - task_db = CompTaskAtDB.from_orm(row) + task_db = CompTaskAtDB.model_validate(row) tasks.append(task_db) return tasks @@ -72,7 +72,7 @@ async def list_computational_tasks( & (comp_tasks.c.node_class == NodeClass.COMPUTATIONAL) ) ): - task_db = CompTaskAtDB.from_orm(row) + task_db = CompTaskAtDB.model_validate(row) tasks.append(task_db) return tasks @@ -166,7 +166,7 @@ async def upsert_tasks_from_project( result = await conn.execute(on_update_stmt) row = await result.fetchone() assert row # nosec - inserted_comp_tasks_db.append(CompTaskAtDB.from_orm(row)) + inserted_comp_tasks_db.append(CompTaskAtDB.model_validate(row)) _logger.debug( "inserted the following tasks in comp_tasks: %s", f"{inserted_comp_tasks_db=}", @@ -193,7 +193,7 @@ async def _update_task( ) row = await result.fetchone() assert row # nosec - return CompTaskAtDB.from_orm(row) + return CompTaskAtDB.model_validate(row) async def mark_project_published_waiting_for_cluster_tasks_as_aborted( self, project_id: ProjectID diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index a33f689e9da..bdb64cbbf99 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -36,7 +36,7 @@ ) from models_library.users import UserID from models_library.wallets import ZERO_CREDITS, WalletInfo -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.rabbitmq import ( RabbitMQRPCClient, RemoteMethodNotRegisteredError, @@ -89,7 +89,7 @@ async def _get_service_details( node.version, product_name, ) - obj: ServiceMetaDataPublished = ServiceMetaDataPublished.construct( + obj: ServiceMetaDataPublished = ServiceMetaDataPublished.model_construct( **service_details ) return obj @@ -105,7 +105,7 @@ def _compute_node_requirements( node_defined_resources[resource_name] = node_defined_resources.get( resource_name, 0 ) + min(resource_value.limit, resource_value.reservation) - return NodeRequirements.parse_obj(node_defined_resources) + return NodeRequirements.model_validate(node_defined_resources) def _compute_node_boot_mode(node_resources: ServiceResourcesDict) -> BootMode: @@ -174,7 +174,9 @@ async def _generate_task_image( } project_nodes_repo = ProjectNodesRepo(project_uuid=project_id) project_node = await project_nodes_repo.get(connection, node_id=node_id) - node_resources = parse_obj_as(ServiceResourcesDict, project_node.required_resources) + node_resources = TypeAdapter(ServiceResourcesDict).validate_python( + project_node.required_resources + ) if not node_resources: node_resources = await catalog_client.get_service_resources( user_id, node.key, node.version @@ -187,7 +189,7 @@ async def _generate_task_image( data.update(envs=_compute_node_envs(node_labels)) if node_extras and node_extras.container_spec: data.update(command=node_extras.container_spec.command) - return Image.parse_obj(data) + return Image.model_validate(data) async def _get_pricing_and_hardware_infos( @@ -287,7 +289,9 @@ def _by_type_name(ec2: EC2InstanceTypeGet) -> bool: # less memory than the machine theoretical amount project_nodes_repo = ProjectNodesRepo(project_uuid=project_id) node = await project_nodes_repo.get(connection, node_id=node_id) - node_resources = parse_obj_as(ServiceResourcesDict, node.required_resources) + node_resources = TypeAdapter(ServiceResourcesDict).validate_python( + node.required_resources + ) if DEFAULT_SINGLE_SERVICE_NAME in node_resources: image_resources: ImageResources = node_resources[ DEFAULT_SINGLE_SERVICE_NAME @@ -322,7 +326,7 @@ def _by_type_name(ec2: EC2InstanceTypeGet) -> bool: except ( RemoteMethodNotRegisteredError, RPCServerError, - asyncio.TimeoutError, + TimeoutError, ) as exc: raise ClustersKeeperNotAvailableError from exc @@ -343,7 +347,7 @@ async def generate_tasks_list_from_project( list_comp_tasks = [] unique_service_key_versions: set[ServiceKeyVersion] = { - ServiceKeyVersion.construct( + ServiceKeyVersion.model_construct( key=node.key, version=node.version ) # the service key version is frozen for node in project.workbench.values() @@ -362,7 +366,7 @@ async def generate_tasks_list_from_project( for internal_id, node_id in enumerate(project.workbench, 1): node: Node = project.workbench[node_id] - node_key_version = ServiceKeyVersion.construct( + node_key_version = ServiceKeyVersion.model_construct( key=node.key, version=node.version ) node_details, node_extras, node_labels = key_version_to_node_infos.get( @@ -430,8 +434,8 @@ async def generate_tasks_list_from_project( task_db = CompTaskAtDB( project_id=project.uuid, node_id=NodeID(node_id), - schema=NodeSchema.parse_obj( - node_details.dict( + schema=NodeSchema.model_validate( + node_details.model_dump( exclude_unset=True, by_alias=True, include={"inputs", "outputs"} ) ), @@ -446,7 +450,7 @@ async def generate_tasks_list_from_project( last_heartbeat=None, created=arrow.utcnow().datetime, modified=arrow.utcnow().datetime, - pricing_info=pricing_info.dict(exclude={"pricing_unit_cost"}) + pricing_info=pricing_info.model_dump(exclude={"pricing_unit_cost"}) if pricing_info else None, hardware_info=hardware_info, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 856c0ec3650..5f5fe5263ff 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -23,7 +23,7 @@ async def get_project(self, project_id: ProjectID) -> ProjectAtDB: ).first() if not row: raise ProjectNotFoundError(project_id) - return ProjectAtDB.from_orm(row) + return ProjectAtDB.model_validate(row) async def is_node_present_in_workbench( self, project_id: ProjectID, node_uuid: NodeID diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py index fe033761128..59334aa0a06 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py @@ -23,17 +23,17 @@ async def get_projects_networks(self, project_id: ProjectID) -> ProjectsNetworks ).first() if not row: raise ProjectNetworkNotFoundError(project_id) - return ProjectsNetworks.from_orm(row) + return ProjectsNetworks.model_validate(row) async def upsert_projects_networks( self, project_id: ProjectID, networks_with_aliases: NetworksWithAliases ) -> None: - projects_networks_to_insert = ProjectsNetworks.parse_obj( + projects_networks_to_insert = ProjectsNetworks.model_validate( {"project_uuid": project_id, "networks_with_aliases": networks_with_aliases} ) async with self.db_engine.acquire() as conn: - row_data = json.loads(projects_networks_to_insert.json()) + row_data = json.loads(projects_networks_to_insert.model_dump_json()) insert_stmt = pg_insert(projects_networks).values(**row_data) upsert_snapshot = insert_stmt.on_conflict_do_update( constraint=projects_networks.primary_key, set_=row_data diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/user_preferences_frontend.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/user_preferences_frontend.py index 01d7fdcce61..0ce81c14bb6 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/user_preferences_frontend.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/user_preferences_frontend.py @@ -31,5 +31,5 @@ async def get_user_preference( return ( None if preference_payload is None - else preference_class.parse_obj(preference_payload) + else preference_class.model_validate(preference_payload) ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py index 94f17b90295..434e523965c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py @@ -1,5 +1,5 @@ from models_library.users import UserID -from pydantic import EmailStr, parse_obj_as +from pydantic import EmailStr, TypeAdapter from simcore_postgres_database.models.users import UserRole from simcore_postgres_database.utils_users import UsersRepo @@ -10,7 +10,7 @@ class UsersRepository(BaseRepository): async def get_user_email(self, user_id: UserID) -> EmailStr: async with self.db_engine.acquire() as conn: email = await UsersRepo.get_email(conn, user_id) - return parse_obj_as(EmailStr, email) + return TypeAdapter(EmailStr).validate_python(email) async def get_user_role(self, user_id: UserID) -> UserRole: async with self.db_engine.acquire() as conn: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py index 0bc8c799dcb..322e5281e46 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py @@ -83,7 +83,7 @@ async def get_service_extras( f"/service_extras/{urllib.parse.quote_plus(service_key)}/{service_version}", ) if resp.status_code == status.HTTP_200_OK: - return ServiceExtras.parse_obj(unenvelope_or_raise_error(resp)) + return ServiceExtras.model_validate(unenvelope_or_raise_error(resp)) raise HTTPException(status_code=resp.status_code, detail=resp.content) @log_decorator(logger=logger) @@ -94,7 +94,7 @@ async def get_running_service_details( "GET", f"running_interactive_services/{service_uuid}" ) if resp.status_code == status.HTTP_200_OK: - return RunningDynamicServiceDetails.parse_obj( + return RunningDynamicServiceDetails.model_validate( unenvelope_or_raise_error(resp) ) raise HTTPException(status_code=resp.status_code, detail=resp.content) @@ -109,7 +109,7 @@ async def get_service_labels( ) resp.raise_for_status() if resp.status_code == status.HTTP_200_OK: - return SimcoreServiceLabels.parse_obj(unenvelope_or_raise_error(resp)) + return SimcoreServiceLabels.model_validate(unenvelope_or_raise_error(resp)) raise HTTPException(status_code=resp.status_code, detail=resp.content) @log_decorator(logger=logger) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index 7ce782c6366..5945e07b8e3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -293,7 +293,7 @@ def _get_client(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> Client: return Client( app=self._app, async_client=self._async_client, - base_url=dynamic_sidecar_endpoint, + base_url=f"{dynamic_sidecar_endpoint}", ) async def _await_for_result( @@ -358,7 +358,7 @@ async def restore_service_state(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> i result: Any | None = await self._await_for_result( task_id, dynamic_sidecar_endpoint, - self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), _debug_progress_callback, ) assert isinstance(result, int) # nosec @@ -392,7 +392,7 @@ async def save_service_state( result: Any | None = await self._await_for_result( task_id, dynamic_sidecar_endpoint, - self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), progress_callback, ) assert isinstance(result, int) # nosec @@ -411,7 +411,7 @@ async def pull_service_input_ports( transferred_bytes = await self._await_for_result( task_id, dynamic_sidecar_endpoint, - self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), _debug_progress_callback, ) return transferred_bytes or 0 @@ -429,7 +429,7 @@ async def pull_service_output_ports( result: Any | None = await self._await_for_result( task_id, dynamic_sidecar_endpoint, - self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), _debug_progress_callback, ) assert isinstance(result, int) # nosec @@ -448,7 +448,7 @@ async def push_service_output_ports( await self._await_for_result( task_id, dynamic_sidecar_endpoint, - self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + self._dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), progress_callback, ) @@ -495,7 +495,9 @@ async def get_service_activity( dynamic_sidecar_endpoint ) decoded_response = response.json() - return ActivityInfo.parse_obj(decoded_response) if decoded_response else None + return ( + ActivityInfo.model_validate(decoded_response) if decoded_response else None + ) async def free_reserved_disk_space( self, dynamic_sidecar_endpoint: AnyHttpUrl diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py index 241f32fe70e..21ef1bbe279 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py @@ -35,7 +35,7 @@ def __init__(self, app: FastAPI): # timeouts self._health_request_timeout = Timeout(1.0, connect=1.0) self._save_restore_timeout = Timeout( - scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT, + scheduler_settings.DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT.total_seconds(), connect=scheduler_settings.DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT, ) self._restart_containers_timeout = Timeout( @@ -63,13 +63,13 @@ def _get_url( no_api_version: bool = False, ) -> str: """formats and returns an url for the request""" - api_version = "" if no_api_version else f"/{self.API_VERSION}" + api_version = "" if no_api_version else f"{self.API_VERSION}/" return f"{dynamic_sidecar_endpoint}{api_version}{postfix}" async def _get_health_common( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/health", no_api_version=True) + url = self._get_url(dynamic_sidecar_endpoint, "health", no_api_version=True) return await self.client.get(url, timeout=self._health_request_timeout) @retry_on_errors() @@ -88,7 +88,7 @@ async def get_health_no_retry( async def get_containers( self, dynamic_sidecar_endpoint: AnyHttpUrl, *, only_status: bool ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers") + url = self._get_url(dynamic_sidecar_endpoint, "containers") return await self.client.get(url, params={"only_status": only_status}) @retry_on_errors() @@ -100,7 +100,7 @@ async def patch_containers_ports_io( enable_outputs: bool, enable_inputs: bool, ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/ports/io") + url = self._get_url(dynamic_sidecar_endpoint, "containers/ports/io") return await self.client.patch( url, json={"enable_outputs": enable_outputs, "enable_inputs": enable_inputs} ) @@ -110,7 +110,7 @@ async def patch_containers_ports_io( async def post_containers_ports_outputs_dirs( self, dynamic_sidecar_endpoint: AnyHttpUrl, *, outputs_labels: dict[str, Any] ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/ports/outputs/dirs") + url = self._get_url(dynamic_sidecar_endpoint, "containers/ports/outputs/dirs") return await self.client.post(url, json={"outputs_labels": outputs_labels}) @retry_on_errors() @@ -125,7 +125,7 @@ async def get_containers_name( } ) url = self._get_url( - dynamic_sidecar_endpoint, f"/containers/name?filters={filters}" + dynamic_sidecar_endpoint, f"containers/name?filters={filters}" ) return await self.client.get(url=url) @@ -140,7 +140,7 @@ async def post_containers_networks_attach( network_aliases: list[str], ) -> Response: url = self._get_url( - dynamic_sidecar_endpoint, f"/containers/{container_id}/networks:attach" + dynamic_sidecar_endpoint, f"containers/{container_id}/networks:attach" ) return await self.client.post( url, @@ -158,7 +158,7 @@ async def post_containers_networks_detach( network_id: str, ) -> Response: url = self._get_url( - dynamic_sidecar_endpoint, f"/containers/{container_id}/networks:detach" + dynamic_sidecar_endpoint, f"containers/{container_id}/networks:detach" ) return await self.client.post( url, @@ -174,7 +174,7 @@ async def post_containers_compose_spec( *, compose_spec: str, ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/compose-spec") + url = self._get_url(dynamic_sidecar_endpoint, "containers/compose-spec") return await self.client.post(url, json={"docker_compose_yaml": compose_spec}) @retry_on_errors() @@ -185,9 +185,9 @@ async def post_containers_tasks( *, metrics_params: CreateServiceMetricsAdditionalParams, ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers") + url = self._get_url(dynamic_sidecar_endpoint, "containers") return await self.client.post( - url, json={"metrics_params": metrics_params.dict()} + url, json={"metrics_params": metrics_params.model_dump()} ) @retry_on_errors() @@ -195,7 +195,7 @@ async def post_containers_tasks( async def post_containers_tasks_down( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers:down") + url = self._get_url(dynamic_sidecar_endpoint, "containers:down") return await self.client.post(url) @retry_on_errors() @@ -203,7 +203,7 @@ async def post_containers_tasks_down( async def post_containers_tasks_state_restore( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/state:restore") + url = self._get_url(dynamic_sidecar_endpoint, "containers/state:restore") return await self.client.post(url) @retry_on_errors() @@ -211,7 +211,7 @@ async def post_containers_tasks_state_restore( async def post_containers_tasks_state_save( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/state:save") + url = self._get_url(dynamic_sidecar_endpoint, "containers/state:save") return await self.client.post(url) @retry_on_errors() @@ -219,7 +219,7 @@ async def post_containers_tasks_state_save( async def post_containers_images_pull( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/images:pull") + url = self._get_url(dynamic_sidecar_endpoint, "containers/images:pull") return await self.client.post(url) @retry_on_errors() @@ -230,7 +230,7 @@ async def post_containers_tasks_ports_inputs_pull( port_keys: list[str] | None = None, ) -> Response: port_keys = [] if port_keys is None else port_keys - url = self._get_url(dynamic_sidecar_endpoint, "/containers/ports/inputs:pull") + url = self._get_url(dynamic_sidecar_endpoint, "containers/ports/inputs:pull") return await self.client.post(url, json=port_keys) @retry_on_errors() @@ -241,7 +241,7 @@ async def post_containers_tasks_ports_outputs_pull( port_keys: list[str] | None = None, ) -> Response: port_keys = [] if port_keys is None else port_keys - url = self._get_url(dynamic_sidecar_endpoint, "/containers/ports/outputs:pull") + url = self._get_url(dynamic_sidecar_endpoint, "containers/ports/outputs:pull") return await self.client.post(url, json=port_keys) @retry_on_errors() @@ -249,7 +249,7 @@ async def post_containers_tasks_ports_outputs_pull( async def post_containers_tasks_ports_outputs_push( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/ports/outputs:push") + url = self._get_url(dynamic_sidecar_endpoint, "containers/ports/outputs:push") return await self.client.post(url) @retry_on_errors() @@ -257,7 +257,7 @@ async def post_containers_tasks_ports_outputs_push( async def post_containers_tasks_restart( self, dynamic_sidecar_endpoint: AnyHttpUrl ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers:restart") + url = self._get_url(dynamic_sidecar_endpoint, "containers:restart") return await self.client.post(url) @retry_on_errors() @@ -268,7 +268,7 @@ async def put_volumes( volume_category: VolumeCategory, volume_status: VolumeStatus, ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, f"/volumes/{volume_category}") + url = self._get_url(dynamic_sidecar_endpoint, f"volumes/{volume_category}") return await self.client.put(url, json={"status": volume_status}) @@ -277,7 +277,7 @@ async def put_volumes( async def proxy_config_load( self, proxy_endpoint: AnyHttpUrl, proxy_configuration: dict[str, Any] ) -> Response: - url = self._get_url(proxy_endpoint, "/load", no_api_version=True) + url = self._get_url(proxy_endpoint, "load", no_api_version=True) return await self.client.post(url, json=proxy_configuration) @retry_on_errors() @@ -286,7 +286,7 @@ async def get_containers_activity( self, dynamic_sidecar_endpoint: AnyHttpUrl, ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/containers/activity") + url = self._get_url(dynamic_sidecar_endpoint, "containers/activity") return await self.client.get(url) @retry_on_errors() @@ -295,5 +295,5 @@ async def post_disk_reserved_free( self, dynamic_sidecar_endpoint: AnyHttpUrl, ) -> Response: - url = self._get_url(dynamic_sidecar_endpoint, "/disk/reserved:free") + url = self._get_url(dynamic_sidecar_endpoint, "disk/reserved:free") return await self.client.post(url) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py index b9d38ca6502..98ba1ea2f40 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py @@ -239,7 +239,7 @@ def _update_container_labels( spec_service_key, default_limits ) - label_keys = StandardSimcoreDockerLabels.construct( + label_keys = StandardSimcoreDockerLabels.model_construct( user_id=user_id, project_id=project_id, node_id=node_id, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py index 5100d63bab0..78a1201a714 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py @@ -304,7 +304,7 @@ def _merge_resources_in_settings( # merge all resources empty_resource_entry: SimcoreServiceSettingLabelEntry = ( - SimcoreServiceSettingLabelEntry.parse_obj( + SimcoreServiceSettingLabelEntry.model_validate( { "name": "Resources", "type": "Resources", @@ -399,14 +399,14 @@ def _format_env_var(env_var: str, destination_container: list[str]) -> str: def _get_boot_options( service_labels: SimcoreServiceLabels, ) -> dict[EnvVarKey, BootOption] | None: - as_dict = service_labels.dict() + as_dict = service_labels.model_dump() boot_options_encoded = as_dict.get("io.simcore.boot-options", None) if boot_options_encoded is None: return None boot_options = json.loads(boot_options_encoded)["boot-options"] log.debug("got boot_options=%s", boot_options) - return {k: BootOption.parse_obj(v) for k, v in boot_options.items()} + return {k: BootOption.model_validate(v) for k, v in boot_options.items()} def _assemble_env_vars_for_boot_options( @@ -423,7 +423,7 @@ def _assemble_env_vars_for_boot_options( env_vars.append(f"{env_var_name}={value}") return SimcoreServiceSettingsLabel( - __root__=[ + root=[ SimcoreServiceSettingLabelEntry( name="env", type="string", value=list(env_vars) ) @@ -511,7 +511,7 @@ async def merge_settings_before_use( ) settings = _patch_target_service_into_env_vars(settings) - return SimcoreServiceSettingsLabel.parse_obj(settings) + return SimcoreServiceSettingsLabel.model_validate(settings) __all__ = ["merge_settings_before_use", "update_service_params_from_settings"] diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 001e549bf57..f344c93422f 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -3,6 +3,7 @@ from typing import Any, NamedTuple from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from models_library.aiodocker_api import AioDockerServiceSpec from models_library.basic_types import BootModeEnum, PortInt from models_library.callbacks_mapping import CallbacksMapping @@ -15,13 +16,10 @@ ) from models_library.resource_tracker import HardwareInfo from models_library.service_settings_labels import SimcoreServiceSettingsLabel -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.efs_guardian import efs_manager from servicelib.utils import unused_port -from settings_library.aws_s3_cli import AwsS3CliSettings -from settings_library.docker_registry import RegistrySettings -from settings_library.utils_encoders import create_json_encoder_wo_secrets from ....constants import DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL from ....core.dynamic_services_settings.scheduler import ( @@ -101,8 +99,11 @@ def _get_environment_variables( app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_AWS_S3_CLI_SETTINGS and app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_AWS_S3_CLI_SETTINGS.AWS_S3_CLI_S3 ): - dy_sidecar_aws_s3_cli_settings = app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_AWS_S3_CLI_SETTINGS.json( - encoder=create_json_encoder_wo_secrets(AwsS3CliSettings), + dy_sidecar_aws_s3_cli_settings = json_dumps( + model_dump_with_secrets( + app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_AWS_S3_CLI_SETTINGS, + show_secrets=True, + ) ) state_exclude = set() @@ -133,7 +134,7 @@ def _get_environment_variables( "DY_SIDECAR_USER_SERVICES_HAVE_INTERNET_ACCESS": f"{allow_internet_access}", "DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE": f"{telemetry_enabled}", "DY_SIDECAR_STATE_EXCLUDE": json_dumps(f"{x}" for x in state_exclude), - "DY_SIDECAR_CALLBACKS_MAPPING": callbacks_mapping.json(), + "DY_SIDECAR_CALLBACKS_MAPPING": callbacks_mapping.model_dump_json(), "DY_SIDECAR_STATE_PATHS": json_dumps( f"{x}" for x in scheduler_data.paths_mapping.state_paths ), @@ -157,14 +158,22 @@ def _get_environment_variables( "RABBIT_PORT": f"{rabbit_settings.RABBIT_PORT}", "RABBIT_USER": f"{rabbit_settings.RABBIT_USER}", "RABBIT_SECURE": f"{rabbit_settings.RABBIT_SECURE}", - "DY_DEPLOYMENT_REGISTRY_SETTINGS": app_settings.DIRECTOR_V2_DOCKER_REGISTRY.json( - encoder=create_json_encoder_wo_secrets(RegistrySettings), - exclude={"resolved_registry_url", "api_url"}, + "DY_DEPLOYMENT_REGISTRY_SETTINGS": ( + json_dumps( + model_dump_with_secrets( + app_settings.DIRECTOR_V2_DOCKER_REGISTRY, + show_secrets=True, + exclude={"resolved_registry_url", "api_url"}, + ) + ) ), "DY_DOCKER_HUB_REGISTRY_SETTINGS": ( - app_settings.DIRECTOR_V2_DOCKER_HUB_REGISTRY.json( - encoder=create_json_encoder_wo_secrets(RegistrySettings), - exclude={"resolved_registry_url", "api_url"}, + json_dumps( + model_dump_with_secrets( + app_settings.DIRECTOR_V2_DOCKER_HUB_REGISTRY, + show_secrets=True, + exclude={"resolved_registry_url", "api_url"}, + ) ) if app_settings.DIRECTOR_V2_DOCKER_HUB_REGISTRY else "null" @@ -190,7 +199,7 @@ def _get_environment_variables( "NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS": f"{app_settings.DIRECTOR_V2_NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS}", } if r_clone_settings.R_CLONE_S3.S3_ENDPOINT is not None: - envs["S3_ENDPOINT"] = r_clone_settings.R_CLONE_S3.S3_ENDPOINT + envs["S3_ENDPOINT"] = f"{r_clone_settings.R_CLONE_S3.S3_ENDPOINT}" return envs @@ -471,8 +480,7 @@ async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: if hardware_info and len(hardware_info.aws_ec2_instances) == 1: ec2_instance_type: str = hardware_info.aws_ec2_instances[0] placement_constraints.append( - parse_obj_as( - DockerPlacementConstraint, + TypeAdapter(DockerPlacementConstraint).validate_python( f"node.labels.{DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY}=={ec2_instance_type}", ) ) @@ -554,4 +562,4 @@ async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: create_service_params=create_service_params, ) - return AioDockerServiceSpec.parse_obj(create_service_params) + return AioDockerServiceSpec.model_validate(create_service_params) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py index 5a503f8b8a8..afd44dc0f59 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_states.py @@ -74,10 +74,10 @@ def extract_task_state(task_status: dict[str, str]) -> tuple[ServiceState, str]: def _extract_container_status( container_state: ContainerState, ) -> tuple[ServiceState, ServiceMessage]: - assert container_state.Status # nosec + assert container_state.status # nosec return ( - _CONTAINER_STATE_TO_SERVICE_STATE[container_state.Status], - container_state.Error if container_state.Error else "", + _CONTAINER_STATE_TO_SERVICE_STATE[container_state.status], + container_state.error if container_state.error else "", ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py index ecb86e9a6aa..8b40a4e0f35 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py @@ -1,6 +1,6 @@ from aiodocker.exceptions import DockerError +from common_library.errors_classes import OsparcErrorMixin from models_library.projects_nodes_io import NodeID -from pydantic.errors import PydanticErrorMixin from ...core.errors import DirectorError @@ -39,6 +39,6 @@ class LegacyServiceIsNotSupportedError(DirectorError): """This API is not implemented by the director-v0""" -class UnexpectedContainerStatusError(PydanticErrorMixin, DynamicSidecarError): - code = "dynamic_sidecar.container_status" +class UnexpectedContainerStatusError(OsparcErrorMixin, DynamicSidecarError): + code = "dynamic_sidecar.container_status" # type: ignore msg_template = "Unexpected status from containers: {containers_with_error}" diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py index 6475e324950..166d4562186 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py @@ -242,19 +242,19 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: scheduler_data.user_id, scheduler_data.key, scheduler_data.version ) ).get("sidecar", {}) or {} - user_specific_service_spec = AioDockerServiceSpec.parse_obj( + user_specific_service_spec = AioDockerServiceSpec.model_validate( user_specific_service_spec ) # NOTE: since user_specific_service_spec follows Docker Service Spec and not Aio # we do not use aliases when exporting dynamic_sidecar_service_spec_base - dynamic_sidecar_service_final_spec = AioDockerServiceSpec.parse_obj( + dynamic_sidecar_service_final_spec = AioDockerServiceSpec.model_validate( nested_update( jsonable_encoder(dynamic_sidecar_service_spec_base, exclude_unset=True), jsonable_encoder(user_specific_service_spec, exclude_unset=True), include=_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS, ) ) - rabbit_message = ProgressRabbitMessageNode.construct( + rabbit_message = ProgressRabbitMessageNode.model_construct( user_id=scheduler_data.user_id, project_id=scheduler_data.project_id, node_id=scheduler_data.node_uuid, @@ -272,7 +272,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: ) ) - rabbit_message = ProgressRabbitMessageNode.construct( + rabbit_message = ProgressRabbitMessageNode.model_construct( user_id=scheduler_data.user_id, project_id=scheduler_data.project_id, node_id=scheduler_data.node_uuid, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py index f708c1cb22c..f8416b4809b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py @@ -7,7 +7,7 @@ from models_library.service_settings_labels import SimcoreServiceLabels from models_library.services import ServiceKeyVersion, ServiceVersion from models_library.services_creation import CreateServiceMetricsAdditionalParams -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.fastapi.long_running_tasks.client import TaskId from tenacity import RetryError from tenacity.asyncio import AsyncRetrying @@ -168,7 +168,7 @@ async def progress_create_containers( project_name=project_name, node_name=node_name, service_key=scheduler_data.key, - service_version=parse_obj_as(ServiceVersion, scheduler_data.version), + service_version=TypeAdapter(ServiceVersion).validate_python(scheduler_data.version), service_resources=scheduler_data.service_resources, service_additional_metadata={}, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index 3071cde1060..e861ad9f30c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -535,7 +535,7 @@ async def _restore_service_state_with_metrics() -> None: ) ) service_outputs_labels = json.loads( - simcore_service_labels.dict().get("io.simcore.outputs", "{}") + simcore_service_labels.model_dump().get("io.simcore.outputs", "{}") ).get("outputs", {}) _logger.debug( "Creating dirs from service outputs labels: %s", @@ -563,7 +563,7 @@ async def get_allow_metrics_collection( bool, AllowMetricsCollectionFrontendUserPreference.get_default_value() ) - allow_metrics_collection = AllowMetricsCollectionFrontendUserPreference.parse_obj( - preference + allow_metrics_collection = ( + AllowMetricsCollectionFrontendUserPreference.model_validate(preference) ) return allow_metrics_collection.value diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index b68467a572d..04853661c47 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -40,7 +40,7 @@ from models_library.services_types import ServicePortKey from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import AnyHttpUrl, NonNegativeFloat +from pydantic import NonNegativeFloat from servicelib.background_task import ( cancel_task, start_periodic_task, @@ -455,7 +455,7 @@ async def retrieve_service_inputs( service_name = self._inverse_search_mapping[node_uuid] scheduler_data: SchedulerData = self._to_observe[service_name] - dynamic_sidecar_endpoint: AnyHttpUrl = scheduler_data.endpoint + dynamic_sidecar_endpoint = scheduler_data.endpoint sidecars_client: SidecarsClient = await get_sidecars_client(self.app, node_uuid) started = time.time() diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index b0335677084..5a4a011a874 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -76,7 +76,7 @@ def create_model_from_scheduler_data( service_state: ServiceState, service_message: str, ) -> RunningDynamicServiceDetails: - return RunningDynamicServiceDetails.parse_obj( + return RunningDynamicServiceDetails.model_validate( { "boot_type": ServiceBootType.V2, "user_id": scheduler_data.user_id, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py index 2f1ac9b6548..85e56d52e63 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/instrumentation/_models.py @@ -2,7 +2,7 @@ from typing import Final from prometheus_client import CollectorRegistry, Histogram -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.instrumentation import MetricsBase, get_metrics_namespace from ..._meta import PROJECT_NAME @@ -31,7 +31,7 @@ _RATE_BPS_BUCKETS: Final[tuple[float, ...]] = tuple( - parse_obj_as(ByteSize, f"{m}MiB") + TypeAdapter(ByteSize).validate_python(f"{m}MiB") for m in ( 1, 30, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/_api_auth_rpc.py b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/_api_auth_rpc.py index d86f09ec9c2..c9edc8c0f1c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/_api_auth_rpc.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables/_api_auth_rpc.py @@ -6,7 +6,7 @@ from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from ..rabbitmq import get_rabbitmq_rpc_client @@ -26,10 +26,10 @@ async def get_or_create_api_key_and_secret( rpc_client = get_rabbitmq_rpc_client(app) result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_or_create_api_keys"), + TypeAdapter(RPCMethodName).validate_python("get_or_create_api_keys"), product_name=product_name, user_id=user_id, name=name, expiration=expiration, ) - return ApiKeyGet.parse_obj(result) + return ApiKeyGet.model_validate(result) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/projects_networks.py b/services/director-v2/src/simcore_service_director_v2/modules/projects_networks.py index cba005a92ae..e18dfc24121 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/projects_networks.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/projects_networks.py @@ -17,7 +17,7 @@ from models_library.service_settings_labels import SimcoreServiceLabels from models_library.services import ServiceKeyVersion from models_library.users import UserID -from pydantic import ValidationError, parse_obj_as +from pydantic import TypeAdapter, ValidationError from servicelib.rabbitmq import RabbitMQClient from servicelib.utils import logged_gather @@ -45,7 +45,7 @@ class _ToAdd(NamedTuple): def _network_name(project_id: ProjectID, user_defined: str) -> DockerNetworkName: network_name = f"{PROJECT_NETWORK_PREFIX}_{project_id}_{user_defined}" - return parse_obj_as(DockerNetworkName, network_name) + return TypeAdapter(DockerNetworkName).validate_python(network_name) async def requires_dynamic_sidecar( @@ -64,7 +64,7 @@ async def requires_dynamic_sidecar( simcore_service_labels: SimcoreServiceLabels = ( await director_v0_client.get_service_labels( - service=ServiceKeyVersion.parse_obj( + service=ServiceKeyVersion.model_validate( {"key": decoded_service_key, "version": service_version} ) ) @@ -184,10 +184,10 @@ async def _get_networks_with_aliases_for_default_network( be on the same network. Return an updated version of the projects_networks """ - new_networks_with_aliases: NetworksWithAliases = NetworksWithAliases.parse_obj({}) + new_networks_with_aliases: NetworksWithAliases = NetworksWithAliases.model_validate({}) default_network = _network_name(project_id, "default") - new_networks_with_aliases[default_network] = ContainerAliases.parse_obj({}) + new_networks_with_aliases[default_network] = ContainerAliases.model_validate({}) for node_uuid, node_content in new_workbench.items(): # only add dynamic-sidecar nodes @@ -200,7 +200,7 @@ async def _get_networks_with_aliases_for_default_network( # only add if network label is valid, otherwise it will be skipped try: - network_alias = parse_obj_as(DockerNetworkAlias, node_content.label) + network_alias = TypeAdapter(DockerNetworkAlias).validate_python(node_content.label) except ValidationError: message = LoggerRabbitMessage( user_id=user_id, @@ -248,7 +248,7 @@ async def update_from_workbench( ) ) except ProjectNetworkNotFoundError: - existing_projects_networks = ProjectsNetworks.parse_obj( + existing_projects_networks = ProjectsNetworks.model_validate( {"project_uuid": project_id, "networks_with_aliases": {}} ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py b/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py index 2563a4133d7..dcda51ad0e5 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py @@ -22,7 +22,7 @@ async def handler_out_of_credits(app: FastAPI, data: bytes) -> bool: - message = WalletCreditsLimitReachedMessage.parse_raw(data) + message = WalletCreditsLimitReachedMessage.model_validate_json(data) scheduler: "DynamicSidecarsScheduler" = app.state.dynamic_sidecar_scheduler # type: ignore[name-defined] # noqa: F821 settings: AppSettings = app.state.settings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py b/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py index 2c546ea3d84..f1b4280bdcc 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py @@ -23,7 +23,6 @@ ) from models_library.services import ServiceKey, ServiceVersion from models_library.wallets import WalletID -from pydantic import parse_obj_as from ..core.errors import PricingPlanUnitNotFoundError from ..core.settings import AppSettings @@ -92,7 +91,7 @@ async def get_default_service_pricing_plan( raise PricingPlanUnitNotFoundError(msg) response.raise_for_status() - return parse_obj_as(PricingPlanGet, response.json()) + return PricingPlanGet.model_validate(response.json()) async def get_default_pricing_and_hardware_info( self, @@ -130,7 +129,7 @@ async def get_pricing_unit( }, ) response.raise_for_status() - return parse_obj_as(PricingUnitGet, response.json()) + return PricingUnitGet.model_validate(response.json()) async def get_wallet_credits( self, @@ -142,7 +141,7 @@ async def get_wallet_credits( params={"product_name": product_name, "wallet_id": wallet_id}, ) response.raise_for_status() - return parse_obj_as(WalletTotalCredits, response.json()) + return WalletTotalCredits.model_validate(response.json()) # # app diff --git a/services/director-v2/src/simcore_service_director_v2/modules/storage.py b/services/director-v2/src/simcore_service_director_v2/modules/storage.py index 98e18845333..b0cbb5e9629 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/storage.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/storage.py @@ -72,5 +72,5 @@ async def get_s3_access(self, user_id: UserID) -> S3Settings: ) resp.raise_for_status() if resp.status_code == status.HTTP_200_OK: - return S3Settings.parse_obj(unenvelope_or_raise_error(resp)) + return S3Settings.model_validate(unenvelope_or_raise_error(resp)) raise HTTPException(status_code=resp.status_code, detail=resp.content) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/clients.py b/services/director-v2/src/simcore_service_director_v2/utils/clients.py index d01d38a1907..e12cf2d09f0 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/clients.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/clients.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Union +from typing import Any import httpx from fastapi import HTTPException @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -def unenvelope_or_raise_error(resp: httpx.Response) -> Union[list[Any], dict[str, Any]]: +def unenvelope_or_raise_error(resp: httpx.Response) -> list[Any] | dict[str, Any]: """ Director responses are enveloped If successful response, we un-envelop it and return data as a dict diff --git a/services/director-v2/src/simcore_service_director_v2/utils/comp_scheduler.py b/services/director-v2/src/simcore_service_director_v2/utils/comp_scheduler.py index e2310c4914a..15f3481da10 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/comp_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/comp_scheduler.py @@ -66,7 +66,7 @@ def create_service_resources_from_task(task: CompTaskAtDB) -> ServiceResourcesDi DockerGenericTag(f"{task.image.name}:{task.image.tag}"), { res_name: ResourceValue(limit=res_value, reservation=res_value) - for res_name, res_value in task.image.node_requirements.dict( + for res_name, res_value in task.image.node_requirements.model_dump( by_alias=True ).items() if res_value is not None diff --git a/services/director-v2/src/simcore_service_director_v2/utils/computations.py b/services/director-v2/src/simcore_service_director_v2/utils/computations.py index 0abbc18f593..93e3197c71b 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/computations.py @@ -1,12 +1,12 @@ -import datetime +import datetime as dt import logging from typing import Any +import arrow from models_library.projects_state import RunningState from models_library.services import ServiceKeyVersion from models_library.services_regex import SERVICE_KEY_RE from models_library.users import UserID -from pydantic import parse_obj_as from servicelib.utils import logged_gather from ..models.comp_tasks import CompTaskAtDB @@ -123,18 +123,18 @@ async def find_deprecated_tasks( ) ) service_key_version_to_details = { - ServiceKeyVersion.construct( + ServiceKeyVersion.model_construct( key=details["key"], version=details["version"] ): details for details in services_details } - today = datetime.datetime.now(tz=datetime.timezone.utc) + today = dt.datetime.now(tz=dt.UTC) def _is_service_deprecated(service: dict[str, Any]) -> bool: if deprecation_date := service.get("deprecated"): - deprecation_date = parse_obj_as( - datetime.datetime, deprecation_date - ).replace(tzinfo=datetime.timezone.utc) + deprecation_date = arrow.get(deprecation_date).datetime.replace( + tzinfo=dt.UTC + ) is_deprecated: bool = today > deprecation_date return is_deprecated return False diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 452f1ba50a9..36075eb3bf3 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -2,7 +2,7 @@ import collections import logging from collections.abc import Awaitable, Callable, Coroutine, Generator -from typing import Any, Final, NoReturn, Optional, ParamSpec, TypeVar, cast, get_args +from typing import Any, Final, NoReturn, ParamSpec, TypeVar, cast, get_args from uuid import uuid4 import dask_gateway # type: ignore[import-untyped] @@ -30,7 +30,7 @@ from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, ValidationError, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter, ValidationError from servicelib.logging_utils import log_catch, log_context from simcore_sdk import node_ports_v2 from simcore_sdk.node_ports_common.exceptions import ( @@ -61,7 +61,7 @@ ServiceKeyStr = str ServiceVersionStr = str -_PVType = Optional[_NPItemValue] +_PVType = _NPItemValue | None assert len(get_args(_PVType)) == len( # nosec get_args(PortValue) @@ -73,7 +73,7 @@ def _get_port_validation_errors(port_key: str, err: ValidationError) -> list[Err for error in errors: assert error["loc"][-1] != (port_key,) error["loc"] = error["loc"] + (port_key,) - return errors + return list(errors) def generate_dask_job_id( @@ -134,7 +134,7 @@ async def create_node_ports( db_manager=db_manager, ) except ValidationError as err: - raise PortsValidationError(project_id, node_id, err.errors()) from err + raise PortsValidationError(project_id, node_id, list(err.errors())) from err async def parse_output_data( @@ -229,7 +229,7 @@ async def compute_input_data( if ports_errors: raise PortsValidationError(project_id, node_id, ports_errors) - return TaskInputData.parse_obj(input_data) + return TaskInputData.model_validate(input_data) async def compute_output_data_schema( @@ -276,7 +276,7 @@ async def compute_output_data_schema( } ) - return TaskOutputDataSchema.parse_obj(output_data_schema) + return TaskOutputDataSchema.model_validate(output_data_schema) _LOGS_FILE_NAME = "logs.zip" @@ -314,7 +314,7 @@ def compute_task_labels( ValidationError """ product_name = run_metadata.get("product_name", UNDEFINED_DOCKER_LABEL) - standard_simcore_labels = StandardSimcoreDockerLabels.construct( + standard_simcore_labels = StandardSimcoreDockerLabels.model_construct( user_id=user_id, project_id=project_id, node_id=node_id, @@ -326,8 +326,7 @@ def compute_task_labels( memory_limit=node_requirements.ram, cpu_limit=node_requirements.cpu, ).to_simcore_runtime_docker_labels() - return standard_simcore_labels | parse_obj_as( - ContainerLabelsDict, + return standard_simcore_labels | TypeAdapter(ContainerLabelsDict).validate_python( { DockerLabelKey.from_key(k): f"{v}" for k, v in run_metadata.items() @@ -470,7 +469,7 @@ def from_node_reqs_to_dask_resources( node_reqs: NodeRequirements, ) -> dict[str, int | float]: """Dask resources are set such as {"CPU": X.X, "GPU": Y.Y, "RAM": INT}""" - dask_resources: dict[str, int | float] = node_reqs.dict( + dask_resources: dict[str, int | float] = node_reqs.model_dump( exclude_unset=True, by_alias=True, exclude_none=True, @@ -552,9 +551,9 @@ def _to_human_readable_resource_values(resources: dict[str, Any]) -> dict[str, A for res_name, res_value in resources.items(): if "RAM" in res_name: try: - human_readable_resources[res_name] = parse_obj_as( - ByteSize, res_value - ).human_readable() + human_readable_resources[res_name] = ( + TypeAdapter(ByteSize).validate_python(res_value).human_readable() + ) except ValidationError: _logger.warning( "could not parse %s:%s, please check what changed in how Dask prepares resources!", diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py b/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py index 2deb203780b..15e6e98dfce 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py @@ -216,7 +216,7 @@ async def test_scheduler_endpoint( try: if _is_dask_scheduler(authentication): async with distributed.Client( - address=endpoint, timeout=f"{_PING_TIMEOUT_S}", asynchronous=True + address=f"{endpoint}", timeout=f"{_PING_TIMEOUT_S}", asynchronous=True ) as dask_client: if dask_client.status != _DASK_SCHEDULER_RUNNING_STATE: msg = "internal scheduler is not running!" diff --git a/services/director-v2/src/simcore_service_director_v2/utils/db.py b/services/director-v2/src/simcore_service_director_v2/utils/db.py index b4240a1289b..af944c11dff 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/db.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/db.py @@ -1,11 +1,10 @@ -import json import logging from typing import Any +from common_library.serialization import model_dump_with_secrets from fastapi import FastAPI from models_library.clusters import BaseCluster from models_library.projects_state import RunningState -from settings_library.utils_encoders import create_json_encoder_wo_secrets from simcore_postgres_database.models.comp_pipeline import StateType from ..api.dependencies.database import RepoType, get_base_repository @@ -30,14 +29,13 @@ def to_clusters_db(cluster: BaseCluster, *, only_update: bool) -> dict[str, Any]: - db_model: dict[str, Any] = json.loads( - cluster.json( - by_alias=True, - exclude={"id", "access_rights"}, - exclude_unset=only_update, - exclude_none=only_update, - encoder=create_json_encoder_wo_secrets(BaseCluster), - ) + db_model: dict[str, Any] = model_dump_with_secrets( + cluster, + show_secrets=True, + by_alias=True, + exclude={"id", "access_rights"}, + exclude_unset=only_update, + exclude_none=only_update, ) return db_model diff --git a/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py index 0dfef24cfef..6704c8369ef 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py @@ -4,7 +4,7 @@ from typing import Any, Final, NamedTuple, TypeAlias from models_library.utils.specs_substitution import SubstitutionValue -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt, TypeAdapter from servicelib.utils import logged_gather ContextDict: TypeAlias = dict[str, Any] @@ -89,7 +89,7 @@ def copy( return {k: self._variables_getters[k] for k in selection} -_HANDLERS_TIMEOUT: Final[NonNegativeInt] = parse_obj_as(NonNegativeInt, 4) +_HANDLERS_TIMEOUT: Final[NonNegativeInt] = 4 async def resolve_variables_from_context( diff --git a/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py b/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py index 57d014a3c0f..70249d3c1da 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py @@ -31,7 +31,7 @@ async def publish_service_started_metrics( simcore_user_agent: str, task: CompTaskAtDB, ) -> None: - message = InstrumentationRabbitMessage.construct( + message = InstrumentationRabbitMessage.model_construct( metrics="service_started", user_id=user_id, project_id=task.project_id, @@ -53,7 +53,7 @@ async def publish_service_stopped_metrics( task: CompTaskAtDB, task_final_state: RunningState, ) -> None: - message = InstrumentationRabbitMessage.construct( + message = InstrumentationRabbitMessage.model_construct( metrics="service_stopped", user_id=user_id, project_id=task.project_id, @@ -153,7 +153,7 @@ async def publish_service_log( log: str, log_level: LogLevelInt, ) -> None: - message = LoggerRabbitMessage.construct( + message = LoggerRabbitMessage.model_construct( user_id=user_id, project_id=project_id, node_id=node_id, @@ -172,7 +172,7 @@ async def publish_service_progress( node_id: NodeID, progress: NonNegativeFloat, ) -> None: - message = ProgressRabbitMessageNode.construct( + message = ProgressRabbitMessageNode.model_construct( user_id=user_id, project_id=project_id, node_id=node_id, @@ -188,7 +188,7 @@ async def publish_project_log( log: str, log_level: LogLevelInt, ) -> None: - message = LoggerRabbitMessage.construct( + message = LoggerRabbitMessage.model_construct( user_id=user_id, project_id=project_id, node_id=None, diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index db64158d6d5..d04ff9134aa 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -198,7 +198,7 @@ def mock_env( async def client(mock_env: EnvVarsDict) -> AsyncIterator[TestClient]: settings = AppSettings.create_from_envs() app = init_app(settings) - print("Application settings\n", settings.json(indent=2)) + print("Application settings\n", settings.model_dump_json(indent=2)) # NOTE: this way we ensure the events are run in the application # since it starts the app on a test server with TestClient(app, raise_server_exceptions=True) as test_client: @@ -209,7 +209,7 @@ async def client(mock_env: EnvVarsDict) -> AsyncIterator[TestClient]: async def initialized_app(mock_env: EnvVarsDict) -> AsyncIterable[FastAPI]: settings = AppSettings.create_from_envs() app = init_app(settings) - print("Application settings\n", settings.json(indent=2)) + print("Application settings\n", settings.model_dump_json(indent=2)) async with LifespanManager(app): yield app @@ -239,7 +239,7 @@ def fake_workbench(fake_workbench_file: Path) -> NodesDict: workbench_dict = json.loads(fake_workbench_file.read_text()) workbench = {} for node_id, node_data in workbench_dict.items(): - workbench[node_id] = Node.parse_obj(node_data) + workbench[node_id] = Node.model_validate(node_data) return workbench @@ -336,7 +336,9 @@ async def wrapper(*args, **kwargs): @pytest.fixture def mock_osparc_variables_api_auth_rpc(mocker: MockerFixture) -> None: - fake_data = ApiKeyGet.parse_obj(ApiKeyGet.Config.schema_extra["examples"][0]) + fake_data = ApiKeyGet.model_validate( + ApiKeyGet.model_config["json_schema_extra"]["examples"][0] + ) async def _create( app: FastAPI, diff --git a/services/director-v2/tests/helpers/shared_comp_utils.py b/services/director-v2/tests/helpers/shared_comp_utils.py index ad7185e8fa7..8ee507f4a2b 100644 --- a/services/director-v2/tests/helpers/shared_comp_utils.py +++ b/services/director-v2/tests/helpers/shared_comp_utils.py @@ -43,8 +43,8 @@ async def assert_computation_task_out_obj( assert task_out.iteration == iteration assert task_out.cluster_id == cluster_id # check pipeline details contents - received_task_out_pipeline = task_out.pipeline_details.dict() - expected_task_out_pipeline = exp_pipeline_details.dict() + received_task_out_pipeline = task_out.pipeline_details.model_dump() + expected_task_out_pipeline = exp_pipeline_details.model_dump() assert received_task_out_pipeline == expected_task_out_pipeline @@ -64,11 +64,11 @@ async def assert_and_wait_for_pipeline_status( MAX_TIMEOUT_S = 5 * MINUTE async def check_pipeline_state() -> ComputationGet: - response = await client.get(url, params={"user_id": user_id}) + response = await client.get(f"{url}", params={"user_id": user_id}) assert ( response.status_code == status.HTTP_200_OK ), f"response code is {response.status_code}, error: {response.text}" - task_out = ComputationGet.parse_obj(response.json()) + task_out = ComputationGet.model_validate(response.json()) assert task_out.id == project_uuid assert task_out.url.path == f"/v2/computations/{project_uuid}" print( @@ -100,4 +100,5 @@ async def check_pipeline_state() -> ComputationGet: return task_out # this is only to satisfy pylance - raise AssertionError("No computation task generated!") + msg = "No computation task generated!" + raise AssertionError(msg) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 16a6311da1b..47f130a98be 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -67,7 +67,7 @@ def mock_env( "COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED": "1", "COMPUTATIONAL_BACKEND_ENABLED": "1", "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL": dask_scheduler_service, - "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": dask_scheduler_auth.json(), + "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": dask_scheduler_auth.model_dump_json(), "DYNAMIC_SIDECAR_IMAGE": dynamic_sidecar_docker_image_name, "SIMCORE_SERVICES_NETWORK_NAME": "test_swarm_network_name", "SWARM_STACK_NAME": "test_mocked_stack_name", @@ -109,7 +109,7 @@ def fake_workbench_computational_pipeline_details( ) -> PipelineDetails: adjacency_list = json.loads(fake_workbench_computational_adjacency_file.read_text()) node_states = json.loads(fake_workbench_node_states_file.read_text()) - return PipelineDetails.parse_obj( + return PipelineDetails.model_validate( {"adjacency_list": adjacency_list, "node_states": node_states, "progress": 0} ) @@ -720,7 +720,7 @@ async def test_abort_computation( assert ( response.status_code == status.HTTP_202_ACCEPTED ), f"response code is {response.status_code}, error: {response.text}" - task_out = ComputationGet.parse_obj(response.json()) + task_out = ComputationGet.model_validate(response.json()) assert task_out.url.path == f"/v2/computations/{sleepers_project.uuid}:stop" assert task_out.stop_url is None diff --git a/services/director-v2/tests/integration/02/conftest.py b/services/director-v2/tests/integration/02/conftest.py index 0d0df8a402f..1cc8e4fd64e 100644 --- a/services/director-v2/tests/integration/02/conftest.py +++ b/services/director-v2/tests/integration/02/conftest.py @@ -5,6 +5,7 @@ from uuid import uuid4 import aiodocker +from pydantic import TypeAdapter import pytest from models_library.api_schemas_resource_usage_tracker.pricing_plans import ( PricingPlanGet, @@ -14,7 +15,6 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import parse_obj_as from pytest_mock.plugin import MockerFixture @@ -64,7 +64,7 @@ def mock_projects_networks_repository(mocker: MockerFixture) -> None: "simcore_service_director_v2.modules.db.repositories." "projects_networks.ProjectsNetworksRepository.get_projects_networks" ), - return_value=ProjectsNetworks.parse_obj( + return_value=ProjectsNetworks.model_validate( {"project_uuid": uuid4(), "networks_with_aliases": {}} ), ) @@ -72,17 +72,16 @@ def mock_projects_networks_repository(mocker: MockerFixture) -> None: @pytest.fixture def service_resources() -> ServiceResourcesDict: - return parse_obj_as( - ServiceResourcesDict, - ServiceResourcesDictHelpers.Config.schema_extra["examples"][0], + return TypeAdapter(ServiceResourcesDict).validate_python( + ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], ) @pytest.fixture def mock_resource_usage_tracker(mocker: MockerFixture) -> None: base_module = "simcore_service_director_v2.modules.resource_usage_tracker_client" - service_pricing_plan = PricingPlanGet.parse_obj( - PricingPlanGet.Config.schema_extra["examples"][1] + service_pricing_plan = PricingPlanGet.model_validate( + PricingPlanGet.model_config["json_schema_extra"]["examples"][1] ) for unit in service_pricing_plan.pricing_units: unit.specific_info.aws_ec2_instances.clear() diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 720e7d0c3e1..a160afd664f 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -47,7 +47,7 @@ from models_library.projects_pipeline import PipelineDetails from models_library.projects_state import RunningState from models_library.users import UserID -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -403,7 +403,7 @@ def mock_env( "COMPUTATIONAL_BACKEND_ENABLED": "true", "COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED": "true", "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL": dask_scheduler_service, - "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": dask_scheduler_auth.json(), + "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": dask_scheduler_auth.model_dump_json(), "DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED": "1", }, ) @@ -446,13 +446,13 @@ async def projects_networks_db( # NOTE: director-v2 does not have access to the webserver which creates this # injecting all dynamic-sidecar started services on a default networks - container_aliases: ContainerAliases = ContainerAliases.parse_obj({}) + container_aliases: ContainerAliases = ContainerAliases.model_validate({}) for k, (node_uuid, node) in enumerate(current_study.workbench.items()): if not is_legacy(node): container_aliases[node_uuid] = f"networkable_alias_{k}" - networks_with_aliases: NetworksWithAliases = NetworksWithAliases.parse_obj({}) + networks_with_aliases: NetworksWithAliases = NetworksWithAliases.model_validate({}) default_network_name = f"{PROJECT_NETWORK_PREFIX}_{current_study.uuid}_test" networks_with_aliases[default_network_name] = container_aliases @@ -463,7 +463,7 @@ async def projects_networks_db( engine: Engine = initialized_app.state.engine async with engine.acquire() as conn: - row_data = projects_networks_to_insert.dict() + row_data = projects_networks_to_insert.model_dump() insert_stmt = pg_insert(projects_networks).values(**row_data) upsert_snapshot = insert_stmt.on_conflict_do_update( constraint=projects_networks.primary_key, set_=row_data @@ -841,7 +841,9 @@ async def _debug_progress_callback( Client( app=initialized_app, async_client=director_v2_client, - base_url=parse_obj_as(AnyHttpUrl, f"{director_v2_client.base_url}"), + base_url=TypeAdapter(AnyHttpUrl).validate_python( + f"{director_v2_client.base_url}" + ), ), task_id, task_timeout=60, @@ -972,7 +974,7 @@ async def test_nodeports_integration( task_out, project=current_study, exp_task_state=RunningState.SUCCESS, - exp_pipeline_details=PipelineDetails.parse_obj(fake_dy_success), + exp_pipeline_details=PipelineDetails.model_validate(fake_dy_success), iteration=1, cluster_id=DEFAULT_CLUSTER_ID, ) diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 0c5f10c07bd..02f6358e426 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -20,7 +20,7 @@ ServiceResourcesDictHelpers, ) from models_library.users import UserID -from pydantic import PositiveInt, parse_obj_as +from pydantic import PositiveInt, TypeAdapter from pytest_simcore.helpers.host import get_localhost_ip from servicelib.common_headers import ( X_DYNAMIC_SIDECAR_REQUEST_DNS, @@ -263,7 +263,9 @@ async def patch_dynamic_service_url(app: FastAPI, node_uuid: str) -> str: proxy_service_name, target_port=dynamic_sidecar_proxy_settings.DYNAMIC_SIDECAR_CADDY_ADMIN_API_PORT, ) - assert proxy_published_port is not None, f"{sidecar_settings.json()=}" + assert ( + proxy_published_port is not None + ), f"{sidecar_settings.model_dump_json()=}" async with scheduler.scheduler._lock: # noqa: SLF001 localhost_ip = get_localhost_ip() @@ -303,7 +305,7 @@ async def _get_service_resources( url = f"{catalog_url}/v0/services/{encoded_key}/{service_version}/resources" async with httpx.AsyncClient() as client: response = await client.get(f"{url}") - return parse_obj_as(ServiceResourcesDict, response.json()) + return TypeAdapter(ServiceResourcesDict).validate_python(response.json()) async def _handle_redirection( @@ -458,7 +460,7 @@ async def assert_retrieve_service( size_bytes = json_result["data"]["size_bytes"] assert size_bytes > 0 - assert type(size_bytes) == int + assert isinstance(size_bytes, int) async def assert_stop_service( diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 0e6f8632094..424cb128257 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -98,7 +98,7 @@ async def _creator( response.raise_for_status() assert response.status_code == status.HTTP_201_CREATED - computation_task = ComputationGet.parse_obj(response.json()) + computation_task = ComputationGet.model_validate(response.json()) created_comp_tasks.append((user_id, computation_task)) return computation_task diff --git a/services/director-v2/tests/mocks/fake_task.json b/services/director-v2/tests/mocks/fake_task.json index b26ebfa9ba5..00a9dfe3501 100644 --- a/services/director-v2/tests/mocks/fake_task.json +++ b/services/director-v2/tests/mocks/fake_task.json @@ -60,6 +60,7 @@ "end": "2008-03-24T07:02:09.279Z", "created": "1961-07-06T11:24:30.877Z", "modified": "2008-03-24T07:02:09.279Z", + "pricing_info": null, "last_heartbeat": null, "hardware_info": { "aws_ec2_instances": [] diff --git a/services/director-v2/tests/unit/conftest.py b/services/director-v2/tests/unit/conftest.py index ecd7da59544..2123a506ad7 100644 --- a/services/director-v2/tests/unit/conftest.py +++ b/services/director-v2/tests/unit/conftest.py @@ -27,7 +27,8 @@ from models_library.service_settings_labels import SimcoreServiceLabels from models_library.services import RunID, ServiceKey, ServiceKeyVersion, ServiceVersion from models_library.services_enums import ServiceState -from pydantic import parse_obj_as +from models_library.utils._original_fastapi_encoders import jsonable_encoder +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.s3 import S3Settings @@ -52,17 +53,17 @@ def simcore_services_network_name() -> str: @pytest.fixture def simcore_service_labels() -> SimcoreServiceLabels: - simcore_service_labels = SimcoreServiceLabels.parse_obj( - SimcoreServiceLabels.Config.schema_extra["examples"][1] + simcore_service_labels = SimcoreServiceLabels.model_validate( + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][1] ) - simcore_service_labels.callbacks_mapping = parse_obj_as(CallbacksMapping, {}) + simcore_service_labels.callbacks_mapping = CallbacksMapping.model_validate({}) return simcore_service_labels @pytest.fixture def dynamic_service_create() -> DynamicServiceCreate: - return DynamicServiceCreate.parse_obj( - DynamicServiceCreate.Config.schema_extra["example"] + return DynamicServiceCreate.model_validate( + DynamicServiceCreate.model_config["json_schema_extra"]["example"] ) @@ -123,7 +124,7 @@ def scheduler_data_from_http_request( def mock_service_inspect( scheduler_data_from_http_request: ServiceDetails, ) -> Mapping[str, Any]: - service_details = json.loads(scheduler_data_from_http_request.json()) + service_details = json.loads(scheduler_data_from_http_request.model_dump_json()) service_details["compose_spec"] = json.dumps(service_details["compose_spec"]) return { "Spec": { @@ -200,7 +201,7 @@ def mocked_storage_service_api( respx_mock.post( "/simcore-s3:access", name="get_or_create_temporary_s3_access", - ).respond(json={"data": fake_s3_settings.dict(by_alias=True)}) + ).respond(json=jsonable_encoder({"data": fake_s3_settings}, by_alias=True)) yield respx_mock @@ -211,8 +212,10 @@ def mocked_storage_service_api( @pytest.fixture def mock_service_key_version() -> ServiceKeyVersion: return ServiceKeyVersion( - key=parse_obj_as(ServiceKey, "simcore/services/dynamic/myservice"), - version=parse_obj_as(ServiceVersion, "1.4.5"), + key=TypeAdapter(ServiceKey).validate_python( + "simcore/services/dynamic/myservice" + ), + version=TypeAdapter(ServiceVersion).validate_python("1.4.5"), ) @@ -221,7 +224,7 @@ def fake_service_specifications(faker: Faker) -> dict[str, Any]: # the service specifications follow the Docker service creation available # https://docs.docker.com/engine/api/v1.41/#operation/ServiceCreate return { - "sidecar": DockerServiceSpec.parse_obj( + "sidecar": DockerServiceSpec.model_validate( { "Labels": {"label_one": faker.pystr(), "label_two": faker.pystr()}, "TaskTemplate": { diff --git a/services/director-v2/tests/unit/test_core_settings.py b/services/director-v2/tests/unit/test_core_settings.py index 84d99057f3a..2151d64cfa5 100644 --- a/services/director-v2/tests/unit/test_core_settings.py +++ b/services/director-v2/tests/unit/test_core_settings.py @@ -5,9 +5,10 @@ from typing import Any import pytest -from models_library.basic_types import LogLevel +from models_library.basic_types import BootModeEnum, LogLevel from pydantic import ValidationError from pytest_simcore.helpers.typing_env import EnvVarsDict +from settings_library.base import DefaultFromEnvFactoryError from settings_library.r_clone import S3Provider from simcore_service_director_v2.core.dynamic_services_settings.egress_proxy import ( EnvoyLogLevel, @@ -17,7 +18,7 @@ PlacementSettings, RCloneSettings, ) -from simcore_service_director_v2.core.settings import AppSettings, BootModeEnum +from simcore_service_director_v2.core.settings import AppSettings def _get_backend_type_options() -> set[str]: @@ -43,7 +44,7 @@ def test_enforce_r_clone_requirement(monkeypatch: pytest.MonkeyPatch) -> None: def test_settings_with_project_env_devel(project_env_devel_environment: dict[str, Any]): # loads from environ settings = AppSettings.create_from_envs() - print("captured settings: \n", settings.json(indent=2)) + print("captured settings: \n", settings.model_dump_json(indent=2)) assert settings.SC_BOOT_MODE == BootModeEnum.DEBUG assert settings.LOG_LEVEL == LogLevel.DEBUG @@ -60,7 +61,7 @@ def test_settings_with_repository_env_devel( ) # defined in docker-compose settings = AppSettings.create_from_envs() - print("captured settings: \n", settings.json(indent=2)) + print("captured settings: \n", settings.model_dump_json(indent=2)) assert settings @@ -185,7 +186,7 @@ def test_services_custom_constraint_failures( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS", custom_constraints) - with pytest.raises(Exception): + with pytest.raises(DefaultFromEnvFactoryError): AppSettings.create_from_envs() diff --git a/services/director-v2/tests/unit/test_models_clusters.py b/services/director-v2/tests/unit/test_models_clusters.py index 0a5d29283bb..b08a988fc68 100644 --- a/services/director-v2/tests/unit/test_models_clusters.py +++ b/services/director-v2/tests/unit/test_models_clusters.py @@ -13,7 +13,7 @@ WorkerMetrics, ) from models_library.clusters import ClusterTypeInModel -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, ByteSize, TypeAdapter from simcore_postgres_database.models.clusters import ClusterType @@ -61,13 +61,12 @@ def test_scheduler_constructor_with_no_workers_has_correct_dict(faker: Faker): def test_worker_constructor_corrects_negative_used_resources(faker: Faker): worker = Worker( - id=faker.pyint(min_value=1), + id=f"{faker.pyint(min_value=1)}", name=faker.name(), - resources=parse_obj_as(AvailableResources, {}), - used_resources=parse_obj_as(UsedResources, {"CPU": -0.0000234}), - memory_limit=faker.pyint(min_value=1), - metrics=parse_obj_as( - WorkerMetrics, + resources=TypeAdapter(AvailableResources).validate_python({}), + used_resources=TypeAdapter(UsedResources).validate_python({"CPU": -0.0000234}), + memory_limit=ByteSize(faker.pyint(min_value=1)), + metrics=WorkerMetrics.model_validate( { "cpu": faker.pyfloat(min_value=0), "memory": faker.pyint(min_value=0), diff --git a/services/director-v2/tests/unit/test_models_dynamic_services.py b/services/director-v2/tests/unit/test_models_dynamic_services.py index dd0df8a0eed..99a22ece3bb 100644 --- a/services/director-v2/tests/unit/test_models_dynamic_services.py +++ b/services/director-v2/tests/unit/test_models_dynamic_services.py @@ -138,7 +138,7 @@ def test_running_service_details_make_status( print(running_service_details) assert running_service_details - running_service_details_dict = running_service_details.dict( + running_service_details_dict = running_service_details.model_dump( exclude_unset=True, by_alias=True ) @@ -219,7 +219,7 @@ def test_regression_legacy_service_compatibility() -> None: "user_id": "1", "project_id": "b1ec5c8e-f5bb-11eb-b1d5-02420a000006", } - service_details = RunningDynamicServiceDetails.parse_obj(api_response) + service_details = RunningDynamicServiceDetails.model_validate(api_response) assert service_details diff --git a/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py b/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py index 607b8231f78..0bbd9bca526 100644 --- a/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py +++ b/services/director-v2/tests/unit/test_models_schemas_dynamic_services_scheduler.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest -from pydantic import parse_file_as +from pydantic import TypeAdapter from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData @@ -20,13 +20,13 @@ def test_regression_as_label_data(scheduler_data: SchedulerData) -> None: # old tested implementation scheduler_data_copy = deepcopy(scheduler_data) scheduler_data_copy.compose_spec = json.dumps(scheduler_data_copy.compose_spec) - json_encoded = scheduler_data_copy.json() + json_encoded = scheduler_data_copy.model_dump_json() # using pydantic's internals label_data = scheduler_data.as_label_data() - parsed_json_encoded = SchedulerData.parse_raw(json_encoded) - parsed_label_data = SchedulerData.parse_raw(label_data) + parsed_json_encoded = SchedulerData.model_validate_json(json_encoded) + parsed_label_data = SchedulerData.model_validate_json(label_data) assert parsed_json_encoded == parsed_label_data @@ -35,4 +35,6 @@ def test_ensure_legacy_format_compatibility(legacy_scheduler_data_format: Path): # PRs applying changes to the legacy format: # - https://github.com/ITISFoundation/osparc-simcore/pull/3610 - assert parse_file_as(list[SchedulerData], legacy_scheduler_data_format) + assert TypeAdapter(list[SchedulerData]).validate_json( + legacy_scheduler_data_format.read_text() + ) diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 68f8464b829..f45040c143a 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -55,7 +55,6 @@ from models_library.resource_tracker import HardwareInfo from models_library.users import UserID from pydantic import AnyUrl, ByteSize, SecretStr, TypeAdapter -from pydantic.tools import parse_obj_as from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.background_task import periodic_task @@ -164,7 +163,7 @@ async def factory() -> DaskClient: client = await DaskClient.create( app=minimal_app, settings=minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND, - endpoint=parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address), + endpoint=TypeAdapter(AnyUrl).validate_python(dask_spec_local_cluster.scheduler_address), authentication=NoAuthentication(), tasks_file_link_type=tasks_file_link_type, cluster_type=ClusterTypeInModel.ON_PREMISE, @@ -205,7 +204,7 @@ async def factory() -> DaskClient: client = await DaskClient.create( app=minimal_app, settings=minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND, - endpoint=parse_obj_as(AnyUrl, local_dask_gateway_server.address), + endpoint=TypeAdapter(AnyUrl).validate_python(local_dask_gateway_server.address), authentication=SimpleAuthentication( username="pytest_user", password=SecretStr(local_dask_gateway_server.password), @@ -299,7 +298,7 @@ def cpu_image(node_id: NodeID) -> ImageParams: tag="1.5.5", node_requirements=NodeRequirements( CPU=1, - RAM=parse_obj_as(ByteSize, "128 MiB"), + RAM=TypeAdapter(ByteSize).validate_python("128 MiB"), GPU=None, ), ) # type: ignore @@ -327,7 +326,7 @@ def gpu_image(node_id: NodeID) -> ImageParams: node_requirements=NodeRequirements( CPU=1, GPU=1, - RAM=parse_obj_as(ByteSize, "256 MiB"), + RAM=TypeAdapter(ByteSize).validate_python("256 MiB"), ), ) # type: ignore return ImageParams( @@ -367,11 +366,11 @@ def _mocked_node_ports(mocker: MockerFixture) -> None: mocker.patch( "simcore_service_director_v2.modules.dask_client.dask_utils.compute_input_data", - return_value=TaskInputData.parse_obj({}), + return_value=TaskInputData.model_validate({}), ) mocker.patch( "simcore_service_director_v2.modules.dask_client.dask_utils.compute_output_data_schema", - return_value=TaskOutputDataSchema.parse_obj({}), + return_value=TaskOutputDataSchema.model_validate({}), ) mocker.patch( "simcore_service_director_v2.modules.dask_client.dask_utils.compute_service_log_file_upload_link", @@ -470,8 +469,7 @@ def comp_run_metadata(faker: Faker) -> RunMetadataDict: @pytest.fixture def task_labels(comp_run_metadata: RunMetadataDict) -> ContainerLabelsDict: - return parse_obj_as( - ContainerLabelsDict, + return TypeAdapter(ContainerLabelsDict).validate_python( { k.replace("_", "-").lower(): v for k, v in comp_run_metadata.items() @@ -482,7 +480,9 @@ def task_labels(comp_run_metadata: RunMetadataDict) -> ContainerLabelsDict: @pytest.fixture def hardware_info() -> HardwareInfo: - return HardwareInfo.parse_obj(HardwareInfo.Config.schema_extra["examples"][0]) + return HardwareInfo.model_validate( + HardwareInfo.model_config["json_schema_extra"]["examples"][0] + ) @pytest.fixture @@ -529,7 +529,7 @@ def fake_sidecar_fct( event = distributed.Event(_DASK_EVENT_NAME) event.wait(timeout=25) - return TaskOutputData.parse_obj({"some_output_key": 123}) + return TaskOutputData.model_validate({"some_output_key": 123}) # NOTE: We pass another fct so it can run in our localy created dask cluster # NOTE2: since there is only 1 task here, it's ok to pass the nodeID @@ -645,7 +645,7 @@ def fake_sidecar_fct( task = worker.state.tasks.get(worker.get_current_task()) assert task is not None - return TaskOutputData.parse_obj({"some_output_key": 123}) + return TaskOutputData.model_validate({"some_output_key": 123}) # NOTE: We pass another fct so it can run in our localy created dask cluster published_computation_task = await dask_client.send_computation_tasks( @@ -737,7 +737,7 @@ def fake_remote_fct( print("--> raising cancellation error now") raise TaskCancelledError - return TaskOutputData.parse_obj({"some_output_key": 123}) + return TaskOutputData.model_validate({"some_output_key": 123}) published_computation_task = await dask_client.send_computation_tasks( user_id=user_id, @@ -947,7 +947,7 @@ async def test_too_many_resources_send_computation_task( tag="1.4.5", node_requirements=NodeRequirements( CPU=10000000000000000, - RAM=parse_obj_as(ByteSize, "128 MiB"), + RAM=TypeAdapter(ByteSize).validate_python("128 MiB"), GPU=None, ), ) # type: ignore @@ -1083,7 +1083,7 @@ def fake_remote_fct( if fail_remote_fct: err_msg = "We fail because we're told to!" raise ValueError(err_msg) - return TaskOutputData.parse_obj({"some_output_key": 123}) + return TaskOutputData.model_validate({"some_output_key": 123}) published_computation_task = await dask_client.send_computation_tasks( user_id=user_id, @@ -1174,7 +1174,7 @@ def fake_remote_fct( published_event = Event(name=_DASK_START_EVENT) published_event.set() - return TaskOutputData.parse_obj({"some_output_key": 123}) + return TaskOutputData.model_validate({"some_output_key": 123}) # run the computation published_computation_task = await dask_client.send_computation_tasks( @@ -1250,7 +1250,7 @@ def fake_sidecar_fct( event = distributed.Event(_DASK_EVENT_NAME) event.wait(timeout=25) - return TaskOutputData.parse_obj({"some_output_key": 123}) + return TaskOutputData.model_validate({"some_output_key": 123}) # NOTE: We pass another fct so it can run in our localy created dask cluster published_computation_task = await dask_client.send_computation_tasks( diff --git a/services/director-v2/tests/unit/test_modules_dask_clients_pool.py b/services/director-v2/tests/unit/test_modules_dask_clients_pool.py index 3dd97cc4753..3bd1e318878 100644 --- a/services/director-v2/tests/unit/test_modules_dask_clients_pool.py +++ b/services/director-v2/tests/unit/test_modules_dask_clients_pool.py @@ -9,6 +9,8 @@ import pytest from _dask_helpers import DaskGatewayServer +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from distributed.deploy.spec import SpecCluster from faker import Faker from models_library.clusters import ( @@ -24,7 +26,6 @@ from pydantic import SecretStr from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from settings_library.utils_encoders import create_json_encoder_wo_secrets from simcore_postgres_database.models.clusters import ClusterType from simcore_service_director_v2.core.application import init_app from simcore_service_director_v2.core.errors import ( @@ -87,7 +88,7 @@ def creator(num_clusters: int) -> list[Cluster]: fake_clusters = [] for n in range(num_clusters): fake_clusters.append( - Cluster.parse_obj( + Cluster.model_validate( { "id": faker.pyint(), "name": faker.name(), @@ -126,10 +127,15 @@ def creator(): ) monkeypatch.setenv( "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH", - SimpleAuthentication( - username=faker.user_name(), - password=SecretStr(local_dask_gateway_server.password), - ).json(encoder=create_json_encoder_wo_secrets(SimpleAuthentication)), + json_dumps( + model_dump_with_secrets( + SimpleAuthentication( + username=faker.user_name(), + password=SecretStr(local_dask_gateway_server.password), + ), + show_secrets=True, + ) + ), ) return creator @@ -194,11 +200,11 @@ async def test_dask_clients_pool_acquisition_creates_client_on_demand( cluster_type=ClusterTypeInModel.ON_PREMISE, ) ) - async with clients_pool.acquire(cluster) as dask_client: + async with clients_pool.acquire(cluster): # on start it is created mocked_dask_client.create.assert_has_calls(mocked_creation_calls) - async with clients_pool.acquire(cluster) as dask_client: + async with clients_pool.acquire(cluster): # the connection already exists, so there is no new call to create mocked_dask_client.create.assert_has_calls(mocked_creation_calls) @@ -278,5 +284,5 @@ def just_a_quick_fct(x, y): ) future = dask_client.backend.client.submit(just_a_quick_fct, 12, 23) assert future - result = await future.result(timeout=10) # type: ignore + result = await future.result(timeout=10) assert result == 35 diff --git a/services/director-v2/tests/unit/test_modules_db_repositories_services_environments.py b/services/director-v2/tests/unit/test_modules_db_repositories_services_environments.py index 82e397bd3f1..32c39f416ee 100644 --- a/services/director-v2/tests/unit/test_modules_db_repositories_services_environments.py +++ b/services/director-v2/tests/unit/test_modules_db_repositories_services_environments.py @@ -1,5 +1,5 @@ from models_library.osparc_variable_identifier import OsparcVariableIdentifier -from pydantic import parse_obj_as +from pydantic import TypeAdapter from simcore_postgres_database.models.services_environments import VENDOR_SECRET_PREFIX @@ -8,4 +8,4 @@ def test_vendor_secret_names_are_osparc_environments(): # packages simcore_postgres_database and models_library which are indenpendent assert VENDOR_SECRET_PREFIX.endswith("_") - parse_obj_as(OsparcVariableIdentifier, f"${VENDOR_SECRET_PREFIX}FAKE_SECRET") + TypeAdapter(OsparcVariableIdentifier).validate_python(f"${VENDOR_SECRET_PREFIX}FAKE_SECRET") diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py index a38a5a06197..c748fc1cd1b 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py @@ -4,6 +4,9 @@ from contextlib import contextmanager from typing import Any, AsyncIterable, Callable, Iterator from unittest.mock import AsyncMock +from models_library.api_schemas_dynamic_sidecar.containers import ( + ActivityInfoOrNone +) import pytest from common_library.json_serialization import json_dumps @@ -11,7 +14,7 @@ from fastapi import FastAPI, status from httpx import HTTPError, Response from models_library.sidecar_volumes import VolumeCategory, VolumeStatus -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.http_client_thin import ClientHttpError, UnexpectedStatusError @@ -33,7 +36,7 @@ @pytest.fixture def dynamic_sidecar_endpoint() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://missing-host:1111") + return TypeAdapter(AnyHttpUrl).validate_python("http://missing-host:1111") @pytest.fixture @@ -352,21 +355,21 @@ async def test_update_volume_state( @pytest.mark.parametrize( - "mock_json", + "mock_dict", [{"seconds_inactive": 1}, {"seconds_inactive": 0}, None], ) async def test_get_service_activity( get_patched_client: Callable, dynamic_sidecar_endpoint: AnyHttpUrl, - mock_json: dict[str, Any], + mock_dict: dict[str, Any], ) -> None: with get_patched_client( "get_containers_activity", return_value=Response( - status_code=status.HTTP_200_OK, text=json_dumps(mock_json) + status_code=status.HTTP_200_OK, text=json_dumps(mock_dict) ), ) as client: - assert await client.get_service_activity(dynamic_sidecar_endpoint) == mock_json + assert await client.get_service_activity(dynamic_sidecar_endpoint) == TypeAdapter(ActivityInfoOrNone).validate_python(mock_dict) async def test_free_reserved_disk_space( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py index 6403be5a78e..6584020dcb6 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py @@ -11,7 +11,7 @@ from httpx import Response from models_library.services_creation import CreateServiceMetricsAdditionalParams from models_library.sidecar_volumes import VolumeCategory, VolumeStatus -from pydantic import AnyHttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, TypeAdapter from pytest_simcore.helpers.typing_env import EnvVarsDict from respx import MockRouter, Route from respx.types import SideEffectTypes @@ -63,7 +63,7 @@ async def thin_client(mocked_app: FastAPI) -> AsyncIterable[ThinSidecarsClient]: @pytest.fixture def dynamic_sidecar_endpoint() -> AnyHttpUrl: - return parse_obj_as(AnyHttpUrl, "http://missing-host:1111") + return TypeAdapter(AnyHttpUrl).validate_python("http://missing-host:1111") @pytest.fixture @@ -116,7 +116,7 @@ async def test_get_containers( mock_response = Response(status.HTTP_200_OK) mock_request( "GET", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers?only_status={str(only_status).lower()}", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers?only_status={str(only_status).lower()}", mock_response, None, ) @@ -139,7 +139,7 @@ async def test_post_patch_containers_ports_io( mock_response = Response(status.HTTP_204_NO_CONTENT) mock_request( "PATCH", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers/ports/io", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers/ports/io", mock_response, None, ) @@ -162,7 +162,7 @@ async def test_post_containers_ports_outputs_dirs( mock_response = Response(status.HTTP_204_NO_CONTENT) mock_request( "POST", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers/ports/outputs/dirs", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers/ports/outputs/dirs", mock_response, None, ) @@ -191,7 +191,7 @@ async def test_get_containers_name( mock_request( "GET", ( - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}" + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}" f"/containers/name?filters={encoded_filters}" ), mock_response, @@ -216,7 +216,7 @@ async def test_post_containers_networks_attach( container_id = "a_container_id" mock_request( "POST", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers/{container_id}/networks:attach", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers/{container_id}/networks:attach", mock_response, None, ) @@ -239,7 +239,7 @@ async def test_post_containers_networks_detach( container_id = "a_container_id" mock_request( "POST", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers/{container_id}/networks:detach", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers/{container_id}/networks:detach", mock_response, None, ) @@ -262,7 +262,7 @@ async def test_put_volumes( mock_response = Response(status.HTTP_204_NO_CONTENT) mock_request( "PUT", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/volumes/{volume_category}", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/volumes/{volume_category}", mock_response, None, ) @@ -353,7 +353,7 @@ async def test_post_containers_tasks( mock_response = Response(status.HTTP_202_ACCEPTED, json="mocked_task_id") mock_request( "POST", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}{mock_endpoint}", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}{mock_endpoint}", mock_response, None, ) @@ -371,7 +371,7 @@ async def test_get_containers_inactivity( mock_response = Response(status.HTTP_200_OK, json={}) mock_request( "GET", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers/activity", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers/activity", mock_response, None, ) @@ -388,7 +388,7 @@ async def test_post_disk_reserved_free( mock_response = Response(status.HTTP_204_NO_CONTENT) mock_request( "POST", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/disk/reserved:free", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/disk/reserved:free", mock_response, None, ) @@ -405,7 +405,7 @@ async def test_post_containers_compose_spec( mock_response = Response(status.HTTP_202_ACCEPTED) mock_request( "POST", - f"{dynamic_sidecar_endpoint}/{thin_client.API_VERSION}/containers/compose-spec", + f"{dynamic_sidecar_endpoint}{thin_client.API_VERSION}/containers/compose-spec", mock_response, None, ) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py index 8b390e7b973..340c7ad3e44 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_compose_specs.py @@ -6,6 +6,7 @@ from typing import Any from uuid import uuid4 +from pydantic import TypeAdapter import pytest import yaml from models_library.docker import to_simcore_runtime_docker_label_key @@ -21,7 +22,6 @@ ServiceResourcesDict, ) from models_library.users import UserID -from pydantic import parse_obj_as from servicelib.resources import CPU_RESOURCE_LIMIT_KEY, MEM_RESOURCE_LIMIT_KEY from simcore_service_director_v2.modules.dynamic_sidecar import docker_compose_specs @@ -74,8 +74,7 @@ def test_parse_and_export_of_compose_environment_section(): [ pytest.param( {"version": "2.3", "services": {DEFAULT_SINGLE_SERVICE_NAME: {}}}, - parse_obj_as( - ServiceResourcesDict, + TypeAdapter(ServiceResourcesDict).validate_python( { DEFAULT_SINGLE_SERVICE_NAME: { "image": "simcore/services/dynamic/jupyter-math:2.0.5", @@ -90,8 +89,7 @@ def test_parse_and_export_of_compose_environment_section(): ), pytest.param( {"version": "3.7", "services": {DEFAULT_SINGLE_SERVICE_NAME: {}}}, - parse_obj_as( - ServiceResourcesDict, + TypeAdapter(ServiceResourcesDict).validate_python( { DEFAULT_SINGLE_SERVICE_NAME: { "image": "simcore/services/dynamic/jupyter-math:2.0.5", @@ -156,7 +154,7 @@ async def test_inject_resource_limits_and_reservations( [ pytest.param( json.loads( - SimcoreServiceLabels.Config.schema_extra["examples"][2][ + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][2][ "simcore.service.compose-spec" ] ), @@ -200,7 +198,7 @@ def test_regression_service_has_no_reservations(): "version": "3.7", "services": {DEFAULT_SINGLE_SERVICE_NAME: {}}, } - service_resources: ServiceResourcesDict = parse_obj_as(ServiceResourcesDict, {}) + service_resources: ServiceResourcesDict = TypeAdapter(ServiceResourcesDict).validate_python({}) spec_before = deepcopy(service_spec) docker_compose_specs._update_resource_limits_and_reservations( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 13d617ed82d..84711cf0936 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -162,7 +162,11 @@ def mocked_director_v0( ), name="service labels", ).respond( - json={"data": SimcoreServiceLabels.Config.schema_extra["examples"][0]} + json={ + "data": SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][0] + } ) yield mock @@ -525,9 +529,9 @@ async def test_mark_all_services_in_wallet_for_removal( wallet_id = scheduler_data.wallet_info.wallet_id can_remove = scheduler_data.dynamic_sidecar.service_removal_state.can_remove match wallet_id: - case WalletID(1): + case 1: assert can_remove is True - case WalletID(2): + case 2: assert can_remove is False case _: pytest.fail("unexpected case") diff --git a/services/director-v2/tests/unit/test_modules_notifier.py b/services/director-v2/tests/unit/test_modules_notifier.py index 46d0879cebc..cf6d8e1b01c 100644 --- a/services/director-v2/tests/unit/test_modules_notifier.py +++ b/services/director-v2/tests/unit/test_modules_notifier.py @@ -18,7 +18,7 @@ from models_library.projects_nodes_io import NodeID from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import NonNegativeInt, parse_obj_as +from pydantic import NonNegativeInt from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.utils import logged_gather @@ -111,7 +111,7 @@ def _get_on_no_more_credits_event( # emulates front-end receiving message async def on_no_more_credits(data): - assert parse_obj_as(ServiceNoMoreCredits, data) is not None + assert ServiceNoMoreCredits.model_validate(data) is not None on_event_spy = AsyncMock(wraps=on_no_more_credits) socketio_client.on(SOCKET_IO_SERVICE_NO_MORE_CREDITS_EVENT, on_event_spy) diff --git a/services/director-v2/tests/unit/test_modules_osparc_variables.py b/services/director-v2/tests/unit/test_modules_osparc_variables.py index 9ed659f00ad..635904292b8 100644 --- a/services/director-v2/tests/unit/test_modules_osparc_variables.py +++ b/services/director-v2/tests/unit/test_modules_osparc_variables.py @@ -21,7 +21,7 @@ from models_library.users import UserID from models_library.utils.specs_substitution import SubstitutionValue from models_library.utils.string_substitution import OSPARC_IDENTIFIER_PREFIX -from pydantic import parse_obj_as +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_compose_specs import generate_fake_docker_compose from simcore_postgres_database.models.services_environments import VENDOR_SECRET_PREFIX @@ -48,8 +48,8 @@ def session_context(faker: Faker) -> ContextDict: return ContextDict( app=FastAPI(), - service_key=parse_obj_as(ServiceKey, "simcore/services/dynamic/foo"), - service_version=parse_obj_as(ServiceVersion, "1.2.3"), + service_key=TypeAdapter(ServiceKey).validate_python("simcore/services/dynamic/foo"), + service_version=TypeAdapter(ServiceVersion).validate_python("1.2.3"), compose_spec=generate_fake_docker_compose(faker), product_name=faker.word(), project_id=faker.uuid4(), @@ -101,7 +101,7 @@ async def request_user_email(app: FastAPI, user_id: UserID) -> SubstitutionValue # All values extracted from the context MUST be SubstitutionValue assert { - key: parse_obj_as(SubstitutionValue, value) for key, value in environs.items() + key: TypeAdapter(SubstitutionValue).validate_python(value) for key, value in environs.items() } for osparc_variable_name, context_name in [ diff --git a/services/director-v2/tests/unit/test_modules_project_networks.py b/services/director-v2/tests/unit/test_modules_project_networks.py index 585d8131b8e..848b3629e10 100644 --- a/services/director-v2/tests/unit/test_modules_project_networks.py +++ b/services/director-v2/tests/unit/test_modules_project_networks.py @@ -40,8 +40,8 @@ def using( attach: list[Any], ) -> "Example": return cls( - existing_networks_with_aliases=NetworksWithAliases.parse_obj(existing), - new_networks_with_aliases=NetworksWithAliases.parse_obj(new), + existing_networks_with_aliases=NetworksWithAliases.model_validate(existing), + new_networks_with_aliases=NetworksWithAliases.model_validate(new), expected_calls=MockedCalls(detach=detach, attach=attach), ) @@ -184,7 +184,7 @@ def dy_workbench_with_networkable_labels(mocks_dir: Path) -> NodesDict: for node_uuid, node_data in dy_workbench.items(): node_data["label"] = f"label_{uuid4()}" - parsed_workbench[node_uuid] = Node.parse_obj(node_data) + parsed_workbench[node_uuid] = Node.model_validate(node_data) return parsed_workbench diff --git a/services/director-v2/tests/unit/test_modules_rabbitmq.py b/services/director-v2/tests/unit/test_modules_rabbitmq.py index 1d557d673a8..972f836f575 100644 --- a/services/director-v2/tests/unit/test_modules_rabbitmq.py +++ b/services/director-v2/tests/unit/test_modules_rabbitmq.py @@ -44,7 +44,7 @@ def message(faker: Faker) -> WalletCreditsLimitReachedMessage: async def test_handler_out_of_credits( mock_app: FastAPI, message: WalletCreditsLimitReachedMessage, ignore_limits ): - await handler_out_of_credits(mock_app, message.json().encode()) + await handler_out_of_credits(mock_app, message.model_dump_json().encode()) removal_mark_count = ( mock_app.state.dynamic_sidecar_scheduler.mark_all_services_in_wallet_for_removal.call_count diff --git a/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py b/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py index 8d58d96f675..6347ebab5f4 100644 --- a/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py +++ b/services/director-v2/tests/unit/test_schemas_dynamic_services_scheduler.py @@ -40,11 +40,11 @@ def assert_copy_has_changes(original: SchedulerData) -> Iterator[SchedulerData]: async def test_parse_saved_fake_scheduler_data(fake_scheduler_data: str) -> None: - assert SchedulerData.parse_raw(fake_scheduler_data) + assert SchedulerData.model_validate_json(fake_scheduler_data) def test_nested_compare(fake_scheduler_data: str) -> None: - scheduler_data = SchedulerData.parse_raw(fake_scheduler_data) + scheduler_data = SchedulerData.model_validate_json(fake_scheduler_data) with assert_copy_has_changes(scheduler_data) as to_change: to_change.paths_mapping.inputs_path = Path("/tmp") diff --git a/services/director-v2/tests/unit/test_utils_client_decorators.py b/services/director-v2/tests/unit/test_utils_client_decorators.py index 066bedad11b..5b630f788c7 100644 --- a/services/director-v2/tests/unit/test_utils_client_decorators.py +++ b/services/director-v2/tests/unit/test_utils_client_decorators.py @@ -35,10 +35,10 @@ async def a_request(method: str, **kwargs) -> Response: await a_request( "POST", url=url, - params=dict(kettle="boiling"), - data=dict(kettle_number="royal_01"), + params={"kettle": "boiling"}, + data={"kettle_number": "royal_01"}, ) - assert status.HTTP_503_SERVICE_UNAVAILABLE == exec_info.value.status_code + assert exec_info.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE # ERROR test_utils_client_decorators:client_decorators.py:76 AService service error: # |Request| diff --git a/services/director-v2/tests/unit/test_utils_comp_scheduler.py b/services/director-v2/tests/unit/test_utils_comp_scheduler.py index 970cdad75b7..dfb7c0326b1 100644 --- a/services/director-v2/tests/unit/test_utils_comp_scheduler.py +++ b/services/director-v2/tests/unit/test_utils_comp_scheduler.py @@ -78,8 +78,8 @@ def test_get_resource_tracking_run_id( @pytest.mark.parametrize( "task", [ - CompTaskAtDB.parse_obj(example) - for example in CompTaskAtDB.Config.schema_extra["examples"] + CompTaskAtDB.model_validate(example) + for example in CompTaskAtDB.model_config["json_schema_extra"]["examples"] ], ids=str, ) diff --git a/services/director-v2/tests/unit/test_utils_computation.py b/services/director-v2/tests/unit/test_utils_computation.py index 184a65d0db7..14c9ffa34f3 100644 --- a/services/director-v2/tests/unit/test_utils_computation.py +++ b/services/director-v2/tests/unit/test_utils_computation.py @@ -27,7 +27,7 @@ def fake_task_file(mocks_dir: Path): @pytest.fixture(scope="session") def fake_task(fake_task_file: Path) -> CompTaskAtDB: - return CompTaskAtDB.parse_file(fake_task_file) + return CompTaskAtDB.model_validate_json(fake_task_file.read_text()) # NOTE: these parametrizations are made to mimic something like a sleepers project diff --git a/services/director-v2/tests/unit/test_utils_dags.py b/services/director-v2/tests/unit/test_utils_dags.py index 3ab2c68fea1..125f3153db4 100644 --- a/services/director-v2/tests/unit/test_utils_dags.py +++ b/services/director-v2/tests/unit/test_utils_dags.py @@ -476,7 +476,7 @@ def pipeline_test_params( for x in range(_MANY_NODES) }, [ - CompTaskAtDB.construct( + CompTaskAtDB.model_construct( project_id=uuid4(), node_id=f"node_{x}", schema=NodeSchema(inputs={}, outputs={}), @@ -493,7 +493,7 @@ def pipeline_test_params( ) for x in range(_MANY_NODES) ], - PipelineDetails.construct( + PipelineDetails.model_construct( adjacency_list={f"node_{x}": [] for x in range(_MANY_NODES)}, progress=1.0, node_states={ @@ -527,7 +527,7 @@ def pipeline_test_params( }, [ # NOTE: we use construct here to be able to use non uuid names to simplify test setup - CompTaskAtDB.construct( + CompTaskAtDB.model_construct( project_id=uuid4(), node_id="node_1", schema=NodeSchema(inputs={}, outputs={}), @@ -541,7 +541,7 @@ def pipeline_test_params( modified=datetime.datetime.now(tz=datetime.timezone.utc), last_heartbeat=None, ), - CompTaskAtDB.construct( + CompTaskAtDB.model_construct( project_id=uuid4(), node_id="node_2", schema=NodeSchema(inputs={}, outputs={}), @@ -555,7 +555,7 @@ def pipeline_test_params( modified=datetime.datetime.now(tz=datetime.timezone.utc), last_heartbeat=None, ), - CompTaskAtDB.construct( + CompTaskAtDB.model_construct( project_id=uuid4(), node_id="node_3", schema=NodeSchema(inputs={}, outputs={}), @@ -571,7 +571,7 @@ def pipeline_test_params( progress=1.00, ), ], - PipelineDetails.construct( + PipelineDetails.model_construct( adjacency_list={ "node_1": ["node_2", "node_3"], "node_2": ["node_3"], @@ -597,5 +597,6 @@ async def test_compute_pipeline_details( pipeline_test_params.comp_tasks, ) assert ( - received_details.dict() == pipeline_test_params.expected_pipeline_details.dict() + received_details.model_dump() + == pipeline_test_params.expected_pipeline_details.model_dump() ) diff --git a/services/director-v2/tests/unit/test_utils_distributed_identifier.py b/services/director-v2/tests/unit/test_utils_distributed_identifier.py index ce200feef97..518552af1e1 100644 --- a/services/director-v2/tests/unit/test_utils_distributed_identifier.py +++ b/services/director-v2/tests/unit/test_utils_distributed_identifier.py @@ -10,7 +10,7 @@ from uuid import UUID, uuid4 import pytest -from pydantic import BaseModel, NonNegativeInt, StrBytes +from pydantic import BaseModel, NonNegativeInt from pytest_mock import MockerFixture from servicelib.redis import RedisClientSDK from servicelib.utils import logged_gather @@ -132,14 +132,16 @@ def _serialize_identifier(cls, identifier: UserDefinedID) -> str: return f"{identifier._id}" # noqa: SLF001 @classmethod - def _deserialize_cleanup_context(cls, raw: StrBytes) -> AnEmptyTextCleanupContext: - return AnEmptyTextCleanupContext.parse_raw(raw) + def _deserialize_cleanup_context( + cls, raw: str | bytes + ) -> AnEmptyTextCleanupContext: + return AnEmptyTextCleanupContext.model_validate_json(raw) @classmethod def _serialize_cleanup_context( cls, cleanup_context: AnEmptyTextCleanupContext ) -> str: - return cleanup_context.json() + return cleanup_context.model_dump_json() async def is_used( self, identifier: UserDefinedID, cleanup_context: AnEmptyTextCleanupContext diff --git a/services/director-v2/tests/unit/with_dbs/conftest.py b/services/director-v2/tests/unit/with_dbs/conftest.py index 8dd5527f00a..a3234328c9f 100644 --- a/services/director-v2/tests/unit/with_dbs/conftest.py +++ b/services/director-v2/tests/unit/with_dbs/conftest.py @@ -6,7 +6,6 @@ import datetime -import json from collections.abc import Awaitable, Callable, Iterator from typing import Any, cast from uuid import uuid4 @@ -59,7 +58,7 @@ def creator(**pipeline_kwargs) -> CompPipelineAtDB: ) assert result - new_pipeline = CompPipelineAtDB.from_orm(result.first()) + new_pipeline = CompPipelineAtDB.model_validate(result.first()) created_pipeline_ids.append(f"{new_pipeline.project_id}") return new_pipeline @@ -92,7 +91,9 @@ def creator( "inputs": ( { key: ( - json.loads(value.json(by_alias=True, exclude_unset=True)) + value.model_dump( + mode="json", by_alias=True, exclude_unset=True + ) if isinstance(value, BaseModel) else value ) @@ -104,7 +105,9 @@ def creator( "outputs": ( { key: ( - json.loads(value.json(by_alias=True, exclude_unset=True)) + value.model_dump( + mode="json", by_alias=True, exclude_unset=True + ) if isinstance(value, BaseModel) else value ) @@ -113,9 +116,9 @@ def creator( if node_data.outputs else {} ), - "image": Image(name=node_data.key, tag=node_data.version).dict( # type: ignore + "image": Image(name=node_data.key, tag=node_data.version).model_dump( by_alias=True, exclude_unset=True - ), # type: ignore + ), "node_class": to_node_class(node_data.key), "internal_id": internal_id + 1, "submit": datetime.datetime.now(tz=datetime.UTC), @@ -134,7 +137,7 @@ def creator( .values(**task_config) .returning(sa.literal_column("*")) ) - new_task = CompTaskAtDB.from_orm(result.first()) + new_task = CompTaskAtDB.model_validate(result.first()) created_tasks.append(new_task) created_task_ids.extend([t.task_id for t in created_tasks if t.task_id]) return created_tasks @@ -205,7 +208,7 @@ def creator( .values(**jsonable_encoder(run_config)) .returning(sa.literal_column("*")) ) - new_run = CompRunsAtDB.from_orm(result.first()) + new_run = CompRunsAtDB.model_validate(result.first()) created_run_ids.append(new_run.run_id) return new_run @@ -223,10 +226,10 @@ def cluster( created_cluster_ids: list[str] = [] def creator(user: dict[str, Any], **cluster_kwargs) -> Cluster: - cluster_config = Cluster.Config.schema_extra["examples"][1] + cluster_config = Cluster.model_config["json_schema_extra"]["examples"][1] cluster_config["owner"] = user["primary_gid"] cluster_config.update(**cluster_kwargs) - new_cluster = Cluster.parse_obj(cluster_config) + new_cluster = Cluster.model_validate(cluster_config) assert new_cluster with postgres_db.connect() as conn: diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py index 63457484613..19ab0ea2df3 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters.py @@ -2,7 +2,6 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import json import random from collections.abc import Callable, Iterator from typing import Any @@ -11,6 +10,7 @@ import pytest import sqlalchemy as sa from _dask_helpers import DaskGatewayServer +from common_library.serialization import model_dump_with_secrets from distributed.deploy.spec import SpecCluster from faker import Faker from httpx import URL @@ -30,9 +30,8 @@ ClusterAuthentication, SimpleAuthentication, ) -from pydantic import AnyHttpUrl, SecretStr, parse_obj_as +from pydantic import AnyHttpUrl, SecretStr, TypeAdapter from pytest_simcore.helpers.typing_env import EnvVarsDict -from settings_library.utils_encoders import create_json_encoder_wo_secrets from simcore_postgres_database.models.clusters import ClusterType, clusters from starlette import status @@ -70,7 +69,7 @@ def creator() -> dict[str, Any]: "username": faker.user_name(), "password": faker.password(), } - assert SimpleAuthentication.parse_obj(simple_auth) + assert SimpleAuthentication.model_validate(simple_auth) return simple_auth return creator @@ -94,7 +93,9 @@ async def test_list_clusters( # there is no cluster at the moment, the list shall contain the default cluster response = await async_client.get(list_clusters_url) assert response.status_code == status.HTTP_200_OK - returned_clusters_list = parse_obj_as(list[ClusterGet], response.json()) + returned_clusters_list = TypeAdapter(list[ClusterGet]).validate_python( + response.json() + ) assert ( len(returned_clusters_list) == 1 ), f"no default cluster in {returned_clusters_list=}" @@ -109,7 +110,9 @@ async def test_list_clusters( response = await async_client.get(list_clusters_url) assert response.status_code == status.HTTP_200_OK - returned_clusters_list = parse_obj_as(list[ClusterGet], response.json()) + returned_clusters_list = TypeAdapter(list[ClusterGet]).validate_python( + response.json() + ) assert ( len(returned_clusters_list) == NUM_CLUSTERS + 1 ) # the default cluster comes on top of the NUM_CLUSTERS @@ -121,7 +124,9 @@ async def test_list_clusters( user_2 = registered_user() response = await async_client.get(f"/v2/clusters?user_id={user_2['id']}") assert response.status_code == status.HTTP_200_OK - returned_clusters_list = parse_obj_as(list[ClusterGet], response.json()) + returned_clusters_list = TypeAdapter(list[ClusterGet]).validate_python( + response.json() + ) assert ( len(returned_clusters_list) == 1 ), f"no default cluster in {returned_clusters_list=}" @@ -147,7 +152,7 @@ async def test_list_clusters( response = await async_client.get(f"/v2/clusters?user_id={user_2['id']}") assert response.status_code == status.HTTP_200_OK - user_2_clusters = parse_obj_as(list[ClusterGet], response.json()) + user_2_clusters = TypeAdapter(list[ClusterGet]).validate_python(response.json()) # we should find 3 clusters + the default cluster assert len(user_2_clusters) == 3 + 1 for name in [ @@ -187,11 +192,11 @@ async def test_get_cluster( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}" ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - returned_cluster = parse_obj_as(ClusterGet, response.json()) + returned_cluster = ClusterGet.model_validate(response.json()) assert returned_cluster - assert the_cluster.dict(exclude={"authentication"}) == returned_cluster.dict( + assert the_cluster.model_dump( exclude={"authentication"} - ) + ) == returned_cluster.model_dump(exclude={"authentication"}) user_2 = registered_user() # getting the same cluster for user 2 shall return 403 @@ -283,7 +288,7 @@ async def test_get_default_cluster( get_cluster_url = URL(f"/v2/clusters/default?user_id={user_1['id']}") response = await async_client.get(get_cluster_url) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - returned_cluster = parse_obj_as(ClusterGet, response.json()) + returned_cluster = ClusterGet.model_validate(response.json()) assert returned_cluster assert returned_cluster.id == 0 assert returned_cluster.name == "Default cluster" @@ -307,24 +312,24 @@ async def test_create_cluster( authentication=cluster_simple_authentication(), name=faker.name(), type=random.choice(list(ClusterType)), + owner=faker.pyint(min_value=1), ) response = await async_client.post( create_cluster_url, - json=json.loads( - cluster_data.json( - by_alias=True, - exclude_unset=True, - encoder=create_json_encoder_wo_secrets(ClusterCreate), - ) + json=model_dump_with_secrets( + cluster_data, + show_secrets=True, + by_alias=True, + exclude_unset=True, ), ) assert response.status_code == status.HTTP_201_CREATED, f"received: {response.text}" - created_cluster = parse_obj_as(ClusterGet, response.json()) + created_cluster = ClusterGet.model_validate(response.json()) assert created_cluster - assert cluster_data.dict( + assert cluster_data.model_dump( exclude={"id", "owner", "access_rights", "authentication"} - ) == created_cluster.dict( + ) == created_cluster.model_dump( exclude={"id", "owner", "access_rights", "authentication"} ) @@ -354,10 +359,8 @@ async def test_update_own_cluster( # try to modify one that does not exist response = await async_client.patch( f"/v2/clusters/15615165165165?user_id={user_1['id']}", - json=json.loads( - ClusterPatch().json( - **_PATCH_EXPORT, encoder=create_json_encoder_wo_secrets(ClusterPatch) - ) + json=model_dump_with_secrets( + ClusterPatch(), show_secrets=True, **_PATCH_EXPORT ), ) assert response.status_code == status.HTTP_404_NOT_FOUND @@ -371,23 +374,21 @@ async def test_update_own_cluster( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}" ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - original_cluster = parse_obj_as(ClusterGet, response.json()) + original_cluster = ClusterGet.model_validate(response.json()) # now we modify nothing response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}", - json=json.loads( - ClusterPatch().json( - **_PATCH_EXPORT, encoder=create_json_encoder_wo_secrets(ClusterPatch) - ) + json=model_dump_with_secrets( + ClusterPatch(), show_secrets=True, **_PATCH_EXPORT ), ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - returned_cluster = parse_obj_as(ClusterGet, response.json()) - assert returned_cluster.dict() == original_cluster.dict() + returned_cluster = ClusterGet.model_validate(response.json()) + assert returned_cluster.model_dump() == original_cluster.model_dump() # modify some simple things - expected_modified_cluster = original_cluster.copy() + expected_modified_cluster = original_cluster.model_copy() for cluster_patch in [ ClusterPatch(name=faker.name()), ClusterPatch(description=faker.text()), @@ -396,10 +397,8 @@ async def test_update_own_cluster( ClusterPatch(endpoint=faker.uri()), ClusterPatch(authentication=cluster_simple_authentication()), ]: - jsonable_cluster_patch = json.loads( - cluster_patch.json( - **_PATCH_EXPORT, encoder=create_json_encoder_wo_secrets(ClusterPatch) - ) + jsonable_cluster_patch = model_dump_with_secrets( + cluster_patch, show_secrets=True, **_PATCH_EXPORT ) print(f"--> patching cluster with {jsonable_cluster_patch}") response = await async_client.patch( @@ -407,13 +406,15 @@ async def test_update_own_cluster( json=jsonable_cluster_patch, ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - returned_cluster = parse_obj_as(ClusterGet, response.json()) - expected_modified_cluster = expected_modified_cluster.copy( - update=cluster_patch.dict(**_PATCH_EXPORT) + returned_cluster = ClusterGet.model_validate(response.json()) + expected_modified_cluster = expected_modified_cluster.model_copy( + update=cluster_patch.model_dump(**_PATCH_EXPORT) ) - assert returned_cluster.dict( + assert returned_cluster.model_dump( exclude={"authentication": {"password"}} - ) == expected_modified_cluster.dict(exclude={"authentication": {"password"}}) + ) == expected_modified_cluster.model_dump( + exclude={"authentication": {"password"}} + ) # we can change the access rights, the owner rights are always kept user_2 = registered_user() @@ -427,34 +428,32 @@ async def test_update_own_cluster( cluster_patch = ClusterPatch(accessRights={user_2["primary_gid"]: rights}) response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}", - json=cluster_patch.dict(**_PATCH_EXPORT), + json=cluster_patch.model_dump(**_PATCH_EXPORT), ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - returned_cluster = ClusterGet.parse_obj(response.json()) + returned_cluster = ClusterGet.model_validate(response.json()) expected_modified_cluster.access_rights[user_2["primary_gid"]] = rights - assert returned_cluster.dict( + assert returned_cluster.model_dump( + exclude={"authentication": {"password"}} + ) == expected_modified_cluster.model_dump( exclude={"authentication": {"password"}} - ) == expected_modified_cluster.dict(exclude={"authentication": {"password"}}) + ) # we can change the owner since we are admin cluster_patch = ClusterPatch(owner=user_2["primary_gid"]) response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}", - json=json.loads( - cluster_patch.json( - **_PATCH_EXPORT, encoder=create_json_encoder_wo_secrets(ClusterPatch) - ) - ), + json=model_dump_with_secrets(cluster_patch, show_secrets=True, **_PATCH_EXPORT), ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - returned_cluster = ClusterGet.parse_obj(response.json()) + returned_cluster = ClusterGet.model_validate(response.json()) expected_modified_cluster.owner = user_2["primary_gid"] expected_modified_cluster.access_rights[ user_2["primary_gid"] ] = CLUSTER_ADMIN_RIGHTS - assert returned_cluster.dict( + assert returned_cluster.model_dump( exclude={"authentication": {"password"}} - ) == expected_modified_cluster.dict(exclude={"authentication": {"password"}}) + ) == expected_modified_cluster.model_dump(exclude={"authentication": {"password"}}) # we should not be able to reduce the rights of the new owner cluster_patch = ClusterPatch( @@ -462,11 +461,7 @@ async def test_update_own_cluster( ) response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}", - json=json.loads( - cluster_patch.json( - **_PATCH_EXPORT, encoder=create_json_encoder_wo_secrets(ClusterPatch) - ) - ), + json=model_dump_with_secrets(cluster_patch, show_secrets=True, **_PATCH_EXPORT), ) assert ( response.status_code == status.HTTP_403_FORBIDDEN @@ -486,10 +481,8 @@ async def test_update_default_cluster_fails( # try to modify one that does not exist response = await async_client.patch( f"/v2/clusters/default?user_id={user_1['id']}", - json=json.loads( - ClusterPatch().json( - **_PATCH_EXPORT, encoder=create_json_encoder_wo_secrets(ClusterPatch) - ) + json=model_dump_with_secrets( + ClusterPatch(), show_secrets=True, **_PATCH_EXPORT ), ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY @@ -545,7 +538,7 @@ async def test_update_another_cluster( f"/v2/clusters/{the_cluster.id}?user_id={user_1['id']}" ) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" - parse_obj_as(ClusterGet, response.json()) + ClusterGet.model_validate(response.json()) # let's try to modify stuff as we are user 2 for cluster_patch in [ @@ -558,11 +551,8 @@ async def test_update_another_cluster( ]: response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_2['id']}", - json=json.loads( - cluster_patch.json( - **_PATCH_EXPORT, - encoder=create_json_encoder_wo_secrets(ClusterPatch), - ) + json=model_dump_with_secrets( + cluster_patch, show_secrets=True, **_PATCH_EXPORT ), ) assert ( @@ -581,11 +571,8 @@ async def test_update_another_cluster( cluster_patch = ClusterPatch(accessRights={user_3["primary_gid"]: rights}) response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_2['id']}", - json=json.loads( - cluster_patch.json( - **_PATCH_EXPORT, - encoder=create_json_encoder_wo_secrets(ClusterPatch), - ) + json=model_dump_with_secrets( + cluster_patch, show_secrets=True, **_PATCH_EXPORT ), ) assert ( @@ -602,11 +589,8 @@ async def test_update_another_cluster( cluster_patch = ClusterPatch(accessRights={user_3["primary_gid"]: rights}) response = await async_client.patch( f"/v2/clusters/{the_cluster.id}?user_id={user_2['id']}", - json=json.loads( - cluster_patch.json( - **_PATCH_EXPORT, - encoder=create_json_encoder_wo_secrets(ClusterPatch), - ) + json=model_dump_with_secrets( + cluster_patch, show_secrets=True, **_PATCH_EXPORT ), ) assert ( @@ -729,16 +713,14 @@ async def test_ping_invalid_cluster_raises_422( # calling with correct data but non existing cluster also raises some_fake_cluster = ClusterPing( endpoint=faker.url(), - authentication=parse_obj_as( - ClusterAuthentication, cluster_simple_authentication() + authentication=TypeAdapter(ClusterAuthentication).validate_python( + cluster_simple_authentication() ), ) response = await async_client.post( "/v2/clusters:ping", - json=json.loads( - some_fake_cluster.json( - by_alias=True, encoder=create_json_encoder_wo_secrets(ClusterPing) - ) + json=model_dump_with_secrets( + some_fake_cluster, show_secrets=True, by_alias=True ), ) with pytest.raises(httpx.HTTPStatusError): @@ -751,20 +733,19 @@ async def test_ping_cluster( local_dask_gateway_server: DaskGatewayServer, ): valid_cluster = ClusterPing( - endpoint=parse_obj_as(AnyHttpUrl, local_dask_gateway_server.address), + endpoint=TypeAdapter(AnyHttpUrl).validate_python( + local_dask_gateway_server.address + ), authentication=SimpleAuthentication( username="pytest_user", - password=parse_obj_as(SecretStr, local_dask_gateway_server.password), + password=TypeAdapter(SecretStr).validate_python( + local_dask_gateway_server.password + ), ), ) response = await async_client.post( "/v2/clusters:ping", - json=json.loads( - valid_cluster.json( - by_alias=True, - encoder=create_json_encoder_wo_secrets(SimpleAuthentication), - ) - ), + json=model_dump_with_secrets(valid_cluster, show_secrets=True, by_alias=True), ) response.raise_for_status() assert response.status_code == status.HTTP_204_NO_CONTENT @@ -792,7 +773,9 @@ async def test_ping_specific_cluster( endpoint=local_dask_gateway_server.address, authentication=SimpleAuthentication( username="pytest_user", - password=parse_obj_as(SecretStr, local_dask_gateway_server.password), + password=TypeAdapter(SecretStr).validate_python( + local_dask_gateway_server.password + ), ), ) for n in range(111) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py index 2b509ab1a6f..5dd1abaa594 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_clusters_details.py @@ -85,7 +85,7 @@ async def test_local_dask_gateway_server(local_dask_gateway_server: DaskGatewayS async with cluster.get_client() as client: print(f"--> created new client {client=}, submitting a job") - res = await client.submit(lambda x: x + 1, 1) # type: ignore + res = await client.submit(lambda x: x + 1, 1) assert res == 2 print(f"--> scaling cluster {cluster=} back to 0") @@ -114,12 +114,12 @@ async def test_get_default_cluster_details( f"/v2/clusters/default/details?user_id={user_1['id']}" ) assert response.status_code == status.HTTP_200_OK - default_cluster_out = ClusterDetailsGet.parse_obj(response.json()) + default_cluster_out = ClusterDetailsGet.model_validate(response.json()) response = await async_client.get( f"/v2/clusters/{0}/details?user_id={user_1['id']}" ) assert response.status_code == status.HTTP_200_OK - assert default_cluster_out == ClusterDetailsGet.parse_obj(response.json()) + assert default_cluster_out == ClusterDetailsGet.model_validate(response.json()) async def _get_cluster_details( @@ -130,7 +130,7 @@ async def _get_cluster_details( ) assert response.status_code == status.HTTP_200_OK print(f"<-- received cluster details response {response=}") - cluster_out = ClusterDetailsGet.parse_obj(response.json()) + cluster_out = ClusterDetailsGet.model_validate(response.json()) assert cluster_out print(f"<-- received cluster details {cluster_out=}") assert cluster_out.scheduler, "the cluster's scheduler is not started!" @@ -155,7 +155,7 @@ async def test_get_cluster_details( authentication=SimpleAuthentication( username=gateway_username, password=SecretStr(local_dask_gateway_server.password), - ).dict(by_alias=True), + ).model_dump(by_alias=True), ) # in its present state, the cluster should have no workers cluster_out = await _get_cluster_details( diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index 1135465ef61..add9c4d77d3 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -6,7 +6,7 @@ # pylint: disable=unused-variable # pylint:disable=too-many-positional-arguments -import datetime +import datetime as dt import json import re import urllib.parse @@ -23,6 +23,7 @@ import respx from faker import Faker from fastapi import FastAPI, status +from models_library.api_schemas_catalog.services import ServiceGet from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.comp_tasks import ( ComputationCreate, @@ -33,7 +34,6 @@ PricingPlanGet, PricingUnitGet, ) -from models_library.basic_types import VersionStr from models_library.clusters import DEFAULT_CLUSTER_ID, Cluster, ClusterID from models_library.projects import ProjectAtDB from models_library.projects_nodes import NodeID, NodeState @@ -49,7 +49,7 @@ ) from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.wallets import WalletInfo -from pydantic import AnyHttpUrl, ByteSize, PositiveInt, ValidationError, parse_obj_as +from pydantic import AnyHttpUrl, ByteSize, PositiveInt, TypeAdapter, ValidationError from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.rabbit import RabbitSettings @@ -111,23 +111,24 @@ def fake_service_details(mocks_dir: Path) -> ServiceMetaDataPublished: @pytest.fixture def fake_service_extras() -> ServiceExtras: - extra_example = ServiceExtras.Config.schema_extra["examples"][2] - random_extras = ServiceExtras(**extra_example) + extra_example = ServiceExtras.model_config["json_schema_extra"]["examples"][2] # type: ignore + random_extras = ServiceExtras(**extra_example) # type: ignore assert random_extras is not None return random_extras @pytest.fixture def fake_service_resources() -> ServiceResourcesDict: - return parse_obj_as( - ServiceResourcesDict, - ServiceResourcesDictHelpers.Config.schema_extra["examples"][0], + return TypeAdapter(ServiceResourcesDict).validate_python( + ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], # type: ignore ) @pytest.fixture def fake_service_labels() -> dict[str, Any]: - return choice(SimcoreServiceLabels.Config.schema_extra["examples"]) # noqa: S311 + return choice( # noqa: S311 + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"] # type: ignore + ) @pytest.fixture @@ -143,13 +144,14 @@ def mocked_director_service_fcts( assert_all_called=False, assert_all_mocked=True, ) as respx_mock: - assert VersionStr.regex respx_mock.get( re.compile( r"/services/simcore%2Fservices%2F(comp|dynamic|frontend)%2F[^/]+/\d+.\d+.\d+$" ), name="get_service", - ).respond(json={"data": [fake_service_details.dict(by_alias=True)]}) + ).respond( + json={"data": [fake_service_details.model_dump(mode="json", by_alias=True)]} + ) respx_mock.get( re.compile( r"/services/simcore%2Fservices%2F(comp|dynamic|frontend)%2F[^/]+/\d+.\d+.\d+/labels" @@ -162,7 +164,9 @@ def mocked_director_service_fcts( r"/service_extras/(simcore)%2F(services)%2F(comp|dynamic|frontend)%2F.+/(.+)" ), name="get_service_extras", - ).respond(json={"data": fake_service_extras.dict(by_alias=True)}) + ).respond( + json={"data": fake_service_extras.model_dump(mode="json", by_alias=True)} + ) yield respx_mock @@ -175,7 +179,7 @@ def mocked_catalog_service_fcts( ) -> Iterator[respx.MockRouter]: def _mocked_service_resources(request) -> httpx.Response: return httpx.Response( - 200, json=jsonable_encoder(fake_service_resources, by_alias=True) + httpx.codes.OK, json=jsonable_encoder(fake_service_resources, by_alias=True) ) def _mocked_services_details( @@ -184,7 +188,7 @@ def _mocked_services_details( return httpx.Response( 200, json=jsonable_encoder( - fake_service_details.copy( + fake_service_details.model_copy( update={ "key": urllib.parse.unquote(service_key), "version": service_version, @@ -225,19 +229,30 @@ def mocked_catalog_service_fcts_deprecated( def _mocked_services_details( request, service_key: str, service_version: str ) -> httpx.Response: + data_published = fake_service_details.model_copy( + update={ + "key": urllib.parse.unquote(service_key), + "version": service_version, + "deprecated": ( + dt.datetime.now(tz=dt.UTC) - dt.timedelta(days=1) + ).isoformat(), + } + ).model_dump(by_alias=True) + + deprecated = { + "deprecated": ( + dt.datetime.now(tz=dt.UTC) - dt.timedelta(days=1) + ).isoformat() + } + + data = {**ServiceGet.model_config["json_schema_extra"]["examples"][0], **data_published, **deprecated} # type: ignore + + payload = ServiceGet.model_validate(data) + return httpx.Response( - 200, + httpx.codes.OK, json=jsonable_encoder( - fake_service_details.copy( - update={ - "key": urllib.parse.unquote(service_key), - "version": service_version, - "deprecated": ( - datetime.datetime.now(tz=datetime.timezone.utc) - - datetime.timedelta(days=1) - ).isoformat(), - } - ), + payload, by_alias=True, ), ) @@ -259,7 +274,7 @@ def _mocked_services_details( @pytest.fixture( - params=PricingPlanGet.Config.schema_extra["examples"], + params=PricingPlanGet.model_config["json_schema_extra"]["examples"], ids=["with ec2 restriction", "without"], ) def default_pricing_plan(request: pytest.FixtureRequest) -> PricingPlanGet: @@ -303,7 +318,7 @@ def _mocked_get_pricing_unit(request, pricing_plan_id: int) -> httpx.Response: ( default_pricing_plan.pricing_units[0] if default_pricing_plan.pricing_units - else PricingUnitGet.Config.schema_extra["examples"][0] + else PricingUnitGet.model_config["json_schema_extra"]["examples"][0] ), by_alias=True, ), @@ -422,7 +437,7 @@ def fake_ec2_cpus() -> PositiveInt: @pytest.fixture def fake_ec2_ram() -> ByteSize: - return parse_obj_as(ByteSize, "4GiB") + return TypeAdapter(ByteSize).validate_python("4GiB") @pytest.fixture @@ -463,7 +478,9 @@ def mocked_clusters_keeper_service_get_instance_type_details_with_invalid_name( ) -@pytest.fixture(params=ServiceResourcesDictHelpers.Config.schema_extra["examples"]) +@pytest.fixture( + params=ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"] +) def project_nodes_overrides(request: pytest.FixtureRequest) -> dict[str, Any]: return request.param @@ -572,7 +589,11 @@ async def test_create_computation_with_wallet( @pytest.mark.parametrize( "default_pricing_plan", - [PricingPlanGet.construct(**PricingPlanGet.Config.schema_extra["examples"][0])], + [ + PricingPlanGet.model_construct( + **PricingPlanGet.model_config["json_schema_extra"]["examples"][0] + ) + ], ) async def test_create_computation_with_wallet_with_invalid_pricing_unit_name_raises_422( minimal_configuration: None, @@ -610,7 +631,11 @@ async def test_create_computation_with_wallet_with_invalid_pricing_unit_name_rai @pytest.mark.parametrize( "default_pricing_plan", - [PricingPlanGet.construct(**PricingPlanGet.Config.schema_extra["examples"][0])], + [ + PricingPlanGet.model_construct( + **PricingPlanGet.model_config["json_schema_extra"]["examples"][0] # type: ignore + ) + ], ) async def test_create_computation_with_wallet_with_no_clusters_keeper_raises_503( minimal_configuration: None, @@ -711,9 +736,9 @@ async def test_start_computation_with_project_node_resources_defined( proj = await project( user, project_nodes_overrides={ - "required_resources": ServiceResourcesDictHelpers.Config.schema_extra[ - "examples" - ][0] + "required_resources": ServiceResourcesDictHelpers.model_config[ + "json_schema_extra" + ]["examples"][0] }, workbench=fake_workbench_without_outputs, ) @@ -864,7 +889,7 @@ async def test_get_computation_from_empty_project( ) response = await async_client.get(get_computation_url) assert response.status_code == status.HTTP_200_OK, response.text - returned_computation = ComputationGet.parse_obj(response.json()) + returned_computation = ComputationGet.model_validate(response.json()) assert returned_computation expected_computation = ComputationGet( id=proj.uuid, @@ -872,8 +897,8 @@ async def test_get_computation_from_empty_project( pipeline_details=PipelineDetails( adjacency_list={}, node_states={}, progress=None ), - url=parse_obj_as( - AnyHttpUrl, f"{async_client.base_url.join(get_computation_url)}" + url=TypeAdapter(AnyHttpUrl).validate_python( + f"{async_client.base_url.join(get_computation_url)}" ), stop_url=None, result=None, @@ -883,7 +908,7 @@ async def test_get_computation_from_empty_project( stopped=None, submitted=None, ) - assert returned_computation.dict() == expected_computation.dict() + assert returned_computation.model_dump() == expected_computation.model_dump() async def test_get_computation_from_not_started_computation_task( @@ -913,14 +938,14 @@ async def test_get_computation_from_not_started_computation_task( comp_tasks = tasks(user=user, project=proj) response = await async_client.get(get_computation_url) assert response.status_code == status.HTTP_200_OK, response.text - returned_computation = ComputationGet.parse_obj(response.json()) + returned_computation = ComputationGet.model_validate(response.json()) assert returned_computation expected_computation = ComputationGet( id=proj.uuid, state=RunningState.NOT_STARTED, pipeline_details=PipelineDetails( - adjacency_list=parse_obj_as( - dict[NodeID, list[NodeID]], fake_workbench_adjacency + adjacency_list=TypeAdapter(dict[NodeID, list[NodeID]]).validate_python( + fake_workbench_adjacency ), progress=0, node_states={ @@ -938,8 +963,8 @@ async def test_get_computation_from_not_started_computation_task( if t.node_class == NodeClass.COMPUTATIONAL }, ), - url=parse_obj_as( - AnyHttpUrl, f"{async_client.base_url.join(get_computation_url)}" + url=TypeAdapter(AnyHttpUrl).validate_python( + f"{async_client.base_url.join(get_computation_url)}" ), stop_url=None, result=None, @@ -950,12 +975,12 @@ async def test_get_computation_from_not_started_computation_task( submitted=None, ) _CHANGED_FIELDS = {"submitted"} - assert returned_computation.dict( + assert returned_computation.model_dump( exclude=_CHANGED_FIELDS - ) == expected_computation.dict(exclude=_CHANGED_FIELDS) - assert returned_computation.dict( + ) == expected_computation.model_dump(exclude=_CHANGED_FIELDS) + assert returned_computation.model_dump( include=_CHANGED_FIELDS - ) != expected_computation.dict(include=_CHANGED_FIELDS) + ) != expected_computation.model_dump(include=_CHANGED_FIELDS) async def test_get_computation_from_published_computation_task( @@ -983,7 +1008,7 @@ async def test_get_computation_from_published_computation_task( ) response = await async_client.get(get_computation_url) assert response.status_code == status.HTTP_200_OK, response.text - returned_computation = ComputationGet.parse_obj(response.json()) + returned_computation = ComputationGet.model_validate(response.json()) assert returned_computation expected_stop_url = async_client.base_url.join( f"/v2/computations/{proj.uuid}:stop?user_id={user['id']}" @@ -992,8 +1017,8 @@ async def test_get_computation_from_published_computation_task( id=proj.uuid, state=RunningState.PUBLISHED, pipeline_details=PipelineDetails( - adjacency_list=parse_obj_as( - dict[NodeID, list[NodeID]], fake_workbench_adjacency + adjacency_list=TypeAdapter(dict[NodeID, list[NodeID]]).validate_python( + fake_workbench_adjacency ), node_states={ t.node_id: NodeState( @@ -1011,10 +1036,10 @@ async def test_get_computation_from_published_computation_task( }, progress=0, ), - url=parse_obj_as( - AnyHttpUrl, f"{async_client.base_url.join(get_computation_url)}" + url=TypeAdapter(AnyHttpUrl).validate_python( + f"{async_client.base_url.join(get_computation_url)}" ), - stop_url=parse_obj_as(AnyHttpUrl, f"{expected_stop_url}"), + stop_url=TypeAdapter(AnyHttpUrl).validate_python(f"{expected_stop_url}"), result=None, iteration=1, cluster_id=DEFAULT_CLUSTER_ID, @@ -1024,9 +1049,9 @@ async def test_get_computation_from_published_computation_task( ) _CHANGED_FIELDS = {"submitted"} - assert returned_computation.dict( + assert returned_computation.model_dump( exclude=_CHANGED_FIELDS - ) == expected_computation.dict(exclude=_CHANGED_FIELDS) - assert returned_computation.dict( + ) == expected_computation.model_dump(exclude=_CHANGED_FIELDS) + assert returned_computation.model_dump( include=_CHANGED_FIELDS - ) != expected_computation.dict(include=_CHANGED_FIELDS) + ) != expected_computation.model_dump(include=_CHANGED_FIELDS) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py index 6f75f43c59f..10bd1ba3a2f 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations_tasks.py @@ -9,6 +9,7 @@ from uuid import uuid4 import httpx +from pydantic import TypeAdapter import pytest from faker import Faker from fastapi import FastAPI, status @@ -21,7 +22,6 @@ from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID -from pydantic import parse_obj_as, parse_raw_as from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_director_v2.core.settings import AppSettings @@ -69,7 +69,7 @@ def _get_app(async_client: httpx.AsyncClient) -> FastAPI: settings: AppSettings = app.state.settings assert settings - print(settings.json(indent=1)) + print(settings.model_dump_json(indent=1)) return async_client @@ -162,7 +162,7 @@ async def test_get_all_tasks_log_files( # test expected response according to OAS! assert resp.status_code == status.HTTP_200_OK - log_files = parse_raw_as(list[TaskLogFileGet], resp.text) + log_files = TypeAdapter(list[TaskLogFileGet]).validate_json(resp.text) assert log_files assert all(l.download_link for l in log_files) @@ -180,7 +180,7 @@ async def test_get_task_logs_file( ) assert resp.status_code == status.HTTP_200_OK - log_file = TaskLogFileGet.parse_raw(resp.text) + log_file = TaskLogFileGet.model_validate_json(resp.text) assert log_file.download_link @@ -197,7 +197,7 @@ async def test_get_tasks_outputs( assert resp.status_code == status.HTTP_200_OK - tasks_outputs = parse_obj_as(TasksOutputs, resp.json()) + tasks_outputs = TasksOutputs.model_validate(resp.json()) assert selection == set(tasks_outputs.nodes_outputs.keys()) outputs = tasks_outputs.nodes_outputs[node_id] diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index cc0246bfec9..5787fa119e1 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -162,12 +162,18 @@ async def mock_retrieve_features( assert_all_mocked=True, ) as respx_mock: if is_legacy: - service_details = RunningDynamicServiceDetails.parse_obj( - RunningDynamicServiceDetails.Config.schema_extra["examples"][0] + service_details = RunningDynamicServiceDetails.model_validate( + RunningDynamicServiceDetails.model_config["json_schema_extra"][ + "examples" + ][0] ) respx_mock.post( f"{service_details.legacy_service_url}/retrieve", name="retrieve" - ).respond(json=RetrieveDataOutEnveloped.Config.schema_extra["examples"][0]) + ).respond( + json=RetrieveDataOutEnveloped.model_config["json_schema_extra"][ + "examples" + ][0] + ) yield respx_mock # no cleanup required @@ -185,7 +191,7 @@ async def mock_retrieve_features( ] = scheduler_data_from_http_request respx_mock.post( - f"{scheduler_data_from_http_request.endpoint}/v1/containers/ports/inputs:pull", + f"{scheduler_data_from_http_request.endpoint}v1/containers/ports/inputs:pull", name="service_pull_input_ports", ).respond(json="mocked_task_id", status_code=status.HTTP_202_ACCEPTED) @@ -230,7 +236,9 @@ def mocked_director_v0_service_api( name="running interactive service", ).respond( json={ - "data": RunningDynamicServiceDetails.Config.schema_extra["examples"][0] + "data": RunningDynamicServiceDetails.model_config["json_schema_extra"][ + "examples" + ][0] } ) @@ -246,8 +254,10 @@ def get_stack_status(node_uuid: NodeID) -> RunningDynamicServiceDetails: if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: raise DynamicSidecarNotFoundError(node_uuid) - return RunningDynamicServiceDetails.parse_obj( - RunningDynamicServiceDetails.Config.schema_extra["examples"][0] + return RunningDynamicServiceDetails.model_validate( + RunningDynamicServiceDetails.model_config["json_schema_extra"]["examples"][ + 0 + ] ) module_base = "simcore_service_director_v2.modules.dynamic_sidecar.scheduler" @@ -279,8 +289,12 @@ def remove_service(node_uuid: NodeID, *ars: Any, **kwargs: Any) -> None: [ pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][0], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][0], exp_status_code=status.HTTP_307_TEMPORARY_REDIRECT, is_legacy=True, ), @@ -288,8 +302,12 @@ def remove_service(node_uuid: NodeID, *ars: Any, **kwargs: Any) -> None: ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][1], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][1], exp_status_code=status.HTTP_201_CREATED, is_legacy=False, ), @@ -297,8 +315,12 @@ def remove_service(node_uuid: NodeID, *ars: Any, **kwargs: Any) -> None: ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][2], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][2], exp_status_code=status.HTTP_201_CREATED, is_legacy=False, ), @@ -316,12 +338,12 @@ def test_create_dynamic_services( exp_status_code: int, is_legacy: bool, ): - post_data = DynamicServiceCreate.parse_obj(service) + post_data = DynamicServiceCreate.model_validate(service) response = client.post( "/v2/dynamic_services", headers=dynamic_sidecar_headers, - json=json.loads(post_data.json()), + json=json.loads(post_data.model_dump_json()), follow_redirects=False, ) assert ( @@ -351,8 +373,12 @@ def test_create_dynamic_services( [ pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][0], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][0], exp_status_code=status.HTTP_307_TEMPORARY_REDIRECT, is_legacy=True, ), @@ -360,8 +386,12 @@ def test_create_dynamic_services( ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][1], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][1], exp_status_code=status.HTTP_200_OK, is_legacy=False, ), @@ -369,8 +399,12 @@ def test_create_dynamic_services( ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][2], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][2], exp_status_code=status.HTTP_200_OK, is_legacy=False, ), @@ -409,8 +443,12 @@ def test_get_service_status( [ pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][0], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][0], exp_status_code=status.HTTP_307_TEMPORARY_REDIRECT, is_legacy=True, ), @@ -418,8 +456,12 @@ def test_get_service_status( ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][1], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][1], exp_status_code=status.HTTP_204_NO_CONTENT, is_legacy=False, ), @@ -427,8 +469,12 @@ def test_get_service_status( ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][2], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][2], exp_status_code=status.HTTP_204_NO_CONTENT, is_legacy=False, ), @@ -481,8 +527,12 @@ def dynamic_sidecar_scheduler(minimal_app: FastAPI) -> DynamicSidecarsScheduler: [ pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][1], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][1], exp_status_code=status.HTTP_201_CREATED, is_legacy=False, ) @@ -500,12 +550,12 @@ def test_delete_service_waiting_for_manual_intervention( is_legacy: bool, dynamic_sidecar_scheduler: DynamicSidecarsScheduler, ): - post_data = DynamicServiceCreate.parse_obj(service) + post_data = DynamicServiceCreate.model_validate(service) response = client.post( "/v2/dynamic_services", headers=dynamic_sidecar_headers, - json=json.loads(post_data.json()), + json=json.loads(post_data.model_dump_json()), ) assert ( response.status_code == exp_status_code @@ -528,8 +578,12 @@ def test_delete_service_waiting_for_manual_intervention( [ pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][0], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][0], exp_status_code=status.HTTP_200_OK, is_legacy=True, ), @@ -537,8 +591,12 @@ def test_delete_service_waiting_for_manual_intervention( ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][1], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][1], exp_status_code=status.HTTP_200_OK, is_legacy=False, ), @@ -546,8 +604,12 @@ def test_delete_service_waiting_for_manual_intervention( ), pytest.param( *ServiceParams( - service=DynamicServiceCreate.Config.schema_extra["example"], - service_labels=SimcoreServiceLabels.Config.schema_extra["examples"][2], + service=DynamicServiceCreate.model_config["json_schema_extra"][ + "example" + ], + service_labels=SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ][2], exp_status_code=status.HTTP_200_OK, is_legacy=False, ), @@ -571,7 +633,8 @@ def test_retrieve( response.status_code == exp_status_code ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" assert ( - response.json() == RetrieveDataOutEnveloped.Config.schema_extra["examples"][0] + response.json() + == RetrieveDataOutEnveloped.model_config["json_schema_extra"]["examples"][0] ) diff --git a/services/director-v2/tests/unit/with_dbs/test_cli.py b/services/director-v2/tests/unit/with_dbs/test_cli.py index 43beec85900..1892e2a5a38 100644 --- a/services/director-v2/tests/unit/with_dbs/test_cli.py +++ b/services/director-v2/tests/unit/with_dbs/test_cli.py @@ -106,8 +106,10 @@ def node_id(faker: Faker) -> NodeID: def mock_get_node_state(mocker: MockerFixture) -> None: mocker.patch( "simcore_service_director_v2.cli._core._get_dy_service_state", - return_value=DynamicServiceGet.parse_obj( - RunningDynamicServiceDetails.Config.schema_extra["examples"][0] + return_value=DynamicServiceGet.model_validate( + RunningDynamicServiceDetails.model_config["json_schema_extra"]["examples"][ + 0 + ] ), ) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py index fbc90204f83..8567c8ccca0 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py @@ -41,7 +41,7 @@ RabbitResourceTrackingStoppedMessage, ) from models_library.users import UserID -from pydantic import parse_obj_as, parse_raw_as +from pydantic import TypeAdapter from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQClient @@ -124,7 +124,7 @@ async def _assert_comp_run_db( & (comp_runs.c.project_uuid == f"{pub_project.project.uuid}") ) # there is only one entry ) - run_entry = CompRunsAtDB.parse_obj(await result.first()) + run_entry = CompRunsAtDB.model_validate(await result.first()) assert ( run_entry.result == expected_state ), f"comp_runs: expected state '{expected_state}, found '{run_entry.result}'" @@ -146,7 +146,7 @@ async def _assert_comp_tasks_db( & (comp_tasks.c.node_id.in_([f"{n}" for n in task_ids])) ) # there is only one entry ) - tasks = parse_obj_as(list[CompTaskAtDB], await result.fetchall()) + tasks = TypeAdapter(list[CompTaskAtDB]).validate_python(await result.fetchall()) assert all( t.state == expected_state for t in tasks ), f"expected state: {expected_state}, found: {[t.state for t in tasks]}" @@ -376,7 +376,7 @@ async def test_misconfigured_pipeline_is_not_scheduled( & (comp_runs.c.project_uuid == f"{sleepers_project.uuid}") ) # there is only one entry ) - run_entry = CompRunsAtDB.parse_obj(await result.first()) + run_entry = CompRunsAtDB.model_validate(await result.first()) assert run_entry.result == RunningState.PUBLISHED # let the scheduler kick in await run_comp_scheduler(scheduler) @@ -390,7 +390,7 @@ async def test_misconfigured_pipeline_is_not_scheduled( & (comp_runs.c.project_uuid == f"{sleepers_project.uuid}") ) # there is only one entry ) - run_entry = CompRunsAtDB.parse_obj(await result.first()) + run_entry = CompRunsAtDB.model_validate(await result.first()) assert run_entry.result == RunningState.ABORTED assert run_entry.metadata == run_metadata @@ -609,7 +609,7 @@ async def _trigger_progress_event( ), ) await cast(DaskScheduler, scheduler)._task_progress_change_handler( # noqa: SLF001 - event.json() + event.model_dump_json() ) @@ -737,18 +737,20 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta mocked_dask_client.get_tasks_status.reset_mock() mocked_dask_client.get_task_result.assert_not_called() messages = await _assert_message_received( - instrumentation_rabbit_client_parser, 1, InstrumentationRabbitMessage.parse_raw + instrumentation_rabbit_client_parser, + 1, + InstrumentationRabbitMessage.model_validate_json, ) assert messages[0].metrics == "service_started" assert messages[0].service_uuid == exp_started_task.node_id def _parser(x) -> RabbitResourceTrackingMessages: - return parse_raw_as(RabbitResourceTrackingMessages, x) + return TypeAdapter(RabbitResourceTrackingMessages).validate_json(x) messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingStartedMessage.parse_raw, + RabbitResourceTrackingStartedMessage.model_validate_json, ) assert messages[0].node_id == exp_started_task.node_id @@ -767,7 +769,7 @@ async def _return_1st_task_success(job_ids: list[str]) -> list[DaskClientTaskSta mocked_dask_client.get_tasks_status.side_effect = _return_1st_task_success async def _return_random_task_result(job_id) -> TaskOutputData: - return TaskOutputData.parse_obj({"out_1": None, "out_2": 45}) + return TaskOutputData.model_validate({"out_1": None, "out_2": 45}) mocked_dask_client.get_task_result.side_effect = _return_random_task_result await run_comp_scheduler(scheduler) @@ -780,14 +782,16 @@ async def _return_random_task_result(job_id) -> TaskOutputData: expected_progress=1, ) messages = await _assert_message_received( - instrumentation_rabbit_client_parser, 1, InstrumentationRabbitMessage.parse_raw + instrumentation_rabbit_client_parser, + 1, + InstrumentationRabbitMessage.model_validate_json, ) assert messages[0].metrics == "service_stopped" assert messages[0].service_uuid == exp_started_task.node_id messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingStoppedMessage.parse_raw, + RabbitResourceTrackingStoppedMessage.model_validate_json, ) completed_tasks = [exp_started_task] @@ -882,14 +886,16 @@ async def _return_2nd_task_running(job_ids: list[str]) -> list[DaskClientTaskSta mocked_dask_client.get_tasks_status.reset_mock() mocked_dask_client.get_task_result.assert_not_called() messages = await _assert_message_received( - instrumentation_rabbit_client_parser, 1, InstrumentationRabbitMessage.parse_raw + instrumentation_rabbit_client_parser, + 1, + InstrumentationRabbitMessage.model_validate_json, ) assert messages[0].metrics == "service_started" assert messages[0].service_uuid == exp_started_task.node_id messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingStartedMessage.parse_raw, + RabbitResourceTrackingStartedMessage.model_validate_json, ) assert messages[0].node_id == exp_started_task.node_id @@ -926,14 +932,16 @@ async def _return_2nd_task_failed(job_ids: list[str]) -> list[DaskClientTaskStat mocked_parse_output_data_fct.assert_not_called() expected_pending_tasks.remove(exp_started_task) messages = await _assert_message_received( - instrumentation_rabbit_client_parser, 1, InstrumentationRabbitMessage.parse_raw + instrumentation_rabbit_client_parser, + 1, + InstrumentationRabbitMessage.model_validate_json, ) assert messages[0].metrics == "service_stopped" assert messages[0].service_uuid == exp_started_task.node_id messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingStoppedMessage.parse_raw, + RabbitResourceTrackingStoppedMessage.model_validate_json, ) # ------------------------------------------------------------------------------- @@ -970,7 +978,9 @@ async def _return_3rd_task_success(job_ids: list[str]) -> list[DaskClientTaskSta ) mocked_dask_client.get_task_result.assert_called_once_with(exp_started_task.job_id) messages = await _assert_message_received( - instrumentation_rabbit_client_parser, 2, InstrumentationRabbitMessage.parse_raw + instrumentation_rabbit_client_parser, + 2, + InstrumentationRabbitMessage.model_validate_json, ) # NOTE: the service was fast and went directly to success assert messages[0].metrics == "service_started" @@ -1032,7 +1042,7 @@ async def test_task_progress_triggers( await cast( DaskScheduler, scheduler )._task_progress_change_handler( # noqa: SLF001 - progress_event.json() + progress_event.model_dump_json() ) # NOTE: not sure whether it should switch to STARTED.. it would make sense await _assert_comp_tasks_db( @@ -1186,7 +1196,7 @@ class RebootState: pytest.param( RebootState( dask_task_status=DaskClientTaskState.SUCCESS, - task_result=TaskOutputData.parse_obj({"whatever_output": 123}), + task_result=TaskOutputData.model_validate({"whatever_output": 123}), expected_task_state_group1=RunningState.SUCCESS, expected_task_progress_group1=1, expected_task_state_group2=RunningState.SUCCESS, @@ -1339,7 +1349,7 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingStartedMessage.parse_raw, + RabbitResourceTrackingStartedMessage.model_validate_json, ) assert messages[0].node_id == exp_started_task.node_id @@ -1351,7 +1361,7 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingHeartbeatMessage.parse_raw, + RabbitResourceTrackingHeartbeatMessage.model_validate_json, ) assert isinstance(messages[0], RabbitResourceTrackingHeartbeatMessage) @@ -1363,7 +1373,7 @@ async def _return_1st_task_running(job_ids: list[str]) -> list[DaskClientTaskSta messages = await _assert_message_received( resource_tracking_rabbit_client_parser, 1, - RabbitResourceTrackingHeartbeatMessage.parse_raw, + RabbitResourceTrackingHeartbeatMessage.model_validate_json, ) assert isinstance(messages[0], RabbitResourceTrackingHeartbeatMessage) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index 0536261ed62..f36a8f8f7f6 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -195,7 +195,7 @@ def dynamic_sidecar_service_spec( f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", - DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL: scheduler_data_from_http_request.json(), + DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL: scheduler_data_from_http_request.model_dump_json(), }, } @@ -330,8 +330,10 @@ def service_name() -> str: @pytest.fixture( params=[ - SimcoreServiceLabels.parse_obj(example) - for example in SimcoreServiceLabels.Config.schema_extra["examples"] + SimcoreServiceLabels.model_validate(example) + for example in SimcoreServiceLabels.model_config["json_schema_extra"][ + "examples" + ] ], ) def labels_example(request: pytest.FixtureRequest) -> SimcoreServiceLabels: @@ -391,11 +393,11 @@ def test_settings__valid_network_names( monkeypatch: pytest.MonkeyPatch, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, ) -> None: - items = dynamic_services_scheduler_settings.dict() + items = dynamic_services_scheduler_settings.model_dump() items["SIMCORE_SERVICES_NETWORK_NAME"] = simcore_services_network_name # validate network names - DynamicServicesSchedulerSettings.parse_obj(items) + DynamicServicesSchedulerSettings.model_validate(items) async def test_failed_docker_client_request(docker_swarm: None): @@ -727,7 +729,7 @@ async def test_update_scheduler_data_label( # fetch stored data in labels service_inspect = await async_docker_client.services.inspect(mock_service) labels = service_inspect["Spec"]["Labels"] - scheduler_data = SchedulerData.parse_raw( + scheduler_data = SchedulerData.model_validate_json( labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL] ) assert scheduler_data == mock_scheduler_data diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index edca0dfe03c..16032677a98 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -4,7 +4,6 @@ import json -from collections.abc import Mapping from typing import Any, cast from unittest.mock import Mock @@ -49,7 +48,9 @@ @pytest.fixture def mock_s3_settings() -> S3Settings: - return S3Settings.parse_obj(S3Settings.Config.schema_extra["examples"][0]) + return S3Settings.model_validate( + S3Settings.model_config["json_schema_extra"]["examples"][0] + ) @pytest.fixture @@ -115,14 +116,16 @@ def swarm_network_id() -> str: @pytest.fixture def simcore_service_labels() -> SimcoreServiceLabels: # overwrites global fixture - return SimcoreServiceLabels.parse_obj( - SimcoreServiceLabels.Config.schema_extra["examples"][2] + return SimcoreServiceLabels.model_validate( + SimcoreServiceLabels.model_config["json_schema_extra"]["examples"][2] ) @pytest.fixture def hardware_info() -> HardwareInfo: - return HardwareInfo.parse_obj(HardwareInfo.Config.schema_extra["examples"][0]) + return HardwareInfo.model_validate( + HardwareInfo.model_config["json_schema_extra"]["examples"][0] + ) @pytest.fixture @@ -137,7 +140,7 @@ def expected_dynamic_sidecar_spec( return { "endpoint_spec": {}, "labels": { - "io.simcore.scheduler-data": SchedulerData.parse_obj( + "io.simcore.scheduler-data": SchedulerData.model_validate( { "compose_spec": '{"version": "2.3", "services": {"rt-web": {"image": ' '"${SIMCORE_REGISTRY}/simcore/services/dynamic/sim4life:${SERVICE_VERSION}", ' @@ -180,9 +183,9 @@ def expected_dynamic_sidecar_spec( "state_exclude": ["/tmp/strip_me/*", "*.py"], # noqa: S108 "state_paths": ["/tmp/save_1", "/tmp_save_2"], # noqa: S108 }, - "callbacks_mapping": CallbacksMapping.Config.schema_extra[ - "examples" - ][3], + "callbacks_mapping": CallbacksMapping.model_config[ + "json_schema_extra" + ]["examples"][3], "product_name": osparc_product_name, "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe", "proxy_service_name": "dy-proxy_75c7f3f4-18f9-4678-8610-54a2ade78eaa", @@ -190,8 +193,12 @@ def expected_dynamic_sidecar_spec( "request_scheme": "http", "request_simcore_user_agent": request_simcore_user_agent, "restart_policy": "on-inputs-downloaded", - "wallet_info": WalletInfo.Config.schema_extra["examples"][0], - "pricing_info": PricingInfo.Config.schema_extra["examples"][0], + "wallet_info": WalletInfo.model_config["json_schema_extra"][ + "examples" + ][0], + "pricing_info": PricingInfo.model_config["json_schema_extra"][ + "examples" + ][0], "hardware_info": hardware_info, "service_name": "dy-sidecar_75c7f3f4-18f9-4678-8610-54a2ade78eaa", "service_port": 65534, @@ -244,19 +251,19 @@ def expected_dynamic_sidecar_spec( "NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS": "3", "DYNAMIC_SIDECAR_LOG_LEVEL": "DEBUG", "DY_DEPLOYMENT_REGISTRY_SETTINGS": ( - '{"REGISTRY_AUTH": false, "REGISTRY_PATH": null, ' - '"REGISTRY_URL": "foo.bar.com", "REGISTRY_USER": ' - '"test", "REGISTRY_PW": "test", "REGISTRY_SSL": false}' + '{"REGISTRY_AUTH":false,"REGISTRY_PATH":null,' + '"REGISTRY_URL":"foo.bar.com","REGISTRY_USER":' + '"test","REGISTRY_PW":"test","REGISTRY_SSL":false}' ), "DY_DOCKER_HUB_REGISTRY_SETTINGS": "null", "DY_SIDECAR_AWS_S3_CLI_SETTINGS": ( - '{"AWS_S3_CLI_S3": {"S3_ACCESS_KEY": "12345678", "S3_BUCKET_NAME": "simcore", ' - '"S3_ENDPOINT": "http://172.17.0.1:9001", "S3_REGION": "us-east-1", "S3_SECRET_KEY": "12345678"}}' + '{"AWS_S3_CLI_S3":{"S3_ACCESS_KEY":"12345678","S3_BUCKET_NAME":"simcore",' + '"S3_ENDPOINT":"http://172.17.0.1:9001/","S3_REGION":"us-east-1","S3_SECRET_KEY":"12345678"}}' ), "DY_SIDECAR_CALLBACKS_MAPPING": ( - '{"metrics": {"service": "rt-web", "command": "ls", "timeout": 1.0}, "before_shutdown"' - ': [{"service": "rt-web", "command": "ls", "timeout": 1.0}, {"service": "s4l-core", ' - '"command": ["ls", "-lah"], "timeout": 1.0}], "inactivity": null}' + '{"metrics":{"service":"rt-web","command":"ls","timeout":1.0},"before_shutdown"' + ':[{"service":"rt-web","command":"ls","timeout":1.0},{"service":"s4l-core",' + '"command":["ls","-lah"],"timeout":1.0}],"inactivity":null}' ), "DY_SIDECAR_SERVICE_KEY": "simcore/services/dynamic/3dviewer", "DY_SIDECAR_SERVICE_VERSION": "2.4.5", @@ -435,12 +442,12 @@ async def test_get_dynamic_proxy_spec( == minimal_app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR ) - expected_dynamic_sidecar_spec_model = AioDockerServiceSpec.parse_obj( + expected_dynamic_sidecar_spec_model = AioDockerServiceSpec.model_validate( expected_dynamic_sidecar_spec ) - assert expected_dynamic_sidecar_spec_model.TaskTemplate - assert expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec - assert expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec.Env + assert expected_dynamic_sidecar_spec_model.task_template + assert expected_dynamic_sidecar_spec_model.task_template.container_spec + assert expected_dynamic_sidecar_spec_model.task_template.container_spec.env for count in range(1, 11): # loop to check it does not repeat copies print(f"{count:*^50}") @@ -463,7 +470,7 @@ async def test_get_dynamic_proxy_spec( rpc_client=Mock(), ) - exclude_keys: Mapping[int | str, Any] = { + exclude_keys = { "Labels": True, "TaskTemplate": {"ContainerSpec": {"Env": True}}, } @@ -471,62 +478,64 @@ async def test_get_dynamic_proxy_spec( # NOTE: some flakiness here # state_exclude is a set and does not preserve order # when dumping to json it gets converted to a list - assert dynamic_sidecar_spec.TaskTemplate - assert dynamic_sidecar_spec.TaskTemplate.ContainerSpec - assert dynamic_sidecar_spec.TaskTemplate.ContainerSpec.Env - assert dynamic_sidecar_spec.TaskTemplate.ContainerSpec.Env[ + assert dynamic_sidecar_spec.task_template + assert dynamic_sidecar_spec.task_template.container_spec + assert dynamic_sidecar_spec.task_template.container_spec.env + assert dynamic_sidecar_spec.task_template.container_spec.env[ "DY_SIDECAR_STATE_EXCLUDE" ] - dynamic_sidecar_spec.TaskTemplate.ContainerSpec.Env[ + dynamic_sidecar_spec.task_template.container_spec.env[ "DY_SIDECAR_STATE_EXCLUDE" ] = json.dumps( sorted( json.loads( - dynamic_sidecar_spec.TaskTemplate.ContainerSpec.Env[ + dynamic_sidecar_spec.task_template.container_spec.env[ "DY_SIDECAR_STATE_EXCLUDE" ] ) ) ) - assert expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec.Env[ + assert expected_dynamic_sidecar_spec_model.task_template.container_spec.env[ "DY_SIDECAR_STATE_EXCLUDE" ] - expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec.Env[ + expected_dynamic_sidecar_spec_model.task_template.container_spec.env[ "DY_SIDECAR_STATE_EXCLUDE" ] = json.dumps( sorted( json.loads( - expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec.Env[ + expected_dynamic_sidecar_spec_model.task_template.container_spec.env[ "DY_SIDECAR_STATE_EXCLUDE" ] ) ) ) - assert dynamic_sidecar_spec.dict( - exclude=exclude_keys - ) == expected_dynamic_sidecar_spec_model.dict(exclude=exclude_keys) - assert dynamic_sidecar_spec.Labels - assert expected_dynamic_sidecar_spec_model.Labels - assert sorted(dynamic_sidecar_spec.Labels.keys()) == sorted( - expected_dynamic_sidecar_spec_model.Labels.keys() + assert dynamic_sidecar_spec.model_dump( + exclude=exclude_keys # type: ignore[arg-type] + ) == expected_dynamic_sidecar_spec_model.model_dump( + exclude=exclude_keys # type: ignore[arg-type] + ) + assert dynamic_sidecar_spec.labels + assert expected_dynamic_sidecar_spec_model.labels + assert sorted(dynamic_sidecar_spec.labels.keys()) == sorted( + expected_dynamic_sidecar_spec_model.labels.keys() ) assert ( - dynamic_sidecar_spec.Labels["io.simcore.scheduler-data"] - == expected_dynamic_sidecar_spec_model.Labels["io.simcore.scheduler-data"] + dynamic_sidecar_spec.labels["io.simcore.scheduler-data"] + == expected_dynamic_sidecar_spec_model.labels["io.simcore.scheduler-data"] ) - assert dynamic_sidecar_spec.Labels == expected_dynamic_sidecar_spec_model.Labels + assert dynamic_sidecar_spec.labels == expected_dynamic_sidecar_spec_model.labels dynamic_sidecar_spec_accumulated = dynamic_sidecar_spec # check reference after multiple runs assert dynamic_sidecar_spec_accumulated is not None assert ( - dynamic_sidecar_spec_accumulated.dict() - == expected_dynamic_sidecar_spec_model.dict() + dynamic_sidecar_spec_accumulated.model_dump() + == expected_dynamic_sidecar_spec_model.model_dump() ) @@ -561,22 +570,22 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs( rpc_client=Mock(), ) assert dynamic_sidecar_spec - dynamic_sidecar_spec_dict = dynamic_sidecar_spec.dict() - expected_dynamic_sidecar_spec_dict = AioDockerServiceSpec.parse_obj( + dynamic_sidecar_spec_dict = dynamic_sidecar_spec.model_dump() + expected_dynamic_sidecar_spec_dict = AioDockerServiceSpec.model_validate( expected_dynamic_sidecar_spec - ).dict() + ).model_dump() # ensure some entries are sorted the same to prevent flakyness for sorted_dict in [dynamic_sidecar_spec_dict, expected_dynamic_sidecar_spec_dict]: for key in ["DY_SIDECAR_STATE_EXCLUDE", "DY_SIDECAR_STATE_PATHS"]: # this is a json of a list assert isinstance( - sorted_dict["TaskTemplate"]["ContainerSpec"]["Env"][key], str + sorted_dict["task_template"]["container_spec"]["env"][key], str ) unsorted_list = json.loads( - sorted_dict["TaskTemplate"]["ContainerSpec"]["Env"][key] + sorted_dict["task_template"]["container_spec"]["env"][key] ) assert isinstance(unsorted_list, list) - sorted_dict["TaskTemplate"]["ContainerSpec"]["Env"][key] = json.dumps( + sorted_dict["task_template"]["container_spec"]["env"][key] = json.dumps( unsorted_list.sort() ) assert dynamic_sidecar_spec_dict == expected_dynamic_sidecar_spec_dict @@ -591,13 +600,15 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs( ) assert user_service_specs assert "sidecar" in user_service_specs - user_aiodocker_service_spec = AioDockerServiceSpec.parse_obj( + user_aiodocker_service_spec = AioDockerServiceSpec.model_validate( user_service_specs["sidecar"] ) assert user_aiodocker_service_spec - orig_dict = dynamic_sidecar_spec.dict(by_alias=True, exclude_unset=True) - user_dict = user_aiodocker_service_spec.dict(by_alias=True, exclude_unset=True) + orig_dict = dynamic_sidecar_spec.model_dump(by_alias=True, exclude_unset=True) + user_dict = user_aiodocker_service_spec.model_dump( + by_alias=True, exclude_unset=True + ) another_merged_dict = nested_update( orig_dict, diff --git a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py index 9a2a93d3a33..977828e4753 100644 --- a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py +++ b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py @@ -36,9 +36,8 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimCoreFileLink, SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize +from pydantic import ByteSize, TypeAdapter from pydantic.networks import AnyUrl -from pydantic.tools import parse_obj_as from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_sdk.node_ports_v2 import FileLinkType @@ -95,16 +94,15 @@ async def mocked_node_ports_filemanager_fcts( 0, FileUploadSchema( urls=[ - parse_obj_as( - AnyUrl, + TypeAdapter(AnyUrl).validate_python( f"{URL(faker.uri()).with_scheme(choice(tasks_file_link_scheme))}", # noqa: S311 ) ], - chunk_size=parse_obj_as(ByteSize, "5GiB"), + chunk_size=TypeAdapter(ByteSize).validate_python("5GiB"), links=FileUploadLinks( - abort_upload=parse_obj_as(AnyUrl, "https://www.fakeabort.com"), - complete_upload=parse_obj_as( - AnyUrl, "https://www.fakecomplete.com" + abort_upload=TypeAdapter(AnyUrl).validate_python("https://www.fakeabort.com"), + complete_upload=TypeAdapter(AnyUrl).validate_python( + "https://www.fakecomplete.com" ), ), ), @@ -198,7 +196,7 @@ def generate_simcore_file_link() -> dict[str, Any]: path=create_simcore_file_id( faker.uuid4(), faker.uuid4(), faker.file_name() ), - ).dict(by_alias=True, exclude_unset=True) + ).model_dump(by_alias=True, exclude_unset=True) TYPE_TO_FAKE_CALLABLE_MAP = { "number": faker.pyfloat, @@ -234,7 +232,7 @@ def fake_task_output_data( ) for key, value in fake_io_data.items() } - data = parse_obj_as(TaskOutputData, converted_data) + data = TypeAdapter(TaskOutputData).validate_python(converted_data) assert data return data @@ -318,7 +316,7 @@ async def test_compute_input_data( sleeper_task.node_id, faker.file_name(), ), - ).dict(by_alias=True, exclude_unset=True) + ).model_dump(by_alias=True, exclude_unset=True) if value_type["type"] == "data:*/*" else fake_io_data[key] ) @@ -334,7 +332,7 @@ def return_fake_input_value(*args, **kwargs): fake_inputs.values(), fake_io_schema.values(), strict=True ): if value_type["type"] == "data:*/*": - yield parse_obj_as(AnyUrl, faker.url()) + yield TypeAdapter(AnyUrl).validate_python(faker.url()) else: yield value @@ -448,7 +446,7 @@ async def test_clean_task_output_and_log_files_if_invalid( path=create_simcore_file_id( published_project.project.uuid, sleeper_task.node_id, faker.file_name() ), - ).dict(by_alias=True, exclude_unset=True) + ).model_dump(by_alias=True, exclude_unset=True) for key, value_type in fake_io_schema.items() if value_type["type"] == "data:*/*" } @@ -494,7 +492,7 @@ def _add_is_directory(entry: mock._Call) -> mock._Call: # noqa: SLF001 @pytest.mark.parametrize( - "req_example", NodeRequirements.Config.schema_extra["examples"] + "req_example", NodeRequirements.model_config["json_schema_extra"]["examples"] ) def test_node_requirements_correctly_convert_to_dask_resources( req_example: dict[str, Any] diff --git a/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py b/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py index ccd3f304a0a..a041f70ecc7 100644 --- a/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py +++ b/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py @@ -129,7 +129,7 @@ async def test_publish_service_started_metrics( task=random.choice(tasks), # noqa: S311 ) await _assert_message_received( - mocked_message_parser, 1, InstrumentationRabbitMessage.parse_raw + mocked_message_parser, 1, InstrumentationRabbitMessage.model_validate_json ) @@ -154,7 +154,7 @@ async def test_publish_service_stopped_metrics( task_final_state=random.choice(list(RunningState)), # noqa: S311 ) await _assert_message_received( - mocked_message_parser, 1, InstrumentationRabbitMessage.parse_raw + mocked_message_parser, 1, InstrumentationRabbitMessage.model_validate_json ) @@ -177,7 +177,7 @@ async def test_publish_service_resource_tracking_started( RabbitResourceTrackingBaseMessage.get_channel_name(), mocked_message_parser ) random_service_run_id = faker.pystr() - before_publication_time = datetime.datetime.now(datetime.timezone.utc) + before_publication_time = datetime.datetime.now(datetime.UTC) await publish_service_resource_tracking_started( publisher, service_run_id=random_service_run_id, @@ -205,9 +205,11 @@ async def test_publish_service_resource_tracking_started( service_resources={}, service_additional_metadata=faker.pydict(), ) - after_publication_time = datetime.datetime.now(datetime.timezone.utc) + after_publication_time = datetime.datetime.now(datetime.UTC) received_messages = await _assert_message_received( - mocked_message_parser, 1, RabbitResourceTrackingStartedMessage.parse_raw + mocked_message_parser, + 1, + RabbitResourceTrackingStartedMessage.model_validate_json, ) assert isinstance(received_messages[0], RabbitResourceTrackingStartedMessage) assert received_messages[0].service_run_id == random_service_run_id @@ -231,7 +233,7 @@ async def test_publish_service_resource_tracking_stopped( RabbitResourceTrackingBaseMessage.get_channel_name(), mocked_message_parser ) random_service_run_id = faker.pystr() - before_publication_time = datetime.datetime.now(datetime.timezone.utc) + before_publication_time = datetime.datetime.now(datetime.UTC) await publish_service_resource_tracking_stopped( publisher, service_run_id=random_service_run_id, @@ -239,9 +241,11 @@ async def test_publish_service_resource_tracking_stopped( list(SimcorePlatformStatus) ), ) - after_publication_time = datetime.datetime.now(datetime.timezone.utc) + after_publication_time = datetime.datetime.now(datetime.UTC) received_messages = await _assert_message_received( - mocked_message_parser, 1, RabbitResourceTrackingStoppedMessage.parse_raw + mocked_message_parser, + 1, + RabbitResourceTrackingStoppedMessage.model_validate_json, ) assert isinstance(received_messages[0], RabbitResourceTrackingStoppedMessage) assert received_messages[0].service_run_id == random_service_run_id @@ -265,14 +269,16 @@ async def test_publish_service_resource_tracking_heartbeat( RabbitResourceTrackingBaseMessage.get_channel_name(), mocked_message_parser ) random_service_run_id = faker.pystr() - before_publication_time = datetime.datetime.now(datetime.timezone.utc) + before_publication_time = datetime.datetime.now(datetime.UTC) await publish_service_resource_tracking_heartbeat( publisher, service_run_id=random_service_run_id, ) - after_publication_time = datetime.datetime.now(datetime.timezone.utc) + after_publication_time = datetime.datetime.now(datetime.UTC) received_messages = await _assert_message_received( - mocked_message_parser, 1, RabbitResourceTrackingHeartbeatMessage.parse_raw + mocked_message_parser, + 1, + RabbitResourceTrackingHeartbeatMessage.model_validate_json, ) assert isinstance(received_messages[0], RabbitResourceTrackingHeartbeatMessage) assert received_messages[0].service_run_id == random_service_run_id diff --git a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py index a1b64635e58..cb5631ec20d 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py @@ -313,7 +313,7 @@ async def s3_client(s3_settings: S3Settings) -> AsyncIterable[S3Client]: session = aioboto3.Session() session_client = session.client( "s3", - endpoint_url=s3_settings.S3_ENDPOINT, + endpoint_url=f"{s3_settings.S3_ENDPOINT}", aws_access_key_id=s3_settings.S3_ACCESS_KEY, aws_secret_access_key=s3_settings.S3_SECRET_KEY, region_name=s3_settings.S3_REGION, From a2d3742592adcab044fddf2c8f8a2d751e7ebfd8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Nov 2024 10:20:23 +0100 Subject: [PATCH 35/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Fix=20unit=20tests?= =?UTF-8?q?=20after=20master=20merge=20(#6727)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/models_library/folders.py | 23 +++++--- .../src/models_library/workspaces.py | 23 +++++--- .../tests/test_projects_state.py | 2 +- .../services/director_v2.py | 10 +++- .../core/settings.py | 6 +- .../folders/_folders_handlers.py | 8 +-- .../folders/_models.py | 50 ++++++++--------- .../tests/unit/with_dbs/03/test_trash.py | 56 +++++++++---------- .../04/folders/test_folders__full_search.py | 2 +- 9 files changed, 97 insertions(+), 83 deletions(-) diff --git a/packages/models-library/src/models_library/folders.py b/packages/models-library/src/models_library/folders.py index fad48dfb90a..094ea25be92 100644 --- a/packages/models-library/src/models_library/folders.py +++ b/packages/models-library/src/models_library/folders.py @@ -2,7 +2,14 @@ from enum import auto from typing import TypeAlias -from pydantic import BaseModel, ConfigDict, Field, PositiveInt, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PositiveInt, + ValidationInfo, + field_validator, +) from .access_rights import AccessRights from .users import GroupID, UserID @@ -24,16 +31,14 @@ class FolderQuery(BaseModel): @field_validator("folder_id", mode="before") @classmethod - def validate_folder_id(cls, value, values): - scope = values.get("folder_scope") + def validate_folder_id(cls, value, info: ValidationInfo): + scope = info.data.get("folder_scope") if scope == FolderScope.SPECIFIC and value is None: - raise ValueError( - "folder_id must be provided when folder_scope is SPECIFIC." - ) + msg = "folder_id must be provided when folder_scope is SPECIFIC." + raise ValueError(msg) if scope != FolderScope.SPECIFIC and value is not None: - raise ValueError( - "folder_id should be None when folder_scope is not SPECIFIC." - ) + msg = "folder_id should be None when folder_scope is not SPECIFIC." + raise ValueError(msg) return value diff --git a/packages/models-library/src/models_library/workspaces.py b/packages/models-library/src/models_library/workspaces.py index 24bae0e7d5d..5d1d206e8dd 100644 --- a/packages/models-library/src/models_library/workspaces.py +++ b/packages/models-library/src/models_library/workspaces.py @@ -2,7 +2,14 @@ from enum import auto from typing import TypeAlias -from pydantic import BaseModel, ConfigDict, Field, PositiveInt, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PositiveInt, + ValidationInfo, + field_validator, +) from .access_rights import AccessRights from .users import GroupID @@ -23,16 +30,14 @@ class WorkspaceQuery(BaseModel): @field_validator("workspace_id", mode="before") @classmethod - def validate_workspace_id(cls, value, values): - scope = values.get("workspace_scope") + def validate_workspace_id(cls, value, info: ValidationInfo): + scope = info.data.get("workspace_scope") if scope == WorkspaceScope.SHARED and value is None: - raise ValueError( - "workspace_id must be provided when workspace_scope is SHARED." - ) + msg = "workspace_id must be provided when workspace_scope is SHARED." + raise ValueError(msg) if scope != WorkspaceScope.SHARED and value is not None: - raise ValueError( - "workspace_id should be None when workspace_scope is not SHARED." - ) + msg = "workspace_id should be None when workspace_scope is not SHARED." + raise ValueError(msg) return value diff --git a/packages/models-library/tests/test_projects_state.py b/packages/models-library/tests/test_projects_state.py index a5e7f5ce7ea..08493f9f3b1 100644 --- a/packages/models-library/tests/test_projects_state.py +++ b/packages/models-library/tests/test_projects_state.py @@ -9,7 +9,7 @@ def test_project_locked_with_missing_owner_raises(): def test_project_locked_with_missing_owner_ok_during_maintaining(): - ProjectLocked.parse_obj({"value": True, "status": ProjectStatus.MAINTAINING}) + ProjectLocked.model_validate({"value": True, "status": ProjectStatus.MAINTAINING}) @pytest.mark.parametrize( diff --git a/services/api-server/src/simcore_service_api_server/services/director_v2.py b/services/api-server/src/simcore_service_api_server/services/director_v2.py index abf112212b4..e225a8adef7 100644 --- a/services/api-server/src/simcore_service_api_server/services/director_v2.py +++ b/services/api-server/src/simcore_service_api_server/services/director_v2.py @@ -7,7 +7,15 @@ from models_library.projects_nodes_io import NodeID from models_library.projects_pipeline import ComputationTask from models_library.projects_state import RunningState -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, PositiveInt, TypeAdapter +from pydantic import ( + AnyHttpUrl, + AnyUrl, + BaseModel, + ConfigDict, + Field, + PositiveInt, + TypeAdapter, +) from settings_library.tracing import TracingSettings from starlette import status diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py index e2b2bc8791b..cc7261e73be 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py @@ -4,6 +4,7 @@ from pydantic import AliasChoices, Field, TypeAdapter, field_validator from pydantic.networks import AnyUrl from servicelib.logging_utils_filtering import LoggerName, MessageSubstring +from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings from settings_library.tracing import TracingSettings from settings_library.utils_logging import MixinLoggingSettings @@ -19,10 +20,7 @@ class PennsieveSettings(BaseCustomSettings): PENNSIEVE_HEALTCHCHECK_TIMEOUT: float = 1.0 -class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): - # DOCKER - SC_BOOT_MODE: BootModeEnum | None - +class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): LOG_LEVEL: LogLevel = Field( default=LogLevel.INFO.value, validation_alias=AliasChoices( diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py index 64b69994f51..c7904930897 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py @@ -82,10 +82,10 @@ async def list_folders(request: web.Request): trashed=query_params.filters.trashed, offset=query_params.offset, limit=query_params.limit, - order_by=parse_obj_as(OrderBy, query_params.order_by), + order_by=OrderBy.model_validate(query_params.order_by), ) - page = Page[FolderGet].parse_obj( + page = Page[FolderGet].model_validate( paginate_data( chunk=folders.items, request_url=request.url, @@ -95,7 +95,7 @@ async def list_folders(request: web.Request): ) ) return web.Response( - text=page.json(**RESPONSE_MODEL_POLICY), + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) @@ -105,7 +105,7 @@ async def list_folders(request: web.Request): @permission_required("folder.read") @handle_plugin_requests_exceptions async def list_folders_full_search(request: web.Request): - req_ctx = FoldersRequestContext.parse_obj(request) + req_ctx = FoldersRequestContext.model_validate(request) query_params: FolderListFullSearchWithJsonStrQueryParams = ( parse_request_query_parameters_as( FolderListFullSearchWithJsonStrQueryParams, request diff --git a/services/web/server/src/simcore_service_webserver/folders/_models.py b/services/web/server/src/simcore_service_webserver/folders/_models.py index 899514a271b..7e90b91e412 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_models.py +++ b/services/web/server/src/simcore_service_webserver/folders/_models.py @@ -1,4 +1,5 @@ import logging +from typing import Annotated from models_library.basic_types import IDStr from models_library.folders import FolderID @@ -11,7 +12,14 @@ null_or_none_str_to_none_validator, ) from models_library.workspaces import WorkspaceID -from pydantic import BaseModel, Extra, Field, Json, validator +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + Field, + Json, + field_validator, +) from servicelib.aiohttp.requests_validation import RequestParams, StrictRequestParams from servicelib.request_keys import RQT_USERID_KEY @@ -41,11 +49,11 @@ class FolderListSortParams(BaseModel): order_by: Json[OrderBy] = Field( default=OrderBy(field=IDStr("modified"), direction=OrderDirection.DESC), description="Order by field (modified_at|name|description) and direction (asc|desc). The default sorting order is ascending.", - example='{"field": "name", "direction": "desc"}', + examples=['{"field": "name", "direction": "desc"}'], alias="order_by", ) - @validator("order_by", check_fields=False) + @field_validator("order_by", check_fields=False) @classmethod def _validate_order_by_field(cls, v): if v.field not in { @@ -59,51 +67,41 @@ def _validate_order_by_field(cls, v): v.field = "modified" return v - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class FolderListWithJsonStrQueryParams( PageQueryParameters, FolderListSortParams, FiltersQueryParameters[FolderFilters] ): - folder_id: FolderID | None = Field( + folder_id: Annotated[ + FolderID | None, BeforeValidator(null_or_none_str_to_none_validator) + ] = Field( default=None, description="List the subfolders of this folder. By default, list the subfolders of the root directory (Folder ID is None).", ) - workspace_id: WorkspaceID | None = Field( + workspace_id: Annotated[ + WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator) + ] = Field( default=None, description="List folders in specific workspace. By default, list in the user private workspace", ) - class Config: - extra = Extra.forbid - - # validators - _null_or_none_str_to_none_validator = validator( - "folder_id", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) - - _null_or_none_str_to_none_validator2 = validator( - "workspace_id", allow_reuse=True, pre=True - )(null_or_none_str_to_none_validator) + model_config = ConfigDict(extra="forbid") class FolderListFullSearchWithJsonStrQueryParams( PageQueryParameters, FolderListSortParams, FiltersQueryParameters[FolderFilters] ): - text: str | None = Field( + text: Annotated[ + str | None, BeforeValidator(empty_str_to_none_pre_validator) + ] = Field( default=None, description="Multi column full text search, across all folders and workspaces", max_length=100, - example="My Project", - ) - - _empty_is_none = validator("text", allow_reuse=True, pre=True)( - empty_str_to_none_pre_validator + examples=["My Project"], ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class RemoveQueryParams(BaseModel): diff --git a/services/web/server/tests/unit/with_dbs/03/test_trash.py b/services/web/server/tests/unit/with_dbs/03/test_trash.py index 7d6c701c522..16a5f9dc147 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_trash.py +++ b/services/web/server/tests/unit/with_dbs/03/test_trash.py @@ -103,7 +103,7 @@ async def test_trash_projects( # noqa: PLR0915 resp = await client.get("/v0/projects") await assert_status(resp, status.HTTP_200_OK) - page = Page[ProjectListItem].parse_obj(await resp.json()) + page = Page[ProjectListItem].model_validate(await resp.json()) assert page.meta.total == 1 got = page.data[0] @@ -114,7 +114,7 @@ async def test_trash_projects( # noqa: PLR0915 resp = await client.get("/v0/projects", params={"filters": '{"trashed": true}'}) await assert_status(resp, status.HTTP_200_OK) - page = Page[ProjectListItem].parse_obj(await resp.json()) + page = Page[ProjectListItem].model_validate(await resp.json()) assert page.meta.total == 0 # TRASH @@ -138,7 +138,7 @@ async def test_trash_projects( # noqa: PLR0915 # GET resp = await client.get(f"/v0/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = ProjectGet.parse_obj(data) + got = ProjectGet.model_validate(data) assert got.uuid == project_uuid if could_not_trash: @@ -152,7 +152,7 @@ async def test_trash_projects( # noqa: PLR0915 resp = await client.get("/v0/projects", params={"filters": '{"trashed": true}'}) await assert_status(resp, status.HTTP_200_OK) - page = Page[ProjectListItem].parse_obj(await resp.json()) + page = Page[ProjectListItem].model_validate(await resp.json()) if could_not_trash: assert page.meta.total == 0 else: @@ -166,7 +166,7 @@ async def test_trash_projects( # noqa: PLR0915 # GET resp = await client.get(f"/v0/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = ProjectGet.parse_obj(data) + got = ProjectGet.model_validate(data) assert got.uuid == project_uuid assert got.trashed_at is None @@ -187,7 +187,7 @@ async def test_trash_single_folder(client: TestClient, logged_user: UserInfoDict # CREATE a folder resp = await client.post("/v0/folders", json={"name": "My first folder"}) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - folder = FolderGet.parse_obj(data) + folder = FolderGet.model_validate(data) # --------------------------------------------------------------------- @@ -195,7 +195,7 @@ async def test_trash_single_folder(client: TestClient, logged_user: UserInfoDict resp = await client.get("/v0/folders") await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 1 assert page.data[0] == folder @@ -204,7 +204,7 @@ async def test_trash_single_folder(client: TestClient, logged_user: UserInfoDict resp = await client.get("/v0/folders", params={"filters": '{"trashed": true}'}) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 0 # TRASH @@ -222,7 +222,7 @@ async def test_trash_single_folder(client: TestClient, logged_user: UserInfoDict # GET resp = await client.get(f"/v0/folders/{folder.folder_id}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = FolderGet.parse_obj(data) + got = FolderGet.model_validate(data) assert got.folder_id == folder.folder_id assert got.trashed_at @@ -233,7 +233,7 @@ async def test_trash_single_folder(client: TestClient, logged_user: UserInfoDict resp = await client.get("/v0/folders", params={"filters": '{"trashed": true}'}) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 1 assert page.data[0].folder_id == folder.folder_id @@ -249,7 +249,7 @@ async def test_trash_single_folder(client: TestClient, logged_user: UserInfoDict # GET resp = await client.get(f"/v0/folders/{folder.folder_id}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = FolderGet.parse_obj(data) + got = FolderGet.model_validate(data) assert got.folder_id == folder.folder_id assert got.trashed_at is None @@ -271,7 +271,7 @@ async def test_trash_folder_with_content( # CREATE a folder resp = await client.post("/v0/folders", json={"name": "My first folder"}) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - folder = FolderGet.parse_obj(data) + folder = FolderGet.model_validate(data) # CREATE a SUB-folder resp = await client.post( @@ -279,7 +279,7 @@ async def test_trash_folder_with_content( json={"name": "My subfolder 1", "parentFolderId": folder.folder_id}, ) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - subfolder = FolderGet.parse_obj(data) + subfolder = FolderGet.model_validate(data) # MOVE project to SUB-folder resp = await client.put( @@ -290,13 +290,13 @@ async def test_trash_folder_with_content( # CHECK created resp = await client.get("/v0/folders") await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 1 assert page.data[0] == folder resp = await client.get("/v0/folders", params={"folder_id": f"{folder.folder_id}"}) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 1 assert page.data[0] == subfolder @@ -304,7 +304,7 @@ async def test_trash_folder_with_content( "/v0/projects", params={"folder_id": f"{subfolder.folder_id}"} ) await assert_status(resp, status.HTTP_200_OK) - page = Page[ProjectListItem].parse_obj(await resp.json()) + page = Page[ProjectListItem].model_validate(await resp.json()) assert page.meta.total == 1 assert page.data[0].uuid == project_uuid assert page.data[0].folder_id == subfolder.folder_id @@ -318,7 +318,7 @@ async def test_trash_folder_with_content( # ONLY folder listed in trash. The rest is not listed anymore! resp = await client.get("/v0/folders", params={"filters": '{"trashed": true}'}) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 1 assert page.data[0].folder_id == folder.folder_id @@ -327,7 +327,7 @@ async def test_trash_folder_with_content( params={"filters": '{"trashed": true}', "folder_id": f"{folder.folder_id}"}, ) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 0 resp = await client.get( @@ -335,23 +335,23 @@ async def test_trash_folder_with_content( params={"filters": '{"trashed": true}', "folder_id": f"{subfolder.folder_id}"}, ) await assert_status(resp, status.HTTP_200_OK) - page = Page[ProjectListItem].parse_obj(await resp.json()) + page = Page[ProjectListItem].model_validate(await resp.json()) assert page.meta.total == 0 # CHECK marked as trashed resp = await client.get(f"/v0/folders/{folder.folder_id}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = FolderGet.parse_obj(data) + got = FolderGet.model_validate(data) assert got.trashed_at is not None resp = await client.get(f"/v0/folders/{subfolder.folder_id}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = FolderGet.parse_obj(data) + got = FolderGet.model_validate(data) assert got.trashed_at is not None resp = await client.get(f"/v0/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = ProjectGet.parse_obj(data) + got = ProjectGet.model_validate(data) assert got.trashed_at is not None # UNTRASH folder @@ -361,7 +361,7 @@ async def test_trash_folder_with_content( # NO folders listed in trash. resp = await client.get("/v0/folders", params={"filters": '{"trashed": true}'}) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 0 resp = await client.get( @@ -369,7 +369,7 @@ async def test_trash_folder_with_content( params={"filters": '{"trashed": true}', "folder_id": f"{folder.folder_id}"}, ) await assert_status(resp, status.HTTP_200_OK) - page = Page[FolderGet].parse_obj(await resp.json()) + page = Page[FolderGet].model_validate(await resp.json()) assert page.meta.total == 0 resp = await client.get( @@ -377,21 +377,21 @@ async def test_trash_folder_with_content( params={"filters": '{"trashed": true}', "folder_id": f"{subfolder.folder_id}"}, ) await assert_status(resp, status.HTTP_200_OK) - page = Page[ProjectListItem].parse_obj(await resp.json()) + page = Page[ProjectListItem].model_validate(await resp.json()) assert page.meta.total == 0 # CHECK marked as trashed resp = await client.get(f"/v0/folders/{folder.folder_id}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = FolderGet.parse_obj(data) + got = FolderGet.model_validate(data) assert got.trashed_at is None resp = await client.get(f"/v0/folders/{subfolder.folder_id}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = FolderGet.parse_obj(data) + got = FolderGet.model_validate(data) assert got.trashed_at is None resp = await client.get(f"/v0/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - got = ProjectGet.parse_obj(data) + got = ProjectGet.model_validate(data) assert got.trashed_at is None diff --git a/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py b/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py index 74126da042f..e9bde5d9ec5 100644 --- a/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py +++ b/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py @@ -95,7 +95,7 @@ async def test_folders_full_search( }, ) data, _ = await assert_status(resp, status.HTTP_200_OK) - assert FolderGet.parse_obj(data) + assert FolderGet.model_validate(data) # list full folder search url = client.app.router["list_folders_full_search"].url_for() From 327beab8a3cf0d3779e86b0141ae0815a0e19abd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 15 Nov 2024 13:23:56 +0100 Subject: [PATCH 36/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Align=20versions=20f?= =?UTF-8?q?or=20pydantic/fastapi=20(#6730)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/aws-library/requirements/_base.txt | 4 ++-- packages/aws-library/requirements/_test.txt | 4 ++-- .../requirements/_base.txt | 4 ++-- .../notifications-library/requirements/_base.txt | 4 ++-- packages/postgres-database/requirements/_base.txt | 14 +++++++------- packages/service-library/requirements/_fastapi.txt | 4 ++-- services/agent/requirements/_base.txt | 3 ++- services/api-server/requirements/_base.txt | 4 ++-- services/autoscaling/requirements/_base.txt | 4 ++-- services/catalog/requirements/_base.txt | 2 +- services/clusters-keeper/requirements/_base.txt | 4 ++-- services/datcore-adapter/requirements/_base.txt | 2 +- services/director-v2/requirements/_base.txt | 2 +- services/dynamic-scheduler/requirements/_base.txt | 2 +- services/dynamic-sidecar/requirements/_base.txt | 4 ++-- services/efs-guardian/requirements/_base.txt | 4 ++-- services/invitations/requirements/_base.txt | 4 ++-- services/payments/requirements/_base.txt | 2 +- .../resource-usage-tracker/requirements/_base.txt | 4 ++-- 19 files changed, 38 insertions(+), 37 deletions(-) diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index c6500144e42..a3a10ea494a 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -218,7 +218,7 @@ protobuf==4.25.5 # opentelemetry-proto psutil==6.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -pydantic==2.9.1 +pydantic==2.9.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -247,7 +247,7 @@ pydantic==2.9.1 # fast-depends # pydantic-extra-types # pydantic-settings -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt index 562f30e3486..eef079aef0f 100644 --- a/packages/aws-library/requirements/_test.txt +++ b/packages/aws-library/requirements/_test.txt @@ -158,12 +158,12 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==2.9.1 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via # -c requirements/_base.txt # pydantic diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt index 2fa04d4835b..82af72057cd 100644 --- a/packages/dask-task-models-library/requirements/_base.txt +++ b/packages/dask-task-models-library/requirements/_base.txt @@ -76,7 +76,7 @@ partd==1.4.2 # via dask psutil==6.0.0 # via distributed -pydantic==2.9.1 +pydantic==2.9.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -92,7 +92,7 @@ pydantic==2.9.1 # -r requirements/_base.in # pydantic-extra-types # pydantic-settings -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt index 3fc598588d3..560e3e1e3b6 100644 --- a/packages/notifications-library/requirements/_base.txt +++ b/packages/notifications-library/requirements/_base.txt @@ -102,7 +102,7 @@ packaging==24.2 # via opentelemetry-instrumentation psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==2.9.1 +pydantic==2.9.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -121,7 +121,7 @@ pydantic==2.9.1 # -r requirements/../../../packages/settings-library/requirements/_base.in # pydantic-extra-types # pydantic-settings -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.9.0 # via diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index 258f5340638..4eddd14e0e4 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -25,11 +25,6 @@ markupsafe==2.1.5 # via mako multidict==6.1.0 # via yarl -orjson==3.10.11 - # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/common-library/requirements/_base.in opentelemetry-api==1.27.0 # via # opentelemetry-instrumentation @@ -41,16 +36,21 @@ opentelemetry-instrumentation-asyncpg==0.48b0 # via -r requirements/_base.in opentelemetry-semantic-conventions==0.48b0 # via opentelemetry-instrumentation-asyncpg +orjson==3.10.11 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==2.9.1 +pydantic==2.9.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in # pydantic-extra-types -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.10.0 # via -r requirements/../../../packages/common-library/requirements/_base.in diff --git a/packages/service-library/requirements/_fastapi.txt b/packages/service-library/requirements/_fastapi.txt index c1aa437bdea..b93f8be5cc4 100644 --- a/packages/service-library/requirements/_fastapi.txt +++ b/packages/service-library/requirements/_fastapi.txt @@ -22,7 +22,7 @@ deprecated==1.2.14 # via # opentelemetry-api # opentelemetry-semantic-conventions -fastapi==0.115.0 +fastapi==0.115.5 # via # -r requirements/_fastapi.in # prometheus-fastapi-instrumentator @@ -98,7 +98,7 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.38.5 +starlette==0.41.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index dd7a2ade645..887b8ce278e 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -86,7 +86,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.2 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -424,6 +424,7 @@ wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis yarl==1.15.3 # via # aio-pika diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 8b08e429db1..50bb56b4e69 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -182,7 +182,7 @@ email-validator==2.1.1 # pydantic fast-depends==2.4.2 # via faststream -fastapi==0.115.2 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -753,7 +753,7 @@ sqlalchemy==1.4.52 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # aiopg # alembic -starlette==0.38.5 +starlette==0.41.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index ba3c992df23..39676c07d5b 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -157,7 +157,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.0 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -631,7 +631,7 @@ sortedcontainers==2.4.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -starlette==0.38.6 +starlette==0.41.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 6b8253888a8..7f61e93f32a 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -99,7 +99,7 @@ email-validator==2.1.1 # pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.3 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 24cf9913155..c642e30aa64 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -155,7 +155,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.0 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -629,7 +629,7 @@ sortedcontainers==2.4.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -starlette==0.38.6 +starlette==0.41.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index ef6b7c14ebb..476901f832b 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -93,7 +93,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.3 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 4ce57c6439b..15a4e37ffc5 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -192,7 +192,7 @@ email-validator==2.1.1 # pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.4 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 2259db3f8c3..f8f20669b02 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -98,7 +98,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.2 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index e87c25b3aa3..58a33dc57b6 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -152,7 +152,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.2 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -679,7 +679,7 @@ sqlalchemy==1.4.52 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in # aiopg # alembic -starlette==0.39.2 +starlette==0.41.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index b627b9f2146..f0ac604d836 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -150,7 +150,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.0 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -645,7 +645,7 @@ sqlalchemy==1.4.54 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.38.6 +starlette==0.41.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 2f391b7a2c8..a0868ca0ed8 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -100,7 +100,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.0 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -383,7 +383,7 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.38.6 +starlette==0.41.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index 9c37389b68b..e520a81b67c 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -120,7 +120,7 @@ email-validator==2.2.0 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.2 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index b1c8e1f6878..0f0c9c3592e 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -156,7 +156,7 @@ email-validator==2.1.1 # via pydantic fast-depends==2.4.12 # via faststream -fastapi==0.115.0 +fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in @@ -692,7 +692,7 @@ sqlalchemy==1.4.52 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.38.6 +starlette==0.41.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt From 84c4c85573003d04848bcc483fd4901c149bf984 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:33:06 +0100 Subject: [PATCH 37/53] =?UTF-8?q?=E2=99=BB=EF=B8=8FFixed=20Unit=20test=20d?= =?UTF-8?q?irector=20v2=20(#6733)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/models_library/clusters.py | 29 +++++++------------ .../models/comp_runs.py | 28 ++++++++++-------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index 8b38913c246..c857073dead 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -1,8 +1,7 @@ from enum import auto from pathlib import Path -from typing import Final, Literal, TypeAlias +from typing import Final, Literal, Self, TypeAlias -from models_library.utils._original_fastapi_encoders import jsonable_encoder from pydantic import ( AnyUrl, BaseModel, @@ -222,29 +221,23 @@ class Cluster(BaseCluster): }, ) - @model_validator(mode="before") - @classmethod - def check_owner_has_access_rights(cls, values): - values = jsonable_encoder(values) - - is_default_cluster = bool(values["id"] == DEFAULT_CLUSTER_ID) - owner_gid = values["owner"] + @model_validator(mode="after") + def check_owner_has_access_rights(self: Self) -> Self: + is_default_cluster = bool(self.id == DEFAULT_CLUSTER_ID) + owner_gid = self.owner # check owner is in the access rights, if not add it - access_rights = values.get("access_rights", values.get("accessRights", {})) + access_rights = self.access_rights.copy() if owner_gid not in access_rights: access_rights[owner_gid] = ( - CLUSTER_USER_RIGHTS.model_dump() - if is_default_cluster - else CLUSTER_ADMIN_RIGHTS.model_dump() + CLUSTER_USER_RIGHTS if is_default_cluster else CLUSTER_ADMIN_RIGHTS ) # check owner has the expected access if access_rights[owner_gid] != ( - CLUSTER_USER_RIGHTS.model_dump() - if is_default_cluster - else CLUSTER_ADMIN_RIGHTS.model_dump() + CLUSTER_USER_RIGHTS if is_default_cluster else CLUSTER_ADMIN_RIGHTS ): msg = f"the cluster owner access rights are incorrectly set: {access_rights[owner_gid]}" raise ValueError(msg) - values["access_rights"] = access_rights - return values + # NOTE: overcomes frozen configuration (far fetched in ClusterGet model of webserver) + object.__setattr__(self, "access_rights", access_rights) + return self diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index 2c671d2c146..aba02269415 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -95,8 +95,10 @@ def convert_null_to_empty_metadata(cls, v): "cluster_id": 0, "iteration": 42, "result": "UNKNOWN", - "created": "2021-03-01 13:07:34.19161", - "modified": "2021-03-01 13:07:34.19161", + "started": None, + "ended": None, + "created": "2021-03-01T13:07:34.191610", + "modified": "2021-03-01T13:07:34.191610", "cancelled": None, "use_on_demand_clusters": False, }, @@ -107,8 +109,10 @@ def convert_null_to_empty_metadata(cls, v): "cluster_id": None, # this default to DEFAULT_CLUSTER_ID "iteration": 42, "result": "NOT_STARTED", - "created": "2021-03-01 13:07:34.19161", - "modified": "2021-03-01 13:07:34.19161", + "started": None, + "ended": None, + "created": "2021-03-01T13:07:34.191610", + "modified": "2021-03-01T13:07:34.191610", "cancelled": None, "use_on_demand_clusters": False, }, @@ -119,10 +123,10 @@ def convert_null_to_empty_metadata(cls, v): "cluster_id": 123, "iteration": 12, "result": "SUCCESS", - "created": "2021-03-01 13:07:34.19161", - "modified": "2021-03-01 13:07:34.19161", - "started": "2021-03-01 08:07:34.19161", - "ended": "2021-03-01 13:07:34.10", + "created": "2021-03-01T13:07:34.191610", + "modified": "2021-03-01T13:07:34.191610", + "started": "2021-03-01T08:07:34.191610", + "ended": "2021-03-01T13:07:34.10", "cancelled": None, "metadata": { "node_id_names_map": {}, @@ -140,10 +144,10 @@ def convert_null_to_empty_metadata(cls, v): "cluster_id": 123, "iteration": 12, "result": "SUCCESS", - "created": "2021-03-01 13:07:34.19161", - "modified": "2021-03-01 13:07:34.19161", - "started": "2021-03-01 8:07:34.19161", - "ended": "2021-03-01 13:07:34.10", + "created": "2021-03-01T13:07:34.191610", + "modified": "2021-03-01T13:07:34.191610", + "started": "2021-03-01T08:07:34.191610", + "ended": "2021-03-01T13:07:34.10", "cancelled": None, "metadata": None, "use_on_demand_clusters": False, From 8eac56bf59ae0e5bf73f9c7455c03104d91c8b74 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 18 Nov 2024 11:57:07 +0100 Subject: [PATCH 38/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Fix=20deprecated=20s?= =?UTF-8?q?tuff=20(Pydantic=20v2)=20(#6732)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/web-server/_common.py | 5 +++-- .../function_services_catalog/api.py | 2 +- .../services_api_mocks_for_aiohttp_clients.py | 4 ++-- .../servicelib/aiohttp/db_asyncpg_engine.py | 2 +- .../servicelib/aiohttp/requests_validation.py | 2 +- .../src/servicelib/project_lock.py | 2 +- .../src/servicelib/services_utils.py | 4 ++-- ...st_long_running_tasks_with_task_context.py | 2 +- .../src/settings_library/utils_encoders.py | 22 ------------------- .../agent/tests/unit/test_api_rest__health.py | 2 +- .../modules/comp_scheduler/_base_scheduler.py | 2 +- services/director-v2/tests/unit/conftest.py | 2 +- .../test_modules_dynamic_sidecar_scheduler.py | 2 +- .../tests/unit/test_utils_computation.py | 2 +- .../director-v2/tests/unit/test_utils_dags.py | 4 ++-- .../tests/unit/with_dbs/conftest.py | 9 ++++++-- .../tests/unit/test_api_rest_containers.py | 2 +- .../tests/unit/api/test_api_dependencies.py | 2 +- services/invitations/tests/unit/test_cli.py | 2 +- .../osparc_gateway_server/backend/osparc.py | 2 +- .../scripts/example_payment_gateway.py | 2 +- .../services/payments_gateway.py | 2 +- .../cli.py | 2 +- .../api_keys/_api.py | 2 +- .../catalog/_api_units.py | 4 ++-- .../catalog/_models.py | 4 ++-- .../groups/_handlers.py | 4 ++-- .../simcore_service_webserver/groups/api.py | 4 ++-- .../login/_registration.py | 2 +- .../projects/projects_api.py | 12 +++++++--- .../socketio/models.py | 5 +++-- .../studies_dispatcher/_redirects_handlers.py | 7 +++--- .../users/_tokens.py | 6 ++--- .../01/clusters/test_clusters_handlers.py | 6 +++-- .../test_meta_modeling_iterations.py | 2 +- 35 files changed, 67 insertions(+), 73 deletions(-) delete mode 100644 packages/settings-library/src/settings_library/utils_encoders.py diff --git a/api/specs/web-server/_common.py b/api/specs/web-server/_common.py index f3dcd66bc5c..25341fd2a31 100644 --- a/api/specs/web-server/_common.py +++ b/api/specs/web-server/_common.py @@ -8,6 +8,7 @@ from typing import Any, ClassVar, NamedTuple import yaml +from common_library.pydantic_fields_extension import get_type from fastapi import FastAPI from models_library.basic_types import LogLevel from pydantic import BaseModel, Field @@ -116,8 +117,8 @@ def assert_handler_signature_against_model( # query and path parameters implemented_params = [ - ParamSpec(field.name, field.type_, field.field_info) - for field in model_cls.__fields__.values() + ParamSpec(name, get_type(info), info) + for name, info in model_cls.model_fields.items() ] assert {p.name for p in implemented_params}.issubset( # nosec diff --git a/packages/models-library/src/models_library/function_services_catalog/api.py b/packages/models-library/src/models_library/function_services_catalog/api.py index 48488cbf567..0b99e4d6682 100644 --- a/packages/models-library/src/models_library/function_services_catalog/api.py +++ b/packages/models-library/src/models_library/function_services_catalog/api.py @@ -24,7 +24,7 @@ def iter_service_docker_data() -> Iterator[ServiceMetaDataPublished]: for meta_obj in catalog.iter_metadata(): # NOTE: the originals are this way not modified from outside - copied_meta_obj = meta_obj.copy(deep=True) + copied_meta_obj = meta_obj.model_copy(deep=True) assert is_function_service(copied_meta_obj.key) # nosec yield copied_meta_obj diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index 429783e7061..aea927de4d6 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -109,7 +109,7 @@ def create_computation_cb(url, **kwargs) -> CallbackResult: } returned_computation = ComputationTask.model_validate( ComputationTask.model_config["json_schema_extra"]["examples"][0] - ).copy( + ).model_copy( update={ "id": f"{kwargs['json']['project_id']}", "state": state, @@ -133,7 +133,7 @@ def get_computation_cb(url, **kwargs) -> CallbackResult: node_states = FULL_PROJECT_NODE_STATES returned_computation = ComputationTask.model_validate( ComputationTask.model_config["json_schema_extra"]["examples"][0] - ).copy( + ).model_copy( update={ "id": Path(url.path).name, "state": state, diff --git a/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py b/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py index 2ca9d431075..1163a479c68 100644 --- a/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py +++ b/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py @@ -46,7 +46,7 @@ async def connect_to_db(app: web.Application, settings: PostgresSettings) -> Non - sets an engine in app state (use `get_async_engine(app)` to retrieve) """ if settings.POSTGRES_CLIENT_NAME: - settings = settings.copy( + settings = settings.model_copy( update={"POSTGRES_CLIENT_NAME": settings.POSTGRES_CLIENT_NAME + "-asyncpg"} ) diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index 0dd40949ba8..a776ad918f4 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -171,7 +171,7 @@ def parse_request_query_parameters_as( # query parameters with the same key. However, we are not using such cases anywhere at the moment. data = dict(request.query) - if hasattr(parameters_schema_cls, "parse_obj"): + if hasattr(parameters_schema_cls, "model_validate"): return parameters_schema_cls.model_validate(data) model: ModelClass = TypeAdapter(parameters_schema_cls).validate_python(data) return model diff --git a/packages/service-library/src/servicelib/project_lock.py b/packages/service-library/src/servicelib/project_lock.py index e1a275dcfc7..f2ae6ce6ddd 100644 --- a/packages/service-library/src/servicelib/project_lock.py +++ b/packages/service-library/src/servicelib/project_lock.py @@ -49,7 +49,7 @@ async def lock_project( value=True, owner=owner, status=status, - ).json(), + ).model_dump_json(), ): msg = f"Lock for project {project_uuid!r} owner {owner!r} could not be acquired" raise ProjectLockError(msg) diff --git a/packages/service-library/src/servicelib/services_utils.py b/packages/service-library/src/servicelib/services_utils.py index 98aace49c6c..889594cbf0c 100644 --- a/packages/service-library/src/servicelib/services_utils.py +++ b/packages/service-library/src/servicelib/services_utils.py @@ -22,7 +22,7 @@ def get_status_as_dict( ) -> dict: """shared between different backend services to guarantee same result to frontend""" return ( - status.dict(by_alias=True) + status.model_dump(by_alias=True) if isinstance(status, DynamicServiceGet) - else status.dict() + else status.model_dump() ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index 5671eda108f..4d5e41dd920 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -53,7 +53,7 @@ async def _test_task_context_decorator( ) -> web.StreamResponse: """this task context callback tries to get the user_id from the query if available""" query_param = parse_request_query_parameters_as(query_model, request) - request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] = query_param.dict() + request[RQT_LONG_RUNNING_TASKS_CONTEXT_KEY] = query_param.model_dump() return await handler(request) return _test_task_context_decorator diff --git a/packages/settings-library/src/settings_library/utils_encoders.py b/packages/settings-library/src/settings_library/utils_encoders.py deleted file mode 100644 index f38e156b6a5..00000000000 --- a/packages/settings-library/src/settings_library/utils_encoders.py +++ /dev/null @@ -1,22 +0,0 @@ -from functools import partial - -from pydantic import BaseModel, SecretStr -from pydantic.json import custom_pydantic_encoder - - -def create_json_encoder_wo_secrets(model_cls: type[BaseModel]): - """Use to reveal secrtes when seriaizng a model via `.dict()` or `.json()` - - Example: - model.dict()['my_secret'] == "********" - show_secrets_encoder = create_json_encoder_wo_secrets(type(model)) - model.dict(encoder=show_secrets_encoder)['my_secret'] == "secret" - """ - current_encoders = getattr(model_cls.model_config, "json_encoders", {}) - return partial( - custom_pydantic_encoder, - { - SecretStr: lambda v: v.get_secret_value(), - **current_encoders, - }, - ) diff --git a/services/agent/tests/unit/test_api_rest__health.py b/services/agent/tests/unit/test_api_rest__health.py index 6e690daa788..9f0904c182e 100644 --- a/services/agent/tests/unit/test_api_rest__health.py +++ b/services/agent/tests/unit/test_api_rest__health.py @@ -14,4 +14,4 @@ def test_health_ok(test_client: TestClient): response = test_client.get("/health") assert response.status_code == status.HTTP_200_OK - assert HealthCheckGet.parse_obj(response.json()) + assert HealthCheckGet.model_validate(response.json()) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py index 097afd95288..c3a8e7ee1b1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py @@ -460,7 +460,7 @@ async def _get_changed_tasks_from_backend( return [ ( task, - task.copy(update={"state": backend_state}), + task.model_copy(update={"state": backend_state}), ) for task, backend_state in zip( processing_tasks, tasks_backend_status, strict=True diff --git a/services/director-v2/tests/unit/conftest.py b/services/director-v2/tests/unit/conftest.py index 2123a506ad7..1375795f0cb 100644 --- a/services/director-v2/tests/unit/conftest.py +++ b/services/director-v2/tests/unit/conftest.py @@ -267,7 +267,7 @@ def fake_service_specifications(faker: Faker) -> dict[str, Any]: }, }, } - ).dict(by_alias=True, exclude_unset=True) + ).model_dump(by_alias=True, exclude_unset=True) } diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 84711cf0936..560794460de 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -502,7 +502,7 @@ async def test_mark_all_services_in_wallet_for_removal( ) -> None: for wallet_id in [WalletID(1), WalletID(2)]: for _ in range(2): - new_scheduler_data = scheduler_data.copy(deep=True) + new_scheduler_data = scheduler_data.model_copy(deep=True) new_scheduler_data.node_uuid = faker.uuid4(cast_to=None) new_scheduler_data.service_name = ServiceName( f"fake_{new_scheduler_data.node_uuid}" diff --git a/services/director-v2/tests/unit/test_utils_computation.py b/services/director-v2/tests/unit/test_utils_computation.py index 14c9ffa34f3..046326a296c 100644 --- a/services/director-v2/tests/unit/test_utils_computation.py +++ b/services/director-v2/tests/unit/test_utils_computation.py @@ -265,7 +265,7 @@ def test_get_pipeline_state_from_task_states( fake_task: CompTaskAtDB, ): tasks: list[CompTaskAtDB] = [ - fake_task.copy(deep=True, update={"state": s}) for s in task_states + fake_task.model_copy(deep=True, update={"state": s}) for s in task_states ] pipeline_state: RunningState = get_pipeline_state_from_task_states(tasks) diff --git a/services/director-v2/tests/unit/test_utils_dags.py b/services/director-v2/tests/unit/test_utils_dags.py index 125f3153db4..11975ac9e88 100644 --- a/services/director-v2/tests/unit/test_utils_dags.py +++ b/services/director-v2/tests/unit/test_utils_dags.py @@ -422,13 +422,13 @@ def pipeline_test_params( # resolve the comp_tasks resolved_list_comp_tasks = [ - c.copy(update={"node_id": node_name_to_uuid_map[c.node_id]}) + c.model_copy(update={"node_id": node_name_to_uuid_map[c.node_id]}) for c in list_comp_tasks ] # resolved the expected output - resolved_expected_pipeline_details = expected_pipeline_details_output.copy( + resolved_expected_pipeline_details = expected_pipeline_details_output.model_copy( update={ "adjacency_list": { NodeID(node_name_to_uuid_map[node_a]): [ diff --git a/services/director-v2/tests/unit/with_dbs/conftest.py b/services/director-v2/tests/unit/with_dbs/conftest.py index ff55e862abb..fdb3b7d5a64 100644 --- a/services/director-v2/tests/unit/with_dbs/conftest.py +++ b/services/director-v2/tests/unit/with_dbs/conftest.py @@ -245,9 +245,14 @@ def creator(user: dict[str, Any], **cluster_kwargs) -> Cluster: for gid, rights in cluster_kwargs["access_rights"].items(): conn.execute( pg_insert(cluster_to_groups) - .values(cluster_id=created_cluster.id, gid=gid, **rights.dict()) + .values( + cluster_id=created_cluster.id, + gid=gid, + **rights.model_dump(), + ) .on_conflict_do_update( - index_elements=["gid", "cluster_id"], set_=rights.dict() + index_elements=["gid", "cluster_id"], + set_=rights.model_dump(), ) ) access_rights_in_db = {} diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index 8e4efa04c2c..719d2f38c6c 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -576,7 +576,7 @@ async def test_container_create_outputs_dirs( assert mock_event_filter_enqueue.call_count == 0 json_outputs_labels = { - k: v.dict(by_alias=True) for k, v in mock_outputs_labels.items() + k: v.model_dump(by_alias=True) for k, v in mock_outputs_labels.items() } response = await test_client.post( f"/{API_VTAG}/containers/ports/outputs/dirs", diff --git a/services/invitations/tests/unit/api/test_api_dependencies.py b/services/invitations/tests/unit/api/test_api_dependencies.py index 6104a57236c..38e94a52d74 100644 --- a/services/invitations/tests/unit/api/test_api_dependencies.py +++ b/services/invitations/tests/unit/api/test_api_dependencies.py @@ -18,7 +18,7 @@ def test_invalid_http_basic_auth( ): response = client.post( f"/{API_VTAG}/invitations", - json=invitation_data.dict(), + json=invitation_data.model_dump(), auth=invalid_basic_auth, ) assert response.status_code == status.HTTP_401_UNAUTHORIZED, f"{response.json()=}" diff --git a/services/invitations/tests/unit/test_cli.py b/services/invitations/tests/unit/test_cli.py index 0ca095f1915..0c4bf15c7a8 100644 --- a/services/invitations/tests/unit/test_cli.py +++ b/services/invitations/tests/unit/test_cli.py @@ -46,7 +46,7 @@ def test_invite_user_and_check_invitation( } expected = { - **invitation_data.dict(exclude={"product"}), + **invitation_data.model_dump(exclude={"product"}), "product": environs["INVITATIONS_DEFAULT_PRODUCT"], } diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py index c6e5c0b58f3..f905dfc83a4 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py @@ -67,7 +67,7 @@ async def do_setup(self) -> None: assert isinstance(self.log, logging.Logger) # nosec self.log.info( "osparc-gateway-server application settings:\n%s", - self.settings.json(indent=2), + self.settings.model_dump_json(indent=2), ) if self.settings.SC_BOOT_MODE in [BootModeEnum.DEBUG]: diff --git a/services/payments/scripts/example_payment_gateway.py b/services/payments/scripts/example_payment_gateway.py index bc1e93bc59f..f3e3b64627b 100755 --- a/services/payments/scripts/example_payment_gateway.py +++ b/services/payments/scripts/example_payment_gateway.py @@ -429,7 +429,7 @@ def create_app(): override_fastapi_openapi_method(app) app.state.settings = Settings.create_from_envs() - logging.info(app.state.settings.json(indent=2)) + logging.info(app.state.settings.model_dump_json(indent=2)) for factory in ( create_payment_router, diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index b854c915d41..34af4fa2268 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -70,7 +70,7 @@ def from_http_status_error( def get_detailed_message(self) -> str: err_json = "null" if model := getattr(self, "model", None): - err_json = model.json(indent=1) + err_json = model.model_dump_json(indent=1) curl_cmd = "null" if http_status_error := getattr(self, "http_status_error", None): diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py index e71b614c411..fefb9df5dd7 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/cli.py @@ -42,5 +42,5 @@ def evaluate(ctx: typer.Context) -> None: assert ctx # nosec settings = MinimalApplicationSettings.create_from_envs() err_console.print( - f"[yellow]running with configuration:\n{settings.json()}[/yellow]" + f"[yellow]running with configuration:\n{settings.model_dump_json()}[/yellow]" ) diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_api.py b/services/web/server/src/simcore_service_webserver/api_keys/_api.py index 9bbe56f7c6f..6cdc15e2f24 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_api.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_api.py @@ -93,7 +93,7 @@ async def get_or_create_api_key( api_key=api_key, api_secret=api_secret, ) - return ApiKeyGet.construct( + return ApiKeyGet.model_construct( display_name=row.display_name, api_key=row.api_key, api_secret=row.api_secret ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_api_units.py b/services/web/server/src/simcore_service_webserver/catalog/_api_units.py index a8558e674ec..65e435f6886 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_api_units.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_api_units.py @@ -57,10 +57,10 @@ async def replace_service_input_outputs( # replace if above is successful for input_key, new_input in zip(service["inputs"], new_inputs, strict=True): - service["inputs"][input_key] = new_input.dict(**export_options) + service["inputs"][input_key] = new_input.model_dump(**export_options) for output_key, new_output in zip(service["outputs"], new_outputs, strict=True): - service["outputs"][output_key] = new_output.dict(**export_options) + service["outputs"][output_key] = new_output.model_dump(**export_options) def can_connect( diff --git a/services/web/server/src/simcore_service_webserver/catalog/_models.py b/services/web/server/src/simcore_service_webserver/catalog/_models.py index 2432e98bc96..af137ba11d8 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_models.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_models.py @@ -86,7 +86,7 @@ async def from_catalog_service_api_model( if ureg and (unit_html := get_html_formatted_unit(port, ureg)): # we know data is ok since it was validated above - return ServiceInputGet.construct( + return ServiceInputGet.model_construct( key_id=input_key, unit_long=unit_html.long, unit_short=unit_html.short, @@ -123,7 +123,7 @@ async def from_catalog_service_api_model( unit_html: UnitHtmlFormat | None if ureg and (unit_html := get_html_formatted_unit(port, ureg)): # we know data is ok since it was validated above - return ServiceOutputGet.construct( + return ServiceOutputGet.model_construct( key_id=output_key, unit_long=unit_html.long, unit_short=unit_html.short, diff --git a/services/web/server/src/simcore_service_webserver/groups/_handlers.py b/services/web/server/src/simcore_service_webserver/groups/_handlers.py index 428769b725d..fac761aaf25 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/groups/_handlers.py @@ -166,7 +166,7 @@ async def update_group(request: web.Request): req_ctx = _GroupsRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_GroupPathParams, request) update: GroupUpdate = await parse_request_body_as(GroupUpdate, request) - new_group_values = update.dict(exclude_unset=True) + new_group_values = update.model_dump(exclude_unset=True) updated_group = await api.update_user_group( request.app, req_ctx.user_id, path_params.gid, new_group_values @@ -270,7 +270,7 @@ async def update_group_user(request: web.Request): user_id=req_ctx.user_id, gid=path_params.gid, the_user_id_in_group=path_params.uid, - access_rights=update.access_rights.dict(), + access_rights=update.access_rights.model_dump(), ) assert GroupUserGet.model_validate(user) is not None # nosec return envelope_json_response(user) diff --git a/services/web/server/src/simcore_service_webserver/groups/api.py b/services/web/server/src/simcore_service_webserver/groups/api.py index 37cb859a89f..503eee73839 100644 --- a/services/web/server/src/simcore_service_webserver/groups/api.py +++ b/services/web/server/src/simcore_service_webserver/groups/api.py @@ -32,7 +32,7 @@ async def list_all_user_groups(app: web.Application, user_id: UserID) -> list[Gr async with get_database_engine(app).acquire() as conn: groups_db = await _db.get_all_user_groups(conn, user_id=user_id) - return [Group.construct(**group.model_dump()) for group in groups_db] + return [Group.model_construct(**group.model_dump()) for group in groups_db] async def get_user_group( @@ -199,5 +199,5 @@ async def get_group_from_gid(app: web.Application, gid: GroupID) -> Group | None group_db = await _db.get_group_from_gid(conn, gid=gid) if group_db: - return Group.construct(**group_db.model_dump()) + return Group.model_construct(**group_db.model_dump()) return None diff --git a/services/web/server/src/simcore_service_webserver/login/_registration.py b/services/web/server/src/simcore_service_webserver/login/_registration.py index 1a66b5ba4e5..6471757d183 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration.py @@ -192,7 +192,7 @@ async def create_invitation_token( return await db.create_confirmation( user_id=user_id, action=ConfirmationAction.INVITATION.name, - data=data_model.json(), + data=data_model.model_dump_json(), ) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index d86be96058d..6876c63718d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -380,7 +380,9 @@ async def _get_default_pricing_and_hardware_info( _MACHINE_TOTAL_RAM_SAFE_MARGIN_RATIO: Final[ float ] = 0.1 # NOTE: machines always have less available RAM than advertised -_SIDECARS_OPS_SAFE_RAM_MARGIN: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("1GiB") +_SIDECARS_OPS_SAFE_RAM_MARGIN: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( + "1GiB" +) _CPUS_SAFE_MARGIN: Final[float] = 1.4 _MIN_NUM_CPUS: Final[float] = 0.5 @@ -840,7 +842,7 @@ async def start_project_node( workbench = project.get("workbench", {}) if not workbench.get(f"{node_id}"): raise NodeNotFoundError(project_uuid=f"{project_id}", node_uuid=f"{node_id}") - node_details = Node.construct(**workbench[f"{node_id}"]) + node_details = Node.model_construct(**workbench[f"{node_id}"]) await _start_dynamic_service( request, @@ -1511,7 +1513,11 @@ async def is_service_deprecated( app, user_id, service_key, service_version, product_name ) if deprecation_date := service.get("deprecated"): - deprecation_date_bool: bool = datetime.datetime.now(datetime.UTC) > datetime.datetime.fromisoformat(deprecation_date).replace(tzinfo=datetime.UTC) + deprecation_date_bool: bool = datetime.datetime.now( + datetime.UTC + ) > datetime.datetime.fromisoformat(deprecation_date).replace( + tzinfo=datetime.UTC + ) return deprecation_date_bool return False diff --git a/services/web/server/src/simcore_service_webserver/socketio/models.py b/services/web/server/src/simcore_service_webserver/socketio/models.py index 37bb942298b..63f071b2ab8 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/models.py +++ b/services/web/server/src/simcore_service_webserver/socketio/models.py @@ -12,7 +12,7 @@ from models_library.socketio import SocketMessageDict from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ConfigDict, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class WebSocketMessageBase(BaseModel): @@ -26,6 +26,7 @@ def get_event_type(cls) -> str: @abstractmethod def to_socket_dict(self) -> SocketMessageDict: ... + model_config = ConfigDict(frozen=True) @@ -58,7 +59,7 @@ class WebSocketProjectProgress( def from_rabbit_message( cls, message: ProgressRabbitMessageProject ) -> "WebSocketProjectProgress": - return cls.construct( + return cls.model_construct( user_id=message.user_id, project_id=message.project_id, progress_type=message.progress_type, diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py index 647a523f23c..18bd6e96e8f 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py @@ -12,7 +12,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services import ServiceKey, ServiceVersion -from pydantic import field_validator, ConfigDict, BaseModel, ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -81,7 +81,7 @@ def _create_service_info_from(service: ValidService) -> ServiceInfo: ) if service.thumbnail: values_map["thumbnail"] = service.thumbnail - return ServiceInfo.construct(_fields_set=set(values_map.keys()), **values_map) + return ServiceInfo.model_construct(_fields_set=set(values_map.keys()), **values_map) def _handle_errors_with_error_page(handler: Handler): @@ -176,7 +176,8 @@ class ServiceAndFileParams(FileQueryParams, ServiceParams): {"$ref": "#/definitions/FileParams"}, {"$ref": "#/definitions/ServiceParams"}, ] - }) + } + ) class ViewerQueryParams(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/users/_tokens.py b/services/web/server/src/simcore_service_webserver/users/_tokens.py index 3453309d200..6b4e58c8443 100644 --- a/services/web/server/src/simcore_service_webserver/users/_tokens.py +++ b/services/web/server/src/simcore_service_webserver/users/_tokens.py @@ -34,7 +34,7 @@ async def list_tokens(app: web.Application, user_id: UserID) -> list[ThirdPartyT async for row in conn.execute( sa.select(tokens.c.token_data).where(tokens.c.user_id == user_id) ): - user_tokens.append(ThirdPartyToken.construct(**row["token_data"])) + user_tokens.append(ThirdPartyToken.model_construct(**row["token_data"])) return user_tokens @@ -48,7 +48,7 @@ async def get_token( ) ) if row := await result.first(): - return ThirdPartyToken.construct(**row["token_data"]) + return ThirdPartyToken.model_construct(**row["token_data"]) raise TokenNotFoundError(service_id=service_id) @@ -78,7 +78,7 @@ async def update_token( assert resp.rowcount == 1 # nosec updated_token = await resp.fetchone() assert updated_token # nosec - return ThirdPartyToken.construct(**updated_token["token_data"]) + return ThirdPartyToken.model_construct(**updated_token["token_data"]) async def delete_token(app: web.Application, user_id: UserID, service_id: str) -> None: diff --git a/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py b/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py index edef380dc86..e75aee0866f 100644 --- a/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py +++ b/services/web/server/tests/unit/with_dbs/01/clusters/test_clusters_handlers.py @@ -246,7 +246,7 @@ async def test_update_cluster( url = client.app.router["update_cluster"].url_for(cluster_id=f"{25}") rsp = await client.patch( f"{url}", - json=json.loads(cluster_patch.json(**_PATCH_EXPORT)), + json=json.loads(cluster_patch.model_dump_json(**_PATCH_EXPORT)), ) data, error = await assert_status(rsp, expected.ok) if not error: @@ -497,7 +497,9 @@ async def test_ping_cluster_with_error( ) assert client.app url = client.app.router["ping_cluster"].url_for() - rsp = await client.post(f"{url}", json=json.loads(cluster_ping.json(by_alias=True))) + rsp = await client.post( + f"{url}", json=json.loads(cluster_ping.model_dump_json(by_alias=True)) + ) data, error = await assert_status(rsp, expected_http_error) assert not data assert error diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py index 1572f862421..862a0db06e8 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_iterations.py @@ -262,7 +262,7 @@ async def _mock_catalog_get(*args, **kwarg): assert node.inputs node.inputs["linspace_stop"] = 4 - _new_project_data = new_project.dict(**REQUEST_MODEL_POLICY) + _new_project_data = new_project.model_dump(**REQUEST_MODEL_POLICY) _new_project_data.pop("state") await db.replace_project( json_loads(json_dumps(_new_project_data)), From 65665ec01fe3468dadd6f3a58422b169ea5f60d4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 18 Nov 2024 11:57:56 +0100 Subject: [PATCH 39/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Migrate=20scripts=20?= =?UTF-8?q?(Pydantic=20v2)=20(#6741)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/demo-meta/meta_modeling_results.py | 9 ++-- scripts/demo-meta/osparc_webapi.py | 52 +++++++++++----------- scripts/demo-meta/requirements.txt | 1 + 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/scripts/demo-meta/meta_modeling_results.py b/scripts/demo-meta/meta_modeling_results.py index 4ff2a3a869b..04a19de3463 100644 --- a/scripts/demo-meta/meta_modeling_results.py +++ b/scripts/demo-meta/meta_modeling_results.py @@ -3,7 +3,6 @@ """ from collections import defaultdict -from typing import List from uuid import UUID import httpx @@ -24,24 +23,24 @@ def print_checkpoints(client: httpx.Client): - repos: List[ProjectRepo] = list(iter_repos(client)) + repos: list[ProjectRepo] = list(iter_repos(client)) project_id = repos[0].project_uuid for checkpoint in iter_checkpoints(client, project_id): - print(checkpoint.json(exclude_unset=True, indent=1)) + print(checkpoint.model_dump_json(exclude_unset=True, indent=1)) def print_iterations(client: httpx.Client, project_id: UUID, checkpoint: CheckPoint): # print-iterations print("Metaproject at", f"{project_id=}", f"{checkpoint=}") for project_iteration in iter_project_iteration(client, project_id, checkpoint.id): - print(project_iteration.json(exclude_unset=True, indent=1)) + print(project_iteration.model_dump_json(exclude_unset=True, indent=1)) def select_project_head(client: httpx.Client, project_id: UUID): # get head r = client.get(f"/repos/projects/{project_id}/checkpoints/HEAD") - head = Envelope[CheckPoint].parse_obj(r.json()).data + head = Envelope[CheckPoint].model_validate(r.json()).data assert head # nosec return project_id, head diff --git a/scripts/demo-meta/osparc_webapi.py b/scripts/demo-meta/osparc_webapi.py index d64915fd7d7..8d76e8be369 100644 --- a/scripts/demo-meta/osparc_webapi.py +++ b/scripts/demo-meta/osparc_webapi.py @@ -7,7 +7,7 @@ from contextlib import contextmanager from datetime import datetime from pathlib import Path -from typing import Any, Generic, Iterator, Optional, Type, TypeVar +from typing import Annotated, Any, Generic, Iterator, TypeVar from uuid import UUID import httpx @@ -16,15 +16,13 @@ AnyHttpUrl, AnyUrl, BaseModel, - BaseSettings, EmailStr, Field, NonNegativeInt, SecretStr, ValidationError, - conint, ) -from pydantic.generics import GenericModel +from pydantic_settings import BaseSettings, SettingsConfigDict log = logging.getLogger(__name__) logging.basicConfig(level=getattr(logging, os.environ.get("LOG_LEVEL", "INFO"))) @@ -46,32 +44,32 @@ class Meta(BaseModel): class PageLinks(BaseModel): self: AnyHttpUrl first: AnyHttpUrl - prev: Optional[AnyHttpUrl] - next: Optional[AnyHttpUrl] + prev: AnyHttpUrl | None + next: AnyHttpUrl | None last: AnyHttpUrl -class Page(GenericModel, Generic[ItemT]): +class Page(BaseModel, Generic[ItemT]): meta: Meta = Field(..., alias="_meta") data: list[ItemT] links: PageLinks = Field(..., alias="_links") -class Envelope(GenericModel, Generic[DataT]): - data: Optional[DataT] - error: Optional[Any] +class Envelope(BaseModel, Generic[DataT]): + data: DataT | None + error: Any | None @classmethod def parse_data(cls, obj): - return cls.parse_obj({"data": obj}) + return cls.model_validate({"data": obj}) class CheckPoint(BaseModel): id: NonNegativeInt checksum: str - tag: Optional[str] = None - message: Optional[str] = None - parent: Optional[NonNegativeInt] = None + tag: str | None = None + message: str | None = None + parent: NonNegativeInt | None = None created_at: datetime @@ -98,7 +96,7 @@ class ProjectIteration(BaseModel): class ExtractedResults(BaseModel): - progress: dict[NodeIDStr, conint(ge=0, le=100)] = Field( + progress: dict[NodeIDStr, Annotated[int, Field(ge=0, le=100)]] = Field( ..., description="Progress in each computational node" ) labels: dict[NodeIDStr, str] = Field( @@ -140,19 +138,19 @@ def login(client: httpx.Client, user: str, password: str): def get_profile(client: httpx.Client): r = client.get("/me") - assert r.status_code == 200 + assert r.status_code == httpx.codes.OK return r.json()["data"] def iter_items( - client: httpx.Client, url_path: str, item_cls: Type[ItemT] + client: httpx.Client, url_path: str, item_cls: type[ItemT] ) -> Iterator[ItemT]: """iterates items returned by a List std-method SEE https://google.aip.dev/132 """ - def _relative_url_path(page_link: Optional[AnyHttpUrl]) -> Optional[str]: + def _relative_url_path(page_link: AnyHttpUrl | None) -> str | None: if page_link: return f"{page_link.path}".replace(client.base_url.path, "") return None @@ -165,9 +163,8 @@ def _relative_url_path(page_link: Optional[AnyHttpUrl]) -> Optional[str]: r = client.get(next_url) r.raise_for_status() - page = Page[item_cls].parse_raw(r.text) - for item in page.data: - yield item + page = Page[item_cls].model_validate_json(r.text) + yield from page.data next_url = _relative_url_path(page.links.next) last_url = _relative_url_path(page.links.last) @@ -198,16 +195,17 @@ def iter_project_iteration( # SETUP ------------------------------------------ class ClientSettings(BaseSettings): - OSPARC_API_URL: AnyUrl = Field(default="http://127.0.0.1.nip.io:9081/v0") # NOSONAR + OSPARC_API_URL: AnyUrl = Field( + default="http://127.0.0.1.nip.io:9081/v0" + ) # NOSONAR OSPARC_USER_EMAIL: EmailStr OSPARC_USER_PASSWORD: SecretStr - class Config: - env_file = ".env-osparc-web.ignore" + model_config = SettingsConfigDict(env_file=".env-osparc-web.ignore") def init(): - env_file = Path(ClientSettings.Config.env_file) + env_file = Path(ClientSettings.model_config.env_file) log.info("Creating %s", f"{env_file}") kwargs = {} kwargs["OSPARC_API_URL"] = input("OSPARC_API_URL: ").strip() or None @@ -215,7 +213,7 @@ def init(): input("OSPARC_USER_EMAIL: ") or getpass.getuser() + "@itis.swiss" ) kwargs["OSPARC_USER_PASSWORD"] = getpass.getpass() - with open(env_file, "wt") as fh: + with env_file.open("w") as fh: for key, value in kwargs.items(): print(key, value) if value is not None: @@ -234,7 +232,7 @@ def query_if_invalid_config(): def setup_client() -> Iterator[httpx.Client]: settings = ClientSettings() - client = httpx.Client(base_url=settings.OSPARC_API_URL) + client = httpx.Client(base_url=f"{settings.OSPARC_API_URL}") try: # check if online and login print(ping(client)) diff --git a/scripts/demo-meta/requirements.txt b/scripts/demo-meta/requirements.txt index 4aa7c6af281..da07e4c3fbe 100644 --- a/scripts/demo-meta/requirements.txt +++ b/scripts/demo-meta/requirements.txt @@ -1,4 +1,5 @@ httpx pandas pydantic[dotenv,email] +pydantic-settings tabulate From fa57c9e2d3a0ca6e147eb923ae4f0350b5a2de4a Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:44:58 +0100 Subject: [PATCH 40/53] pydantic2 migration made integration tests green (#6719) Co-authored-by: Andrei Neagu Co-authored-by: Giancarlo Romeo --- .../container_tasks/io.py | 25 +++++++++-------- .../api_schemas_catalog/services.py | 8 ++++-- .../src/models_library/clusters.py | 6 +++-- .../src/pytest_simcore/docker_compose.py | 26 +++++++++++++++++- .../src/pytest_simcore/docker_registry.py | 2 +- .../helpers/httpx_calls_capture_parameters.py | 14 ++++++++-- .../src/pytest_simcore/helpers/storage.py | 11 ++++++++ .../pytest_simcore/simcore_dask_service.py | 5 ++-- .../pytest_simcore/simcore_storage_service.py | 19 ++++--------- .../aiohttp/long_running_tasks/_server.py | 9 +++---- .../fastapi/long_running_tasks/_client.py | 17 +++++++----- ...test_long_running_tasks_context_manager.py | 22 ++++++++------- .../tests/fastapi/test_http_client_thin.py | 9 +++---- .../node_ports_common/_filemanager.py | 19 ++++++------- .../node_ports_common/r_clone_utils.py | 4 +-- .../simcore-sdk/tests/integration/conftest.py | 4 +-- services/agent/requirements/prod.txt | 1 + services/api-server/requirements/prod.txt | 1 + .../models/domain/groups.py | 2 +- services/autoscaling/requirements/prod.txt | 1 + services/catalog/requirements/prod.txt | 2 +- .../api/rest/_services.py | 2 +- .../clusters-keeper/requirements/prod.txt | 1 + services/dask-sidecar/requirements/prod.txt | 1 + .../datcore-adapter/requirements/prod.txt | 1 + services/director-v2/requirements/prod.txt | 1 + .../api/errors/validation_error.py | 2 -- .../core/dynamic_services_settings/sidecar.py | 18 ++++++++----- .../core/settings.py | 27 ++++++++++--------- .../models/dynamic_services_scheduler.py | 9 +++---- .../modules/clusters_keeper.py | 1 + .../dynamic_sidecar/docker_api/_core.py | 8 ++++++ .../simcore_service_director_v2/utils/dask.py | 15 ++++------- .../utils/osparc_variables.py | 2 +- .../integration/01/test_computation_api.py | 4 +-- ...t_dynamic_sidecar_nodeports_integration.py | 10 +++---- .../director-v2/tests/integration/02/utils.py | 2 +- .../director-v2/tests/integration/conftest.py | 2 +- .../mocks/fake_dy_workbench_template.json | 4 +-- .../dynamic-scheduler/requirements/prod.txt | 1 + .../dynamic-sidecar/requirements/prod.txt | 1 + .../core/settings.py | 3 ++- .../test_modules_long_running_tasks.py | 20 ++++---------- ..._api_rest_containers_long_running_tasks.py | 5 +--- .../unit/test_api_rest_prometheus_metrics.py | 5 +--- .../test_api_rest_workflow_service_metrics.py | 5 +--- services/efs-guardian/requirements/prod.txt | 1 + services/invitations/requirements/prod.txt | 1 + services/payments/requirements/prod.txt | 1 + .../requirements/prod.txt | 1 + services/storage/requirements/prod.txt | 1 + .../src/simcore_service_storage/models.py | 2 +- .../_formatter/xlsx/code_description.py | 4 +-- .../01/test_exporter_requests_handlers.py | 7 ++++- tests/environment-setup/requirements/ci.txt | 1 + tests/environment-setup/requirements/dev.txt | 1 + tests/public-api/requirements/ci.txt | 1 + tests/public-api/requirements/dev.txt | 1 + tests/public-api/test_users_api.py | 3 ++- tests/swarm-deploy/requirements/ci.txt | 1 + tests/swarm-deploy/requirements/dev.txt | 1 + 61 files changed, 226 insertions(+), 158 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/helpers/storage.py diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py index 0bb95130723..b9ed0e9f6de 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py @@ -1,7 +1,7 @@ import json from contextlib import suppress from pathlib import Path -from typing import Any, TypeAlias, Union +from typing import Annotated, Any, TypeAlias from models_library.basic_regex import MIME_TYPE_RE from models_library.generics import DictModel @@ -84,15 +84,16 @@ class FileUrl(BaseModel): ) -PortValue: TypeAlias = Union[ - StrictBool, - StrictInt, - StrictFloat, - StrictStr, - FileUrl, - list[Any], - dict[str, Any], - None, +PortValue: TypeAlias = Annotated[ + StrictBool + | StrictInt + | StrictFloat + | StrictStr + | FileUrl + | list[Any] + | dict[str, Any] + | None, + Field(union_mode="left_to_right"), ] @@ -112,7 +113,9 @@ class TaskInputData(DictModel[ServicePortKey, PortValue]): ) -PortSchemaValue: TypeAlias = Union[PortSchema, FilePortSchema] +PortSchemaValue: TypeAlias = Annotated[ + PortSchema | FilePortSchema, Field(union_mode="left_to_right") +] class TaskOutputDataSchema(DictModel[ServicePortKey, PortSchemaValue]): diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index bce573a2494..8090edf0ebd 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -205,7 +205,9 @@ class ServiceUpdate(ServiceMetaDataEditable, ServiceAccessRights): class ServiceGet( ServiceMetaDataPublished, ServiceAccessRights, ServiceMetaDataEditable ): # pylint: disable=too-many-ancestors - owner: LowerCaseEmailStr | None + owner: LowerCaseEmailStr | None = Field( + description="None when the owner email cannot be found in the database" + ) model_config = ConfigDict( extra="ignore", @@ -230,7 +232,9 @@ class ServiceGetV2(BaseModel): contact: LowerCaseEmailStr | None authors: list[Author] = Field(..., min_length=1) - owner: LowerCaseEmailStr | None + owner: LowerCaseEmailStr | None = Field( + description="None when the owner email cannot be found in the database" + ) inputs: ServiceInputsDict outputs: ServiceOutputsDict diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index 07a5f0dc6d9..911b709a1f6 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -1,6 +1,6 @@ from enum import auto from pathlib import Path -from typing import Final, Literal, Self, TypeAlias +from typing import Annotated, Final, Literal, Self, TypeAlias from pydantic import ( AnyUrl, @@ -142,7 +142,9 @@ class BaseCluster(BaseModel): authentication: ClusterAuthentication = Field( ..., description="Dask gateway authentication", discriminator="type" ) - access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict) + access_rights: Annotated[ + dict[GroupID, ClusterAccessRights], Field(default_factory=dict) + ] _from_equivalent_enums = field_validator("type", mode="before")( create_enums_pre_validator(ClusterTypeInModel) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 581fcf30187..9a80acd4d55 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -94,13 +94,37 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict: # ensure we do not use the bucket of simcore or so env_devel["S3_BUCKET_NAME"] = "pytestbucket" + # ensure OpenTelemetry is not enabled + env_devel |= { + tracing_setting: "null" + for tracing_setting in ( + "AGENT_TRACING", + "API_SERVER_TRACING", + "AUTOSCALING_TRACING", + "CATALOG_TRACING", + "CLUSTERS_KEEPER_TRACING", + "DATCORE_ADAPTER_TRACING", + "DIRECTOR_TRACING", + "DIRECTOR_V2_TRACING", + "DYNAMIC_SCHEDULER_TRACING", + "EFS_GUARDIAN_TRACING", + "INVITATIONS_TRACING", + "PAYMENTS_TRACING", + "RESOURCE_USAGE_TRACKER_TRACING", + "STORAGE_TRACING", + "WB_DB_EL_TRACING", + "WB_GC_TRACING", + "WEBSERVER_TRACING", + ) + } + return {key: value for key, value in env_devel.items() if value is not None} @pytest.fixture(scope="module") def env_file_for_testing( temp_folder: Path, - testing_environ_vars: dict[str, str], + testing_environ_vars: EnvVarsDict, osparc_simcore_root_dir: Path, ) -> Iterator[Path]: """Dumps all the environment variables into an $(temp_folder)/.env.test file diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index 91cd5e2d428..84b4d1e4b24 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -261,7 +261,7 @@ def jupyter_service(docker_registry: str, node_meta_schema: dict) -> dict[str, A ) -@pytest.fixture(scope="session", params=["2.0.4"]) +@pytest.fixture(scope="session", params=["2.0.7"]) def dy_static_file_server_version(request: pytest.FixtureRequest): return request.param diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py index 6c1a00c2302..80170682430 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py @@ -67,9 +67,19 @@ def regex_pattern(self) -> str: msg = "Current version cannot compute regex patterns in case of oneOf. Please go ahead and implement it yourself." raise NotImplementedError(msg) if self.anyOf: - return "|".join([elm.regex_pattern for elm in self.anyOf]) + return "|".join( + [ + elm.regex_pattern + for elm in self.anyOf # pylint:disable=not-an-iterable + ] + ) if self.allOf: - return "&".join([elm.regex_pattern for elm in self.allOf]) + return "&".join( + [ + elm.regex_pattern + for elm in self.allOf # pylint:disable=not-an-iterable + ] + ) # now deal with non-recursive cases pattern: str | None = None diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/storage.py b/packages/pytest-simcore/src/pytest_simcore/helpers/storage.py new file mode 100644 index 00000000000..b07006a24dc --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/storage.py @@ -0,0 +1,11 @@ +from collections.abc import Callable + +from yarl import URL + + +def replace_storage_endpoint(host: str, port: int) -> Callable[[str], str]: + def _(url: str) -> str: + url_obj = URL(url).with_host(host).with_port(port) + return f"{url_obj}" + + return _ diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py index 26f16b02a41..c2900bf3e4f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py @@ -27,9 +27,10 @@ async def dask_scheduler_service( ) # override the port monkeypatch.setenv("DASK_SCHEDULER_PORT", f"{dask_scheduler_api_port}") - return AnyUrl.build( - scheme="tls", host=get_localhost_ip(), port=dask_scheduler_api_port + url = AnyUrl.build( + scheme="tls", host=get_localhost_ip(), port=int(dask_scheduler_api_port) ) + return f"{url}" @pytest.fixture diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index e2f7654d3d0..3f451972086 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -10,13 +10,14 @@ import tenacity from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from pydantic import AnyUrl, TypeAdapter +from pydantic import TypeAdapter from pytest_mock import MockerFixture from servicelib.minio_utils import ServiceRetryPolicyUponInitialization from yarl import URL from .helpers.docker import get_service_published_port from .helpers.host import get_localhost_ip +from .helpers.storage import replace_storage_endpoint @pytest.fixture(scope="module") @@ -45,22 +46,12 @@ async def storage_service( ) -> URL: await wait_till_storage_responsive(storage_endpoint) - def correct_ip(url: AnyUrl): - assert storage_endpoint.host is not None - assert storage_endpoint.port is not None - - return AnyUrl.build( - scheme=url.scheme, - host=storage_endpoint.host, - port=f"{storage_endpoint.port}", - path=url.path, - query=url.query, - ) - # NOTE: Mock to ensure container IP agrees with host IP when testing + assert storage_endpoint.host is not None + assert storage_endpoint.port is not None mocker.patch( "simcore_sdk.node_ports_common._filemanager._get_https_link_if_storage_secure", - correct_ip, + replace_storage_endpoint(storage_endpoint.host, storage_endpoint.port), ) return storage_endpoint diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 1d97f9320c8..d0c96699462 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -6,8 +6,7 @@ from aiohttp import web from common_library.json_serialization import json_dumps -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy -from pydantic import PositiveFloat, TypeAdapter +from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from ...aiohttp import status from ...long_running_tasks._models import TaskGet @@ -68,13 +67,13 @@ async def start_long_running_task( ip_addr, port = request_.transport.get_extra_info( "sockname" ) # https://docs.python.org/3/library/asyncio-protocol.html#asyncio.BaseTransport.get_extra_info - status_url = TypeAdapter(AnyHttpUrlLegacy).validate_python( + status_url = TypeAdapter(AnyHttpUrl).validate_python( f"http://{ip_addr}:{port}{request_.app.router['get_task_status'].url_for(task_id=task_id)}" # NOSONAR ) - result_url = TypeAdapter(AnyHttpUrlLegacy).validate_python( + result_url = TypeAdapter(AnyHttpUrl).validate_python( f"http://{ip_addr}:{port}{request_.app.router['get_task_result'].url_for(task_id=task_id)}" # NOSONAR ) - abort_url = TypeAdapter(AnyHttpUrlLegacy).validate_python( + abort_url = TypeAdapter(AnyHttpUrl).validate_python( f"http://{ip_addr}:{port}{request_.app.router['cancel_and_delete_task'].url_for(task_id=task_id)}" # NOSONAR ) task_get = TaskGet( diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index 3aa82e76a46..a00c2417e49 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -4,10 +4,9 @@ import warnings from typing import Any, Awaitable, Callable, Final -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI, status from httpx import AsyncClient, HTTPError -from pydantic import PositiveFloat, TypeAdapter +from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from tenacity import RetryCallState from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -106,6 +105,9 @@ async def request_wrapper(zelf: "Client", *args, **kwargs) -> Any: with attempt: return await request_func(zelf, *args, **kwargs) + msg = "Unexpected" + raise RuntimeError(msg) + return request_wrapper @@ -131,12 +133,13 @@ def _client_configuration(self) -> ClientConfiguration: return output def _get_url(self, path: str) -> str: - url = f"{self._base_url}{self._client_configuration.router_prefix}{path}" - return f"{TypeAdapter(AnyHttpUrlLegacy).validate_python(url)}" + url_path = f"{self._client_configuration.router_prefix}{path}".lstrip("/") + url = TypeAdapter(AnyHttpUrl).validate_python(f"{self._base_url}{url_path}") + return f"{url}" @retry_on_http_errors async def get_task_status( - self, task_id: TaskId, *, timeout: PositiveFloat | None = None + self, task_id: TaskId, *, timeout: PositiveFloat | None = None # noqa: ASYNC109 ) -> TaskStatus: timeout = timeout or self._client_configuration.default_timeout result = await self._async_client.get( @@ -155,7 +158,7 @@ async def get_task_status( @retry_on_http_errors async def get_task_result( - self, task_id: TaskId, *, timeout: PositiveFloat | None = None + self, task_id: TaskId, *, timeout: PositiveFloat | None = None # noqa: ASYNC109 ) -> Any | None: timeout = timeout or self._client_configuration.default_timeout result = await self._async_client.get( @@ -177,7 +180,7 @@ async def get_task_result( @retry_on_http_errors async def cancel_and_delete_task( - self, task_id: TaskId, *, timeout: PositiveFloat | None = None + self, task_id: TaskId, *, timeout: PositiveFloat | None = None # noqa: ASYNC109 ) -> None: timeout = timeout or self._client_configuration.default_timeout result = await self._async_client.delete( diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 0dc440bf33e..985cfca2de6 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -4,12 +4,11 @@ import asyncio from typing import AsyncIterable, Final -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy import pytest from asgi_lifespan import LifespanManager from fastapi import APIRouter, Depends, FastAPI, status from httpx import AsyncClient -from pydantic import PositiveFloat, TypeAdapter +from pydantic import AnyHttpUrl, PositiveFloat, TypeAdapter from servicelib.fastapi.long_running_tasks._context_manager import _ProgressManager from servicelib.fastapi.long_running_tasks.client import ( Client, @@ -50,7 +49,8 @@ async def a_test_task(task_progress: TaskProgress) -> int: async def a_failing_test_task(task_progress: TaskProgress) -> None: await asyncio.sleep(TASK_SLEEP_INTERVAL) - raise RuntimeError("I am failing as requested") + msg = "I am failing as requested" + raise RuntimeError(msg) @pytest.fixture @@ -101,7 +101,7 @@ async def test_task_result( assert result.status_code == status.HTTP_200_OK, result.text task_id = result.json() - url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io") + url = TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io/") client = Client(app=bg_task_app, async_client=async_client, base_url=url) async with periodic_task_result( client, @@ -121,7 +121,7 @@ async def test_task_result_times_out( assert result.status_code == status.HTTP_200_OK, result.text task_id = result.json() - url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io") + url = TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io/") client = Client(app=bg_task_app, async_client=async_client, base_url=url) timeout = TASK_SLEEP_INTERVAL / 10 with pytest.raises(TaskClientTimeoutError) as exec_info: @@ -147,7 +147,7 @@ async def test_task_result_task_result_is_an_error( assert result.status_code == status.HTTP_200_OK, result.text task_id = result.json() - url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://backgroud.testserver.io") + url = TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io/") client = Client(app=bg_task_app, async_client=async_client, base_url=url) with pytest.raises(TaskClientResultError) as exec_info: async with periodic_task_result( @@ -158,7 +158,7 @@ async def test_task_result_task_result_is_an_error( ): pass assert f"{exec_info.value}".startswith(f"Task {task_id} finished with exception:") - assert 'raise RuntimeError("I am failing as requested")' in f"{exec_info.value}" + assert "I am failing as requested" in f"{exec_info.value}" await _assert_task_removed(async_client, task_id, router_prefix) @@ -186,13 +186,17 @@ async def progress_update( assert received == ("", None) for _ in range(repeat): - await progress_updater.update(mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(0.0)) + await progress_updater.update( + mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(0.0) + ) assert counter == 2 assert received == ("", 0.0) for _ in range(repeat): await progress_updater.update( - mock_task_id, percent=TypeAdapter(ProgressPercent).validate_python(1.0), message="done" + mock_task_id, + percent=TypeAdapter(ProgressPercent).validate_python(1.0), + message="done", ) assert counter == 3 assert received == ("done", 1.0) diff --git a/packages/service-library/tests/fastapi/test_http_client_thin.py b/packages/service-library/tests/fastapi/test_http_client_thin.py index daff1f8deaf..0a1f800f510 100644 --- a/packages/service-library/tests/fastapi/test_http_client_thin.py +++ b/packages/service-library/tests/fastapi/test_http_client_thin.py @@ -6,7 +6,6 @@ import arrow import pytest -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from httpx import ( HTTPError, PoolTimeout, @@ -16,7 +15,7 @@ TransportError, codes, ) -from pydantic import TypeAdapter +from pydantic import AnyHttpUrl, TypeAdapter from respx import MockRouter from servicelib.fastapi.http_client_thin import ( BaseThinClient, @@ -80,7 +79,7 @@ async def thick_client(request_timeout: int) -> AsyncIterable[FakeThickClient]: @pytest.fixture def test_url() -> str: - url = TypeAdapter(AnyHttpUrlLegacy).validate_python("http://missing-host:1111") + url = TypeAdapter(AnyHttpUrl).validate_python("http://missing-host:1111") return f"{url}" @@ -192,8 +191,8 @@ async def test_expect_state_decorator( respx_mock: MockRouter, request_timeout: int, ) -> None: - url_get_200_ok = f"{test_url}/ok" - get_wrong_state = f"{test_url}/wrong-state" + url_get_200_ok = f"{test_url}ok" + get_wrong_state = f"{test_url}wrong-state" error_status = codes.NOT_FOUND class ATestClient(BaseThinClient): diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py index a5305dd5b93..db77b2269f8 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py @@ -69,18 +69,18 @@ async def _complete_upload( :rtype: ETag """ async with session.post( - _get_https_link_if_storage_secure(str(upload_completion_link)), + _get_https_link_if_storage_secure(f"{upload_completion_link}"), json=jsonable_encoder(FileUploadCompletionBody(parts=parts)), auth=get_basic_auth(), ) as resp: resp.raise_for_status() # now poll for state - file_upload_complete_response = TypeAdapter(Envelope[FileUploadCompleteResponse]).validate_python( - await resp.json() - ) + file_upload_complete_response = TypeAdapter( + Envelope[FileUploadCompleteResponse] + ).validate_python(await resp.json()) assert file_upload_complete_response.data # nosec state_url = _get_https_link_if_storage_secure( - str(file_upload_complete_response.data.links.state) + f"{file_upload_complete_response.data.links.state}" ) _logger.info("completed upload of %s", f"{len(parts)} parts, received {state_url}") @@ -96,9 +96,9 @@ async def _complete_upload( with attempt: async with session.post(state_url, auth=get_basic_auth()) as resp: resp.raise_for_status() - future_enveloped = TypeAdapter(Envelope[FileUploadCompleteFutureResponse]).validate_python( - await resp.json() - ) + future_enveloped = TypeAdapter( + Envelope[FileUploadCompleteFutureResponse] + ).validate_python(await resp.json()) assert future_enveloped.data # nosec if future_enveloped.data.state == FileUploadCompleteState.NOK: msg = "upload not ready yet" @@ -142,7 +142,8 @@ async def _abort_upload( # abort the upload correctly, so it can revert back to last version try: async with session.post( - _get_https_link_if_storage_secure(str(abort_upload_link)), auth=get_basic_auth() + _get_https_link_if_storage_secure(f"{abort_upload_link}"), + auth=get_basic_auth(), ) as resp: resp.raise_for_status() except ClientError: diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py index 7470c394028..c32e9f75888 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py @@ -78,9 +78,7 @@ async def __call__(self, logs: str) -> None: with log_catch(_logger, reraise=False): rclone_message: _RCloneSyncMessages = TypeAdapter( _RCloneSyncMessages - ).validate_strings( - logs, - ) + ).validate_json(logs) if isinstance(rclone_message, _RCloneSyncTransferringMessage): await self.progress_bar.set_(rclone_message.stats.bytes) diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index 552a629565a..923a373f720 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -80,7 +80,7 @@ def project_id(user_id: int, postgres_db: sa.engine.Engine) -> Iterable[str]: @pytest.fixture(scope="module") def node_uuid() -> NodeIDStr: - return NodeIDStr(f"{uuid4()}") + return TypeAdapter(NodeIDStr).validate_python(f"{uuid4()}") @pytest.fixture(scope="session") @@ -156,7 +156,7 @@ async def _create(file_path: Path) -> dict[str, Any]: "Content-Type": "application/binary", } async with session.put( - link, data=file_path.read_bytes(), headers=extra_hdr + f"{link}", data=file_path.read_bytes(), headers=extra_hdr ) as resp: resp.raise_for_status() diff --git a/services/agent/requirements/prod.txt b/services/agent/requirements/prod.txt index aad1cc7a2bb..fb4e4e1e8fa 100644 --- a/services/agent/requirements/prod.txt +++ b/services/agent/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/api-server/requirements/prod.txt b/services/api-server/requirements/prod.txt index a8fc2fb9e80..9d4d747507e 100644 --- a/services/api-server/requirements/prod.txt +++ b/services/api-server/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/api-server/src/simcore_service_api_server/models/domain/groups.py b/services/api-server/src/simcore_service_api_server/models/domain/groups.py index b358d08ae30..59e253e6cf1 100644 --- a/services/api-server/src/simcore_service_api_server/models/domain/groups.py +++ b/services/api-server/src/simcore_service_api_server/models/domain/groups.py @@ -2,7 +2,7 @@ class UsersGroup(BaseModel): - gid: str + gid: str = Field(..., coerce_numbers_to_str=True) label: str description: str | None = None diff --git a/services/autoscaling/requirements/prod.txt b/services/autoscaling/requirements/prod.txt index ee043c82eab..b404473767f 100644 --- a/services/autoscaling/requirements/prod.txt +++ b/services/autoscaling/requirements/prod.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library simcore-dask-task-models-library @ ../../packages/dask-task-models-library diff --git a/services/catalog/requirements/prod.txt b/services/catalog/requirements/prod.txt index c72846d7091..96a80690986 100644 --- a/services/catalog/requirements/prod.txt +++ b/services/catalog/requirements/prod.txt @@ -10,7 +10,7 @@ --requirement _base.txt # installs this repo's packages -simcore-common-library @ ../../packages/common-library +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index f8d4bddabb0..e2abc23d179 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -203,7 +203,7 @@ async def get_service( ], x_simcore_products_name: str = Header(None), ): - service_data: dict[str, Any] = {} + service_data: dict[str, Any] = {"owner": None} # get the user groups user_groups = await groups_repository.list_user_groups(user_id) diff --git a/services/clusters-keeper/requirements/prod.txt b/services/clusters-keeper/requirements/prod.txt index a6058884f33..a0337e60a07 100644 --- a/services/clusters-keeper/requirements/prod.txt +++ b/services/clusters-keeper/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/dask-sidecar/requirements/prod.txt b/services/dask-sidecar/requirements/prod.txt index bc5b9b96a69..27de101557c 100644 --- a/services/dask-sidecar/requirements/prod.txt +++ b/services/dask-sidecar/requirements/prod.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-dask-task-models-library @ ../../packages/dask-task-models-library/ simcore-models-library @ ../../packages/models-library/ +simcore-common-library @ ../../packages/common-library/ simcore-service-library @ ../../packages/service-library/ simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/datcore-adapter/requirements/prod.txt b/services/datcore-adapter/requirements/prod.txt index 84a2c86a0b9..2ca94d67b8b 100644 --- a/services/datcore-adapter/requirements/prod.txt +++ b/services/datcore-adapter/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/director-v2/requirements/prod.txt b/services/director-v2/requirements/prod.txt index db23e34b436..8a770919b4f 100644 --- a/services/director-v2/requirements/prod.txt +++ b/services/director-v2/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-dask-task-models-library @ ../../packages/dask-task-models-library/ +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py b/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py index b3509cbbec9..cbdc2243701 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py +++ b/services/director-v2/src/simcore_service_director_v2/api/errors/validation_error.py @@ -1,5 +1,3 @@ -from typing import Union - from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from fastapi.openapi.constants import REF_PREFIX diff --git a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py index 434c3e0941f..fa0d0e670b7 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py @@ -2,6 +2,7 @@ import warnings from enum import Enum from pathlib import Path +from typing import Annotated from models_library.basic_types import BootModeEnum, PortInt from models_library.docker import DockerPlacementConstraint @@ -104,12 +105,17 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings): ), description="dynamic-sidecar's service 'endpoint_spec' with {'Mode': 'dnsrr'}", ) - DYNAMIC_SIDECAR_SC_BOOT_MODE: BootModeEnum = Field( - ..., - description="Boot mode used for the dynamic-sidecar services" - "By defaults, it uses the same boot mode set for the director-v2", - validation_alias=AliasChoices("DYNAMIC_SIDECAR_SC_BOOT_MODE", "SC_BOOT_MODE"), - ) + DYNAMIC_SIDECAR_SC_BOOT_MODE: Annotated[ + BootModeEnum, + Field( + ..., + description="Boot mode used for the dynamic-sidecar services" + "By defaults, it uses the same boot mode set for the director-v2", + validation_alias=AliasChoices( + "DYNAMIC_SIDECAR_SC_BOOT_MODE", "SC_BOOT_MODE" + ), + ), + ] DYNAMIC_SIDECAR_LOG_LEVEL: str = Field( "WARNING", diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 062e1ec4d5f..0ccdce64de1 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -4,6 +4,7 @@ import datetime from functools import cached_property +from typing import Annotated from common_library.pydantic_validators import validate_numeric_string_as_timedelta from models_library.basic_types import LogLevel, PortInt, VersionTag @@ -108,6 +109,7 @@ def default_cluster(self) -> Cluster: authentication=self.COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH, owner=1, # NOTE: currently this is a soft hack (the group of everyone is the group 1) type=ClusterTypeInModel.ON_PREMISE, + access_rights={}, ) @field_validator("COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH", mode="before") @@ -194,28 +196,29 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings): ) # App modules settings --------------------- - DIRECTOR_V2_STORAGE: StorageSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) + DIRECTOR_V2_STORAGE: Annotated[ + StorageSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH: StorageAuthSettings | None = Field( json_schema_extra={"auto_default_from_env": True} ) - DIRECTOR_V2_CATALOG: CatalogSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) + DIRECTOR_V2_CATALOG: Annotated[ + CatalogSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] DIRECTOR_V0: DirectorV0Settings = Field( json_schema_extra={"auto_default_from_env": True} ) - DYNAMIC_SERVICES: DynamicServicesSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) + DYNAMIC_SERVICES: Annotated[ + DynamicServicesSettings, + Field(json_schema_extra={"auto_default_from_env": True}), + ] - POSTGRES: PostgresSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) + POSTGRES: Annotated[ + PostgresSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] REDIS: RedisSettings = Field(json_schema_extra={"auto_default_from_env": True}) diff --git a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py index 0a2322c11c1..5e68dba227d 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py @@ -119,9 +119,9 @@ def create_as_initially_ok(cls) -> "Status": class DockerContainerInspect(BaseModel): - container_state: ContainerState = Field( - ..., description="current state of container" - ) + container_state: Annotated[ + ContainerState, Field(..., description="current state of container") + ] name: str = Field(..., description="docker name of the container") id: str = Field(..., description="docker id of the container") @@ -391,10 +391,9 @@ class SchedulerData(CommonServiceDetails, DynamicSidecarServiceLabels): @property def endpoint(self) -> AnyHttpUrl: """endpoint where all the services are exposed""" - url = AnyHttpUrl.build( # pylint: disable=no-member + return AnyHttpUrl.build( # pylint: disable=no-member scheme="http", host=self.hostname, port=self.port ) - return url dynamic_sidecar: DynamicSidecar = Field( ..., diff --git a/services/director-v2/src/simcore_service_director_v2/modules/clusters_keeper.py b/services/director-v2/src/simcore_service_director_v2/modules/clusters_keeper.py index 01f5586fc35..2e62c414d86 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/clusters_keeper.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/clusters_keeper.py @@ -48,6 +48,7 @@ async def get_or_create_on_demand_cluster( owner=user_id, endpoint=returned_cluster.endpoint, authentication=returned_cluster.authentication, + access_rights={}, ) except RemoteMethodNotRegisteredError as exc: # no clusters-keeper, that is not going to work! diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index dd80c484279..ed834581b03 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -1,5 +1,6 @@ import json import logging +import re from collections.abc import Mapping from typing import Any, Final @@ -91,6 +92,11 @@ async def create_network(network_config: dict[str, Any]) -> NetworkId: raise DynamicSidecarError(msg) from e +def _to_snake_case(string: str) -> str: + # Convert camelCase or PascalCase to snake_case + return re.sub(r"(? ServiceId: @@ -100,6 +106,8 @@ async def create_service_and_get_id( kwargs = jsonable_encoder( create_service_data, by_alias=True, exclude_unset=True ) + kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} + logging.debug("Creating service with\n%s", json.dumps(kwargs, indent=1)) service_start_result = await client.services.create(**kwargs) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 36075eb3bf3..d76596b5bf1 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -2,7 +2,7 @@ import collections import logging from collections.abc import Awaitable, Callable, Coroutine, Generator -from typing import Any, Final, NoReturn, ParamSpec, TypeVar, cast, get_args +from typing import Any, Final, NoReturn, ParamSpec, TypeVar, cast from uuid import uuid4 import dask_gateway # type: ignore[import-untyped] @@ -11,7 +11,6 @@ from common_library.json_serialization import json_dumps from dask_task_models_library.container_tasks.io import ( FileUrl, - PortValue, TaskInputData, TaskOutputData, TaskOutputDataSchema, @@ -63,10 +62,6 @@ _PVType = _NPItemValue | None -assert len(get_args(_PVType)) == len( # nosec - get_args(PortValue) -), "Types returned by port.get_value() -> _PVType MUST map one-to-one to PortValue. See compute_input_data" - def _get_port_validation_errors(port_key: str, err: ValidationError) -> list[ErrorDict]: errors = err.errors() @@ -104,7 +99,7 @@ def parse_dask_job_id( return ( parts[0], parts[1], - UserID(parts[2][len("userid_") :]), + TypeAdapter(UserID).validate_python(parts[2][len("userid_") :]), ProjectID(parts[3][len("projectid_") :]), NodeID(parts[4][len("nodeid_") :]), ) @@ -130,7 +125,7 @@ async def create_node_ports( return await node_ports_v2.ports( user_id=user_id, project_id=ProjectIDStr(f"{project_id}"), - node_uuid=NodeIDStr(f"{node_id}"), + node_uuid=TypeAdapter(NodeIDStr).validate_python(f"{node_id}"), db_manager=db_manager, ) except ValidationError as err: @@ -350,8 +345,8 @@ async def compute_task_envs( vendor_substituted_envs = await substitute_vendor_secrets_in_specs( app, cast(dict[str, Any], node_image.envs), - service_key=ServiceKey(node_image.name), - service_version=ServiceVersion(node_image.tag), + service_key=TypeAdapter(ServiceKey).validate_python(node_image.name), + service_version=TypeAdapter(ServiceVersion).validate_python(node_image.tag), product_name=product_name, ) resolved_envs = await resolve_and_substitute_session_variables_in_specs( diff --git a/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py index 6704c8369ef..2715858965b 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py @@ -4,7 +4,7 @@ from typing import Any, Final, NamedTuple, TypeAlias from models_library.utils.specs_substitution import SubstitutionValue -from pydantic import NonNegativeInt, TypeAdapter +from pydantic import NonNegativeInt from servicelib.utils import logged_gather ContextDict: TypeAlias = dict[str, Any] diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 47f130a98be..053431fc34d 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -848,7 +848,7 @@ async def test_update_and_delete_computation( # try to delete the pipeline, is expected to be forbidden if force parameter is false (default) response = await async_client.request( - "DELETE", task_out.url, json={"user_id": user["id"]} + "DELETE", f"{task_out.url}", json={"user_id": user["id"]} ) assert ( response.status_code == status.HTTP_403_FORBIDDEN @@ -856,7 +856,7 @@ async def test_update_and_delete_computation( # try again with force=True this should abort and delete the pipeline response = await async_client.request( - "DELETE", task_out.url, json={"user_id": user["id"], "force": True} + "DELETE", f"{task_out.url}", json={"user_id": user["id"], "force": True} ) assert ( response.status_code == status.HTTP_204_NO_CONTENT diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index bb50fb1fa5d..e43f23bc9dd 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -495,7 +495,7 @@ async def _get_mapped_nodeports_values( PORTS: Nodeports = await node_ports_v2.ports( user_id=user_id, project_id=ProjectIDStr(project_id), - node_uuid=NodeIDStr(node_uuid), + node_uuid=TypeAdapter(NodeIDStr).validate_python(node_uuid), db_manager=db_manager, ) result[str(node_uuid)] = InputsOutputs( @@ -699,7 +699,7 @@ async def _fetch_data_via_aioboto( r_clone_settings: RCloneSettings, dir_tag: str, temp_dir: Path, - node_id: NodeID, + node_id: NodeIDStr, project_id: ProjectID, ) -> Path: save_to = temp_dir / f"aioboto_{dir_tag}_{uuid4()}" @@ -840,7 +840,7 @@ async def _assert_push_non_file_outputs( logger.debug("Going to poll task %s", task_id) async def _debug_progress_callback( - message: ProgressMessage, percent: ProgressPercent, task_id: TaskId + message: ProgressMessage, percent: ProgressPercent | None, task_id: TaskId ) -> None: logger.debug("%s: %.2f %s", task_id, percent, message) @@ -1118,7 +1118,7 @@ async def test_nodeports_integration( dir_tag="dy", user_id=current_user["id"], project_id=current_study.uuid, - service_uuid=services_node_uuids.dy, + service_uuid=NodeID(services_node_uuids.dy), temp_dir=tmp_path, io_log_redirect_cb=mock_io_log_redirect_cb, faker=faker, @@ -1139,7 +1139,7 @@ async def test_nodeports_integration( dir_tag="dy_compose_spec", user_id=current_user["id"], project_id=current_study.uuid, - service_uuid=services_node_uuids.dy_compose_spec, + service_uuid=NodeID(services_node_uuids.dy_compose_spec), temp_dir=tmp_path, io_log_redirect_cb=mock_io_log_redirect_cb, faker=faker, diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 02f6358e426..e63e82ff426 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -278,7 +278,7 @@ async def patch_dynamic_service_url(app: FastAPI, node_uuid: str) -> str: scheduler_data.proxy_service_name = localhost_ip scheduler_data.proxy_admin_api_port = proxy_published_port - endpoint = scheduler_data.endpoint + endpoint = f"{scheduler_data.endpoint}".rstrip("/") assert endpoint == f"http://{localhost_ip}:{sidecar_published_port}" return endpoint diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 424cb128257..b28d0bb3b8d 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -108,7 +108,7 @@ async def _creator( responses: list[httpx.Response] = await asyncio.gather( *( async_client.request( - "DELETE", task.url, json={"user_id": user_id, "force": True} + "DELETE", f"{task.url}", json={"user_id": user_id, "force": True} ) for user_id, task in created_comp_tasks ) diff --git a/services/director-v2/tests/mocks/fake_dy_workbench_template.json b/services/director-v2/tests/mocks/fake_dy_workbench_template.json index e395ad9859f..b6685974173 100644 --- a/services/director-v2/tests/mocks/fake_dy_workbench_template.json +++ b/services/director-v2/tests/mocks/fake_dy_workbench_template.json @@ -17,7 +17,7 @@ }, "80103e12-6b01-40f2-94b8-556bd6c3dd98": { "key": "simcore/services/dynamic/dy-static-file-server-dynamic-sidecar", - "version": "2.0.4", + "version": "2.0.7", "label": "dy-static-file-server-dynamic-sidecar", "inputs": { "string_input": "not the default value", @@ -41,7 +41,7 @@ }, "78f06db4-5feb-4ea3-ad1b-176310ac71a7": { "key": "simcore/services/dynamic/dy-static-file-server-dynamic-sidecar-compose-spec", - "version": "2.0.4", + "version": "2.0.7", "label": "dy-static-file-server-dynamic-sidecar-compose-spec", "inputs": { "string_input": { diff --git a/services/dynamic-scheduler/requirements/prod.txt b/services/dynamic-scheduler/requirements/prod.txt index 70d03a1a7c9..6150ebc8780 100644 --- a/services/dynamic-scheduler/requirements/prod.txt +++ b/services/dynamic-scheduler/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ simcore-postgres-database @ ../../packages/postgres-database simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/dynamic-sidecar/requirements/prod.txt b/services/dynamic-sidecar/requirements/prod.txt index 90f40cf30d3..11aba2a4b8c 100644 --- a/services/dynamic-sidecar/requirements/prod.txt +++ b/services/dynamic-sidecar/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library/ +simcore-common-library @ ../../packages/common-library/ simcore-postgres-database @ ../../packages/postgres-database/ simcore-sdk @ ../../packages/simcore-sdk simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index 4db0148dda6..af477d39ce4 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -199,7 +199,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ) DYNAMIC_SIDECAR_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + json_schema_extra={"auto_default_from_env": True}, + description="settings for opentelemetry tracing", ) @property diff --git a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py index cb5631ec20d..6fffd8fabc1 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py @@ -23,11 +23,12 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID -from pydantic import AnyUrl, TypeAdapter +from pydantic import TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_project from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.postgres_tools import PostgresTestConfig +from pytest_simcore.helpers.storage import replace_storage_endpoint from servicelib.fastapi.long_running_tasks.server import TaskProgress from servicelib.utils import logged_gather from settings_library.s3 import S3Settings @@ -161,24 +162,13 @@ def state_paths_to_legacy_archives( async def simcore_storage_service(mocker: MockerFixture, app: FastAPI) -> None: storage_host: Final[str] | None = os.environ.get("STORAGE_HOST") storage_port: Final[str] | None = os.environ.get("STORAGE_PORT") - - def correct_ip(url: AnyUrl): - - assert storage_host is not None - assert storage_port is not None - - return AnyUrl.build( - scheme=url.scheme, - host=storage_host, - port=storage_port, - path=url.path, - query=url.query, - ) + assert storage_host is not None + assert storage_port is not None # NOTE: Mock to ensure container IP agrees with host IP when testing mocker.patch( "simcore_sdk.node_ports_common._filemanager._get_https_link_if_storage_secure", - correct_ip, + replace_storage_endpoint(storage_host, int(storage_port)), ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 98122863397..26fb9048578 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -16,7 +16,6 @@ from aiodocker.containers import DockerContainer from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI from fastapi.routing import APIRoute from httpx import AsyncClient @@ -158,9 +157,7 @@ def compose_spec(request: pytest.FixtureRequest) -> str: @pytest.fixture def backend_url() -> AnyHttpUrl: - return TypeAdapter(AnyHttpUrlLegacy).validate_python( - "http://backgroud.testserver.io" - ) + return TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io") @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index 1767e403991..57e09c5df96 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -10,7 +10,6 @@ import pytest from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI, status from httpx import AsyncClient from models_library.callbacks_mapping import CallbacksMapping @@ -60,9 +59,7 @@ async def app(mock_rabbitmq_envs: EnvVarsDict, app: FastAPI) -> AsyncIterable[Fa @pytest.fixture def backend_url() -> AnyHttpUrl: - return TypeAdapter(AnyHttpUrlLegacy).validate_python( - "http://backgroud.testserver.io" - ) + return TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io") @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index 1b2bb2a16fa..4bf50116ab3 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -16,7 +16,6 @@ from aiodocker.utils import clean_filters from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from fastapi import FastAPI from httpx import AsyncClient from models_library.generated_models.docker_rest_api import ContainerState @@ -80,9 +79,7 @@ def compose_spec(raw_compose_spec: dict[str, Any]) -> str: @pytest.fixture def backend_url() -> AnyHttpUrl: - return TypeAdapter(AnyHttpUrlLegacy).validate_python( - "http://backgroud.testserver.io" - ) + return TypeAdapter(AnyHttpUrl).validate_python("http://backgroud.testserver.io") @pytest.fixture diff --git a/services/efs-guardian/requirements/prod.txt b/services/efs-guardian/requirements/prod.txt index 486d5e9f7a4..efcaf5fb9d5 100644 --- a/services/efs-guardian/requirements/prod.txt +++ b/services/efs-guardian/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/invitations/requirements/prod.txt b/services/invitations/requirements/prod.txt index d9d82ca6a03..9956e844cd3 100644 --- a/services/invitations/requirements/prod.txt +++ b/services/invitations/requirements/prod.txt @@ -13,5 +13,6 @@ simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ # installs current package simcore-service-invitations @ . diff --git a/services/payments/requirements/prod.txt b/services/payments/requirements/prod.txt index 40cb40fdc64..0da328d59e4 100644 --- a/services/payments/requirements/prod.txt +++ b/services/payments/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ simcore-postgres-database @ ../../packages/postgres-database simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/resource-usage-tracker/requirements/prod.txt b/services/resource-usage-tracker/requirements/prod.txt index 12f20069255..b4ea10941d6 100644 --- a/services/resource-usage-tracker/requirements/prod.txt +++ b/services/resource-usage-tracker/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/storage/requirements/prod.txt b/services/storage/requirements/prod.txt index cd48217e0da..5e0703f83a7 100644 --- a/services/storage/requirements/prod.txt +++ b/services/storage/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library/ +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[aiohttp] @ ../../packages/service-library diff --git a/services/storage/src/simcore_service_storage/models.py b/services/storage/src/simcore_service_storage/models.py index 2e21fc2483c..672694b4fc7 100644 --- a/services/storage/src/simcore_service_storage/models.py +++ b/services/storage/src/simcore_service_storage/models.py @@ -191,7 +191,7 @@ def convert_from_lower_case(cls, v: str) -> str: def when_directory_force_link_type_and_file_size(cls, data: Any) -> Any: assert isinstance(data, dict) - if bool(data.get("is_directory", False)) is True: + if TypeAdapter(bool).validate_python(data.get("is_directory", "false")): # sets directory size by default to undefined if int(data.get("file_size", -1)) < 0: data["file_size"] = None diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/xlsx/code_description.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/xlsx/code_description.py index 9c9a2ff484b..15ee7bac3b0 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/xlsx/code_description.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/xlsx/code_description.py @@ -28,8 +28,8 @@ class RRIDEntry(BaseModel): class TSREntry(BaseModel): references: list[str] - target_level: int | None # max value allowed - current_level: int | None # current selection + target_level: int | None = None # max value allowed + current_level: int | None = None # current selection class CodeDescriptionModel(BaseModel): diff --git a/services/web/server/tests/integration/01/test_exporter_requests_handlers.py b/services/web/server/tests/integration/01/test_exporter_requests_handlers.py index 56da358d7c3..7aac0859900 100644 --- a/services/web/server/tests/integration/01/test_exporter_requests_handlers.py +++ b/services/web/server/tests/integration/01/test_exporter_requests_handlers.py @@ -43,7 +43,12 @@ from simcore_service_webserver.projects.models import ProjectDict from yarl import URL -pytest_simcore_core_services_selection = ["migration", "postgres", "redis", "rabbit"] +pytest_simcore_core_services_selection = [ + "migration", + "postgres", + "redis", + "rabbit", +] _logger = logging.getLogger(__name__) diff --git a/tests/environment-setup/requirements/ci.txt b/tests/environment-setup/requirements/ci.txt index 66b3d90e7f9..72a68ee9108 100644 --- a/tests/environment-setup/requirements/ci.txt +++ b/tests/environment-setup/requirements/ci.txt @@ -12,4 +12,5 @@ # installs this repo's packages pytest-simcore @ ../../packages/pytest-simcore/ simcore-settings-library @ ../../packages/settings-library/ +simcore-common-library @ ../../packages/common-library/ simcore-service-library @ ../../packages/service-library/ diff --git a/tests/environment-setup/requirements/dev.txt b/tests/environment-setup/requirements/dev.txt index 9588230cbc2..55b802c5aa2 100644 --- a/tests/environment-setup/requirements/dev.txt +++ b/tests/environment-setup/requirements/dev.txt @@ -12,4 +12,5 @@ # installs this repo's packages --editable ../../packages/pytest-simcore/ --editable ../../packages/settings-library/ +--editable ../../packages/common-library/ --editable ../../packages/service-library/ diff --git a/tests/public-api/requirements/ci.txt b/tests/public-api/requirements/ci.txt index b1b52acbf2a..546fd268f93 100644 --- a/tests/public-api/requirements/ci.txt +++ b/tests/public-api/requirements/ci.txt @@ -11,5 +11,6 @@ --requirement _test.txt --requirement _tools.txt +simcore-common-library @ ../../packages/common-library/ simcore-settings-library @ ../../packages/settings-library/ pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/tests/public-api/requirements/dev.txt b/tests/public-api/requirements/dev.txt index 1a9478c0941..44602b07260 100644 --- a/tests/public-api/requirements/dev.txt +++ b/tests/public-api/requirements/dev.txt @@ -20,5 +20,6 @@ --requirement _test.txt --requirement _tools.txt +--editable ../../packages/common-library/ --editable ../../packages/settings-library/ --editable ../../packages/pytest-simcore/ diff --git a/tests/public-api/test_users_api.py b/tests/public-api/test_users_api.py index 0b953ddf182..273777f8290 100644 --- a/tests/public-api/test_users_api.py +++ b/tests/public-api/test_users_api.py @@ -32,6 +32,7 @@ class ProfileDict(TypedDict): first_name: str last_name: str email: str + login: str role: osparc.UserRoleEnum groups: ProfileGroupsDict gravatar_id: str @@ -44,7 +45,7 @@ def expected_profile(registered_user: RegisteredUserDict) -> ProfileDict: username = email.split("@")[0] return ProfileDict( - **{ + **{ # noqa: PIE804 "first_name": first_name, "last_name": registered_user["last_name"], "login": email, diff --git a/tests/swarm-deploy/requirements/ci.txt b/tests/swarm-deploy/requirements/ci.txt index 1e1ecd0bf10..1ca4969cdbc 100644 --- a/tests/swarm-deploy/requirements/ci.txt +++ b/tests/swarm-deploy/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _tools.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database[migration] @ ../../packages/postgres-database/ pytest-simcore @ ../../packages/pytest-simcore/ diff --git a/tests/swarm-deploy/requirements/dev.txt b/tests/swarm-deploy/requirements/dev.txt index ed2c7cee70a..12d2ec2b30f 100644 --- a/tests/swarm-deploy/requirements/dev.txt +++ b/tests/swarm-deploy/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/common-library/ --editable ../../packages/models-library/ --editable ../../packages/postgres-database[migration] --editable ../../packages/pytest-simcore/ From a6802c96cf254266da9c4400fbe00a4ae552d3a6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 18 Nov 2024 19:46:19 +0100 Subject: [PATCH 41/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=E2=9C=85=20Fixes=20set?= =?UTF-8?q?tings=20tests=20(#6753)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test__pydantic_models.py | 35 ++++- .../src/settings_library/base.py | 15 +- .../tests/test__pydantic_settings.py | 10 +- packages/settings-library/tests/test_base.py | 9 +- .../tests/unit/test_core_settings.py | 140 +++++++++++++----- 5 files changed, 163 insertions(+), 46 deletions(-) diff --git a/packages/models-library/tests/test__pydantic_models.py b/packages/models-library/tests/test__pydantic_models.py index 645dc1ffe21..d4724972d00 100644 --- a/packages/models-library/tests/test__pydantic_models.py +++ b/packages/models-library/tests/test__pydantic_models.py @@ -6,7 +6,7 @@ """ -from typing import Union, get_args, get_origin +from typing import Any, Union, get_args, get_origin import pytest from models_library.projects_nodes import InputTypes, OutputTypes @@ -173,3 +173,36 @@ class Func(BaseModel): print(model.model_dump_json(indent=1)) assert model.input == {"w": 42, "z": False} assert model.output == [1, 2, 3, None] + + +def test_nullable_fields_from_pydantic_v1(): + # Tests issue found during migration. Pydantic v1 would default to None all nullable fields when they were not **explicitly** set with `...` as required + # SEE https://github.com/ITISFoundation/osparc-simcore/pull/6751 + class MyModel(BaseModel): + # pydanticv1 would add a default to fields set as nullable + nullable_required: str | None # <--- This was default to =None in pydantic 1 !!! + nullable_required_with_hyphen: str | None = Field(default=...) + nullable_optional: str | None = None + + # but with non-nullable "required" worked both ways + non_nullable_required: int + non_nullable_required_with_hyphen: int = Field(default=...) + non_nullable_optional: int = 42 + + data: dict[str, Any] = { + "nullable_required_with_hyphen": "foo", + "non_nullable_required_with_hyphen": 1, + "non_nullable_required": 2, + } + + with pytest.raises(ValidationError) as err_info: + MyModel.model_validate(data) + + assert err_info.value.error_count() == 1 + error = err_info.value.errors()[0] + assert error["type"] == "missing" + assert error["loc"] == ("nullable_required",) + + data["nullable_required"] = None + model = MyModel.model_validate(data) + assert model.model_dump(exclude_unset=True) == data diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index 6d436ea93a9..95f76f12219 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -15,9 +15,9 @@ _logger = logging.getLogger(__name__) -_DEFAULTS_TO_NONE_MSG: Final[ +_AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING: Final[ str -] = "%s auto_default_from_env unresolved, defaulting to None" +] = "{field_name} auto_default_from_env unresolved, defaulting to None" class DefaultFromEnvFactoryError(ValueError): @@ -40,10 +40,10 @@ def _default_factory(): except ValidationError as err: if is_nullable(info): # e.g. Optional[PostgresSettings] would warn if defaults to None - _logger.warning( - _DEFAULTS_TO_NONE_MSG, - field_name, + msg = _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format( + field_name=field_name ) + _logger.warning(msg) return None _logger.warning("Validation errors=%s", err.errors()) raise DefaultFromEnvFactoryError(errors=err.errors()) from err @@ -58,6 +58,9 @@ def _is_auto_default_from_env_enabled(field: FieldInfo) -> bool: ) +ENABLED: Final = {} + + class EnvSettingsWithAutoDefaultSource(EnvSettingsSource): def __init__( self, settings_cls: type[BaseSettings], env_settings: EnvSettingsSource @@ -86,7 +89,7 @@ def prepare_field_value( _is_auto_default_from_env_enabled(field) and field.default_factory and field.default is None - and prepared_value == {} + and prepared_value == ENABLED ): prepared_value = field.default_factory() return prepared_value diff --git a/packages/settings-library/tests/test__pydantic_settings.py b/packages/settings-library/tests/test__pydantic_settings.py index 976395b1b10..bc3da5494da 100644 --- a/packages/settings-library/tests/test__pydantic_settings.py +++ b/packages/settings-library/tests/test__pydantic_settings.py @@ -56,9 +56,13 @@ class Settings(BaseSettings): @classmethod def parse_none(cls, v, info: ValidationInfo): # WARNING: In nullable fields, envs equal to null or none are parsed as None !! - if info.field_name and is_nullable(cls.model_fields[info.field_name]): - if isinstance(v, str) and v.lower() in ("null", "none"): - return None + if ( + info.field_name + and is_nullable(cls.model_fields[info.field_name]) + and isinstance(v, str) + and v.lower() in ("null", "none") + ): + return None return v diff --git a/packages/settings-library/tests/test_base.py b/packages/settings-library/tests/test_base.py index 24b0fbbaa0b..879bf5868be 100644 --- a/packages/settings-library/tests/test_base.py +++ b/packages/settings-library/tests/test_base.py @@ -17,7 +17,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_envfile from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.base import ( - _DEFAULTS_TO_NONE_MSG, + _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING, BaseCustomSettings, DefaultFromEnvFactoryError, ) @@ -223,7 +223,12 @@ class SettingsClass(BaseCustomSettings): # Defaulting to None also logs a warning assert logger_warn.call_count == 1 - assert _DEFAULTS_TO_NONE_MSG in logger_warn.call_args[0][0] + assert ( + _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format( + field_name="VALUE_NULLABLE_DEFAULT_ENV" + ) + in logger_warn.call_args[0][0] + ) def test_auto_default_to_not_none( diff --git a/services/autoscaling/tests/unit/test_core_settings.py b/services/autoscaling/tests/unit/test_core_settings.py index 5b30abe878c..bc63be64cff 100644 --- a/services/autoscaling/tests/unit/test_core_settings.py +++ b/services/autoscaling/tests/unit/test_core_settings.py @@ -5,12 +5,15 @@ import datetime import json +import logging import os +from typing import Final import pytest from faker import Faker from pydantic import ValidationError from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from settings_library.base import _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING from simcore_service_autoscaling.core.settings import ( ApplicationSettings, EC2InstancesSettings, @@ -141,11 +144,11 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_valid( # noqa: N802 assert settings.AUTOSCALING_EC2_INSTANCES -@pytest.mark.xfail( - reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" -) def test_EC2_INSTANCES_ALLOWED_TYPES_passing_invalid_image_tags( # noqa: N802 - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + faker: Faker, + caplog: pytest.LogCaptureFixture, ): # passing an invalid image tag name will fail setenvs_from_dict( @@ -161,8 +164,18 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_passing_invalid_image_tags( # noqa: N802 ) }, ) - with pytest.raises(ValidationError): - ApplicationSettings.create_from_envs() + + with caplog.at_level(logging.WARNING): + + settings = ApplicationSettings.create_from_envs() + assert settings.AUTOSCALING_EC2_INSTANCES is None + + assert ( + _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format( + field_name="AUTOSCALING_EC2_INSTANCES" + ) + in caplog.text + ) def test_EC2_INSTANCES_ALLOWED_TYPES_passing_valid_image_tags( # noqa: N802 @@ -199,54 +212,98 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_passing_valid_image_tags( # noqa: N802 ] -@pytest.mark.xfail( - reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" -) +ENABLED_VALUE: Final = "{}" + + def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed( # noqa: N802 app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch ): - assert app_environment["AUTOSCALING_EC2_INSTANCES"] == "{}" - monkeypatch.setenv("EC2_INSTANCES_ALLOWED_TYPES", "{}") + assert ( + os.environ["AUTOSCALING_EC2_INSTANCES"] == ENABLED_VALUE + ) # parent field in ApplicationSettings + monkeypatch.setenv( + "EC2_INSTANCES_ALLOWED_TYPES", "{}" + ) # child field in EC2InstancesSettings - # test child settings with pytest.raises(ValidationError) as err_info: + # test **child** EC2InstancesSettings EC2InstancesSettings.create_from_envs() assert err_info.value.errors()[0]["loc"] == ("EC2_INSTANCES_ALLOWED_TYPES",) def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed_with_main_field_env_var( # noqa: N802 - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ): - assert os.environ["AUTOSCALING_EC2_INSTANCES"] == "{}" - monkeypatch.setenv("EC2_INSTANCES_ALLOWED_TYPES", "{}") - - # now as part of AUTOSCALING_EC2_INSTANCES: EC2InstancesSettings | None - with pytest.raises(ValidationError) as exc_before: + assert ( + os.environ["AUTOSCALING_EC2_INSTANCES"] == ENABLED_VALUE + ) # parent field in ApplicationSettings + monkeypatch.setenv( + "EC2_INSTANCES_ALLOWED_TYPES", "{}" + ) # child field in EC2InstancesSettings + + # explicit init of parent -> fails + with pytest.raises(ValidationError) as exc_info: + # NOTE: input captured via InitSettingsSource ApplicationSettings.create_from_envs(AUTOSCALING_EC2_INSTANCES={}) - with pytest.raises(ValidationError) as exc_after: - ApplicationSettings.create_from_envs() + assert exc_info.value.error_count() == 1 + error = exc_info.value.errors()[0] + + assert error["type"] == "value_error" + assert error["input"] == {} + assert error["loc"] == ("AUTOSCALING_EC2_INSTANCES", "EC2_INSTANCES_ALLOWED_TYPES") + + # NOTE: input captured via EnvSettingsWithAutoDefaultSource + # default env factory -> None + with caplog.at_level(logging.WARNING): - assert exc_before.value.errors() == exc_after.value.errors() + settings = ApplicationSettings.create_from_envs() + assert settings.AUTOSCALING_EC2_INSTANCES is None + + assert ( + _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format( + field_name="AUTOSCALING_EC2_INSTANCES" + ) + in caplog.text + ) def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed_without_main_field_env_var( # noqa: N802 - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ): - monkeypatch.delenv("AUTOSCALING_EC2_INSTANCES") - monkeypatch.setenv("EC2_INSTANCES_ALLOWED_TYPES", "{}") + assert os.environ["AUTOSCALING_EC2_INSTANCES"] == ENABLED_VALUE + monkeypatch.delenv( + "AUTOSCALING_EC2_INSTANCES" + ) # parent field in ApplicationSettings + monkeypatch.setenv( + "EC2_INSTANCES_ALLOWED_TYPES", "{}" + ) # child field in EC2InstancesSettings # removing any value for AUTOSCALING_EC2_INSTANCES - settings = ApplicationSettings.create_from_envs() - assert settings.AUTOSCALING_EC2_INSTANCES is None + caplog.clear() + with caplog.at_level(logging.WARNING): + settings = ApplicationSettings.create_from_envs() + assert settings.AUTOSCALING_EC2_INSTANCES is None -@pytest.mark.xfail( - reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705" -) -def test_invalid_instance_names( - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker + assert ( + _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format( + field_name="AUTOSCALING_EC2_INSTANCES" + ) + in caplog.text + ) + + +def test_EC2_INSTANCES_ALLOWED_TYPES_invalid_instance_names( # noqa: N802 + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + faker: Faker, + caplog: pytest.LogCaptureFixture, ): settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES @@ -256,9 +313,24 @@ def test_invalid_instance_names( monkeypatch, { "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( - {faker.pystr(): {"ami_id": faker.pystr(), "pre_pull_images": []}} + { + faker.pystr(): { + "ami_id": faker.pystr(), + "pre_pull_images": [], + } + } ) }, ) - with pytest.raises(ValidationError): - ApplicationSettings.create_from_envs() + caplog.clear() + with caplog.at_level(logging.WARNING): + + settings = ApplicationSettings.create_from_envs() + assert settings.AUTOSCALING_EC2_INSTANCES is None + + assert ( + _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format( + field_name="AUTOSCALING_EC2_INSTANCES" + ) + in caplog.text + ) From 75f06e4ae1423b3c635d629affda14ae040eff29 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Tue, 19 Nov 2024 07:15:42 +0100 Subject: [PATCH 42/53] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20webserver=20API=20?= =?UTF-8?q?endpoints=20(#6751)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../src/models_library/api_schemas_webserver/projects.py | 8 ++++---- .../api_schemas_webserver/projects_nodes.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index a7d63904350..7c4116a136c 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -33,8 +33,8 @@ class ProjectCreateNew(InputSchema): uuid: ProjectID | None = None # NOTE: suggested uuid! but could be different! name: str - description: str | None - thumbnail: HttpUrl | None + description: str | None = None + thumbnail: HttpUrl | None = None workbench: NodesDict access_rights: dict[GroupIDStr, AccessRights] tags: list[int] = Field(default_factory=list) @@ -55,8 +55,8 @@ class ProjectCreateNew(InputSchema): # NOTE: based on OVERRIDABLE_DOCUMENT_KEYS class ProjectCopyOverride(InputSchema): name: str - description: str | None - thumbnail: HttpUrl | None + description: str | None = None + thumbnail: HttpUrl | None = None prj_owner: LowerCaseEmailStr _empty_is_none = field_validator("thumbnail", mode="before")( diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 7b65b84cb5f..81eaa893d60 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -19,7 +19,7 @@ class NodeCreate(InputSchemaWithoutCamelCase): service_key: ServiceKey service_version: ServiceVersion - service_id: str | None + service_id: str | None = None BootOptions: TypeAlias = dict From f442e634015d4a0ec4c32cb0d5fb64d43f45d925 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:27:30 +0100 Subject: [PATCH 43/53] =?UTF-8?q?=E2=AC=86=EF=B8=8FPydantic=20V2:=20Migrat?= =?UTF-8?q?e=20director=20v0=20+=20some=20fixes=20from=20query=20PR=20(#67?= =?UTF-8?q?55)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/specs/web-server/_common.py | 31 ++--- .../pydantic_fields_extension.py | 2 +- .../src/models_library/basic_types.py | 4 +- .../src/models_library/rest_ordering.py | 55 ++++---- .../models_library/utils/common_validators.py | 3 +- .../tests/test_rest_ordering.py | 45 ++++--- .../servicelib/aiohttp/requests_validation.py | 2 +- .../tests/aiohttp/test_requests_validation.py | 2 +- .../src/settings_library/application.py | 5 +- .../maintenance/migrate_project/src/models.py | 9 +- .../core/application.py | 3 +- services/director/requirements/_base.in | 1 + services/director/requirements/_base.txt | 125 ++++++++++++++++-- services/director/requirements/ci.txt | 1 + services/director/requirements/dev.txt | 1 + services/director/requirements/prod.txt | 1 + .../client_session.py | 2 +- .../core/application.py | 2 +- .../simcore_service_director/core/errors.py | 2 +- .../simcore_service_director/core/settings.py | 52 +++++--- .../services_common.py | 10 +- .../tests/unit/api/test_rest_services.py | 2 +- .../tests/unit/test__model_examples.py | 2 +- services/director/tests/unit/test_cli.py | 5 +- .../director/tests/unit/test_core_settings.py | 8 +- .../resource_usage/_service_runs_handlers.py | 2 +- .../test_usage_services__list.py | 2 +- .../with_dbs/04/workspaces/test_workspaces.py | 4 +- 28 files changed, 262 insertions(+), 121 deletions(-) diff --git a/api/specs/web-server/_common.py b/api/specs/web-server/_common.py index ef2cf88ddc2..06ba6872800 100644 --- a/api/specs/web-server/_common.py +++ b/api/specs/web-server/_common.py @@ -8,10 +8,10 @@ from typing import Any, ClassVar, NamedTuple import yaml +from common_library.json_serialization import json_dumps from common_library.pydantic_fields_extension import get_type from fastapi import FastAPI, Query from models_library.basic_types import LogLevel -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, Field, create_model from pydantic.fields import FieldInfo from servicelib.fastapi.openapi import override_fastapi_openapi_method @@ -38,31 +38,28 @@ def __modify_schema__(cls, field_schema: dict[str, Any]) -> None: def as_query(model_class: type[BaseModel]) -> type[BaseModel]: fields = {} - for field_name, model_field in model_class.__fields__.items(): + for field_name, field_info in model_class.model_fields.items(): - field_type = model_field.type_ - default_value = model_field.default + field_type = get_type(field_info) + default_value = field_info.default kwargs = { - "alias": model_field.field_info.alias, - "title": model_field.field_info.title, - "description": model_field.field_info.description, - "gt": model_field.field_info.gt, - "ge": model_field.field_info.ge, - "lt": model_field.field_info.lt, - "le": model_field.field_info.le, - "min_length": model_field.field_info.min_length, - "max_length": model_field.field_info.max_length, - "regex": model_field.field_info.regex, - **model_field.field_info.extra, + "alias": field_info.alias, + "title": field_info.title, + "description": field_info.description, + "metadata": field_info.metadata, + "json_schema_extra": field_info.json_schema_extra, } if issubclass(field_type, BaseModel): # Complex fields + assert "json_schema_extra" in kwargs # nosec + assert kwargs["json_schema_extra"] # nosec field_type = _create_json_type( description=kwargs["description"], - example=kwargs.get("example_json"), + example=kwargs.get("json_schema_extra", {}).get("example_json"), ) + default_value = json_dumps(default_value) if default_value else None fields[field_name] = (field_type, Query(default=default_value, **kwargs)) @@ -148,7 +145,7 @@ def create_and_save_openapi_specs( ) with file_path.open("wt") as fh: yaml.safe_dump(openapi, fh, indent=1, sort_keys=False) - print("Saved OAS to", file_path) + print("Saved OAS to", file_path) # noqa: T201 class ParamSpec(NamedTuple): diff --git a/packages/common-library/src/common_library/pydantic_fields_extension.py b/packages/common-library/src/common_library/pydantic_fields_extension.py index f2f6d59a5f4..59303b0a1b3 100644 --- a/packages/common-library/src/common_library/pydantic_fields_extension.py +++ b/packages/common-library/src/common_library/pydantic_fields_extension.py @@ -7,7 +7,7 @@ def get_type(info: FieldInfo) -> Any: field_type = info.annotation if args := get_args(info.annotation): - field_type = next(a for a in args if a != type(None)) + field_type = next(a for a in args if a is not type(None)) return field_type diff --git a/packages/models-library/src/models_library/basic_types.py b/packages/models-library/src/models_library/basic_types.py index d47a60514da..fac95b7c2a0 100644 --- a/packages/models-library/src/models_library/basic_types.py +++ b/packages/models-library/src/models_library/basic_types.py @@ -1,7 +1,7 @@ from decimal import Decimal from enum import StrEnum from re import Pattern -from typing import Annotated, Final, TypeAlias +from typing import Annotated, ClassVar, Final, TypeAlias from pydantic import Field, HttpUrl, PositiveInt, StringConstraints from pydantic_core import core_schema @@ -137,7 +137,7 @@ class LongTruncatedStr(ConstrainedStr): # https e.g. https://techterms.com/definition/https class HttpSecureUrl(HttpUrl): - allowed_schemes = {"https"} + allowed_schemes: ClassVar[set[str]] = {"https"} class HttpUrlWithCustomMinLength(HttpUrl): diff --git a/packages/models-library/src/models_library/rest_ordering.py b/packages/models-library/src/models_library/rest_ordering.py index 9e3e61bff38..3b355ae51ef 100644 --- a/packages/models-library/src/models_library/rest_ordering.py +++ b/packages/models-library/src/models_library/rest_ordering.py @@ -1,8 +1,8 @@ from enum import Enum -from typing import Any, ClassVar +from typing import Annotated -from models_library.utils.json_serialization import json_dumps -from pydantic import BaseModel, Extra, Field, field_validator +from common_library.json_serialization import json_dumps +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, field_validator from .basic_types import IDStr from .rest_base import RequestParameters @@ -62,17 +62,19 @@ def create_ordering_query_model_classes( msg_direction_options = "|".join(sorted(OrderDirection)) class _OrderBy(OrderBy): - class Config: - schema_extra: ClassVar[dict[str, Any]] = { - "example": { - "field": next(iter(ordering_fields)), - "direction": OrderDirection.DESC.value, - } - } - extra = Extra.forbid - # Necessary to run _check_ordering_field_and_map in defaults and assignments - validate_all = True - validate_assignment = True + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ + "examples": [ + { + "field": next(iter(ordering_fields)), + "direction": OrderDirection.DESC.value, + } + ] + }, + validate_assignment=True, # Necessary to run _check_ordering_field_and_map in defaults and assignments + validate_default=True, + ) @field_validator("field", mode="before") @classmethod @@ -87,28 +89,31 @@ def _check_ordering_field_and_map(cls, v): # API field name -> DB column_name conversion return _ordering_fields_api_to_column_map.get(v) or v - order_by_example: dict[str, Any] = _OrderBy.Config.schema_extra["example"] + assert "json_schema_extra" in _OrderBy.model_config # nosec + assert isinstance(_OrderBy.model_config["json_schema_extra"], dict) # nosec + assert isinstance( + _OrderBy.model_config["json_schema_extra"]["examples"], list + ) # nosec + order_by_example = _OrderBy.model_config["json_schema_extra"]["examples"][0] order_by_example_json = json_dumps(order_by_example) - assert _OrderBy.parse_obj(order_by_example), "Example is invalid" # nosec + assert _OrderBy.model_validate(order_by_example), "Example is invalid" # nosec - converted_default = _OrderBy.parse_obj( + converted_default = _OrderBy.model_validate( # NOTE: enforces ordering_fields_api_to_column_map - default.dict() + default.model_dump() ) class _OrderQueryParams(_BaseOrderQueryParams): - order_by: _OrderBy = Field( + order_by: Annotated[ + _OrderBy, BeforeValidator(parse_json_pre_validator) + ] = Field( default=converted_default, description=( f"Order by field (`{msg_field_options}`) and direction (`{msg_direction_options}`). " f"The default sorting order is `{json_dumps(default)}`." ), - example=order_by_example, - example_json=order_by_example_json, - ) - - _pre_parse_string = field_validator("order_by", mode="before")( - parse_json_pre_validator + examples=[order_by_example], + json_schema_extra={"example_json": order_by_example_json}, ) return _OrderQueryParams diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index 107c81cbc06..59461014965 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -20,10 +20,9 @@ class MyModel(BaseModel): import operator from typing import Any +from common_library.json_serialization import json_loads from orjson import JSONDecodeError -from .json_serialization import json_loads - def empty_str_to_none_pre_validator(value: Any): if isinstance(value, str) and value.strip() == "": diff --git a/packages/models-library/tests/test_rest_ordering.py b/packages/models-library/tests/test_rest_ordering.py index fec004cd01e..a495dc55721 100644 --- a/packages/models-library/tests/test_rest_ordering.py +++ b/packages/models-library/tests/test_rest_ordering.py @@ -1,12 +1,19 @@ import pytest +from common_library.json_serialization import json_dumps from models_library.basic_types import IDStr from models_library.rest_ordering import ( OrderBy, OrderDirection, create_ordering_query_model_classes, ) -from models_library.utils.json_serialization import json_dumps -from pydantic import BaseModel, Extra, Field, Json, ValidationError, validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + Json, + ValidationError, + field_validator, +) class ReferenceOrderQueryParamsClass(BaseModel): @@ -18,10 +25,10 @@ class ReferenceOrderQueryParamsClass(BaseModel): order_by: Json[OrderBy] = Field( default=OrderBy(field=IDStr("modified_at"), direction=OrderDirection.DESC), description="Order by field (modified_at|name|description) and direction (asc|desc). The default sorting order is ascending.", - example='{"field": "name", "direction": "desc"}', + json_schema_extra={"examples": ['{"field": "name", "direction": "desc"}']}, ) - @validator("order_by", check_fields=False) + @field_validator("order_by", check_fields=False) @classmethod def _validate_order_by_field(cls, v): if v.field not in { @@ -35,8 +42,9 @@ def _validate_order_by_field(cls, v): v.field = "modified_column" return v - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra="forbid", + ) def test_ordering_query_model_class_factory(): @@ -52,16 +60,19 @@ class OrderQueryParamsModel(BaseOrderingQueryModel): # normal data = {"order_by": {"field": "modified_at", "direction": "asc"}} - model = OrderQueryParamsModel.parse_obj(data) + model = OrderQueryParamsModel.model_validate(data) assert model.order_by - assert model.order_by.dict() == {"field": "modified_column", "direction": "asc"} + assert model.order_by.model_dump() == { + "field": "modified_column", + "direction": "asc", + } # test against reference - expected = ReferenceOrderQueryParamsClass.parse_obj( + expected = ReferenceOrderQueryParamsClass.model_validate( {"order_by": json_dumps({"field": "modified_at", "direction": "asc"})} ) - assert expected.dict() == model.dict() + assert expected.model_dump() == model.model_dump() def test_ordering_query_model_class__fails_with_invalid_fields(): @@ -73,7 +84,7 @@ def test_ordering_query_model_class__fails_with_invalid_fields(): # fails with invalid field to sort with pytest.raises(ValidationError) as err_info: - OrderQueryParamsModel.parse_obj({"order_by": {"field": "INVALID"}}) + OrderQueryParamsModel.model_validate({"order_by": {"field": "INVALID"}}) error = err_info.value.errors()[0] @@ -89,7 +100,7 @@ def test_ordering_query_model_class__fails_with_invalid_direction(): ) with pytest.raises(ValidationError) as err_info: - OrderQueryParamsModel.parse_obj( + OrderQueryParamsModel.model_validate( {"order_by": {"field": "modified", "direction": "INVALID"}} ) @@ -110,18 +121,19 @@ def test_ordering_query_model_class__defaults(): # checks all defaults model = OrderQueryParamsModel() assert model.order_by + assert isinstance(model.order_by, OrderBy) # nosec assert model.order_by.field == "modified_at" # NOTE that this was mapped! assert model.order_by.direction == OrderDirection.DESC # partial defaults - model = OrderQueryParamsModel.parse_obj({"order_by": {"field": "name"}}) + model = OrderQueryParamsModel.model_validate({"order_by": {"field": "name"}}) assert model.order_by assert model.order_by.field == "name" - assert model.order_by.direction == OrderBy.__fields__["direction"].default + assert model.order_by.direction == OrderBy.model_fields["direction"].default # direction alone is invalid with pytest.raises(ValidationError) as err_info: - OrderQueryParamsModel.parse_obj({"order_by": {"direction": "asc"}}) + OrderQueryParamsModel.model_validate({"order_by": {"direction": "asc"}}) error = err_info.value.errors()[0] assert error["loc"] == ("order_by", "field") @@ -135,5 +147,6 @@ def test_ordering_query_model_with_map(): ordering_fields_api_to_column_map={"modified": "some_db_column_name"}, ) - model = OrderQueryParamsModel.parse_obj({"order_by": {"field": "modified"}}) + model = OrderQueryParamsModel.model_validate({"order_by": {"field": "modified"}}) + assert model.order_by assert model.order_by.field == "some_db_column_name" diff --git a/packages/service-library/src/servicelib/aiohttp/requests_validation.py b/packages/service-library/src/servicelib/aiohttp/requests_validation.py index c899431b227..a61d8d538cd 100644 --- a/packages/service-library/src/servicelib/aiohttp/requests_validation.py +++ b/packages/service-library/src/servicelib/aiohttp/requests_validation.py @@ -13,7 +13,7 @@ from typing import TypeAlias, TypeVar, Union from aiohttp import web -from models_library.utils.json_serialization import json_dumps +from common_library.json_serialization import json_dumps from pydantic import BaseModel, TypeAdapter, ValidationError from ..mimetype_constants import MIMETYPE_APPLICATION_JSON diff --git a/packages/service-library/tests/aiohttp/test_requests_validation.py b/packages/service-library/tests/aiohttp/test_requests_validation.py index 7ffb67a2991..e1d812da677 100644 --- a/packages/service-library/tests/aiohttp/test_requests_validation.py +++ b/packages/service-library/tests/aiohttp/test_requests_validation.py @@ -9,6 +9,7 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient, make_mocked_request +from common_library.json_serialization import json_dumps from faker import Faker from models_library.rest_base import RequestParameters, StrictRequestParameters from models_library.rest_ordering import ( @@ -16,7 +17,6 @@ OrderDirection, create_ordering_query_model_classes, ) -from models_library.utils.json_serialization import json_dumps from pydantic import BaseModel, ConfigDict, Field from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( diff --git a/packages/settings-library/src/settings_library/application.py b/packages/settings-library/src/settings_library/application.py index 43740b69fd5..890067ca518 100644 --- a/packages/settings-library/src/settings_library/application.py +++ b/packages/settings-library/src/settings_library/application.py @@ -1,7 +1,8 @@ +from models_library.basic_types import BootModeEnum from pydantic import Field, PositiveInt from .base import BaseCustomSettings -from .basic_types import BootMode, BuildTargetEnum +from .basic_types import BuildTargetEnum class BaseApplicationSettings(BaseCustomSettings): @@ -16,7 +17,7 @@ class BaseApplicationSettings(BaseCustomSettings): SC_VCS_URL: str | None = None # @Dockerfile - SC_BOOT_MODE: BootMode | None = None + SC_BOOT_MODE: BootModeEnum | None = None SC_BOOT_TARGET: BuildTargetEnum | None = None SC_HEALTHCHECK_TIMEOUT: PositiveInt | None = Field( default=None, diff --git a/scripts/maintenance/migrate_project/src/models.py b/scripts/maintenance/migrate_project/src/models.py index a26cf63e734..68649c2a392 100644 --- a/scripts/maintenance/migrate_project/src/models.py +++ b/scripts/maintenance/migrate_project/src/models.py @@ -1,6 +1,5 @@ import json from pathlib import Path -from typing import Optional from uuid import UUID from pydantic import BaseModel, Field @@ -32,7 +31,7 @@ class SourceConfig(BaseModel): db: DBConfig s3: S3Config project_uuid: UUID = Field(..., description="project to be moved from the source") - hidden_projects_for_user: Optional[int] = Field( + hidden_projects_for_user: int | None = Field( None, description="by default nothing is moved, must provide an user ID for which to move the hidden projects", ) @@ -57,7 +56,7 @@ class Settings(BaseModel): @classmethod def load_from_file(cls, path: Path) -> "Settings": - return Settings.parse_obj(json.loads(path.read_text())) + return Settings.model_validate(json.loads(path.read_text())) class Config: schema_extra = { @@ -92,4 +91,6 @@ class Config: if __name__ == "__main__": # produces an empty configuration to be saved as starting point - print(Settings.parse_obj(Settings.Config.schema_extra["example"]).json(indent=2)) + print( + Settings.model_validate(Settings.Config.schema_extra["example"]).json(indent=2) + ) diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 469d37ba26c..fb9d6094dff 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -9,7 +9,6 @@ from servicelib.fastapi.profiler_middleware import ProfilerMiddleware from servicelib.fastapi.tracing import setup_tracing from servicelib.logging_utils import config_all_loggers -from settings_library.basic_types import BootMode from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY from ..api.entrypoints import api_router @@ -128,7 +127,7 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: logging.getLogger(name).setLevel(quiet_level) app = FastAPI( - debug=settings.SC_BOOT_MODE is BootMode.DEVELOPMENT, + debug=settings.SC_BOOT_MODE.is_devel_mode(), title=PROJECT_NAME, description=SUMMARY, version=API_VERSION, diff --git a/services/director/requirements/_base.in b/services/director/requirements/_base.in index 468bb684525..af6b80486e8 100644 --- a/services/director/requirements/_base.in +++ b/services/director/requirements/_base.in @@ -4,6 +4,7 @@ --constraint ../../../requirements/constraints.txt # intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/director/requirements/_base.txt b/services/director/requirements/_base.txt index f88e7c85550..d9164137fa0 100644 --- a/services/director/requirements/_base.txt +++ b/services/director/requirements/_base.txt @@ -16,10 +16,16 @@ aiohappyeyeballs==2.4.3 # via aiohttp aiohttp==3.11.1 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker @@ -27,6 +33,8 @@ aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.6.2.post1 # via # fast-depends @@ -48,10 +56,16 @@ attrs==24.2.0 # referencing certifi==2024.8.30 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # httpcore @@ -77,17 +91,13 @@ email-validator==2.2.0 # pydantic fast-depends==2.4.12 # via faststream -fastapi==0.99.1 +fastapi==0.115.5 # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator +fastapi-cli==0.0.5 + # via fastapi faststream==0.5.30 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.5.0 @@ -110,10 +120,16 @@ httptools==0.6.4 # via uvicorn httpx==0.27.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in @@ -132,10 +148,16 @@ itsdangerous==2.2.0 # via fastapi jinja2==3.1.4 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -222,14 +244,26 @@ opentelemetry-util-http==0.49b1 # opentelemetry-instrumentation-requests orjson==3.10.11 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # fastapi packaging==24.2 # via opentelemetry-instrumentation @@ -252,23 +286,56 @@ protobuf==5.28.3 # opentelemetry-proto psutil==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -pydantic==1.10.19 +pydantic==2.9.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.10.0 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # fastapi +pydantic-settings==2.6.1 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # fastapi pygments==2.18.0 # via rich pyinstrument==5.0.0 @@ -276,15 +343,23 @@ pyinstrument==5.0.0 python-dateutil==2.9.0.post0 # via arrow python-dotenv==1.0.1 - # via uvicorn + # via + # pydantic-settings + # uvicorn python-multipart==0.0.17 # via fastapi pyyaml==6.0.2 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -292,10 +367,16 @@ pyyaml==6.0.2 # uvicorn redis==5.2.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -325,12 +406,18 @@ sniffio==1.3.1 # via # anyio # httpx -starlette==0.27.0 +starlette==0.41.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi @@ -344,6 +431,7 @@ typer==0.13.0 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in + # fastapi-cli types-python-dateutil==2.9.0.20241003 # via arrow typing-extensions==4.12.2 @@ -353,22 +441,36 @@ typing-extensions==4.12.2 # faststream # opentelemetry-sdk # pydantic + # pydantic-core + # pydantic-extra-types # typer ujson==5.10.0 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi urllib3==2.2.3 # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -376,6 +478,7 @@ uvicorn==0.32.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # fastapi + # fastapi-cli uvloop==0.21.0 # via uvicorn watchfiles==0.24.0 diff --git a/services/director/requirements/ci.txt b/services/director/requirements/ci.txt index e805fec3802..dd8038caf3b 100644 --- a/services/director/requirements/ci.txt +++ b/services/director/requirements/ci.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library pytest-simcore @ ../../packages/pytest-simcore/ simcore-service-library[fastapi] @ ../../packages/service-library diff --git a/services/director/requirements/dev.txt b/services/director/requirements/dev.txt index f278b7206fd..80538c22580 100644 --- a/services/director/requirements/dev.txt +++ b/services/director/requirements/dev.txt @@ -14,6 +14,7 @@ # installs this repo's packages +--editable ../../packages/common-library/ --editable ../../packages/models-library --editable ../../packages/pytest-simcore/ --editable ../../packages/service-library[fastapi] diff --git a/services/director/requirements/prod.txt b/services/director/requirements/prod.txt index 8a8b1d29125..147a5b7ccba 100644 --- a/services/director/requirements/prod.txt +++ b/services/director/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/director/src/simcore_service_director/client_session.py b/services/director/src/simcore_service_director/client_session.py index 74647f13822..b053d06c456 100644 --- a/services/director/src/simcore_service_director/client_session.py +++ b/services/director/src/simcore_service_director/client_session.py @@ -1,6 +1,6 @@ from aiohttp import ClientSession, ClientTimeout +from common_library.json_serialization import json_dumps from fastapi import FastAPI -from models_library.utils.json_serialization import json_dumps from servicelib.utils import ( get_http_client_request_aiohttp_connect_timeout, get_http_client_request_aiohttp_sock_connect_timeout, diff --git a/services/director/src/simcore_service_director/core/application.py b/services/director/src/simcore_service_director/core/application.py index 10fb32b5518..86031f0d935 100644 --- a/services/director/src/simcore_service_director/core/application.py +++ b/services/director/src/simcore_service_director/core/application.py @@ -32,7 +32,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: for name in _NOISY_LOGGERS: logging.getLogger(name).setLevel(quiet_level) - _logger.info("app settings: %s", settings.json(indent=1)) + _logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.DIRECTOR_DEBUG, diff --git a/services/director/src/simcore_service_director/core/errors.py b/services/director/src/simcore_service_director/core/errors.py index ebbf885451b..23ca2fc17fc 100644 --- a/services/director/src/simcore_service_director/core/errors.py +++ b/services/director/src/simcore_service_director/core/errors.py @@ -1,6 +1,6 @@ from typing import Any -from models_library.errors_classes import OsparcErrorMixin +from common_library.errors_classes import OsparcErrorMixin class DirectorRuntimeError(OsparcErrorMixin, RuntimeError): diff --git a/services/director/src/simcore_service_director/core/settings.py b/services/director/src/simcore_service_director/core/settings.py index e0a352aed82..2773f02fb51 100644 --- a/services/director/src/simcore_service_director/core/settings.py +++ b/services/director/src/simcore_service_director/core/settings.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from models_library.basic_types import LogLevel, PortInt, VersionTag -from pydantic import Field, NonNegativeInt, validator +from pydantic import AliasChoices, Field, NonNegativeInt, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.docker_registry import RegistrySettings @@ -21,28 +21,33 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): API_VTAG: VersionTag = API_VTAG DIRECTOR_DEBUG: bool = Field( - default=False, description="Debug mode", env=["DIRECTOR_DEBUG", "DEBUG"] + default=False, + description="Debug mode", + validation_alias=AliasChoices("DIRECTOR_DEBUG", "DEBUG"), ) - DIRECTOR_REMOTE_DEBUG_PORT: PortInt = PortInt(3000) + DIRECTOR_REMOTE_DEBUG_PORT: PortInt = 3000 DIRECTOR_LOGLEVEL: LogLevel = Field( - ..., env=["DIRECTOR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + ..., validation_alias=AliasChoices("DIRECTOR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL") ) DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( ..., - env=[ + validation_alias=AliasChoices( "DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) DIRECTOR_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( default_factory=dict, - env=["DIRECTOR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + validation_alias=AliasChoices( + "DIRECTOR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ) DIRECTOR_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + description="settings for opentelemetry tracing", + json_schema_extra={"auto_default_from_env": True}, ) DIRECTOR_DEFAULT_MAX_NANO_CPUS: NonNegativeInt = Field(default=0) @@ -64,37 +69,50 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DIRECTOR_TRAEFIK_SIMCORE_ZONE: str = Field( ..., - env=["DIRECTOR_TRAEFIK_SIMCORE_ZONE", "TRAEFIK_SIMCORE_ZONE"], + validation_alias=AliasChoices( + "DIRECTOR_TRAEFIK_SIMCORE_ZONE", "TRAEFIK_SIMCORE_ZONE" + ), ) DIRECTOR_REGISTRY: RegistrySettings = Field( - auto_default_from_env=True, description="settings for the private registry deployed with the platform", + json_schema_extra={"auto_default_from_env": True}, ) - DIRECTOR_POSTGRES: PostgresSettings = Field(auto_default_from_env=True) + DIRECTOR_POSTGRES: PostgresSettings = Field( + ..., json_schema_extra={"auto_default_from_env": True} + ) STORAGE_ENDPOINT: str = Field(..., description="storage endpoint without scheme") DIRECTOR_PUBLISHED_HOST_NAME: str = Field( - ..., env=["DIRECTOR_PUBLISHED_HOST_NAME", "PUBLISHED_HOST_NAME"] + ..., + validation_alias=AliasChoices( + "DIRECTOR_PUBLISHED_HOST_NAME", "PUBLISHED_HOST_NAME" + ), ) DIRECTOR_SWARM_STACK_NAME: str = Field( ..., - env=["DIRECTOR_SWARM_STACK_NAME", "SWARM_STACK_NAME"], + validation_alias=AliasChoices("DIRECTOR_SWARM_STACK_NAME", "SWARM_STACK_NAME"), ) DIRECTOR_SIMCORE_SERVICES_NETWORK_NAME: str | None = Field( # used to find the right network name ..., - env=["DIRECTOR_SIMCORE_SERVICES_NETWORK_NAME", "SIMCORE_SERVICES_NETWORK_NAME"], + validation_alias=AliasChoices( + "DIRECTOR_SIMCORE_SERVICES_NETWORK_NAME", + "SIMCORE_SERVICES_NETWORK_NAME", + ), ) DIRECTOR_MONITORING_ENABLED: bool = Field( - ..., env=["DIRECTOR_MONITORING_ENABLED", "MONITORING_ENABLED"] + ..., + validation_alias=AliasChoices( + "DIRECTOR_MONITORING_ENABLED", "MONITORING_ENABLED" + ), ) - @validator("DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS") + @field_validator("DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS") @classmethod def _validate_substitutions(cls, v): if v: @@ -110,7 +128,7 @@ def _validate_substitutions(cls, v): return v - @validator("DIRECTOR_LOGLEVEL", pre=True) + @field_validator("DIRECTOR_LOGLEVEL", mode="before") @classmethod def _valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) diff --git a/services/director/src/simcore_service_director/services_common.py b/services/director/src/simcore_service_director/services_common.py index b92bbd1ca80..638f6ddefe5 100644 --- a/services/director/src/simcore_service_director/services_common.py +++ b/services/director/src/simcore_service_director/services_common.py @@ -3,7 +3,8 @@ # since this service is frozen and MUST NOT ADD ANY MORE DEPENDENCIES # # -from pydantic import BaseSettings, Field, PositiveInt +from pydantic import Field, PositiveInt +from pydantic_settings import BaseSettings, SettingsConfigDict _BASE_TIMEOUT_FOR_STOPPING_SERVICES = 60 * 60 @@ -34,7 +35,6 @@ class ServicesCommonSettings(BaseSettings): "allow the service to finish the operation." ), ) - - class Config: - env_prefix = "SERVICES_COMMON_" - case_sensitive = False + model_config = SettingsConfigDict( + env_prefix="SERVICES_COMMON_", case_sensitive=False + ) diff --git a/services/director/tests/unit/api/test_rest_services.py b/services/director/tests/unit/api/test_rest_services.py index a34ec6a76d4..e42edea2eff 100644 --- a/services/director/tests/unit/api/test_rest_services.py +++ b/services/director/tests/unit/api/test_rest_services.py @@ -36,7 +36,7 @@ def _assert_services( ] for data in got: - service = ServiceDataGet.parse_obj(data) + service = ServiceDataGet.model_validate(data) assert ( expected_key_version_tuples.count((f"{service.key}", f"{service.version}")) == 1 diff --git a/services/director/tests/unit/test__model_examples.py b/services/director/tests/unit/test__model_examples.py index d9604d738d6..cb38a9a90f7 100644 --- a/services/director/tests/unit/test__model_examples.py +++ b/services/director/tests/unit/test__model_examples.py @@ -21,7 +21,7 @@ def test_director_service_model_examples( model_cls: type[BaseModel], example_name: int, example_data: Any ): try: - assert model_cls.parse_obj(example_data) is not None + assert model_cls.model_validate(example_data) is not None except ValidationError as err: pytest.fail( f"\n{example_name}: {json.dumps(example_data, indent=1)}\nError: {err}" diff --git a/services/director/tests/unit/test_cli.py b/services/director/tests/unit/test_cli.py index 3b42989bcff..92967a3b8a7 100644 --- a/services/director/tests/unit/test_cli.py +++ b/services/director/tests/unit/test_cli.py @@ -24,8 +24,9 @@ def test_settings(cli_runner: CliRunner, app_environment: EnvVarsDict): result = cli_runner.invoke(main, ["settings", "--show-secrets", "--as-json"]) assert result.exit_code == os.EX_OK - settings = ApplicationSettings.parse_raw(result.output) - assert settings.dict() == ApplicationSettings.create_from_envs().dict() + print(result.output) + settings = ApplicationSettings(result.output) + assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() def test_run(cli_runner: CliRunner): diff --git a/services/director/tests/unit/test_core_settings.py b/services/director/tests/unit/test_core_settings.py index 5ac622ba668..dfccd67ce2f 100644 --- a/services/director/tests/unit/test_core_settings.py +++ b/services/director/tests/unit/test_core_settings.py @@ -30,7 +30,7 @@ def test_valid_web_application_settings(app_environment: EnvVarsDict): str( app_environment.get( "DIRECTOR_DEFAULT_MAX_MEMORY", - ApplicationSettings.__fields__["DIRECTOR_DEFAULT_MAX_MEMORY"].default, + ApplicationSettings.model_fields["DIRECTOR_DEFAULT_MAX_MEMORY"].default, ) ) == f"{settings.DIRECTOR_DEFAULT_MAX_MEMORY}" @@ -45,7 +45,7 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch): """ DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS={} DIRECTOR_REGISTRY_CACHING=True - DIRECTOR_REGISTRY_CACHING_TTL=900 + DIRECTOR_REGISTRY_CACHING_TTL=00:15:00 DIRECTOR_SELF_SIGNED_SSL_FILENAME= DIRECTOR_SELF_SIGNED_SSL_SECRET_ID= DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME= @@ -119,7 +119,7 @@ def test_docker_compose_environment_sample( "DEFAULT_MAX_NANO_CPUS": "0", "DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS": '{"VRAM": "node.labels.gpu==true"}', "DIRECTOR_REGISTRY_CACHING": "True", - "DIRECTOR_REGISTRY_CACHING_TTL": "900", + "DIRECTOR_REGISTRY_CACHING_TTL": "00:15:00", "DIRECTOR_SELF_SIGNED_SSL_FILENAME": "", "DIRECTOR_SELF_SIGNED_SSL_SECRET_ID": "", "DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME": "", @@ -150,4 +150,4 @@ def test_docker_compose_environment_sample( }, ) - settings = ApplicationSettings.create_from_envs() + ApplicationSettings.create_from_envs() diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py index 50e14537710..73c75038d52 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py @@ -100,7 +100,7 @@ class ServicesResourceUsagesReportQueryParams( class ServicesResourceUsagesListQueryParams( PageQueryParameters, ServicesResourceUsagesReportQueryParams ): - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid") # type: ignore[misc] class ServicesAggregatedUsagesListQueryParams(PageQueryParameters): diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py index 59994498ec0..86f92d03730 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py @@ -26,7 +26,7 @@ _SERVICE_RUN_GET = ServiceRunPage( items=[ - ServiceRunGet.parse_obj( + ServiceRunGet.model_validate( { "service_run_id": "comp_1_5c2110be-441b-11ee-a0e8-02420a000040_1", "wallet_id": 1, diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py index 60efdef7308..ed2054595dd 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py @@ -26,13 +26,13 @@ def test_workspaces_order_query_model_post_validator(): # on default - query_params = WorkspacesListQueryParams.parse_obj({}) + query_params = WorkspacesListQueryParams.model_validate({}) assert query_params.order_by assert query_params.order_by.field == "modified" assert query_params.order_by.direction == OrderDirection.DESC # on partial default - query_params = WorkspacesListQueryParams.parse_obj( + query_params = WorkspacesListQueryParams.model_validate( {"order_by": {"field": "modified_at"}} ) assert query_params.order_by From c10d7d0eaa1029e2e5ce20510554bb726e7daeb3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:51:27 +0100 Subject: [PATCH 44/53] =?UTF-8?q?=E2=AC=86=EF=B8=8F=E2=99=BB=EF=B8=8F=20Fi?= =?UTF-8?q?x/test=20models=20(#6758)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/models-library/src/models_library/rest_base.py | 5 ++--- .../models-library/src/models_library/rest_ordering.py | 7 ++++--- packages/models-library/tests/test_rest_ordering.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/models-library/src/models_library/rest_base.py b/packages/models-library/src/models_library/rest_base.py index a6b24ef6382..797d8876680 100644 --- a/packages/models-library/src/models_library/rest_base.py +++ b/packages/models-library/src/models_library/rest_base.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict class RequestParameters(BaseModel): @@ -15,5 +15,4 @@ def as_params(self, **export_options) -> dict[str, str]: class StrictRequestParameters(RequestParameters): """Use a base class for context, path and query parameters""" - class Config: - extra = Extra.forbid # strict + model_config = ConfigDict(extra="forbid") diff --git a/packages/models-library/src/models_library/rest_ordering.py b/packages/models-library/src/models_library/rest_ordering.py index 3b355ae51ef..b042950c352 100644 --- a/packages/models-library/src/models_library/rest_ordering.py +++ b/packages/models-library/src/models_library/rest_ordering.py @@ -72,7 +72,8 @@ class _OrderBy(OrderBy): } ] }, - validate_assignment=True, # Necessary to run _check_ordering_field_and_map in defaults and assignments + # Necessary to run _check_ordering_field_and_map in defaults and assignments + validate_assignment=True, validate_default=True, ) @@ -91,9 +92,9 @@ def _check_ordering_field_and_map(cls, v): assert "json_schema_extra" in _OrderBy.model_config # nosec assert isinstance(_OrderBy.model_config["json_schema_extra"], dict) # nosec - assert isinstance( + assert isinstance( # nosec _OrderBy.model_config["json_schema_extra"]["examples"], list - ) # nosec + ) order_by_example = _OrderBy.model_config["json_schema_extra"]["examples"][0] order_by_example_json = json_dumps(order_by_example) assert _OrderBy.model_validate(order_by_example), "Example is invalid" # nosec diff --git a/packages/models-library/tests/test_rest_ordering.py b/packages/models-library/tests/test_rest_ordering.py index a495dc55721..a3f84bf0d2c 100644 --- a/packages/models-library/tests/test_rest_ordering.py +++ b/packages/models-library/tests/test_rest_ordering.py @@ -106,7 +106,7 @@ def test_ordering_query_model_class__fails_with_invalid_direction(): error = err_info.value.errors()[0] - assert error["type"] == "type_error.enum" + assert error["type"] == "enum" assert error["loc"] == ("order_by", "direction") @@ -137,7 +137,7 @@ def test_ordering_query_model_class__defaults(): error = err_info.value.errors()[0] assert error["loc"] == ("order_by", "field") - assert error["type"] == "value_error.missing" + assert error["type"] == "missing" def test_ordering_query_model_with_map(): From 3250e5de90b8c6168d4702edeffd8ca278432693 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 19 Nov 2024 11:40:05 +0100 Subject: [PATCH 45/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Greenify=20pylint=20?= =?UTF-8?q?(after=20Pydantic=20v2=20migration)=20(#6747)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/aws-library/src/aws_library/s3/_errors.py | 5 ----- packages/common-library/tests/test_errors_classes.py | 2 -- .../src/dask_task_models_library/container_tasks/errors.py | 4 +--- .../pytest_simcore/helpers/httpx_calls_capture_parameters.py | 4 ++-- .../src/simcore_sdk/node_ports_v2/port_validation.py | 2 -- .../src/simcore_service_director_v2/core/errors.py | 4 ---- .../modules/dynamic_sidecar/errors.py | 1 - .../src/simcore_service_dynamic_scheduler/core/errors.py | 2 +- .../src/simcore_service_dynamic_sidecar/core/errors.py | 2 +- .../modules/user_services_preferences/_errors.py | 2 +- .../simcore_service_webserver/studies_dispatcher/_errors.py | 4 ---- 11 files changed, 6 insertions(+), 26 deletions(-) diff --git a/packages/aws-library/src/aws_library/s3/_errors.py b/packages/aws-library/src/aws_library/s3/_errors.py index d14105dbd30..3bafa217257 100644 --- a/packages/aws-library/src/aws_library/s3/_errors.py +++ b/packages/aws-library/src/aws_library/s3/_errors.py @@ -10,25 +10,20 @@ class S3NotConnectedError(S3RuntimeError): class S3AccessError(S3RuntimeError): - code = "s3_access.error" # type: ignore[assignment] msg_template: str = "Unexpected error while accessing S3 backend" class S3BucketInvalidError(S3AccessError): - code = "s3_bucket.invalid_error" # type: ignore[assignment] msg_template: str = "The bucket '{bucket}' is invalid" class S3KeyNotFoundError(S3AccessError): - code = "s3_key.not_found_error" # type: ignore[assignment] msg_template: str = "The file {key} in {bucket} was not found" class S3UploadNotFoundError(S3AccessError): - code = "s3_upload.not_found_error" # type: ignore[assignment] msg_template: str = "The upload for {key} in {bucket} was not found" class S3DestinationNotEmptyError(S3AccessError): - code = "s3_destination.not_empty_error" # type: ignore[assignment] msg_template: str = "The destination {dst_prefix} is not empty" diff --git a/packages/common-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py index 3be2532f1ab..efe4c44b86e 100644 --- a/packages/common-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -49,12 +49,10 @@ class MyValueError(MyBaseError, ValueError): assert f"{error}" == "Wrong value 42" class MyTypeError(MyBaseError, TypeError): - code = "i_want_this" msg_template = "Wrong type {type}" error = MyTypeError(type="int") - assert error.code == "i_want_this" assert f"{error}" == "Wrong type int" diff --git a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py index f0a6813ba15..1597ddfb6f4 100644 --- a/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py +++ b/packages/dask-task-models-library/src/dask_task_models_library/container_tasks/errors.py @@ -5,16 +5,14 @@ class TaskValueError(OsparcErrorMixin, ValueError): - code = "task.value_error" # type: ignore[assignment] + ... class TaskCancelledError(OsparcErrorMixin, RuntimeError): - code = "task.cancelled_error" # type: ignore[assignment] msg_template = "The task was cancelled" class ServiceRuntimeError(OsparcErrorMixin, RuntimeError): - code = "service.runtime_error" # type: ignore[assignment] msg_template = ( "The service {service_key}:{service_version}" " running in container {container_id} failed with code" diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py index 80170682430..c2e722b6697 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py @@ -66,14 +66,14 @@ def regex_pattern(self) -> str: if self.oneOf: msg = "Current version cannot compute regex patterns in case of oneOf. Please go ahead and implement it yourself." raise NotImplementedError(msg) - if self.anyOf: + if self.anyOf is not None: return "|".join( [ elm.regex_pattern for elm in self.anyOf # pylint:disable=not-an-iterable ] ) - if self.allOf: + if self.allOf is not None: return "&".join( [ elm.regex_pattern diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py index b33e677c0bf..c6596e669e9 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_validation.py @@ -23,7 +23,6 @@ class PortValueError(OsparcErrorMixin, ValueError): - code = "port_validation.schema_error" # type: ignore msg_template = "Invalid value in port {port_key!r}: {schema_error_message}" # pylint: disable=useless-super-delegation @@ -38,7 +37,6 @@ def __init__(self, *, port_key: str, schema_error: JsonSchemaValidationError): class PortUnitError(OsparcErrorMixin, ValueError): - code = "port_validation.unit_error" # type: ignore msg_template = "Invalid unit in port {port_key!r}: {pint_error_msg}" # pylint: disable=useless-super-delegation diff --git a/services/director-v2/src/simcore_service_director_v2/core/errors.py b/services/director-v2/src/simcore_service_director_v2/core/errors.py index 213cd4744bf..65af83fa28f 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/core/errors.py @@ -167,8 +167,6 @@ def get_errors(self) -> list[ErrorDict]: class MissingComputationalResourcesError(TaskSchedulingError): """A task cannot be scheduled because the cluster does not have the required resources""" - code = "scheduler_error.missing_resources" - def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = None): super().__init__(project_id, node_id, msg=msg) @@ -176,8 +174,6 @@ def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = Non class InsuficientComputationalResourcesError(TaskSchedulingError): """A task cannot be scheduled because the cluster does not have *enough* of the required resources""" - code = "scheduler_error.insuficient_resources" - def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = None): super().__init__(project_id, node_id, msg=msg) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py index 8b40a4e0f35..e3d67da68aa 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py @@ -40,5 +40,4 @@ class LegacyServiceIsNotSupportedError(DirectorError): class UnexpectedContainerStatusError(OsparcErrorMixin, DynamicSidecarError): - code = "dynamic_sidecar.container_status" # type: ignore msg_template = "Unexpected status from containers: {containers_with_error}" diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py index 2677b7bc370..0dd4b43e4bd 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/errors.py @@ -2,4 +2,4 @@ class BaseDynamicSchedulerError(OsparcErrorMixin, ValueError): - code = "simcore.service.dynamic.scheduler" # type:ignore[assignment] + ... diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py index 636260814a1..bd74b68dfb7 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py @@ -36,7 +36,7 @@ def __init__(self, message: str, status_code: int) -> None: class BaseError(OsparcErrorMixin, BaseDynamicSidecarError): - code = "dy_sidecar.error" # type: ignore[assignment] + ... class ContainerExecContainerNotFoundError(BaseError): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py index b2c4a327c83..278ad52b04d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_errors.py @@ -2,7 +2,7 @@ class BaseServicesPreferencesError(OsparcErrorMixin, Exception): - code = "dynamic_sidecar.user_service_preferences" # type: ignore[assignment] + ... class DestinationIsNotADirectoryError(BaseServicesPreferencesError): diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py index 5cff412f415..4c7c0bbce73 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py @@ -6,22 +6,18 @@ class StudyDispatcherError(WebServerBaseError, ValueError): class IncompatibleService(StudyDispatcherError): - code = "studies_dispatcher.incompatible_service" # type: ignore msg_template = "None of the registered services can handle '{file_type}'" class FileToLarge(StudyDispatcherError): - code = "studies_dispatcher.file_to_large" # type: ignore msg_template = "File size {file_size_in_mb} MB is over allowed limit" class ServiceNotFound(StudyDispatcherError): - code = "studies_dispatcher.service_not_found" # type: ignore msg_template = "Service {service_key}:{service_version} not found" class InvalidRedirectionParams(StudyDispatcherError): - code = "studies_dispatcher.invalid_redirection_params" # type: ignore msg_template = ( "The link you provided is invalid because it doesn't contain any information related to data or a service." " Please check the link and make sure it is correct." From dcdc863ec090ca20577259095495bf9105d0294d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:05:07 +0100 Subject: [PATCH 46/53] =?UTF-8?q?=F0=9F=8E=A8=20Ensure=20profiling=20middl?= =?UTF-8?q?eware=20is=20compatible=20with=20fastapi=20(#6760)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/servicelib/fastapi/profiler_middleware.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/profiler_middleware.py b/packages/service-library/src/servicelib/fastapi/profiler_middleware.py index cdf8c77206b..43c46c7ba9d 100644 --- a/packages/service-library/src/servicelib/fastapi/profiler_middleware.py +++ b/packages/service-library/src/servicelib/fastapi/profiler_middleware.py @@ -1,9 +1,9 @@ from typing import Any, Final -from fastapi import FastAPI from servicelib.aiohttp import status from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from starlette.requests import Request +from starlette.types import ASGIApp, Receive, Scope, Send from ..utils_profiling_middleware import ( _is_profiling, @@ -31,11 +31,11 @@ class ProfilerMiddleware: https://fastapi.tiangolo.com/advanced/middleware/#advanced-middleware """ - def __init__(self, app: FastAPI): - self._app: FastAPI = app + def __init__(self, app: ASGIApp): + self._app = app self._profile_header_trigger: Final[str] = "x-profile" - async def __call__(self, scope, receive, send): + async def __call__(self, scope: Scope, receive: Receive, send: Send): if scope["type"] != "http": await self._app(scope, receive, send) return From c50d79cc9c3d3b09dd2898a6590ba64c9db3a64c Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:46:30 +0100 Subject: [PATCH 47/53] =?UTF-8?q?=E2=99=BB=EF=B8=8FPydantic=20V2:=20=20Uni?= =?UTF-8?q?fy=20errors=20dv=202=20(#6763)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/common_library/errors_classes.py | 2 +- .../tests/test_rest_ordering.py | 9 +- .../api/routes/computations.py | 14 +- .../core/application.py | 1 + .../core/errors.py | 206 +++++------------- .../modules/comp_scheduler/_base_scheduler.py | 10 +- .../comp_scheduler/_scheduler_factory.py | 2 +- .../modules/dask_clients_pool.py | 2 +- .../modules/db/repositories/comp_pipelines.py | 6 +- .../db/repositories/comp_tasks/_utils.py | 10 +- .../modules/db/repositories/projects.py | 2 +- .../db/repositories/projects_networks.py | 2 +- .../dynamic_sidecar/docker_api/_core.py | 13 +- .../dynamic_sidecar/docker_api/_utils.py | 3 +- .../docker_service_specs/settings.py | 2 +- .../modules/dynamic_sidecar/errors.py | 30 +-- .../scheduler/_core/_scheduler.py | 16 +- .../modules/dynamic_sidecar/volumes.py | 2 +- .../modules/rabbitmq.py | 4 +- .../modules/resource_usage_tracker_client.py | 4 +- .../simcore_service_director_v2/utils/dask.py | 33 ++- .../utils/dask_client_utils.py | 16 +- .../test_modules_dynamic_sidecar_scheduler.py | 5 +- .../test_api_route_dynamic_services.py | 4 +- ...t_modules_comp_scheduler_dask_scheduler.py | 4 +- ...test_modules_dynamic_sidecar_docker_api.py | 21 +- 26 files changed, 172 insertions(+), 251 deletions(-) diff --git a/packages/common-library/src/common_library/errors_classes.py b/packages/common-library/src/common_library/errors_classes.py index 83e40b2a2b0..dfee557d38c 100644 --- a/packages/common-library/src/common_library/errors_classes.py +++ b/packages/common-library/src/common_library/errors_classes.py @@ -45,7 +45,7 @@ def _get_full_class_name(cls) -> str: ] return ".".join(reversed(relevant_classes)) - def error_context(self): + def error_context(self) -> dict[str, Any]: """Returns context in which error occurred and stored within the exception""" return dict(**self.__dict__) diff --git a/packages/models-library/tests/test_rest_ordering.py b/packages/models-library/tests/test_rest_ordering.py index a3f84bf0d2c..4ceed67dea5 100644 --- a/packages/models-library/tests/test_rest_ordering.py +++ b/packages/models-library/tests/test_rest_ordering.py @@ -120,10 +120,11 @@ def test_ordering_query_model_class__defaults(): # checks all defaults model = OrderQueryParamsModel() - assert model.order_by - assert isinstance(model.order_by, OrderBy) # nosec - assert model.order_by.field == "modified_at" # NOTE that this was mapped! - assert model.order_by.direction == OrderDirection.DESC + assert model.order_by is not None + assert ( + model.order_by.field == "modified_at" # pylint: disable=no-member + ) # NOTE that this was mapped! + assert model.order_by.direction is OrderDirection.DESC # pylint: disable=no-member # partial defaults model = OrderQueryParamsModel.model_validate({"order_by": {"field": "name"}}) diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index e624ed0785c..251e35fa638 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -53,10 +53,10 @@ ClusterNotFoundError, ClustersKeeperNotAvailableError, ComputationalRunNotFoundError, + ComputationalSchedulerError, ConfigurationError, PricingPlanUnitNotFoundError, ProjectNotFoundError, - SchedulerError, WalletNotEnoughCreditsError, ) from ...models.comp_pipelines import CompPipelineAtDB @@ -204,7 +204,9 @@ async def _get_project_node_names( except DBProjectNotFoundError: _logger.exception("Could not find project: %s", f"{project_id=}") except ProjectNotFoundError as exc: - _logger.exception("Could not find parent project: %s", f"{exc.project_id=}") + _logger.exception( + "Could not find parent project: %s", exc.error_context().get("project_id") + ) return {} @@ -510,7 +512,9 @@ async def get_computation( pipeline_details=pipeline_details, url=TypeAdapter(AnyHttpUrl).validate_python(f"{request.url}"), stop_url=( - TypeAdapter(AnyHttpUrl).validate_python(f"{self_url}:stop?user_id={user_id}") + TypeAdapter(AnyHttpUrl).validate_python( + f"{self_url}:stop?user_id={user_id}" + ) if pipeline_state.is_running() else None ), @@ -598,7 +602,7 @@ async def stop_computation( except ProjectNotFoundError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"{e}") from e - except SchedulerError as e: + except ComputationalSchedulerError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"{e}") from e @@ -639,7 +643,7 @@ async def delete_computation( # abort the pipeline first try: await scheduler.stop_pipeline(computation_stop.user_id, project_id) - except SchedulerError as e: + except ComputationalSchedulerError as e: _logger.warning( "Project %s could not be stopped properly.\n reason: %s", project_id, diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index fb9d6094dff..43a9dcc4e03 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -126,6 +126,7 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: for name in _NOISY_LOGGERS: logging.getLogger(name).setLevel(quiet_level) + assert settings.SC_BOOT_MODE # nosec app = FastAPI( debug=settings.SC_BOOT_MODE.is_devel_mode(), title=PROJECT_NAME, diff --git a/services/director-v2/src/simcore_service_director_v2/core/errors.py b/services/director-v2/src/simcore_service_director_v2/core/errors.py index 65af83fa28f..18a5b674ed2 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/core/errors.py @@ -19,134 +19,66 @@ } """ +from typing import Any + from common_library.errors_classes import OsparcErrorMixin from models_library.errors import ErrorDict from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -class DirectorError(Exception): - """Basic exception""" +class DirectorError(OsparcErrorMixin, RuntimeError): + msg_template: str = "Director-v2 unexpected error" class ConfigurationError(DirectorError): - """An error in the director-v2 configuration""" - - def __init__(self, msg: str | None = None): - super().__init__( - msg or "Invalid configuration of the director-v2 application. Please check." - ) - - -class GenericDockerError(DirectorError): - """Generic docker library error""" - - def __init__(self, msg: str, original_exception: Exception): - super().__init__(msg + f": {original_exception}") - self.original_exception = original_exception - - -class ServiceNotAvailableError(DirectorError): - """Service not found""" - - def __init__(self, service_name: str, service_tag: str | None = None): - service_tag = service_tag or "UNDEFINED" - super().__init__(f"The service {service_name}:{service_tag} does not exist") - self.service_name = service_name - self.service_tag = service_tag - - -class ServiceUUIDNotFoundError(DirectorError): - """Service not found""" - - def __init__(self, service_uuid: str): - super().__init__(f"The service with uuid {service_uuid} was not found") - self.service_uuid = service_uuid - - -class ServiceUUIDInUseError(DirectorError): - """Service UUID is already in use""" - - def __init__(self, service_uuid: str): - super().__init__(f"The service uuid {service_uuid} is already in use") - self.service_uuid = service_uuid - - -class ServiceStartTimeoutError(DirectorError): - """The service was created but never run (time-out)""" - - def __init__(self, service_name: str, service_uuid: str): - super().__init__(f"Service {service_name}:{service_uuid} failed to start ") - self.service_name = service_name - self.service_uuid = service_uuid + msg_template: str = "Application misconfiguration: {msg}" class ProjectNotFoundError(DirectorError): - """Project not found error""" - - def __init__(self, project_id: ProjectID): - super().__init__(f"project {project_id} not found") - self.project_id = project_id + msg_template: str = "project {project_id} not found" class ProjectNetworkNotFoundError(DirectorError): - """Project not found error""" - - def __init__(self, project_id: ProjectID): - super().__init__(f"no networks forund for project {project_id}") - self.project_id = project_id + msg_template: str = "no networks found for project {project_id}" class PricingPlanUnitNotFoundError(DirectorError): - """Pricing plan unit not found error""" - - def __init__(self, msg: str): - super().__init__(msg) + msg_template: str = "pricing plan not found {msg}" class PipelineNotFoundError(DirectorError): - """Pipeline not found error""" + msg_template: str = "pipeline {pipeline_id} not found" - def __init__(self, pipeline_id: str): - super().__init__(f"pipeline {pipeline_id} not found") - -class ComputationalRunNotFoundError(OsparcErrorMixin, DirectorError): +class ComputationalRunNotFoundError(DirectorError): msg_template = "Computational run not found" -class ComputationalTaskNotFoundError(OsparcErrorMixin, DirectorError): +class ComputationalTaskNotFoundError(DirectorError): msg_template = "Computational task {node_id} not found" -class WalletNotEnoughCreditsError(OsparcErrorMixin, DirectorError): +class WalletNotEnoughCreditsError(DirectorError): msg_template = "Wallet '{wallet_name}' has {wallet_credit_amount} credits." # # SCHEDULER ERRORS # +class ComputationalSchedulerError(DirectorError): + msg_template = "Computational scheduler unexpected error {msg}" -class SchedulerError(DirectorError): - def __init__(self, msg: str | None = None): - super().__init__(msg or "Unexpected error in the scheduler") - - -class InvalidPipelineError(SchedulerError): - """A pipeline is misconfigured""" - - def __init__(self, pipeline_id: str, msg: str | None = None): - super().__init__(msg or f"Invalid configuration of pipeline {pipeline_id}") +class InvalidPipelineError(ComputationalSchedulerError): + msg_template = "Computational scheduler: Invalid configuration of pipeline {pipeline_id}: {msg}" -class TaskSchedulingError(SchedulerError): - """A task cannot be scheduled""" +class TaskSchedulingError(ComputationalSchedulerError): + msg_template = "Computational scheduler: Task {node_id} in project {project_id} could not be scheduled {msg}" - code: str = "task scheduler error" - - def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = None): - super().__init__(msg=msg) + def __init__(self, project_id: ProjectID, node_id: NodeID, **ctx: Any) -> None: + super().__init__(project_id=project_id, node_id=node_id, **ctx) self.project_id = project_id self.node_id = node_id @@ -164,89 +96,59 @@ def get_errors(self) -> list[ErrorDict]: ] -class MissingComputationalResourcesError(TaskSchedulingError): - """A task cannot be scheduled because the cluster does not have the required resources""" - - def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = None): - super().__init__(project_id, node_id, msg=msg) - +class MissingComputationalResourcesError( + TaskSchedulingError +): # pylint: disable=too-many-ancestors + msg_template = ( + "Service {service_name}:{service_version} cannot be scheduled " + "on cluster {cluster_id}: task needs '{task_resources}', " + "cluster has {cluster_resources}" + ) -class InsuficientComputationalResourcesError(TaskSchedulingError): - """A task cannot be scheduled because the cluster does not have *enough* of the required resources""" - def __init__(self, project_id: ProjectID, node_id: NodeID, msg: str | None = None): - super().__init__(project_id, node_id, msg=msg) +class InsuficientComputationalResourcesError( + TaskSchedulingError +): # pylint: disable=too-many-ancestors + msg_template: str = ( + "Insufficient computational resources to run {service_name}:{service_version} with {service_requested_resources} on cluster {cluster_id}." + "Cluster available workers: {cluster_available_resources}" + "TIP: Reduce service required resources or contact oSparc support" + ) -class PortsValidationError(TaskSchedulingError): - """ - Gathers all validation errors raised while checking input/output - ports in a project's node. - """ +class PortsValidationError(TaskSchedulingError): # pylint: disable=too-many-ancestors + msg_template: str = ( + "Node {node_id} in {project_id} with ports having invalid values {errors_list}" + ) - def __init__(self, project_id: ProjectID, node_id: NodeID, errors: list[ErrorDict]): - super().__init__( - project_id, - node_id, - msg=f"Node with {len(errors)} ports having invalid values", - ) - self.errors = errors - def get_errors(self) -> list[ErrorDict]: - """Returns 'public errors': filters only value_error.port_validation errors for the client. - The rest only shown as number - """ - value_errors: list[ErrorDict] = [] - for error in self.errors: - # NOTE: should I filter? if error["type"].startswith("value_error."): - - loc_tail: list[str] = [] - if port_key := error.get("ctx", {}).get("port_key"): - loc_tail.append(f"{port_key}") - - if schema_error_path := error.get("ctx", {}).get("schema_error_path"): - loc_tail += list(schema_error_path) - - # WARNING: error in a node, might come from the previous node's port - # DO NOT remove project/node/port hiearchy - value_errors.append( - { - "loc": (f"{self.project_id}", f"{self.node_id}", *tuple(loc_tail)), - "msg": error["msg"], - # NOTE: here we list the codes of the PydanticValueErrors collected in ValidationError - "type": error["type"], - } - ) - return value_errors - - -class ComputationalSchedulerChangedError(OsparcErrorMixin, SchedulerError): +class ComputationalSchedulerChangedError(ComputationalSchedulerError): msg_template = "The dask scheduler ID changed from '{original_scheduler_id}' to '{current_scheduler_id}'" -class ComputationalBackendNotConnectedError(OsparcErrorMixin, SchedulerError): +class ComputationalBackendNotConnectedError(ComputationalSchedulerError): msg_template = "The dask computational backend is not connected" -class ComputationalBackendNoS3AccessError(OsparcErrorMixin, SchedulerError): +class ComputationalBackendNoS3AccessError(ComputationalSchedulerError): msg_template = "The S3 backend is not ready, please try again later" -class ComputationalBackendTaskNotFoundError(OsparcErrorMixin, SchedulerError): +class ComputationalBackendTaskNotFoundError(ComputationalSchedulerError): msg_template = ( "The dask computational backend does not know about the task '{job_id}'" ) -class ComputationalBackendTaskResultsNotReadyError(OsparcErrorMixin, SchedulerError): +class ComputationalBackendTaskResultsNotReadyError(ComputationalSchedulerError): msg_template = "The task result is not ready yet for job '{job_id}'" -class ClustersKeeperNotAvailableError(OsparcErrorMixin, SchedulerError): +class ClustersKeeperNotAvailableError(ComputationalSchedulerError): msg_template = "clusters-keeper service is not available!" -class ComputationalBackendOnDemandNotReadyError(OsparcErrorMixin, SchedulerError): +class ComputationalBackendOnDemandNotReadyError(ComputationalSchedulerError): msg_template = ( "The on demand computational cluster is not ready 'est. remaining time: {eta}'" ) @@ -255,15 +157,15 @@ class ComputationalBackendOnDemandNotReadyError(OsparcErrorMixin, SchedulerError # # SCHEDULER/CLUSTER ERRORS # -class ClusterNotFoundError(OsparcErrorMixin, SchedulerError): +class ClusterNotFoundError(ComputationalSchedulerError): msg_template = "The cluster '{cluster_id}' not found" -class ClusterAccessForbiddenError(OsparcErrorMixin, SchedulerError): +class ClusterAccessForbiddenError(ComputationalSchedulerError): msg_template = "Insufficient rights to access cluster '{cluster_id}'" -class ClusterInvalidOperationError(OsparcErrorMixin, SchedulerError): +class ClusterInvalidOperationError(ComputationalSchedulerError): msg_template = "Invalid operation on cluster '{cluster_id}'" @@ -272,21 +174,21 @@ class ClusterInvalidOperationError(OsparcErrorMixin, SchedulerError): # -class DaskClientRequestError(OsparcErrorMixin, SchedulerError): +class DaskClientRequestError(ComputationalSchedulerError): msg_template = ( "The dask client to cluster on '{endpoint}' did an invalid request '{error}'" ) -class DaskClusterError(OsparcErrorMixin, SchedulerError): +class DaskClusterError(ComputationalSchedulerError): msg_template = "The dask cluster on '{endpoint}' encountered an error: '{error}'" -class DaskGatewayServerError(OsparcErrorMixin, SchedulerError): +class DaskGatewayServerError(ComputationalSchedulerError): msg_template = "The dask gateway on '{endpoint}' encountered an error: '{error}'" -class DaskClientAcquisisitonError(OsparcErrorMixin, SchedulerError): +class DaskClientAcquisisitonError(ComputationalSchedulerError): msg_template = ( "The dask client to cluster '{cluster}' encountered an error '{error}'" ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py index 5fd39dda4bc..2d663aec9a1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_base_scheduler.py @@ -45,10 +45,10 @@ ComputationalBackendNotConnectedError, ComputationalBackendOnDemandNotReadyError, ComputationalSchedulerChangedError, + ComputationalSchedulerError, DaskClientAcquisisitonError, InvalidPipelineError, PipelineNotFoundError, - SchedulerError, TaskSchedulingError, ) from ...core.settings import ComputationalBackendSettings @@ -242,7 +242,7 @@ async def stop_pipeline( } if not possible_iterations: msg = f"There are no pipeline scheduled for {user_id}:{project_id}" - raise SchedulerError(msg) + raise ComputationalSchedulerError(msg=msg) current_max_iteration = max(possible_iterations) selected_iteration = current_max_iteration else: @@ -281,7 +281,7 @@ def _get_last_iteration(self, user_id: UserID, project_id: ProjectID) -> Iterati } if not possible_iterations: msg = f"There are no pipeline scheduled for {user_id}:{project_id}" - raise SchedulerError(msg) + raise ComputationalSchedulerError(msg=msg) return max(possible_iterations) def _start_scheduling( @@ -342,10 +342,10 @@ async def _get_pipeline_tasks( } if len(pipeline_comp_tasks) != len(pipeline_dag.nodes()): # type: ignore[arg-type] msg = ( - f"{project_id}The tasks defined for {project_id} do not contain all" + f"The tasks defined for {project_id} do not contain all" f" the tasks defined in the pipeline [{list(pipeline_dag.nodes)}]! Please check." ) - raise InvalidPipelineError(msg) + raise InvalidPipelineError(pipeline_id=project_id, msg=msg) return pipeline_comp_tasks async def _update_run_result_from_tasks( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py index 39b432b9492..524dfc7e8ad 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_factory.py @@ -18,7 +18,7 @@ async def create_from_db(app: FastAPI) -> BaseCompScheduler: if not hasattr(app.state, "engine"): msg = "Database connection is missing. Please check application configuration." - raise ConfigurationError(msg) + raise ConfigurationError(msg=msg) db_engine = app.state.engine with log_context( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_clients_pool.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_clients_pool.py index d246bb35f42..31177b5a616 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_clients_pool.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_clients_pool.py @@ -50,7 +50,7 @@ async def create( def instance(app: FastAPI) -> "DaskClientsPool": if not hasattr(app.state, "dask_clients_pool"): msg = "Dask clients pool is not available. Please check the configuration." - raise ConfigurationError(msg) + raise ConfigurationError(msg=msg) dask_clients_pool: DaskClientsPool = app.state.dask_clients_pool return dask_clients_pool diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py index 37129141f6d..38981b5fa7d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_pipelines.py @@ -25,7 +25,7 @@ async def get_pipeline(self, project_id: ProjectID) -> CompPipelineAtDB: ) row: RowProxy | None = await result.fetchone() if not row: - raise PipelineNotFoundError(str(project_id)) + raise PipelineNotFoundError(pipeline_id=project_id) return CompPipelineAtDB.model_validate(row) async def upsert_pipeline( @@ -39,7 +39,9 @@ async def upsert_pipeline( dag_adjacency_list=nx.to_dict_of_lists(dag_graph), state=RunningState.PUBLISHED if publish else RunningState.NOT_STARTED, ) - insert_stmt = insert(comp_pipeline).values(**pipeline_at_db.model_dump(by_alias=True)) + insert_stmt = insert(comp_pipeline).values( + **pipeline_at_db.model_dump(by_alias=True) + ) # FIXME: This is not a nice thing. this part of the information should be kept in comp_runs. update_exclusion_policy = set() if not dag_graph.nodes(): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index bdb64cbbf99..637e0c7faf6 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -322,7 +322,7 @@ def _by_type_name(ec2: EC2InstanceTypeGet) -> bool: f"invalid EC2 type name selected {set(hardware_info.aws_ec2_instances)}." " TIP: adjust product configuration" ) - raise ConfigurationError(msg) from exc + raise ConfigurationError(msg=msg) from exc except ( RemoteMethodNotRegisteredError, RPCServerError, @@ -450,9 +450,11 @@ async def generate_tasks_list_from_project( last_heartbeat=None, created=arrow.utcnow().datetime, modified=arrow.utcnow().datetime, - pricing_info=pricing_info.model_dump(exclude={"pricing_unit_cost"}) - if pricing_info - else None, + pricing_info=( + pricing_info.model_dump(exclude={"pricing_unit_cost"}) + if pricing_info + else None + ), hardware_info=hardware_info, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 5f5fe5263ff..902f9977574 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -22,7 +22,7 @@ async def get_project(self, project_id: ProjectID) -> ProjectAtDB: ) ).first() if not row: - raise ProjectNotFoundError(project_id) + raise ProjectNotFoundError(project_id=project_id) return ProjectAtDB.model_validate(row) async def is_node_present_in_workbench( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py index 59334aa0a06..12fc7fe2932 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_networks.py @@ -22,7 +22,7 @@ async def get_projects_networks(self, project_id: ProjectID) -> ProjectsNetworks ) ).first() if not row: - raise ProjectNetworkNotFoundError(project_id) + raise ProjectNetworkNotFoundError(project_id=project_id) return ProjectsNetworks.model_validate(row) async def upsert_projects_networks( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index ed834581b03..1e05524b48d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -59,7 +59,7 @@ async def get_swarm_network(simcore_services_network_name: DockerNetworkName) -> f"Swarm network name (searching for '*{simcore_services_network_name}*') " f"is not configured.Found following networks: {networks}" ) - raise DynamicSidecarError(msg) + raise DynamicSidecarError(msg=msg) return networks[0] @@ -89,7 +89,7 @@ async def create_network(network_config: dict[str, Any]) -> NetworkId: # finally raise an error if a network cannot be spawned # pylint: disable=raise-missing-from msg = f"Could not create or recover a network ID for {network_config}" - raise DynamicSidecarError(msg) from e + raise DynamicSidecarError(msg=msg) from e def _to_snake_case(string: str) -> str: @@ -119,7 +119,7 @@ async def create_service_and_get_id( if "ID" not in service_start_result: msg = f"Error while starting service: {service_start_result!s}" - raise DynamicSidecarError(msg) + raise DynamicSidecarError(msg=msg) service_id: ServiceId = service_start_result["ID"] return service_id @@ -159,7 +159,10 @@ async def _get_service_latest_task(service_id: str) -> Mapping[str, Any]: last_task: Mapping[str, Any] = sorted_tasks[-1] return last_task except GenericDockerError as err: - if err.original_exception.status == status.HTTP_404_NOT_FOUND: + if ( + err.error_context()["original_exception"].status + == status.HTTP_404_NOT_FOUND + ): raise DockerServiceNotFoundError(service_id=service_id) from err raise @@ -205,7 +208,7 @@ async def _get_task_data_when_service_running(service_id: str) -> Mapping[str, A docker_node_id: None | str = task.get("NodeID", None) if not docker_node_id: msg = f"Could not find an assigned NodeID for service_id={service_id}. Last task inspect result: {task}" - raise DynamicSidecarError(msg) + raise DynamicSidecarError(msg=msg) return docker_node_id diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py index ceb9d276c13..f625c2ea625 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py @@ -13,8 +13,7 @@ async def docker_client() -> AsyncIterator[aiodocker.docker.Docker]: client = aiodocker.Docker() yield client except aiodocker.exceptions.DockerError as e: - message = "Unexpected error from docker client" - raise GenericDockerError(message, e) from e + raise GenericDockerError(msg=f"{e.message}", original_exception=e) from e finally: if client is not None: await client.close() diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py index 78a1201a714..8c1849064ee 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py @@ -267,7 +267,7 @@ def remap_to_compose_spec_key() -> dict[str, SimcoreServiceLabels]: f"docker_image_name_by_services={docker_image_name_by_services}" ) log.error(message) - raise DynamicSidecarError(message) + raise DynamicSidecarError(msg=message) return remap_to_compose_spec_key() diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py index e3d67da68aa..3b0a400223b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py @@ -1,34 +1,28 @@ -from aiodocker.exceptions import DockerError -from common_library.errors_classes import OsparcErrorMixin -from models_library.projects_nodes_io import NodeID +from typing import Any + +from aiodocker import DockerError from ...core.errors import DirectorError class DynamicSidecarError(DirectorError): - pass + msg_template: str = "Unexpected dynamic sidecar error: {msg}" class GenericDockerError(DynamicSidecarError): - """Generic docker library error""" - - def __init__(self, msg: str, original_exception: DockerError): - super().__init__(msg + f": {original_exception.message}") + def __init__(self, original_exception: DockerError, **ctx: Any) -> None: + super().__init__(original_exception=original_exception, **ctx) self.original_exception = original_exception + msg_template: str = "Unexpected error using docker client: {msg}" -class DynamicSidecarNotFoundError(DirectorError): - """Dynamic sidecar was not found""" - def __init__(self, node_uuid: NodeID): - super().__init__(f"node {node_uuid} not found") +class DynamicSidecarNotFoundError(DirectorError): + msg_template: str = "node {node_uuid} not found" class DockerServiceNotFoundError(DirectorError): - """Raised when an expected docker service is not found""" - - def __init__(self, service_id: str): - super().__init__(f"docker service with {service_id=} not found") + msg_template: str = "docker service with {service_id} not found" class EntrypointContainerNotFoundError(DynamicSidecarError): @@ -39,5 +33,5 @@ class LegacyServiceIsNotSupportedError(DirectorError): """This API is not implemented by the director-v0""" -class UnexpectedContainerStatusError(OsparcErrorMixin, DynamicSidecarError): - msg_template = "Unexpected status from containers: {containers_with_error}" +class UnexpectedContainerStatusError(DynamicSidecarError): + msg_template: str = "Unexpected status from containers: {containers_with_error}" diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 04853661c47..99fa3517130 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -171,7 +171,7 @@ def toggle_observation(self, node_uuid: NodeID, *, disable: bool) -> bool: raises DynamicSidecarNotFoundError """ if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name = self._inverse_search_mapping[node_uuid] service_task = self._service_observation_task.get(service_name) @@ -274,7 +274,7 @@ async def add_service_from_scheduler_data( f"node_uuids at a global level collided. A running service for node {scheduler_data.node_uuid} already exists." " Please checkout other projects which may have this issue." ) - raise DynamicSidecarError(msg) + raise DynamicSidecarError(msg=msg) self._inverse_search_mapping[ scheduler_data.node_uuid @@ -288,7 +288,7 @@ def is_service_tracked(self, node_uuid: NodeID) -> bool: def get_scheduler_data(self, node_uuid: NodeID) -> SchedulerData: if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name = self._inverse_search_mapping[node_uuid] return self._to_observe[service_name] @@ -336,7 +336,7 @@ async def mark_service_for_removal( """Marks service for removal, causing RemoveMarkedService to trigger""" async with self._lock: if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name = self._inverse_search_mapping[node_uuid] if service_name not in self._to_observe: @@ -416,7 +416,7 @@ async def remove_service_from_observation(self, node_uuid: NodeID) -> None: """ async with self._lock: if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name = self._inverse_search_mapping[node_uuid] if service_name not in self._to_observe: @@ -438,7 +438,7 @@ async def get_stack_status(self, node_uuid: NodeID) -> RunningDynamicServiceDeta raises DynamicSidecarNotFoundError """ if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name = self._inverse_search_mapping[node_uuid] scheduler_data: SchedulerData = self._to_observe[service_name] @@ -451,7 +451,7 @@ async def retrieve_service_inputs( ) -> RetrieveDataOutEnveloped: """Pulls data from input ports for the service""" if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name = self._inverse_search_mapping[node_uuid] scheduler_data: SchedulerData = self._to_observe[service_name] @@ -518,7 +518,7 @@ async def detach_project_network( async def restart_containers(self, node_uuid: NodeID) -> None: """Restarts containers without saving or restoring the state or I/O ports""" if node_uuid not in self._inverse_search_mapping: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) service_name: ServiceName = self._inverse_search_mapping[node_uuid] scheduler_data: SchedulerData = self._to_observe[service_name] diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py index d003eec60e6..7f55dc68498 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py @@ -79,7 +79,7 @@ def _get_s3_volume_driver_config( } else: msg = f"Unexpected, all {S3Provider.__name__} should be covered" - raise DynamicSidecarError(msg) + raise DynamicSidecarError(msg=msg) assert extra_options is not None # nosec options: dict[str, Any] = driver_config["Options"] diff --git a/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py b/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py index dcda51ad0e5..a7cb4e1ba27 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/rabbitmq.py @@ -81,7 +81,7 @@ async def on_shutdown() -> None: def get_rabbitmq_client(app: FastAPI) -> RabbitMQClient: if not hasattr(app.state, "rabbitmq_client"): msg = "RabbitMQ client is not available. Please check the configuration." - raise ConfigurationError(msg) + raise ConfigurationError(msg=msg) return cast(RabbitMQClient, app.state.rabbitmq_client) @@ -90,5 +90,5 @@ def get_rabbitmq_rpc_client(app: FastAPI) -> RabbitMQRPCClient: msg = ( "RabbitMQ client for RPC is not available. Please check the configuration." ) - raise ConfigurationError(msg) + raise ConfigurationError(msg=msg) return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_client) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py b/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py index 58d02975fd7..3b75607989d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py @@ -91,7 +91,7 @@ async def get_default_service_pricing_plan( ) if response.status_code == status.HTTP_404_NOT_FOUND: msg = "No pricing plan defined" - raise PricingPlanUnitNotFoundError(msg) + raise PricingPlanUnitNotFoundError(msg=msg) response.raise_for_status() return PricingPlanGet.model_validate(response.json()) @@ -117,7 +117,7 @@ async def get_default_pricing_and_hardware_info( unit.specific_info.aws_ec2_instances, ) msg = "Default pricing plan and unit does not exist" - raise PricingPlanUnitNotFoundError(msg) + raise PricingPlanUnitNotFoundError(msg=msg) async def get_pricing_unit( self, diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index d76596b5bf1..afb1e0b3770 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -129,7 +129,9 @@ async def create_node_ports( db_manager=db_manager, ) except ValidationError as err: - raise PortsValidationError(project_id, node_id, list(err.errors())) from err + raise PortsValidationError( + project_id=project_id, node_id=node_id, errors_list=list(err.errors()) + ) from err async def parse_output_data( @@ -181,7 +183,9 @@ async def parse_output_data( ports_errors.extend(_get_port_validation_errors(port_key, err)) if ports_errors: - raise PortsValidationError(project_id, node_id, ports_errors) + raise PortsValidationError( + project_id=project_id, node_id=node_id, errors_list=ports_errors + ) async def compute_input_data( @@ -218,11 +222,13 @@ async def compute_input_data( else: input_data[port.key] = value - except ValidationError as err: # noqa: PERF203 + except ValidationError as err: ports_errors.extend(_get_port_validation_errors(port.key, err)) if ports_errors: - raise PortsValidationError(project_id, node_id, ports_errors) + raise PortsValidationError( + project_id=project_id, node_id=node_id, errors_list=ports_errors + ) return TaskInputData.model_validate(input_data) @@ -608,18 +614,25 @@ def check_if_cluster_is_able_to_run_pipeline( raise MissingComputationalResourcesError( project_id=project_id, node_id=node_id, - msg=f"Service {node_image.name}:{node_image.tag} cannot be scheduled " - f"on cluster {cluster_id}: task needs '{task_resources}', " - f"cluster has {cluster_resources}", + service_name=node_image.name, + service_version=node_image.tag, + cluster_id=cluster_id, + task_resources=task_resources, + cluster_resources=cluster_resources, ) # well then our workers are not powerful enough raise InsuficientComputationalResourcesError( project_id=project_id, node_id=node_id, - msg=f"Insufficient computational resources to run {node_image.name}:{node_image.tag} with {_to_human_readable_resource_values( task_resources)} on cluster {cluster_id}." - f"Cluster available workers: {[_to_human_readable_resource_values( worker.get('resources', None)) for worker in workers.values()]}" - "TIP: Reduce service required resources or contact oSparc support", + service_name=node_image.name, + service_version=node_image.tag, + service_requested_resources=_to_human_readable_resource_values(task_resources), + cluster_id=cluster_id, + cluster_available_resources=[ + _to_human_readable_resource_values(worker.get("resources", None)) + for worker in workers.values() + ], ) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py b/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py index 15e6e98dfce..964f38e6484 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask_client_utils.py @@ -26,11 +26,11 @@ from pydantic import AnyUrl from ..core.errors import ( + ComputationalSchedulerError, ConfigurationError, DaskClientRequestError, DaskClusterError, DaskGatewayServerError, - SchedulerError, ) from .dask import check_maximize_workers, wrap_client_async_routine @@ -101,7 +101,7 @@ async def _connect_to_dask_scheduler( ) except TypeError as exc: msg = f"Scheduler has invalid configuration: {endpoint=}" - raise ConfigurationError(msg) from exc + raise ConfigurationError(msg=msg) from exc async def _connect_with_gateway_and_create_cluster( @@ -155,7 +155,7 @@ async def _connect_with_gateway_and_create_cluster( except TypeError as exc: msg = f"Cluster has invalid configuration: {endpoint=}, {auth_params=}" - raise ConfigurationError(msg) from exc + raise ConfigurationError(msg=msg) from exc except ValueError as exc: # this is when a 404=NotFound,422=MalformedData comes up raise DaskClientRequestError(endpoint=endpoint, error=exc) from exc @@ -196,10 +196,10 @@ async def get_gateway_auth_from_params( return dask_gateway.JupyterHubAuth(auth_params.api_token) except (TypeError, ValueError) as exc: msg = f"Cluster has invalid configuration: {auth_params}" - raise ConfigurationError(msg) from exc + raise ConfigurationError(msg=msg) from exc msg = f"Cluster has invalid configuration: {auth_params=}" - raise ConfigurationError(msg) + raise ConfigurationError(msg=msg) _PING_TIMEOUT_S: Final[int] = 5 @@ -220,7 +220,7 @@ async def test_scheduler_endpoint( ) as dask_client: if dask_client.status != _DASK_SCHEDULER_RUNNING_STATE: msg = "internal scheduler is not running!" - raise SchedulerError(msg) + raise ComputationalSchedulerError(msg=msg) else: gateway_auth = await get_gateway_auth_from_params(authentication) @@ -247,8 +247,8 @@ async def test_scheduler_endpoint( ClientConnectionError, ClientResponseError, httpx.HTTPError, - SchedulerError, + ComputationalSchedulerError, ) as exc: logger.debug("Pinging %s, failed: %s", f"{endpoint=}", f"{exc=!r}") msg = f"Could not connect to cluster in {endpoint}: error: {exc}" - raise ConfigurationError(msg) from exc + raise ConfigurationError(msg=msg) from exc diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 560794460de..f0a17c5e51c 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -399,9 +399,10 @@ async def test_get_stack_status_missing( mocked_dynamic_scheduler_events: None, mock_docker_api: None, ) -> None: - with pytest.raises(DynamicSidecarNotFoundError) as execinfo: + with pytest.raises( + DynamicSidecarNotFoundError, match=rf"{scheduler_data.node_uuid} not found" + ): await scheduler.get_stack_status(scheduler_data.node_uuid) - assert f"{scheduler_data.node_uuid} not found" in str(execinfo) async def test_get_stack_status_failing_sidecar( diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index 5787fa119e1..2de98368d9a 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -252,7 +252,7 @@ def mocked_director_v2_scheduler(mocker: MockerFixture, exp_status_code: int) -> # MOCKING get_stack_status def get_stack_status(node_uuid: NodeID) -> RunningDynamicServiceDetails: if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) return RunningDynamicServiceDetails.model_validate( RunningDynamicServiceDetails.model_config["json_schema_extra"]["examples"][ @@ -269,7 +269,7 @@ def get_stack_status(node_uuid: NodeID) -> RunningDynamicServiceDetails: # MOCKING remove_service def remove_service(node_uuid: NodeID, *ars: Any, **kwargs: Any) -> None: if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: - raise DynamicSidecarNotFoundError(node_uuid) + raise DynamicSidecarNotFoundError(node_uuid=node_uuid) mocker.patch( f"{module_base}._task.DynamicSidecarsScheduler.mark_service_for_removal", diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py index 495023dbda2..8fbc2d9006b 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py @@ -58,9 +58,9 @@ ComputationalBackendTaskNotFoundError, ComputationalBackendTaskResultsNotReadyError, ComputationalSchedulerChangedError, + ComputationalSchedulerError, ConfigurationError, PipelineNotFoundError, - SchedulerError, ) from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB @@ -1107,7 +1107,7 @@ async def test_handling_of_disconnected_dask_scheduler( aiopg_engine: aiopg.sa.engine.Engine, mocker: MockerFixture, published_project: PublishedProject, - backend_error: SchedulerError, + backend_error: ComputationalSchedulerError, run_metadata: RunMetadataDict, ): # this will create a non connected backend issue that will trigger re-connection diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index f36a8f8f7f6..77c327706fd 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -403,13 +403,12 @@ def test_settings__valid_network_names( async def test_failed_docker_client_request(docker_swarm: None): missing_network_name = "this_network_cannot_be_found" - with pytest.raises(GenericDockerError) as execinfo: + with pytest.raises( + GenericDockerError, + match=f"Unexpected error using docker client: network {missing_network_name} not found", + ): async with docker_client() as client: await client.networks.get(missing_network_name) - assert ( - str(execinfo.value) - == f"Unexpected error from docker client: network {missing_network_name} not found" - ) async def test_get_swarm_network_ok( @@ -428,16 +427,16 @@ async def test_get_swarm_network_missing_network( dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, docker_swarm: None, ): - with pytest.raises(DynamicSidecarError) as excinfo: + with pytest.raises( + DynamicSidecarError, + match=r"Unexpected dynamic sidecar error: " + r"Swarm network name \(searching for \'\*test_network_name\*\'\) is not configured." + r"Found following networks: \[\]", + ): await docker_api.get_swarm_network( dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME ) - assert str(excinfo.value) == ( - "Swarm network name (searching for '*test_network_name*') is not configured." - "Found following networks: []" - ) - async def test_recreate_network_multiple_times( network_config: dict[str, Any], From 0aea337c2462d565d837cae54a94d5e77d32a11e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 19 Nov 2024 20:01:45 +0100 Subject: [PATCH 48/53] Uses typing_extensions.TypedDict in all instances used in pydantic models AND fixes with migration of rest_ordering (#6770) --- .github/CODEOWNERS | 2 +- .../src/models_library/boot_options.py | 4 +++- .../src/models_library/projects_ui.py | 4 +++- .../src/models_library/rest_base.py | 2 +- .../models_library/rest_pagination_utils.py | 8 ++++++-- .../src/models_library/socketio.py | 6 +++++- .../models_library/utils/common_validators.py | 2 +- .../tests/test_rest_ordering.py | 19 +++++++++++++++++++ .../models/products.py | 4 +++- .../servicelib/aiohttp/application_setup.py | 5 ++++- .../src/servicelib/rest_constants.py | 6 +++++- .../tests/aiohttp/test_requests_validation.py | 4 ++-- .../models/comp_runs.py | 4 +++- .../resource_manager/registry.py | 4 +++- .../rest/healthcheck.py | 5 ++++- .../security/_authz_access_roles.py | 5 +++-- .../session/access_policies.py | 5 ++++- .../statics/settings.py | 5 ++++- .../_projects_permalinks.py | 4 +++- .../src/simcore_service_webserver/utils.py | 4 +++- 20 files changed, 80 insertions(+), 22 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index faa7597a371..007676d351c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,7 +14,7 @@ Makefile @pcrespov @sanderegg /ci/ @sanderegg @pcrespov /docs/ @pcrespov /packages/common-library/ @giancarloromeo -/packages/models-library/ @sanderegg @pcrespov @matusdrobuliak66 +/packages/models-library/ @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo /packages/postgres-database/ @matusdrobuliak66 /packages/pytest-simcore/ @pcrespov @sanderegg /packages/service-integration/ @pcrespov @sanderegg @GitHK diff --git a/packages/models-library/src/models_library/boot_options.py b/packages/models-library/src/models_library/boot_options.py index 52756bf1097..8b26f70c210 100644 --- a/packages/models-library/src/models_library/boot_options.py +++ b/packages/models-library/src/models_library/boot_options.py @@ -1,5 +1,7 @@ from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator -from typing_extensions import TypedDict +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .basic_types import EnvVarKey diff --git a/packages/models-library/src/models_library/projects_ui.py b/packages/models-library/src/models_library/projects_ui.py index 0e914de9dba..64556b8c044 100644 --- a/packages/models-library/src/models_library/projects_ui.py +++ b/packages/models-library/src/models_library/projects_ui.py @@ -6,7 +6,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic_extra_types.color import Color -from typing_extensions import TypedDict +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .projects_nodes_io import NodeID, NodeIDStr from .projects_nodes_ui import Marker, Position diff --git a/packages/models-library/src/models_library/rest_base.py b/packages/models-library/src/models_library/rest_base.py index 797d8876680..372b5139ce5 100644 --- a/packages/models-library/src/models_library/rest_base.py +++ b/packages/models-library/src/models_library/rest_base.py @@ -8,7 +8,7 @@ class RequestParameters(BaseModel): """ def as_params(self, **export_options) -> dict[str, str]: - data = self.dict(**export_options) + data = self.model_dump(**export_options) return {k: f"{v}" for k, v in data.items()} diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index d4027583ae7..6399fe652a3 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -1,9 +1,11 @@ from math import ceil from typing import Any, Protocol, runtime_checkable -from typing_extensions import TypedDict from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from pydantic import TypeAdapter +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .rest_pagination import PageLinks, PageMetaInfoLimitOffset @@ -70,7 +72,9 @@ def paginate_data( """ last_page = ceil(total / limit) - 1 - data = [item.model_dump() if hasattr(item, "model_dump") else item for item in chunk] + data = [ + item.model_dump() if hasattr(item, "model_dump") else item for item in chunk + ] return PageDict( _meta=PageMetaInfoLimitOffset( diff --git a/packages/models-library/src/models_library/socketio.py b/packages/models-library/src/models_library/socketio.py index 88b0e9a0beb..abc5cf92c1b 100644 --- a/packages/models-library/src/models_library/socketio.py +++ b/packages/models-library/src/models_library/socketio.py @@ -1,4 +1,8 @@ -from typing import Any, TypedDict +from typing import Any + +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) class SocketMessageDict(TypedDict): diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index 59461014965..5b2cdbf560a 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -48,7 +48,7 @@ def parse_json_pre_validator(value: Any): return json_loads(value) except JSONDecodeError as err: msg = f"Invalid JSON {value=}: {err}" - raise TypeError(msg) from err + raise ValueError(msg) from err return value diff --git a/packages/models-library/tests/test_rest_ordering.py b/packages/models-library/tests/test_rest_ordering.py index 4ceed67dea5..6db89d4bb48 100644 --- a/packages/models-library/tests/test_rest_ordering.py +++ b/packages/models-library/tests/test_rest_ordering.py @@ -151,3 +151,22 @@ def test_ordering_query_model_with_map(): model = OrderQueryParamsModel.model_validate({"order_by": {"field": "modified"}}) assert model.order_by assert model.order_by.field == "some_db_column_name" + + +def test_ordering_query_parse_json_pre_validator(): + + OrderQueryParamsModel = create_ordering_query_model_classes( + ordering_fields={"modified", "name"}, + default=OrderBy(field=IDStr("modified"), direction=OrderDirection.DESC), + ) + + bad_json_value = ",invalid json" + with pytest.raises(ValidationError) as err_info: + OrderQueryParamsModel.model_validate({"order_by": bad_json_value}) + + exc = err_info.value + assert exc.error_count() == 1 + error = exc.errors()[0] + assert error["loc"] == ("order_by",) + assert error["type"] == "value_error" + assert error["input"] == bad_json_value diff --git a/packages/postgres-database/src/simcore_postgres_database/models/products.py b/packages/postgres-database/src/simcore_postgres_database/models/products.py index a71906ace59..70ed22911d7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/products.py @@ -11,7 +11,9 @@ import sqlalchemy as sa from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.sql import func -from typing_extensions import TypedDict +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .base import metadata from .groups import groups diff --git a/packages/service-library/src/servicelib/aiohttp/application_setup.py b/packages/service-library/src/servicelib/aiohttp/application_setup.py index 4da40aa0182..0d52603f965 100644 --- a/packages/service-library/src/servicelib/aiohttp/application_setup.py +++ b/packages/service-library/src/servicelib/aiohttp/application_setup.py @@ -4,11 +4,14 @@ from collections.abc import Callable from copy import deepcopy from enum import Enum -from typing import Any, Protocol, TypedDict +from typing import Any, Protocol import arrow from aiohttp import web from pydantic import TypeAdapter +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .application_keys import APP_CONFIG_KEY, APP_SETTINGS_KEY diff --git a/packages/service-library/src/servicelib/rest_constants.py b/packages/service-library/src/servicelib/rest_constants.py index 25e37af2796..d763657b6c9 100644 --- a/packages/service-library/src/servicelib/rest_constants.py +++ b/packages/service-library/src/servicelib/rest_constants.py @@ -1,6 +1,10 @@ # SEE https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict -from typing import Final, TypedDict +from typing import Final + +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) class PydanticExportParametersDict(TypedDict): diff --git a/packages/service-library/tests/aiohttp/test_requests_validation.py b/packages/service-library/tests/aiohttp/test_requests_validation.py index e1d812da677..7f395bfd654 100644 --- a/packages/service-library/tests/aiohttp/test_requests_validation.py +++ b/packages/service-library/tests/aiohttp/test_requests_validation.py @@ -371,9 +371,9 @@ def test_parse_request_query_parameters_as_with_order_by_query_models(): expected = OrderBy(field="name", direction=OrderDirection.ASC) - url = URL("/test").with_query(order_by=expected.json()) + url = URL("/test").with_query(order_by=expected.model_dump_json()) request = make_mocked_request("GET", path=f"{url}") query_params = parse_request_query_parameters_as(OrderQueryModel, request) - assert query_params.order_by == expected + assert query_params.order_by.model_dump() == expected.model_dump() diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index aba02269415..62270380293 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -8,7 +8,9 @@ from models_library.users import UserID from pydantic import BaseModel, ConfigDict, PositiveInt, field_validator from simcore_postgres_database.models.comp_pipeline import StateType -from typing_extensions import TypedDict +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from ..utils.db import DB_TO_RUNNING_STATE diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/registry.py b/services/web/server/src/simcore_service_webserver/resource_manager/registry.py index f1457308dd8..9ae9b3d2328 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/registry.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/registry.py @@ -14,12 +14,14 @@ """ import logging -from typing import TypedDict import redis.asyncio as aioredis from aiohttp import web from models_library.basic_types import UUIDStr from servicelib.redis_utils import handle_redis_returns_union_types +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from ..redis import get_redis_resources_client from ._constants import APP_CLIENT_SOCKET_REGISTRY_KEY diff --git a/services/web/server/src/simcore_service_webserver/rest/healthcheck.py b/services/web/server/src/simcore_service_webserver/rest/healthcheck.py index 638a99cac96..fd4b5045215 100644 --- a/services/web/server/src/simcore_service_webserver/rest/healthcheck.py +++ b/services/web/server/src/simcore_service_webserver/rest/healthcheck.py @@ -47,10 +47,13 @@ import asyncio import inspect from collections.abc import Awaitable, Callable -from typing import TypeAlias, TypedDict +from typing import TypeAlias from aiohttp import web from aiosignal import Signal +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .._constants import APP_SETTINGS_KEY diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py index c3e54001691..919486962d3 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py @@ -5,9 +5,10 @@ """ -from typing import TypedDict - from simcore_postgres_database.models.users import UserRole +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) class PermissionDict(TypedDict, total=False): diff --git a/services/web/server/src/simcore_service_webserver/session/access_policies.py b/services/web/server/src/simcore_service_webserver/session/access_policies.py index 8fd72f71caf..064dddbc556 100644 --- a/services/web/server/src/simcore_service_webserver/session/access_policies.py +++ b/services/web/server/src/simcore_service_webserver/session/access_policies.py @@ -3,13 +3,16 @@ import time from collections.abc import Iterator from contextlib import contextmanager -from typing import Final, TypedDict +from typing import Final from aiohttp import web from aiohttp_session import Session from pydantic import PositiveInt, validate_call from servicelib.aiohttp import status from servicelib.aiohttp.typing_extension import Handler +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .api import get_session from .settings import SessionSettings, get_plugin_settings diff --git a/services/web/server/src/simcore_service_webserver/statics/settings.py b/services/web/server/src/simcore_service_webserver/statics/settings.py index 7a9c13f4e33..32c3b740220 100644 --- a/services/web/server/src/simcore_service_webserver/statics/settings.py +++ b/services/web/server/src/simcore_service_webserver/statics/settings.py @@ -2,13 +2,16 @@ Typically dumped in statics.json """ -from typing import Any, TypedDict +from typing import Any import pycountry from aiohttp import web from models_library.utils.change_case import snake_to_camel from pydantic import AliasChoices, AnyHttpUrl, Field, TypeAdapter from settings_library.base import BaseCustomSettings +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from .._constants import APP_SETTINGS_KEY diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py index 9308f4e3c81..a3ba8113c7f 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py @@ -1,5 +1,4 @@ import logging -from typing import TypedDict import sqlalchemy as sa from aiohttp import web @@ -7,6 +6,9 @@ from pydantic import HttpUrl, TypeAdapter from simcore_postgres_database.models.project_to_groups import project_to_groups from simcore_postgres_database.models.projects import ProjectType, projects +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) from ..db.plugin import get_database_engine from ..projects.api import ProjectPermalink, register_permalink_factory diff --git a/services/web/server/src/simcore_service_webserver/utils.py b/services/web/server/src/simcore_service_webserver/utils.py index 0ffb59a493b..c6eade6345d 100644 --- a/services/web/server/src/simcore_service_webserver/utils.py +++ b/services/web/server/src/simcore_service_webserver/utils.py @@ -16,7 +16,9 @@ import orjson from common_library.error_codes import ErrorCodeStr from models_library.basic_types import SHA1Str -from typing_extensions import TypedDict +from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict + TypedDict, +) _CURRENT_DIR = ( Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent From c49f421ac29aa66f6483e6fa56af9b75304adadd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 20 Nov 2024 08:56:55 +0100 Subject: [PATCH 49/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Cleanup=20network=20?= =?UTF-8?q?AnyHttpUrlLegacy=20and=20HttpUrlLegacy=20(#6767)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pydantic_networks_extension.py | 20 ---------- .../tests/test_pydantic_networks_extension.py | 39 ------------------- .../src/models_library/rest_pagination.py | 6 +-- .../models_library/rest_pagination_utils.py | 4 +- .../src/settings_library/ssm.py | 7 +--- .../simcore_service_agent/core/settings.py | 5 +-- services/agent/tests/conftest.py | 5 +-- .../simcore_service_invitations/api/_meta.py | 5 +-- .../services/invitations.py | 3 +- .../tests/unit/api/test_api_invitations.py | 6 +-- .../tests/unit/test_invitations.py | 12 ++++-- .../simcore_service_payments/core/settings.py | 6 +-- .../services/payments_gateway.py | 2 +- .../services/stripe.py | 2 +- services/payments/tests/unit/conftest.py | 4 +- .../unit/test_services_payments_gateway.py | 4 +- 16 files changed, 34 insertions(+), 96 deletions(-) delete mode 100644 packages/common-library/src/common_library/pydantic_networks_extension.py delete mode 100644 packages/common-library/tests/test_pydantic_networks_extension.py diff --git a/packages/common-library/src/common_library/pydantic_networks_extension.py b/packages/common-library/src/common_library/pydantic_networks_extension.py deleted file mode 100644 index 1e85a741760..00000000000 --- a/packages/common-library/src/common_library/pydantic_networks_extension.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Annotated, TypeAlias - -from pydantic import AfterValidator, AnyHttpUrl, HttpUrl -from pydantic_core import Url - - -def _strip_last_slash(url: Url) -> str: - return f"{url}".rstrip("/") - - -AnyHttpUrlLegacy: TypeAlias = Annotated[ - AnyHttpUrl, - AfterValidator(_strip_last_slash), -] - - -HttpUrlLegacy: TypeAlias = Annotated[ - HttpUrl, - AfterValidator(_strip_last_slash), -] diff --git a/packages/common-library/tests/test_pydantic_networks_extension.py b/packages/common-library/tests/test_pydantic_networks_extension.py deleted file mode 100644 index 6ab50a42a2b..00000000000 --- a/packages/common-library/tests/test_pydantic_networks_extension.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy -from pydantic import AnyHttpUrl, BaseModel, TypeAdapter, ValidationError -from pydantic_core import Url - - -class A(BaseModel): - url: AnyHttpUrlLegacy - - -def test_any_http_url(): - url = TypeAdapter(AnyHttpUrl).validate_python( - "http://backgroud.testserver.io", - ) - - assert isinstance(url, Url) - assert ( - f"{url}" == "http://backgroud.testserver.io/" - ) # trailing slash added (in Pydantic v2) - - -def test_any_http_url_legacy(): - url = TypeAdapter(AnyHttpUrlLegacy).validate_python( - "http://backgroud.testserver.io", - ) - - assert isinstance(url, str) - assert url == "http://backgroud.testserver.io" # no trailing slash was added - - -def test_valid_any_http_url_legacy_field(): - a = A(url="http://backgroud.testserver.io") # type: ignore - - assert a.url == "http://backgroud.testserver.io" # no trailing slash was added - - -def test_not_valid_any_http_url_legacy_field(): - with pytest.raises(ValidationError): - A(url="htttttp://backgroud.testserver.io") # type: ignore diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index cd6420ca174..b2c82726798 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -1,7 +1,7 @@ from typing import Annotated, Final, Generic, TypeAlias, TypeVar -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from pydantic import ( + AnyHttpUrl, BaseModel, BeforeValidator, ConfigDict, @@ -100,9 +100,7 @@ class PageLinks( PageRefs[ Annotated[ str, - BeforeValidator( - lambda x: str(TypeAdapter(AnyHttpUrlLegacy).validate_python(x)) - ), + BeforeValidator(lambda x: str(TypeAdapter(AnyHttpUrl).validate_python(x))), ] ] ): diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index 6399fe652a3..65c89131fe5 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -1,7 +1,7 @@ from math import ceil from typing import Any, Protocol, runtime_checkable -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy +from pydantic import AnyHttpUrl, TypeAdapter from pydantic import TypeAdapter from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict TypedDict, @@ -44,7 +44,7 @@ def _replace_query(url: _URLType, query: dict[str, Any]) -> str: new_url = url.replace_query_params(**query) new_url_str = f"{new_url}" - return f"{TypeAdapter(AnyHttpUrlLegacy).validate_python(new_url_str)}" + return f"{TypeAdapter(AnyHttpUrl).validate_python(new_url_str)}" class PageDict(TypedDict): diff --git a/packages/settings-library/src/settings_library/ssm.py b/packages/settings-library/src/settings_library/ssm.py index 823ac57c1a2..8bed0906bbe 100644 --- a/packages/settings-library/src/settings_library/ssm.py +++ b/packages/settings-library/src/settings_library/ssm.py @@ -1,7 +1,6 @@ from typing import Annotated -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy -from pydantic import BeforeValidator, Field, SecretStr, TypeAdapter +from pydantic import AnyHttpUrl, BeforeValidator, Field, SecretStr, TypeAdapter from pydantic_settings import SettingsConfigDict from .base import BaseCustomSettings @@ -12,9 +11,7 @@ class SSMSettings(BaseCustomSettings): SSM_ENDPOINT: ( Annotated[ str, - BeforeValidator( - lambda x: str(TypeAdapter(AnyHttpUrlLegacy).validate_python(x)) - ), + BeforeValidator(lambda x: str(TypeAdapter(AnyHttpUrl).validate_python(x))), ] | None ) = Field(default=None, description="do not define if using standard AWS") diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index 3f7af297189..40370257c4a 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -1,8 +1,7 @@ from datetime import timedelta -from common_library.pydantic_networks_extension import AnyHttpUrlLegacy from models_library.basic_types import BootModeEnum, LogLevel -from pydantic import AliasChoices, Field, field_validator +from pydantic import AliasChoices, AnyHttpUrl, Field, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.r_clone import S3Provider @@ -43,7 +42,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field( ..., description="Exactly the same as director-v2's `SWARM_STACK_NAME` env var" ) - AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: AnyHttpUrlLegacy + AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: AnyHttpUrl AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY: str AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY: str AGENT_VOLUMES_CLEANUP_S3_BUCKET: str diff --git a/services/agent/tests/conftest.py b/services/agent/tests/conftest.py index a0c59ca9f37..f421258fd77 100644 --- a/services/agent/tests/conftest.py +++ b/services/agent/tests/conftest.py @@ -3,7 +3,6 @@ import pytest -from common_library.pydantic_networks_extension import HttpUrlLegacy from faker import Faker from models_library.basic_types import BootModeEnum from moto.server import ThreadedMotoServer @@ -65,8 +64,8 @@ def mock_environment( @pytest.fixture(scope="module") -def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrlLegacy: +def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrl: # pylint: disable=protected-access - return TypeAdapter(HttpUrlLegacy).validate_python( + return TypeAdapter(HttpUrl).validate_python( f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # noqa: SLF001 ) diff --git a/services/invitations/src/simcore_service_invitations/api/_meta.py b/services/invitations/src/simcore_service_invitations/api/_meta.py index 49d3f4b48cf..621f3b4445e 100644 --- a/services/invitations/src/simcore_service_invitations/api/_meta.py +++ b/services/invitations/src/simcore_service_invitations/api/_meta.py @@ -2,8 +2,7 @@ from collections.abc import Callable from fastapi import APIRouter, Depends -from pydantic import BaseModel -from common_library.pydantic_networks_extension import HttpUrlLegacy +from pydantic import BaseModel, HttpUrl from .._meta import API_VERSION, PROJECT_NAME from ._dependencies import get_reverse_url_mapper @@ -21,7 +20,7 @@ class _Meta(BaseModel): name: str version: str - docs_url: HttpUrlLegacy + docs_url: HttpUrl # diff --git a/services/invitations/src/simcore_service_invitations/services/invitations.py b/services/invitations/src/simcore_service_invitations/services/invitations.py index 1cdc33f4085..e9d0a2a13ed 100644 --- a/services/invitations/src/simcore_service_invitations/services/invitations.py +++ b/services/invitations/src/simcore_service_invitations/services/invitations.py @@ -3,7 +3,6 @@ import logging from urllib import parse -from common_library.pydantic_networks_extension import HttpUrlLegacy from cryptography.fernet import Fernet, InvalidToken from models_library.invitations import InvitationContent, InvitationInputs from models_library.products import ProductName @@ -67,7 +66,7 @@ def _build_link( # Adds query to fragment base_url = f"{base_url}/" url = URL(base_url).replace(fragment=f"{r}") - return TypeAdapter(HttpUrlLegacy).validate_python(f"{url}") + return TypeAdapter(HttpUrl).validate_python(f"{url}") def _fernet_encrypt_as_urlsafe_code( diff --git a/services/invitations/tests/unit/api/test_api_invitations.py b/services/invitations/tests/unit/api/test_api_invitations.py index 95d5eeeece0..751a0a96bb9 100644 --- a/services/invitations/tests/unit/api/test_api_invitations.py +++ b/services/invitations/tests/unit/api/test_api_invitations.py @@ -100,7 +100,7 @@ def test_check_valid_invitation( # check invitation_url response = client.post( f"/{API_VTAG}/invitations:extract", - json={"invitation_url": invitation_url}, + json={"invitation_url": f"{invitation_url}"}, auth=basic_auth, ) assert response.status_code == 200, f"{response.json()=}" @@ -130,7 +130,7 @@ def test_check_invalid_invitation_with_different_secret( # check invitation_url response = client.post( f"/{API_VTAG}/invitations:extract", - json={"invitation_url": invitation_url}, + json={"invitation_url": f"{invitation_url}"}, auth=basic_auth, ) assert ( @@ -173,7 +173,7 @@ def test_check_invalid_invitation_with_wrong_code( default_product=default_product, ) - invitation_url_with_invalid_code = invitation_url[:-3] + invitation_url_with_invalid_code = f"{invitation_url}"[:-3] # check invitation_url response = client.post( diff --git a/services/invitations/tests/unit/test_invitations.py b/services/invitations/tests/unit/test_invitations.py index 0e6ea34b18c..b37b79f4575 100644 --- a/services/invitations/tests/unit/test_invitations.py +++ b/services/invitations/tests/unit/test_invitations.py @@ -73,8 +73,8 @@ def test_create_and_decrypt_invitation( base_url=faker.url(), default_product=default_product, ) - assert URL(invitation_link).fragment - query_params = dict(parse.parse_qsl((URL(URL(invitation_link).fragment).query))) + assert URL(f"{invitation_link}").fragment + query_params = dict(parse.parse_qsl(URL(URL(f"{invitation_link}").fragment).query)) # will raise TokenError or ValidationError invitation = decrypt_invitation( @@ -196,4 +196,10 @@ class OtherModel(BaseModel): def test_aliases_uniqueness(): - assert not [item for item, count in Counter([field.alias for field in _ContentWithShortNames.model_fields.values()]).items() if count > 1] #nosec + assert not [ + item + for item, count in Counter( + [field.alias for field in _ContentWithShortNames.model_fields.values()] + ).items() + if count > 1 + ] # nosec diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index a5075e8e4ca..41f1bb7e9b3 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -1,11 +1,11 @@ from functools import cached_property -from common_library.pydantic_networks_extension import HttpUrlLegacy from models_library.basic_types import NonNegativeDecimal from pydantic import ( AliasChoices, EmailStr, Field, + HttpUrl, PositiveFloat, SecretStr, TypeAdapter, @@ -69,7 +69,7 @@ class ApplicationSettings(_BaseApplicationSettings): These settings includes extra configuration for the http-API """ - PAYMENTS_GATEWAY_URL: HttpUrlLegacy = Field( + PAYMENTS_GATEWAY_URL: HttpUrl = Field( ..., description="Base url to the payment gateway" ) @@ -134,7 +134,7 @@ class ApplicationSettings(_BaseApplicationSettings): description="settings for postgres service", ) - PAYMENTS_STRIPE_URL: HttpUrlLegacy = Field( + PAYMENTS_STRIPE_URL: HttpUrl = Field( ..., description="Base url to the payment Stripe" ) PAYMENTS_STRIPE_API_SECRET: SecretStr = Field( diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 34af4fa2268..812ab087074 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -209,7 +209,7 @@ def setup_payments_gateway(app: FastAPI): # create api = PaymentsGatewayApi.from_client_kwargs( - base_url=settings.PAYMENTS_GATEWAY_URL, + base_url=f"{settings.PAYMENTS_GATEWAY_URL}", headers={"accept": "application/json"}, auth=_GatewayApiAuth( secret=settings.PAYMENTS_GATEWAY_API_SECRET.get_secret_value() diff --git a/services/payments/src/simcore_service_payments/services/stripe.py b/services/payments/src/simcore_service_payments/services/stripe.py index 9a701965beb..349de908d7e 100644 --- a/services/payments/src/simcore_service_payments/services/stripe.py +++ b/services/payments/src/simcore_service_payments/services/stripe.py @@ -89,7 +89,7 @@ def setup_stripe(app: FastAPI): assert app.state # nosec settings: ApplicationSettings = app.state.settings api = StripeApi.from_client_kwargs( - base_url=settings.PAYMENTS_STRIPE_URL, + base_url=f"{settings.PAYMENTS_STRIPE_URL}", auth=_StripeBearerAuth(settings.PAYMENTS_STRIPE_API_SECRET.get_secret_value()), ) if settings.PAYMENTS_TRACING: diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index be55736c6f6..de408dadf3d 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -198,7 +198,7 @@ def mock_payments_gateway_service_api_base(app: FastAPI) -> Iterator[MockRouter] settings: ApplicationSettings = app.state.settings with respx.mock( - base_url=settings.PAYMENTS_GATEWAY_URL, + base_url=f"{settings.PAYMENTS_GATEWAY_URL}", assert_all_called=False, assert_all_mocked=True, # IMPORTANT: KEEP always True! ) as respx_mock: @@ -410,7 +410,7 @@ def mock_payments_stripe_api_base(app: FastAPI) -> Iterator[MockRouter]: settings: ApplicationSettings = app.state.settings with respx.mock( - base_url=settings.PAYMENTS_STRIPE_URL, + base_url=f"{settings.PAYMENTS_STRIPE_URL}", assert_all_called=False, assert_all_mocked=True, # IMPORTANT: KEEP always True! ) as respx_mock: diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index e89c502894b..c2ba73f4f9a 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -121,7 +121,7 @@ async def test_one_time_payment_workflow( ) app_settings: ApplicationSettings = app.state.settings - assert submission_link.host == URL(app_settings.PAYMENTS_GATEWAY_URL).host + assert submission_link.host == URL(f"{app_settings.PAYMENTS_GATEWAY_URL}").host # cancel payment_canceled = await payment_gateway_api.cancel_payment(payment_initiated) @@ -160,7 +160,7 @@ async def test_payment_methods_workflow( ) app_settings: ApplicationSettings = app.state.settings - assert form_link.host == URL(app_settings.PAYMENTS_GATEWAY_URL).host + assert form_link.host == URL(f"{app_settings.PAYMENTS_GATEWAY_URL}").host # CRUD payment_method_id = initiated.payment_method_id From a0b8a2fae4d055e68d65f25bcd8c9abddfc03949 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:07:53 +0100 Subject: [PATCH 50/53] All errors in dynamic-sidecar inherit from `OsparcErrorMixin` (#6764) Co-authored-by: Andrei Neagu --- .../core/docker_utils.py | 8 +++- .../core/error_handlers.py | 2 +- .../core/errors.py | 41 ++++-------------- .../core/settings.py | 14 +++---- .../tests/unit/test_api_rest_containers.py | 2 +- .../tests/unit/test_core_errors.py | 42 +++++++++++++++++++ 6 files changed, 65 insertions(+), 44 deletions(-) create mode 100644 services/dynamic-sidecar/tests/unit/test_core_errors.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py index 43959d5fba5..baa93fdb4cc 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py @@ -50,7 +50,13 @@ async def get_volume_by_label(label: str, run_id: RunID) -> dict[str, Any]: volumes = data["Volumes"] _logger.debug("volumes query for label=%s volumes=%s", label, volumes) if len(volumes) != 1: - raise VolumeNotFoundError(label, run_id, volumes) + raise VolumeNotFoundError( + volume_count=len(volumes), + source_label=label, + run_id=run_id, + volume_names=" ".join(v.get("Name", "UNKNOWN") for v in volumes), + status_code=http_status.HTTP_404_NOT_FOUND, + ) volume_details: dict[str, Any] = volumes[0] return volume_details diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/error_handlers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/error_handlers.py index 9fbceae96a0..cbd3e4dbe52 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/error_handlers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/error_handlers.py @@ -12,7 +12,7 @@ async def http_error_handler( ) -> JSONResponse: return JSONResponse( content=jsonable_encoder({"errors": [exception.message]}), - status_code=exception.status_code, + status_code=exception.status_code, # type:ignore[attr-defined] ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py index bd74b68dfb7..b9a449ecb36 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/errors.py @@ -1,53 +1,30 @@ -from typing import Any - from common_library.errors_classes import OsparcErrorMixin -from fastapi import status -from models_library.services import RunID -class BaseDynamicSidecarError(Exception): +class BaseDynamicSidecarError(OsparcErrorMixin, Exception): """Used as base for all exceptions""" - def __init__( - self, nessage: str, status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR - ) -> None: - self.message: str = nessage - self.status_code: int = status_code - super().__init__(nessage) - class VolumeNotFoundError(BaseDynamicSidecarError): - def __init__( - self, source_label: str, run_id: RunID, volumes: list[dict[str, Any]] - ) -> None: - super().__init__( - f"Expected 1 got {len(volumes)} volumes labels with {source_label=}, {run_id=}: " - f"Found {' '.join(v.get('Name', 'UNKNOWN') for v in volumes)}", - status_code=status.HTTP_404_NOT_FOUND, - ) + msg_template = ( + "Expected 1 got {volume_count} volumes labels with " + "source_label={source_label}, run_id={run_id}: Found {volume_names}" + ) class UnexpectedDockerError(BaseDynamicSidecarError): - def __init__(self, message: str, status_code: int) -> None: - super().__init__( - f"An unexpected Docker error occurred {status_code=}, {message=}", - status_code=status_code, - ) - - -class BaseError(OsparcErrorMixin, BaseDynamicSidecarError): - ... + msg_template = "An unexpected Docker error occurred status_code={status_code}, message={message}" -class ContainerExecContainerNotFoundError(BaseError): +class ContainerExecContainerNotFoundError(BaseDynamicSidecarError): msg_template = "Container '{container_name}' was not found" -class ContainerExecTimeoutError(BaseError): +class ContainerExecTimeoutError(BaseDynamicSidecarError): msg_template = "Timed out after {timeout} while executing: '{command}'" -class ContainerExecCommandFailedError(BaseError): +class ContainerExecCommandFailedError(BaseDynamicSidecarError): msg_template = ( "Command '{command}' exited with code '{exit_code}'" "and output: '{command_result}'" diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index af477d39ce4..795015e1520 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -5,7 +5,7 @@ from typing import cast from common_library.pydantic_validators import validate_numeric_string_as_timedelta -from models_library.basic_types import BootModeEnum, PortInt +from models_library.basic_types import PortInt from models_library.callbacks_mapping import CallbacksMapping from models_library.products import ProductName from models_library.projects import ProjectID @@ -21,8 +21,8 @@ field_validator, ) from servicelib.logging_utils_filtering import LoggerName, MessageSubstring +from settings_library.application import BaseApplicationSettings from settings_library.aws_s3_cli import AwsS3CliSettings -from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.node_ports import StorageAuthSettings from settings_library.postgres import PostgresSettings @@ -35,7 +35,7 @@ from settings_library.utils_logging import MixinLoggingSettings -class ResourceTrackingSettings(BaseCustomSettings): +class ResourceTrackingSettings(BaseApplicationSettings): RESOURCE_TRACKING_HEARTBEAT_INTERVAL: timedelta = Field( default=DEFAULT_RESOURCE_USAGE_HEARTBEAT_INTERVAL, description="each time the status of the service is propagated", @@ -46,17 +46,13 @@ class ResourceTrackingSettings(BaseCustomSettings): ) -class SystemMonitorSettings(BaseCustomSettings): +class SystemMonitorSettings(BaseApplicationSettings): DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE: bool = Field( default=False, description="enabled/disabled disk usage monitoring" ) -class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): - SC_BOOT_MODE: BootModeEnum = Field( - ..., - description="boot mode helps determine if in development mode or normal operation", - ) +class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR: Path = Field( ..., diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index 719d2f38c6c..7ce7027c6e0 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -480,7 +480,7 @@ async def test_container_docker_error( def _expected_error_string(status_code: int) -> dict[str, Any]: return { "errors": [ - f"An unexpected Docker error occurred status_code={status_code}, message='aiodocker_mocked_error'" + f"An unexpected Docker error occurred status_code={status_code}, message=aiodocker_mocked_error" ] } diff --git a/services/dynamic-sidecar/tests/unit/test_core_errors.py b/services/dynamic-sidecar/tests/unit/test_core_errors.py new file mode 100644 index 00000000000..7b112878c9c --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/test_core_errors.py @@ -0,0 +1,42 @@ +# pylint:disable=broad-exception-caught +# pylint:disable=no-member + +from simcore_service_dynamic_sidecar.core.errors import ( + UnexpectedDockerError, + VolumeNotFoundError, +) +from starlette import status + + +def test_legacy_interface_unexpected_docker_error(): + message = "some_message" + status_code = 42 + try: + raise UnexpectedDockerError( # noqa: TRY301 + message=message, status_code=status_code + ) + except Exception as e: + print(e) + assert e.status_code == status_code # noqa: PT017 + assert message in e.message # noqa: PT017 + + +def test_legacy_interface_volume_not_found_error(): + try: + volumes = [{}, {"Name": "a_volume"}] + volume_names = " ".join(v.get("Name", "UNKNOWN") for v in volumes) + + raise VolumeNotFoundError( # noqa: TRY301 + volume_count=len(volumes), + source_label="some", + run_id="run_id", + volume_names=volume_names, + status_code=status.HTTP_404_NOT_FOUND, + ) + except Exception as e: + print(e) + assert ( # noqa: PT017 + e.message + == "Expected 1 got 2 volumes labels with source_label=some, run_id=run_id: Found UNKNOWN a_volume" + ) + assert e.status_code == status.HTTP_404_NOT_FOUND # noqa: PT017 From 66a492f23de7f58ffbdd9b1f255e2e8f11b42cb5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 20 Nov 2024 10:55:14 +0100 Subject: [PATCH 51/53] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Fix=20agent=20tests?= =?UTF-8?q?=20(#6772)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models-library/src/models_library/rest_pagination_utils.py | 1 - services/agent/tests/conftest.py | 2 +- services/agent/tests/unit/test_services_backup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/rest_pagination_utils.py b/packages/models-library/src/models_library/rest_pagination_utils.py index 65c89131fe5..1bd952cfd12 100644 --- a/packages/models-library/src/models_library/rest_pagination_utils.py +++ b/packages/models-library/src/models_library/rest_pagination_utils.py @@ -2,7 +2,6 @@ from typing import Any, Protocol, runtime_checkable from pydantic import AnyHttpUrl, TypeAdapter -from pydantic import TypeAdapter from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict TypedDict, ) diff --git a/services/agent/tests/conftest.py b/services/agent/tests/conftest.py index f421258fd77..14e8cd1d9e3 100644 --- a/services/agent/tests/conftest.py +++ b/services/agent/tests/conftest.py @@ -48,7 +48,7 @@ def mock_environment( "LOGLEVEL": "DEBUG", "SC_BOOT_MODE": BootModeEnum.DEBUG, "AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME": swarm_stack_name, - "AGENT_VOLUMES_CLEANUP_S3_ENDPOINT": mocked_s3_server_url, + "AGENT_VOLUMES_CLEANUP_S3_ENDPOINT": f"{mocked_s3_server_url}", "AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY": "xxx", "AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY": "xxx", "AGENT_VOLUMES_CLEANUP_S3_BUCKET": bucket, diff --git a/services/agent/tests/unit/test_services_backup.py b/services/agent/tests/unit/test_services_backup.py index 67a1203ea79..c986550da51 100644 --- a/services/agent/tests/unit/test_services_backup.py +++ b/services/agent/tests/unit/test_services_backup.py @@ -77,7 +77,7 @@ async def test_backup_volume( expected_files = _FILES_TO_CREATE_IN_VOLUME * len(VOLUMES_TO_CREATE) - async with session.client("s3", endpoint_url=settings.AGENT_VOLUMES_CLEANUP_S3_ENDPOINT) as s3_client: # type: ignore + async with session.client("s3", endpoint_url=f"{settings.AGENT_VOLUMES_CLEANUP_S3_ENDPOINT}") as s3_client: # type: ignore list_response = await s3_client.list_objects_v2( Bucket=settings.AGENT_VOLUMES_CLEANUP_S3_BUCKET, Prefix=f"{swarm_stack_name}/{project_id}/{node_id}/{run_id}", From c38b064403639afb5894a06a70eb45aa9375c6d5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:15:31 +0100 Subject: [PATCH 52/53] Fixes [unit] webserver 03 and [unit] settings-library (both mypy and tests) (#6773) --- .../src/common_library/basic_types.py | 38 +++++++++++++ .../src/models_library/basic_types.py | 51 +++++------------- .../src/settings_library/application.py | 3 +- .../src/settings_library/base.py | 4 +- .../src/settings_library/basic_types.py | 54 +++++-------------- .../tests/test_utils_logging.py | 4 +- .../test_usage_services__export.py | 4 +- .../test_usage_services__list.py | 8 +-- 8 files changed, 77 insertions(+), 89 deletions(-) create mode 100644 packages/common-library/src/common_library/basic_types.py diff --git a/packages/common-library/src/common_library/basic_types.py b/packages/common-library/src/common_library/basic_types.py new file mode 100644 index 00000000000..ab5278cd4f4 --- /dev/null +++ b/packages/common-library/src/common_library/basic_types.py @@ -0,0 +1,38 @@ +from enum import StrEnum + + +class LogLevel(StrEnum): + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + + +class BootModeEnum(StrEnum): + """ + Values taken by SC_BOOT_MODE environment variable + set in Dockerfile and used during docker/boot.sh + """ + + DEFAULT = "default" + LOCAL = "local-development" + DEBUG = "debug" + PRODUCTION = "production" + DEVELOPMENT = "development" + + def is_devel_mode(self) -> bool: + """returns True if this boot mode is used for development""" + return self in (self.DEBUG, self.DEVELOPMENT, self.LOCAL) + + +class BuildTargetEnum(StrEnum): + """ + Values taken by SC_BUILD_TARGET environment variable + set in Dockerfile that defines the stage targeted in the + docker image build + """ + + BUILD = "build" + CACHE = "cache" + PRODUCTION = "production" + DEVELOPMENT = "development" diff --git a/packages/models-library/src/models_library/basic_types.py b/packages/models-library/src/models_library/basic_types.py index fac95b7c2a0..d56b3037d52 100644 --- a/packages/models-library/src/models_library/basic_types.py +++ b/packages/models-library/src/models_library/basic_types.py @@ -1,8 +1,9 @@ from decimal import Decimal -from enum import StrEnum +from enum import Enum from re import Pattern from typing import Annotated, ClassVar, Final, TypeAlias +from common_library.basic_types import BootModeEnum, BuildTargetEnum, LogLevel from pydantic import Field, HttpUrl, PositiveInt, StringConstraints from pydantic_core import core_schema @@ -13,6 +14,17 @@ UUID_RE, ) +assert issubclass(LogLevel, Enum) # nosec +assert issubclass(BootModeEnum, Enum) # nosec +assert issubclass(BuildTargetEnum, Enum) # nosec + +__all__: tuple[str, ...] = ( + "LogLevel", + "BootModeEnum", + "BuildTargetEnum", +) + + NonNegativeDecimal: TypeAlias = Annotated[Decimal, Field(ge=0)] PositiveDecimal: TypeAlias = Annotated[Decimal, Field(gt=0)] @@ -145,41 +157,4 @@ class HttpUrlWithCustomMinLength(HttpUrl): min_length = 0 -class LogLevel(StrEnum): - DEBUG = "DEBUG" - INFO = "INFO" - WARNING = "WARNING" - ERROR = "ERROR" - - -class BootModeEnum(StrEnum): - """ - Values taken by SC_BOOT_MODE environment variable - set in Dockerfile and used during docker/boot.sh - """ - - DEFAULT = "default" - LOCAL = "local-development" - DEBUG = "debug" - PRODUCTION = "production" - DEVELOPMENT = "development" - - def is_devel_mode(self) -> bool: - """returns True if this boot mode is used for development""" - return self in (self.DEBUG, self.DEVELOPMENT, self.LOCAL) - - -class BuildTargetEnum(StrEnum): - """ - Values taken by SC_BUILD_TARGET environment variable - set in Dockerfile that defines the stage targeted in the - docker image build - """ - - BUILD = "build" - CACHE = "cache" - PRODUCTION = "production" - DEVELOPMENT = "development" - - KeyIDStr = Annotated[str, StringConstraints(pattern=PROPERTY_KEY_RE)] diff --git a/packages/settings-library/src/settings_library/application.py b/packages/settings-library/src/settings_library/application.py index 890067ca518..1af5e142ba8 100644 --- a/packages/settings-library/src/settings_library/application.py +++ b/packages/settings-library/src/settings_library/application.py @@ -1,8 +1,7 @@ -from models_library.basic_types import BootModeEnum from pydantic import Field, PositiveInt from .base import BaseCustomSettings -from .basic_types import BuildTargetEnum +from .basic_types import BootModeEnum, BuildTargetEnum class BaseApplicationSettings(BaseCustomSettings): diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index 95f76f12219..b5bc05f2725 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -58,7 +58,7 @@ def _is_auto_default_from_env_enabled(field: FieldInfo) -> bool: ) -ENABLED: Final = {} +_MARKED_AS_UNSET: Final[dict] = {} class EnvSettingsWithAutoDefaultSource(EnvSettingsSource): @@ -89,7 +89,7 @@ def prepare_field_value( _is_auto_default_from_env_enabled(field) and field.default_factory and field.default is None - and prepared_value == ENABLED + and prepared_value == _MARKED_AS_UNSET ): prepared_value = field.default_factory() return prepared_value diff --git a/packages/settings-library/src/settings_library/basic_types.py b/packages/settings-library/src/settings_library/basic_types.py index 9d1d440d5ae..b96ce428817 100644 --- a/packages/settings-library/src/settings_library/basic_types.py +++ b/packages/settings-library/src/settings_library/basic_types.py @@ -1,57 +1,31 @@ -# -# NOTE: This file copies some of the types from models_library.basic_types -# This is a minor evil to avoid the maintenance burden that creates -# an extra dependency to a larger models_library (intra-repo library) - -from enum import StrEnum +from enum import Enum from typing import Annotated, TypeAlias +from common_library.basic_types import BootModeEnum, BuildTargetEnum, LogLevel from pydantic import Field, StringConstraints +assert issubclass(LogLevel, Enum) # nosec +assert issubclass(BootModeEnum, Enum) # nosec +assert issubclass(BuildTargetEnum, Enum) # nosec + +__all__: tuple[str, ...] = ( + "LogLevel", + "BootModeEnum", + "BuildTargetEnum", +) + + # port number range PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)] +RegisteredPortInt: TypeAlias = Annotated[int, Field(gt=1024, lt=65535)] # e.g. 'v5' VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")] -class LogLevel(StrEnum): - DEBUG = "DEBUG" - INFO = "INFO" - WARNING = "WARNING" - ERROR = "ERROR" - - -class BootMode(StrEnum): - """ - Values taken by SC_BOOT_MODE environment variable - set in Dockerfile and used during docker/boot.sh - """ - - DEFAULT = "default" - LOCAL = "local-development" - DEBUG = "debug" - PRODUCTION = "production" - DEVELOPMENT = "development" - - -class BuildTargetEnum(StrEnum): - """ - Values taken by SC_BUILD_TARGET environment variable - set in Dockerfile that defines the stage targeted in the - docker image build - """ - - BUILD = "build" - CACHE = "cache" - PRODUCTION = "production" - DEVELOPMENT = "development" - - # non-empty bounded string used as identifier # e.g. "123" or "name_123" or "fa327c73-52d8-462a-9267-84eeaf0f90e3" but NOT "" IDStr: TypeAlias = Annotated[ str, StringConstraints(strip_whitespace=True, min_length=1, max_length=50) ] -RegisteredPortInt: TypeAlias = Annotated[int, Field(gt=1024, lt=65535)] diff --git a/packages/settings-library/tests/test_utils_logging.py b/packages/settings-library/tests/test_utils_logging.py index 01673d46f60..d63a8ae8538 100644 --- a/packages/settings-library/tests/test_utils_logging.py +++ b/packages/settings-library/tests/test_utils_logging.py @@ -2,7 +2,7 @@ from pydantic import AliasChoices, Field, field_validator from settings_library.base import BaseCustomSettings -from settings_library.basic_types import BootMode +from settings_library.basic_types import BootModeEnum from settings_library.utils_logging import MixinLoggingSettings @@ -14,7 +14,7 @@ def test_mixin_logging(monkeypatch): class Settings(BaseCustomSettings, MixinLoggingSettings): # DOCKER - SC_BOOT_MODE: BootMode | None = None + SC_BOOT_MODE: BootModeEnum | None = None # LOGGING LOG_LEVEL: str = Field( diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py index 3ac7ffaa665..c91141a0674 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__export.py @@ -115,5 +115,7 @@ async def test_list_service_usage( assert mock_export_usage_services.called args = mock_export_usage_services.call_args[1] - assert args["order_by"] == OrderBy.model_validate(_order_by) + assert ( + args["order_by"].model_dump() == OrderBy.model_validate(_order_by).model_dump() + ) assert args["filters"] == ServiceResourceUsagesFilters.model_validate(_filter) diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py index 86f92d03730..32480e03762 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py @@ -221,7 +221,7 @@ async def test_list_service_usage_with_order_by_query_param( _, error = await assert_status(resp, status.HTTP_422_UNPROCESSABLE_ENTITY) assert mock_list_usage_services.called assert error["status"] == status.HTTP_422_UNPROCESSABLE_ENTITY - assert error["errors"][0]["message"].startswith("Invalid JSON") + assert "Invalid JSON" in error["errors"][0]["message"] # with order by without direction _filter = {"field": "started_at"} @@ -249,7 +249,7 @@ async def test_list_service_usage_with_order_by_query_param( errors = {(e["code"], e["field"]) for e in error["errors"]} assert { ("value_error", "order_by.field"), - ("type_error.enum", "order_by.direction"), + ("enum", "order_by.direction"), } == errors assert len(errors) == 2 @@ -264,8 +264,8 @@ async def test_list_service_usage_with_order_by_query_param( _, error = await assert_status(resp, status.HTTP_422_UNPROCESSABLE_ENTITY) assert mock_list_usage_services.called assert error["status"] == status.HTTP_422_UNPROCESSABLE_ENTITY - assert error["errors"][0]["message"].startswith("field required") - assert error["errors"][0]["code"] == "value_error.missing" + assert error["errors"][0]["message"].startswith("Field required") + assert error["errors"][0]["code"] == "missing" assert error["errors"][0]["field"] == "order_by.field" From 47e357053945e71ee8ecafffefc98016e1e61c35 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:18:51 +0100 Subject: [PATCH 53/53] =?UTF-8?q?=E2=AC=86=EF=B8=8FPydantic=20V2:=20missin?= =?UTF-8?q?g=20import=20in=20director-v0=20(#6781)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/director/src/simcore_service_director/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director/src/simcore_service_director/core/settings.py b/services/director/src/simcore_service_director/core/settings.py index 852e5369fe1..a17f9175bff 100644 --- a/services/director/src/simcore_service_director/core/settings.py +++ b/services/director/src/simcore_service_director/core/settings.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from models_library.basic_types import LogLevel, PortInt, VersionTag -from pydantic import Field, NonNegativeInt, PositiveInt, field_validator +from pydantic import AliasChoices, Field, NonNegativeInt, PositiveInt, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.docker_registry import RegistrySettings