diff --git a/packages/models-library/setup.py b/packages/models-library/setup.py index 1c941a4e906..a36abca1fbf 100644 --- a/packages/models-library/setup.py +++ b/packages/models-library/setup.py @@ -29,30 +29,30 @@ def read_reqs(reqs_path: Path) -> set[str]: ) # STRICK requirements -SETUP = dict( - name="simcore-models-library", - version=Path(CURRENT_DIR / "VERSION").read_text().strip(), - author="Sylvain Anderegg (sanderegg)", - description="Core service library for simcore pydantic models", - python_requires="~=3.10", - classifiers=[ +SETUP = { + "name": "simcore-models-library", + "version": Path(CURRENT_DIR / "VERSION").read_text().strip(), + "author": "Sylvain Anderegg (sanderegg)", + "description": "Core service library for simcore pydantic models", + "python_requires": "~=3.10", + "classifiers": [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python :: 3.10", ], - long_description=Path(CURRENT_DIR / "README.md").read_text(), - license="MIT license", - install_requires=INSTALL_REQUIREMENTS, - packages=find_packages(where="src"), - package_dir={"": "src"}, - include_package_data=True, - test_suite="tests", - tests_require=TEST_REQUIREMENTS, - extras_require={"test": TEST_REQUIREMENTS}, - zip_safe=False, -) + "long_description": Path(CURRENT_DIR / "README.md").read_text(), + "license": "MIT license", + "install_requires": INSTALL_REQUIREMENTS, + "packages": find_packages(where="src"), + "package_dir": {"": "src"}, + "include_package_data": True, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "zip_safe": False, +} if __name__ == "__main__": diff --git a/services/catalog/src/simcore_service_catalog/models/domain/__init__.py b/packages/models-library/src/models_library/api_schemas_catalog/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/models/domain/__init__.py rename to packages/models-library/src/models_library/api_schemas_catalog/__init__.py diff --git a/packages/models-library/src/models_library/api_schemas_catalog/meta.py b/packages/models-library/src/models_library/api_schemas_catalog/meta.py new file mode 100644 index 00000000000..edc2f904000 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_catalog/meta.py @@ -0,0 +1,22 @@ +from typing import Any, ClassVar + +from pydantic import BaseModel, Field + +from ..basic_types import VersionStr + + +class Meta(BaseModel): + name: str + version: VersionStr + released: dict[str, VersionStr] | None = Field( + None, description="Maps every route's path tag with a released version" + ) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "name": "simcore_service_foo", + "version": "2.4.45", + "released": {"v1": "1.3.4", "v2": "2.4.45"}, + } + } diff --git a/packages/models-library/src/models_library/api_schemas_catalog.py b/packages/models-library/src/models_library/api_schemas_catalog/service_access_rights.py similarity index 68% rename from packages/models-library/src/models_library/api_schemas_catalog.py rename to packages/models-library/src/models_library/api_schemas_catalog/service_access_rights.py index c4dac0afbb4..c56edcd7cf9 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/service_access_rights.py @@ -1,7 +1,7 @@ -from models_library.users import GroupID from pydantic import BaseModel -from .services import ServiceKey, ServiceVersion +from ..services import ServiceKey, ServiceVersion +from ..users import GroupID class ServiceAccessRightsGet(BaseModel): diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py similarity index 62% rename from services/catalog/src/simcore_service_catalog/models/schemas/services.py rename to packages/models-library/src/models_library/api_schemas_catalog/services.py index d7bef5f214f..bf3ca20a25c 100644 --- a/services/catalog/src/simcore_service_catalog/models/schemas/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -1,17 +1,16 @@ -from typing import Optional +from typing import Any, ClassVar -from models_library.emails import LowerCaseEmailStr -from models_library.services import ServiceDockerData, ServiceMetaData -from models_library.services_access import ServiceAccessRights -from models_library.services_resources import ServiceResourcesDict from pydantic import Extra -from pydantic.main import BaseModel + +from ..emails import LowerCaseEmailStr +from ..services import ServiceDockerData, ServiceMetaData +from ..services_access import ServiceAccessRights +from ..services_resources import ServiceResourcesDict -# OpenAPI models (contain both service metadata and access rights) class ServiceUpdate(ServiceMetaData, ServiceAccessRights): class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { # ServiceAccessRights "accessRights": { @@ -64,16 +63,16 @@ class Config: class ServiceGet( ServiceDockerData, ServiceAccessRights, ServiceMetaData ): # pylint: disable=too-many-ancestors - owner: Optional[LowerCaseEmailStr] + owner: LowerCaseEmailStr | None class Config: allow_population_by_field_name = True extra = Extra.ignore - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "name": "File Picker", "thumbnail": None, - "description": "File Picker", + "description": "description", "classifiers": [], "quality": {}, "accessRights": { @@ -109,32 +108,4 @@ class Config: } -# TODO: prototype for next iteration -# Items are non-detailed version of resources listed -class ServiceItem(BaseModel): - class Config: - extra = Extra.ignore - schema_extra = { - "example": { - "title": "File Picker", # NEW: rename 'name' as title (so it is not confused with an identifier!) - "thumbnail": None, # optional - "description": "File Picker", - "classifiers_url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/classifiers", - "quality": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/quality", - "access_rights_url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/access_rights", - "key_id": "simcore/services/frontend/file-picker", # NEW: renames key -> key_id - "version": "1.0.0", - "id": "a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4", # NEW: alternative identifier to key_id:version - "integration-version": "1.0.0", - "type": "dynamic", - "badges_url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/badges", - "authors_url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/authors", - "inputs_url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/inputs", - "outputs_url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4/outputs", - "owner": "maiz@itis.swiss", # NEW, replaces "contact": "maiz@itis.swiss" - "url": "https://catalog:8080/services/a8f5a503-01d5-40bc-b416-f5b7cc5d1fa4", # NEW self - } - } - - ServiceResourcesGet = ServiceResourcesDict diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/services_ports.py b/packages/models-library/src/models_library/api_schemas_catalog/services_ports.py similarity index 76% rename from services/catalog/src/simcore_service_catalog/models/schemas/services_ports.py rename to packages/models-library/src/models_library/api_schemas_catalog/services_ports.py index 1df64110e11..ada65d69e28 100644 --- a/services/catalog/src/simcore_service_catalog/models/schemas/services_ports.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services_ports.py @@ -1,22 +1,18 @@ -from typing import Any, Literal, Optional, Union +from typing import Any, ClassVar, Literal -from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE -from models_library.services import ServiceInput, ServiceOutput -from models_library.utils.services_io import ( +from pydantic import BaseModel, Field + +from ..basic_regex import PUBLIC_VARIABLE_NAME_RE +from ..services import ServiceInput, ServiceOutput +from ..utils.services_io import ( get_service_io_json_schema, guess_media_type, update_schema_doc, ) -from pydantic import BaseModel, Field PortKindStr = Literal["input", "output"] -# -# Model ------------------------------------------------------------------------------- -# - - class ServicePortGet(BaseModel): key: str = Field( ..., @@ -25,14 +21,14 @@ class ServicePortGet(BaseModel): title="Key name", ) kind: PortKindStr - content_media_type: Optional[str] = None - content_schema: Optional[dict[str, Any]] = Field( + content_media_type: str | None = None + content_schema: dict[str, Any] | None = Field( None, description="jsonschema for the port's value. SEE https://json-schema.org/understanding-json-schema/", ) class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "key": "input_1", "kind": "input", @@ -51,7 +47,7 @@ def from_service_io( cls, kind: PortKindStr, key: str, - port: Union[ServiceInput, ServiceOutput], + port: ServiceInput | ServiceOutput, ) -> "ServicePortGet": kwargs: dict[str, Any] = {"key": key, "kind": kind} diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/services_specifications.py b/packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py similarity index 74% rename from services/catalog/src/simcore_service_catalog/models/schemas/services_specifications.py rename to packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py index 000903e56d2..aaa2b0489ae 100644 --- a/services/catalog/src/simcore_service_catalog/models/schemas/services_specifications.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services_specifications.py @@ -1,17 +1,14 @@ -from typing import Optional - -from models_library.generated_models.docker_rest_api import ( - ServiceSpec as DockerServiceSpec, -) from pydantic import BaseModel, Field +from ..generated_models.docker_rest_api import ServiceSpec as DockerServiceSpec + class ServiceSpecifications(BaseModel): - sidecar: Optional[DockerServiceSpec] = Field( + sidecar: DockerServiceSpec | None = Field( default=None, description="schedule-time specifications for the service sidecar (follows Docker Service creation API, see https://docs.docker.com/engine/api/v1.25/#operation/ServiceCreate)", ) - service: Optional[DockerServiceSpec] = Field( + service: DockerServiceSpec | None = Field( default=None, description="schedule-time specifications specifications for the service (follows Docker Service creation API (specifically only the Resources part), see https://docs.docker.com/engine/api/v1.41/#tag/Service/operation/ServiceCreate", ) diff --git a/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py b/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py index eb343ef361f..f95d91a6841 100644 --- a/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py +++ b/packages/models-library/src/models_library/api_schemas_long_running_tasks/base.py @@ -1,4 +1,5 @@ import logging +from typing import TypeAlias from pydantic import BaseModel, ConstrainedFloat, Field, validate_arguments, validator @@ -6,7 +7,7 @@ TaskId = str -ProgressMessage = str +ProgressMessage: TypeAlias = str class ProgressPercent(ConstrainedFloat): @@ -21,7 +22,7 @@ class TaskProgress(BaseModel): """ message: ProgressMessage = Field(default="") - percent: ProgressPercent = Field(default=ProgressPercent(0.0)) + percent: ProgressPercent = Field(default=0.0) # type: ignore[assignment] @validate_arguments def update( @@ -35,14 +36,15 @@ def update( self.message = message if percent: if not (0.0 <= percent <= 1.0): - raise ValueError(f"{percent=} must be in range [0.0, 1.0]") + msg = f"percent={percent!r} must be in range [0.0, 1.0]" + raise ValueError(msg) self.percent = percent _logger.debug("Progress update: %s", f"{self}") @classmethod def create(cls) -> "TaskProgress": - return cls.parse_obj(dict(message="", percent=0.0)) + return cls.parse_obj({"message": "", "percent": 0.0}) @validator("percent") @classmethod diff --git a/packages/models-library/src/models_library/api_schemas_storage.py b/packages/models-library/src/models_library/api_schemas_storage.py index a58f2f348c3..c9f3045e9b5 100644 --- a/packages/models-library/src/models_library/api_schemas_storage.py +++ b/packages/models-library/src/models_library/api_schemas_storage.py @@ -13,13 +13,6 @@ from typing import Any, ClassVar from uuid import UUID -from models_library.projects_nodes_io import ( - LocationID, - LocationName, - NodeID, - SimcoreS3FileID, - StorageFileID, -) from pydantic import ( BaseModel, ByteSize, @@ -34,6 +27,13 @@ from .basic_regex import DATCORE_DATASET_NAME_RE, S3_BUCKET_NAME_RE from .generics import ListModel +from .projects_nodes_io import ( + LocationID, + LocationName, + NodeID, + SimcoreS3FileID, + StorageFileID, +) ETag = str diff --git a/packages/models-library/src/models_library/api_schemas_webserver/_base.py b/packages/models-library/src/models_library/api_schemas_webserver/_base.py index 3a77b1b26f6..df5239a5d42 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/_base.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/_base.py @@ -33,6 +33,7 @@ class Config: def data( self, + *, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, @@ -49,6 +50,7 @@ def data( def data_json( self, + *, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 37253b4078d..5b961984f8b 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -10,13 +10,16 @@ from pydantic import Field, validator from ..api_schemas_long_running_tasks.tasks import TaskGet +from ..basic_types import HttpUrlWithCustomMinLength from ..emails import LowerCaseEmailStr from ..projects import ClassifierID, DateTimeStr, NodesDict, ProjectID from ..projects_access import AccessRights, GroupIDStr -from ..projects_nodes import HttpUrlWithCustomMinLength from ..projects_state import ProjectState from ..projects_ui import StudyUI -from ..utils.common_validators import empty_str_to_none, none_to_empty_str +from ..utils.common_validators import ( + empty_str_to_none_pre_validator, + none_to_empty_str_pre_validator, +) from ..utils.pydantic_tools_extension import FieldNotRequired from ._base import EmptyModel, InputSchema, OutputSchema from .permalinks import ProjectPermalink @@ -35,7 +38,7 @@ class ProjectCreateNew(InputSchema): _empty_is_none = validator( "uuid", "thumbnail", "description", allow_reuse=True, pre=True - )(empty_str_to_none) + )(empty_str_to_none_pre_validator) # NOTE: based on OVERRIDABLE_DOCUMENT_KEYS @@ -46,7 +49,7 @@ class ProjectCopyOverride(InputSchema): prj_owner: LowerCaseEmailStr _empty_is_none = validator("thumbnail", allow_reuse=True, pre=True)( - empty_str_to_none + empty_str_to_none_pre_validator ) @@ -69,7 +72,7 @@ class ProjectGet(OutputSchema): permalink: ProjectPermalink = FieldNotRequired() _empty_description = validator("description", allow_reuse=True, pre=True)( - none_to_empty_str + none_to_empty_str_pre_validator ) @@ -99,7 +102,7 @@ class ProjectReplace(InputSchema): ) _empty_is_none = validator("thumbnail", allow_reuse=True, pre=True)( - empty_str_to_none + empty_str_to_none_pre_validator ) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py index 414628258b6..a72a4b5ee8f 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py @@ -1,11 +1,12 @@ from datetime import datetime from enum import Enum -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from models_library.services import ServiceKey, ServiceVersion from pydantic import BaseModel +from ..projects import ProjectID +from ..projects_nodes_io import NodeID +from ..services import ServiceKey, ServiceVersion + # Frontend API diff --git a/packages/models-library/src/models_library/app_diagnostics.py b/packages/models-library/src/models_library/app_diagnostics.py index ca38769699b..ce8c9331eae 100644 --- a/packages/models-library/src/models_library/app_diagnostics.py +++ b/packages/models-library/src/models_library/app_diagnostics.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any from pydantic import AnyUrl, BaseModel, Field @@ -10,16 +10,16 @@ class AppStatusCheck(BaseModel): default={}, description="Other backend services connected from this service" ) - sessions: Optional[dict[str, Any]] = Field( + sessions: dict[str, Any] | None = Field( default={}, description="Client sessions info. If single session per app, then is denoted as main", ) - url: Optional[AnyUrl] = Field( + url: AnyUrl | None = Field( default=None, description="Link to current resource", ) - diagnostics_url: Optional[AnyUrl] = Field( + diagnostics_url: AnyUrl | None = Field( default=None, description="Link to diagnostics report sub-resource. This MIGHT take some time to compute", ) diff --git a/packages/models-library/src/models_library/basic_regex.py b/packages/models-library/src/models_library/basic_regex.py index 067786d3bd6..eb0af9b5646 100644 --- a/packages/models-library/src/models_library/basic_regex.py +++ b/packages/models-library/src/models_library/basic_regex.py @@ -61,7 +61,6 @@ # Accepted characters include both upper- and lower-case Ascii letters, # the digits 0 through 9, and the space character. # They may not be only numerals. -# SEE # https://www.twilio.com/docs/glossary/what-alphanumeric-sender-id # Docker diff --git a/packages/models-library/src/models_library/basic_types.py b/packages/models-library/src/models_library/basic_types.py index 0e47fd17794..71de37eac8e 100644 --- a/packages/models-library/src/models_library/basic_types.py +++ b/packages/models-library/src/models_library/basic_types.py @@ -54,6 +54,11 @@ class HttpSecureUrl(HttpUrl): allowed_schemes = {"https"} +class HttpUrlWithCustomMinLength(HttpUrl): + # Overwriting min length to be back compatible when generating OAS + min_length = 0 + + class LogLevel(str, Enum): DEBUG = "DEBUG" INFO = "INFO" diff --git a/packages/models-library/src/models_library/boot_options.py b/packages/models-library/src/models_library/boot_options.py index 35ca89ebbab..ec1aabd546b 100644 --- a/packages/models-library/src/models_library/boot_options.py +++ b/packages/models-library/src/models_library/boot_options.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Any, ClassVar from pydantic import BaseModel, validator from typing_extensions import TypedDict @@ -15,20 +15,19 @@ class BootOption(BaseModel): label: str description: str default: str - items: Dict[str, BootChoice] + items: dict[str, BootChoice] @validator("items") @classmethod def ensure_default_included(cls, v, values): default = values["default"] if default not in v: - raise ValueError( - f"Expected default={default} to be present a key of items={v}" - ) + msg = f"Expected default={default} to be present a key of items={v}" + raise ValueError(msg) return v class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "label": "Boot mode", @@ -64,4 +63,4 @@ class Config: } -BootOptions = Dict[EnvVarKey, BootOption] +BootOptions = dict[EnvVarKey, BootOption] diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index f1e75939f63..146773e74f6 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -1,10 +1,28 @@ -from typing import Final, Literal, TypeAlias, Union - -from pydantic import AnyUrl, BaseModel, Extra, Field, HttpUrl, SecretStr, root_validator +from enum import auto +from typing import Any, ClassVar, Final, Literal, TypeAlias, Union + +from models_library.utils.common_validators import create_enums_pre_validator +from pydantic import ( + AnyUrl, + BaseModel, + Extra, + Field, + HttpUrl, + SecretStr, + root_validator, + validator, +) from pydantic.types import NonNegativeInt -from simcore_postgres_database.models.clusters import ClusterType from .users import GroupID +from .utils.enums import StrAutoEnum + + +class ClusterTypeInModel(StrAutoEnum): + # This enum is equivalent to `simcore_postgres_database.models.clusters.ClusterType` + # SEE models-library/tests/test__pydantic_models_and_enums.py + AWS = auto() + ON_PREMISE = auto() class ClusterAccessRights(BaseModel): @@ -35,7 +53,7 @@ class SimpleAuthentication(BaseAuthentication): password: SecretStr class Config(BaseAuthentication.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "type": "simple", @@ -51,7 +69,7 @@ class KerberosAuthentication(BaseAuthentication): # NOTE: the entries here still need to be defined class Config(BaseAuthentication.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "type": "kerberos", @@ -65,7 +83,7 @@ class JupyterHubTokenAuthentication(BaseAuthentication): api_token: str class Config(BaseAuthentication.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ {"type": "jupyterhub", "api_token": "some_jupyterhub_token"}, ] @@ -76,11 +94,11 @@ class NoAuthentication(BaseAuthentication): type: Literal["none"] = "none" -InternalClusterAuthentication = NoAuthentication -ExternalClusterAuthentication = Union[ +InternalClusterAuthentication: TypeAlias = NoAuthentication +ExternalClusterAuthentication: TypeAlias = Union[ SimpleAuthentication, KerberosAuthentication, JupyterHubTokenAuthentication ] -ClusterAuthentication = Union[ +ClusterAuthentication: TypeAlias = Union[ ExternalClusterAuthentication, InternalClusterAuthentication, ] @@ -89,7 +107,7 @@ class NoAuthentication(BaseAuthentication): class BaseCluster(BaseModel): name: str = Field(..., description="The human readable name of the cluster") description: str | None = None - type: ClusterType + type: ClusterTypeInModel owner: GroupID thumbnail: HttpUrl | None = Field( None, @@ -102,6 +120,10 @@ class BaseCluster(BaseModel): ) access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict) + _from_equivalent_enums = validator("type", allow_reuse=True, pre=True)( + create_enums_pre_validator(ClusterTypeInModel) + ) + class Config: extra = Extra.forbid use_enum_values = True @@ -115,12 +137,12 @@ class Cluster(BaseCluster): id: ClusterID = Field(..., description="The cluster ID") class Config(BaseCluster.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "id": DEFAULT_CLUSTER_ID, "name": "The default cluster", - "type": ClusterType.ON_PREMISE, + "type": ClusterTypeInModel.ON_PREMISE, "owner": 1456, "endpoint": "tcp://default-dask-scheduler:8786", "authentication": { @@ -132,7 +154,7 @@ class Config(BaseCluster.Config): { "id": 432, "name": "My awesome cluster", - "type": ClusterType.ON_PREMISE, + "type": ClusterTypeInModel.ON_PREMISE, "owner": 12, "endpoint": "https://registry.osparc-development.fake.dev", "authentication": { @@ -145,7 +167,7 @@ class Config(BaseCluster.Config): "id": 432546, "name": "My AWS cluster", "description": "a AWS cluster administered by me", - "type": ClusterType.AWS, + "type": ClusterTypeInModel.AWS, "owner": 154, "endpoint": "https://registry.osparc-development.fake.dev", "authentication": {"type": "kerberos"}, @@ -159,7 +181,7 @@ class Config(BaseCluster.Config): "id": 325436, "name": "My AWS cluster", "description": "a AWS cluster administered by me", - "type": ClusterType.AWS, + "type": ClusterTypeInModel.AWS, "owner": 2321, "endpoint": "https://registry.osparc-development.fake2.dev", "authentication": { @@ -191,8 +213,7 @@ def check_owner_has_access_rights(cls, values): if access_rights[owner_gid] != ( CLUSTER_USER_RIGHTS if is_default_cluster else CLUSTER_ADMIN_RIGHTS ): - raise ValueError( - f"the cluster owner access rights are incorrectly set: {access_rights[owner_gid]}" - ) + msg = f"the cluster owner access rights are incorrectly set: {access_rights[owner_gid]}" + raise ValueError(msg) values["access_rights"] = access_rights return values diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index b5a674a18c4..a6a2d649a16 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -2,11 +2,6 @@ import re from typing import Any, ClassVar, Final -from models_library.generated_models.docker_rest_api import Task -from models_library.products import ProductName -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from models_library.users import UserID from pydantic import ( BaseModel, ByteSize, @@ -18,6 +13,11 @@ ) from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX +from .generated_models.docker_rest_api import Task +from .products import ProductName +from .projects import ProjectID +from .projects_nodes_io import NodeID +from .users import UserID class DockerLabelKey(ConstrainedStr): diff --git a/packages/models-library/src/models_library/errors.py b/packages/models-library/src/models_library/errors.py index 505ad1dc316..7386c6b059a 100644 --- a/packages/models-library/src/models_library/errors.py +++ b/packages/models-library/src/models_library/errors.py @@ -1,6 +1,6 @@ -from typing import Any, TypedDict, Union +from typing import Any, TypedDict -Loc = tuple[Union[int, str], ...] +Loc = tuple[int | str, ...] class _ErrorDictRequired(TypedDict): diff --git a/packages/models-library/src/models_library/function_services_catalog/_registry.py b/packages/models-library/src/models_library/function_services_catalog/_registry.py index ad1d5635308..dc006e9fb43 100644 --- a/packages/models-library/src/models_library/function_services_catalog/_registry.py +++ b/packages/models-library/src/models_library/function_services_catalog/_registry.py @@ -20,7 +20,7 @@ probes, ) -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) catalog = FunctionServices(settings=FunctionServiceSettings()) diff --git a/packages/models-library/src/models_library/function_services_catalog/_utils.py b/packages/models-library/src/models_library/function_services_catalog/_utils.py index 26eaa13e6c8..0535c3096e7 100644 --- a/packages/models-library/src/models_library/function_services_catalog/_utils.py +++ b/packages/models-library/src/models_library/function_services_catalog/_utils.py @@ -1,12 +1,12 @@ import logging +from collections.abc import Callable, Iterator from dataclasses import dataclass -from typing import Callable, Dict, Iterator, Optional, Tuple from urllib.parse import quote from ..services import Author, ServiceDockerData, ServiceKey, ServiceVersion from ._settings import AUTHORS, FunctionServiceSettings -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) _DEFAULT = { @@ -30,32 +30,34 @@ class ServiceNotFound(KeyError): @dataclass class _Record: meta: ServiceDockerData - implementation: Optional[Callable] = None + implementation: Callable | None = None is_under_development: bool = False class FunctionServices: """Used to register a collection of function services""" - def __init__(self, settings: Optional[FunctionServiceSettings] = None): - self._functions: Dict[Tuple[ServiceKey, ServiceVersion], _Record] = {} + def __init__(self, settings: FunctionServiceSettings | None = None): + self._functions: dict[tuple[ServiceKey, ServiceVersion], _Record] = {} self.settings = settings def add( self, meta: ServiceDockerData, - implementation: Optional[Callable] = None, + implementation: Callable | None = None, is_under_development: bool = False, ): """ raises ValueError """ if not isinstance(meta, ServiceDockerData): - raise ValueError(f"Expected ServiceDockerData, got {type(meta)}") + msg = f"Expected ServiceDockerData, got {type(meta)}" + raise ValueError(msg) # ensure unique if (meta.key, meta.version) in self._functions: - raise ValueError(f"{(meta.key, meta.version)} is already registered") + msg = f"{meta.key, meta.version} is already registered" + raise ValueError(msg) # TODO: ensure callable signature fits metadata @@ -77,7 +79,7 @@ def _skip_dev(self): skip = not self.settings.is_dev_feature_enabled() return skip - def _items(self) -> Iterator[Tuple[Tuple[ServiceKey, ServiceVersion], _Record]]: + def _items(self) -> Iterator[tuple[tuple[ServiceKey, ServiceVersion], _Record]]: skip_dev = self._skip_dev() for key, value in self._functions.items(): if value.is_under_development and skip_dev: @@ -89,7 +91,7 @@ def iter_metadata(self) -> Iterator[ServiceDockerData]: for _, f in self._items(): yield f.meta - def iter_services_key_version(self) -> Iterator[Tuple[ServiceKey, ServiceVersion]]: + def iter_services_key_version(self) -> Iterator[tuple[ServiceKey, ServiceVersion]]: """WARNING: this function might skip services makred as 'under development'""" for kv, f in self._items(): assert kv == (f.meta.key, f.meta.version) # nosec @@ -97,14 +99,13 @@ def iter_services_key_version(self) -> Iterator[Tuple[ServiceKey, ServiceVersion def get_implementation( self, service_key: ServiceKey, service_version: ServiceVersion - ) -> Optional[Callable]: + ) -> Callable | None: """raises ServiceNotFound""" try: func = self._functions[(service_key, service_version)] except KeyError as err: - raise ServiceNotFound( - f"{service_key}:{service_version} not found in registry" - ) from err + msg = f"{service_key}:{service_version} not found in registry" + raise ServiceNotFound(msg) from err return func.implementation def get_metadata( @@ -114,9 +115,8 @@ def get_metadata( try: func = self._functions[(service_key, service_version)] except KeyError as err: - raise ServiceNotFound( - f"{service_key}:{service_version} not found in registry" - ) from err + msg = f"{service_key}:{service_version} not found in registry" + raise ServiceNotFound(msg) from err return func.meta def __len__(self): diff --git a/packages/models-library/src/models_library/function_services_catalog/api.py b/packages/models-library/src/models_library/function_services_catalog/api.py index ca75107d587..708671609fb 100644 --- a/packages/models-library/src/models_library/function_services_catalog/api.py +++ b/packages/models-library/src/models_library/function_services_catalog/api.py @@ -6,7 +6,7 @@ director2->catalog, it was decided to share as a library """ -from typing import Iterator, Tuple +from collections.abc import Iterator from ..services import ServiceDockerData from ._key_labels import is_function_service, is_iterator_service @@ -24,7 +24,7 @@ def iter_service_docker_data() -> Iterator[ServiceDockerData]: yield copied_meta_obj -__all__: Tuple[str, ...] = ( +__all__: tuple[str, ...] = ( "catalog", "is_function_service", "is_iterator_service", diff --git a/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py b/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py index 4267ff6881d..b644c71af4f 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/iter_range.py @@ -1,4 +1,4 @@ -from typing import Iterator +from collections.abc import Iterator from ...projects_nodes import OutputID, OutputsDict from ...services import LATEST_INTEGRATION_VERSION, ServiceDockerData, ServiceType diff --git a/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py b/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py index 955735547ea..7df55e8bfb9 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/iter_sensitivity.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from copy import deepcopy -from typing import Any, Iterator +from typing import Any from pydantic import schema_of diff --git a/packages/models-library/src/models_library/function_services_catalog/services/parameters.py b/packages/models-library/src/models_library/function_services_catalog/services/parameters.py index 4d63883f66e..fde17e8dd8f 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/parameters.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/parameters.py @@ -1,12 +1,10 @@ -from typing import Optional - from ...services import LATEST_INTEGRATION_VERSION, ServiceDockerData, ServiceType from .._key_labels import FUNCTION_SERVICE_KEY_PREFIX from .._utils import OM, FunctionServices, create_fake_thumbnail_url def create_metadata( - output_type: str, output_name: Optional[str] = None + output_type: str, output_name: str | None = None ) -> ServiceDockerData: """ Represents a parameter (e.g. "x":5) in a study @@ -46,9 +44,9 @@ def create_metadata( return meta -META_NUMBER, META_BOOL, META_INT, META_STR = [ +META_NUMBER, META_BOOL, META_INT, META_STR = ( create_metadata(output_type=t) for t in ("number", "boolean", "integer", "string") -] +) META_ARRAY = ServiceDockerData.parse_obj( { diff --git a/packages/models-library/src/models_library/function_services_catalog/services/probes.py b/packages/models-library/src/models_library/function_services_catalog/services/probes.py index c41f957523d..8b5d5d03623 100644 --- a/packages/models-library/src/models_library/function_services_catalog/services/probes.py +++ b/packages/models-library/src/models_library/function_services_catalog/services/probes.py @@ -1,11 +1,9 @@ -from typing import Optional - from ...services import LATEST_INTEGRATION_VERSION, ServiceDockerData, ServiceType from .._key_labels import FUNCTION_SERVICE_KEY_PREFIX from .._utils import OM, FunctionServices, create_fake_thumbnail_url -def create_metadata(type_name: str, prefix: Optional[str] = None) -> ServiceDockerData: +def create_metadata(type_name: str, prefix: str | None = None) -> ServiceDockerData: prefix = prefix or type_name LABEL = f"{type_name.capitalize()} probe" @@ -35,9 +33,9 @@ def create_metadata(type_name: str, prefix: Optional[str] = None) -> ServiceDock ) -META_NUMBER, META_BOOL, META_INT, META_STR = [ +META_NUMBER, META_BOOL, META_INT, META_STR = ( create_metadata(t) for t in ("number", "boolean", "integer", "string") -] +) META_ARRAY = ServiceDockerData.parse_obj( { diff --git a/packages/models-library/src/models_library/generated_models/docker_rest_api.py b/packages/models-library/src/models_library/generated_models/docker_rest_api.py index 93434edd9dc..835141ea037 100644 --- a/packages/models-library/src/models_library/generated_models/docker_rest_api.py +++ b/packages/models-library/src/models_library/generated_models/docker_rest_api.py @@ -1883,8 +1883,6 @@ class File1(File): """ - pass - class Config1(BaseModel): File: File1 | None = Field( @@ -2833,7 +2831,7 @@ class Type5(str, Enum): network = "network" node = "node" plugin = "plugin" - secret = "secret" + secret = "secret" # nosec service = "service" volume = "volume" diff --git a/packages/models-library/src/models_library/generics.py b/packages/models-library/src/models_library/generics.py index 07556cb6d27..cb9a73e4e59 100644 --- a/packages/models-library/src/models_library/generics.py +++ b/packages/models-library/src/models_library/generics.py @@ -1,13 +1,5 @@ -from typing import ( - Any, - Generic, - ItemsView, - Iterable, - Iterator, - KeysView, - TypeVar, - ValuesView, -) +from collections.abc import ItemsView, Iterable, Iterator, KeysView, ValuesView +from typing import Any, Generic, TypeVar from pydantic import validator from pydantic.generics import GenericModel diff --git a/packages/models-library/src/models_library/groups.py b/packages/models-library/src/models_library/groups.py index 6de0d571db8..e5ae95fc1aa 100644 --- a/packages/models-library/src/models_library/groups.py +++ b/packages/models-library/src/models_library/groups.py @@ -1 +1,47 @@ -EVERYONE_GROUP_ID = 1 +import enum +from typing import Any, ClassVar, Final + +from models_library.utils.common_validators import create_enums_pre_validator +from pydantic import BaseModel, Field, validator +from pydantic.types import PositiveInt + +EVERYONE_GROUP_ID: Final[int] = 1 + + +class GroupTypeInModel(str, enum.Enum): # noqa: SLOT000 + """ + standard: standard group, e.g. any group that is not a primary group or special group such as the everyone group + primary: primary group, e.g. the primary group is the user own defined group that typically only contain the user (same as in linux) + everyone: the only group for all users + """ + + STANDARD = "standard" + PRIMARY = "primary" + EVERYONE = "everyone" + + +class Group(BaseModel): + gid: PositiveInt + name: str + description: str + group_type: GroupTypeInModel = Field(..., alias="type") + thumbnail: str | None + + _from_equivalent_enums = validator("group_type", allow_reuse=True, pre=True)( + create_enums_pre_validator(GroupTypeInModel) + ) + + +class GroupAtDB(Group): + class Config: + orm_mode = True + + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + "gid": 218, + "name": "Friends group", + "description": "Joey, Ross, Rachel, Monica, Phoeby and Chandler", + "type": "standard", + "thumbnail": "https://image.flaticon.com/icons/png/512/23/23374.png", + } + } diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index c032cc68877..bb1d66ea142 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -8,16 +8,20 @@ from typing import Any, TypeAlias from uuid import UUID -from models_library.utils.common_validators import empty_str_to_none, none_to_empty_str from pydantic import BaseModel, ConstrainedStr, Extra, Field, validator from .basic_regex import DATE_RE, UUID_RE_BASE +from .basic_types import HttpUrlWithCustomMinLength from .emails import LowerCaseEmailStr from .projects_access import AccessRights, GroupIDStr -from .projects_nodes import HttpUrlWithCustomMinLength, Node +from .projects_nodes import Node from .projects_nodes_io import NodeIDStr from .projects_state import ProjectState from .projects_ui import StudyUI +from .utils.common_validators import ( + empty_str_to_none_pre_validator, + none_to_empty_str_pre_validator, +) ProjectID: TypeAlias = UUID ClassifierID: TypeAlias = str @@ -77,11 +81,11 @@ class BaseProjectModel(BaseModel): # validators _empty_thumbnail_is_none = validator("thumbnail", allow_reuse=True, pre=True)( - empty_str_to_none + empty_str_to_none_pre_validator ) _none_description_is_empty = validator("description", allow_reuse=True, pre=True)( - none_to_empty_str + none_to_empty_str_pre_validator ) diff --git a/packages/models-library/src/models_library/projects_comments.py b/packages/models-library/src/models_library/projects_comments.py index a158b60e469..234ec638a4a 100644 --- a/packages/models-library/src/models_library/projects_comments.py +++ b/packages/models-library/src/models_library/projects_comments.py @@ -1,10 +1,11 @@ from datetime import datetime from typing import TypeAlias -from models_library.projects import ProjectID -from models_library.users import UserID from pydantic import BaseModel, Extra, Field, PositiveInt +from .projects import ProjectID +from .users import UserID + CommentID: TypeAlias = PositiveInt diff --git a/packages/models-library/src/models_library/projects_networks.py b/packages/models-library/src/models_library/projects_networks.py index e69cef83fbd..e0775ccb5d5 100644 --- a/packages/models-library/src/models_library/projects_networks.py +++ b/packages/models-library/src/models_library/projects_networks.py @@ -1,10 +1,10 @@ import re -from typing import Final +from typing import Any, ClassVar, Final -from models_library.projects import ProjectID from pydantic import BaseModel, ConstrainedStr, Field from .generics import DictModel +from .projects import ProjectID from .projects_nodes_io import NodeIDStr SERVICE_NETWORK_RE: Final[re.Pattern] = re.compile(r"^[a-zA-Z]([a-zA-Z0-9_-]{0,63})$") @@ -26,7 +26,7 @@ class ContainerAliases(DictModel[NodeIDStr, DockerNetworkAlias]): class NetworksWithAliases(DictModel[DockerNetworkName, ContainerAliases]): class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "network_one": { @@ -50,7 +50,7 @@ class ProjectsNetworks(BaseModel): class Config: orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "project_uuid": "ec5cdfea-f24e-4aa1-83b8-6dccfdc8cf4d", "networks_with_aliases": { diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 74bafc7b579..85f4aa496fb 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -4,14 +4,13 @@ import re from copy import deepcopy -from typing import Any, TypeAlias, Union +from typing import Any, ClassVar, TypeAlias, Union from pydantic import ( BaseModel, ConstrainedStr, Extra, Field, - HttpUrl, Json, StrictBool, StrictFloat, @@ -19,7 +18,7 @@ validator, ) -from .basic_types import EnvVarKey +from .basic_types import EnvVarKey, HttpUrlWithCustomMinLength from .projects_access import AccessEnum from .projects_nodes_io import ( DatCoreFileLink, @@ -41,9 +40,9 @@ Json, # FIXME: remove if OM sends object/array. create project does NOT use pydantic str, PortLink, - Union[SimCoreFileLink, DatCoreFileLink], # *FileLink to service + SimCoreFileLink | DatCoreFileLink, # *FileLink to service DownloadLink, - Union[list[Any], dict[str, Any]], # arrays | object + list[Any] | dict[str, Any], # arrays | object ] OutputTypes = Union[ StrictBool, @@ -51,9 +50,9 @@ StrictFloat, Json, # TODO: remove when OM sends object/array instead of json-formatted strings str, - Union[SimCoreFileLink, DatCoreFileLink], # *FileLink to service + SimCoreFileLink | DatCoreFileLink, # *FileLink to service DownloadLink, - Union[list[Any], dict[str, Any]], # arrays | object + list[Any] | dict[str, Any], # arrays | object ] @@ -94,7 +93,7 @@ class NodeState(BaseModel): class Config: extra = Extra.forbid - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "modified": True, @@ -115,11 +114,6 @@ class Config: } -class HttpUrlWithCustomMinLength(HttpUrl): - # Overwriting min length to be back compatible when generating OAS - min_length = 0 - - class Node(BaseModel): key: ServiceKey = Field( ..., @@ -238,7 +232,6 @@ def convert_from_enum(cls, v): class Config: extra = Extra.forbid - # NOTE: exporting without this trick does not make runHash as nullable. # It is a Pydantic issue see https://github.com/samuelcolvin/pydantic/issues/1270 @staticmethod diff --git a/packages/models-library/src/models_library/projects_pipeline.py b/packages/models-library/src/models_library/projects_pipeline.py index 1f4e2eefa67..2139d182043 100644 --- a/packages/models-library/src/models_library/projects_pipeline.py +++ b/packages/models-library/src/models_library/projects_pipeline.py @@ -1,4 +1,5 @@ import datetime +from typing import Any, ClassVar from uuid import UUID import arrow @@ -58,7 +59,7 @@ class ComputationTask(BaseModel): ) class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "id": "42838344-03de-4ce2-8d93-589a5dcdfd05", diff --git a/packages/models-library/src/models_library/projects_ui.py b/packages/models-library/src/models_library/projects_ui.py index fc6e217cc58..154007a2a6d 100644 --- a/packages/models-library/src/models_library/projects_ui.py +++ b/packages/models-library/src/models_library/projects_ui.py @@ -2,14 +2,14 @@ Models Front-end UI """ -from typing import Literal, TypedDict +from typing import Any, ClassVar, Literal, TypedDict from pydantic import BaseModel, Extra, Field, validator from pydantic.color import Color from .projects_nodes_io import NodeID, NodeIDStr from .projects_nodes_ui import Marker, Position -from .utils.common_validators import empty_str_to_none +from .utils.common_validators import empty_str_to_none_pre_validator class WorkbenchUI(BaseModel): @@ -35,7 +35,7 @@ class Annotation(BaseModel): class Config: extra = Extra.forbid - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "type": "note", @@ -46,7 +46,7 @@ class Config: "width": 117, "height": 26, "destinataryGid": 4, - "text": "ToDo" + "text": "ToDo", }, }, { @@ -72,4 +72,6 @@ class StudyUI(BaseModel): class Config: extra = Extra.allow - _empty_is_none = validator("*", allow_reuse=True, pre=True)(empty_str_to_none) + _empty_is_none = validator("*", allow_reuse=True, pre=True)( + empty_str_to_none_pre_validator + ) diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 413528c045c..af019da6950 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -3,14 +3,15 @@ from enum import Enum, auto from typing import Any, Literal, TypeAlias -from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID -from models_library.projects_state import RunningState -from models_library.users import UserID -from models_library.utils.enums import StrAutoEnum from pydantic import BaseModel, Field from pydantic.types import NonNegativeFloat +from .projects import ProjectID +from .projects_nodes_io import NodeID +from .projects_state import RunningState +from .users import UserID +from .utils.enums import StrAutoEnum + LogLevelInt: TypeAlias = int LogMessageStr: TypeAlias = str diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index 2cff81a89bd..4f08282bdee 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -1,4 +1,4 @@ -from typing import Final, Generic, TypeVar +from typing import Any, ClassVar, Final, Generic, TypeVar from pydantic import ( AnyHttpUrl, @@ -20,6 +20,20 @@ assert DEFAULT_NUMBER_OF_ITEMS_PER_PAGE < MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE # nosec +class PageQueryParameters(BaseModel): + """Use as pagination options in query parameters""" + + limit: int = Field( + default=DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + description="maximum number of items to return (pagination)", + ge=1, + lt=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, + ) + offset: NonNegativeInt = Field( + default=0, description="index to the first item to return (pagination)" + ) + + class PageMetaInfoLimitOffset(BaseModel): limit: PositiveInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE total: NonNegativeInt @@ -53,7 +67,7 @@ def check_count(cls, v, values): class Config: extra = Extra.forbid - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ {"total": 7, "count": 4, "limit": 4, "offset": 0}, ] @@ -95,17 +109,17 @@ def convert_none_to_empty_list(cls, v): def check_data_compatible_with_meta(cls, v, values): if "meta" not in values: # if the validation failed in meta this happens - raise ValueError("meta not in values") + msg = "meta not in values" + raise ValueError(msg) if len(v) != values["meta"].count: - raise ValueError( - f"container size [{len(v)}] must be equal to count [{values['meta'].count}]" - ) + msg = f"container size [{len(v)}] must be equal to count [{values['meta'].count}]" + raise ValueError(msg) return v class Config: extra = Extra.forbid - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ # first page Page[str] { diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 836e7d5c9d9..c6c5535ca6a 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -7,7 +7,7 @@ import re from datetime import datetime from enum import Enum -from typing import Any, Final +from typing import Any, ClassVar, Final from uuid import uuid4 import arrow @@ -276,12 +276,9 @@ class Config: @validator("content_schema") @classmethod def check_type_is_set_to_schema(cls, v, values): - if v is not None: - if (ptype := values["property_type"]) != "ref_contentSchema": - raise ValueError( - "content_schema is defined but set the wrong type." - f"Expected type=ref_contentSchema but got ={ptype}." - ) + if v is not None and (ptype := values["property_type"]) != "ref_contentSchema": + msg = f"content_schema is defined but set the wrong type.Expected type=ref_contentSchema but got ={ptype}." + raise ValueError(msg) return v @validator("content_schema") @@ -293,13 +290,13 @@ def check_valid_json_schema(cls, v): if any_ref_key(v): # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3030 - raise ValueError("Schemas with $ref are still not supported") + msg = "Schemas with $ref are still not supported" + raise ValueError(msg) except InvalidJsonSchema as err: failed_path = "->".join(map(str, err.path)) - raise ValueError( - f"Invalid json-schema at {failed_path}: {err.message}" - ) from err + msg = f"Invalid json-schema at {failed_path}: {err.message}" + raise ValueError(msg) from err return v @classmethod @@ -307,13 +304,12 @@ def _from_json_schema_base_implementation( cls, port_schema: dict[str, Any] ) -> dict[str, Any]: description = port_schema.pop("description", port_schema["title"]) - data = { + return { "label": port_schema["title"], "description": description, "type": "ref_contentSchema", "contentSchema": port_schema, } - return data class ServiceInput(BaseServiceIOModel): @@ -332,7 +328,7 @@ class ServiceInput(BaseServiceIOModel): ) class Config(BaseServiceIOModel.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ # file-wo-widget: { @@ -403,7 +399,7 @@ class ServiceOutput(BaseServiceIOModel): ) class Config(BaseServiceIOModel.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "displayOrder": 2, @@ -545,7 +541,7 @@ class Config: extra = Extra.forbid frozen = False # it inherits from ServiceKeyVersion. - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { "name": "oSparc Python Runner", @@ -659,7 +655,7 @@ class ServiceMetaData(_BaseServiceCommonDataModel): quality: dict[str, Any] = {} class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", diff --git a/packages/models-library/src/models_library/services_access.py b/packages/models-library/src/models_library/services_access.py index c8daca69255..9e121fad95a 100644 --- a/packages/models-library/src/models_library/services_access.py +++ b/packages/models-library/src/models_library/services_access.py @@ -1,7 +1,6 @@ """Service access rights models """ -from typing import Optional from pydantic import BaseModel, Field from pydantic.types import PositiveInt @@ -20,7 +19,7 @@ class ServiceGroupAccessRights(BaseModel): class ServiceAccessRights(BaseModel): - access_rights: Optional[dict[GroupId, ServiceGroupAccessRights]] = Field( + access_rights: dict[GroupId, ServiceGroupAccessRights] | None = Field( None, alias="accessRights", description="service access rights per group id", diff --git a/packages/models-library/src/models_library/services_db.py b/packages/models-library/src/models_library/services_db.py index bdfbd96a832..0e5353353ae 100644 --- a/packages/models-library/src/models_library/services_db.py +++ b/packages/models-library/src/models_library/services_db.py @@ -3,7 +3,8 @@ NOTE: to dump json-schema from CLI use python -c "from models_library.services import ServiceDockerData as cls; print(cls.schema_json(indent=2))" > services-schema.json """ -from typing import Optional + +from typing import Any, ClassVar from pydantic import Field from pydantic.types import PositiveInt @@ -19,12 +20,12 @@ class ServiceMetaDataAtDB(ServiceKeyVersion, ServiceMetaData): # for a partial update all members must be Optional - classifiers: Optional[list[str]] = Field([]) - owner: Optional[PositiveInt] + classifiers: list[str] | None = Field([]) + owner: PositiveInt | None class Config: orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", @@ -64,7 +65,7 @@ class ServiceAccessRightsAtDB(ServiceKeyVersion, ServiceGroupAccessRights): class Config: orm_mode = True - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "key": "simcore/services/dynamic/sim4life", "version": "1.0.9", diff --git a/packages/models-library/src/models_library/services_enums.py b/packages/models-library/src/models_library/services_enums.py new file mode 100644 index 00000000000..8a55c0a960b --- /dev/null +++ b/packages/models-library/src/models_library/services_enums.py @@ -0,0 +1,42 @@ +import functools +from enum import Enum, unique + + +@unique +class ServiceBootType(str, Enum): + V0 = "V0" + V2 = "V2" + + +@functools.total_ordering +@unique +class ServiceState(Enum): + PENDING = "pending" + PULLING = "pulling" + STARTING = "starting" + RUNNING = "running" + COMPLETE = "complete" + FAILED = "failed" + STOPPING = "stopping" + + def __lt__(self, other): + if self.__class__ is other.__class__: + comparison_order = ServiceState.comparison_order() + self_index = comparison_order[self] + other_index = comparison_order[other] + return self_index < other_index + return NotImplemented + + @staticmethod + @functools.lru_cache(maxsize=2) + def comparison_order() -> dict["ServiceState", int]: + """States are comparable to supportmin() on a list of ServiceState""" + return { + ServiceState.FAILED: 0, + ServiceState.PENDING: 1, + ServiceState.PULLING: 2, + ServiceState.STARTING: 3, + ServiceState.RUNNING: 4, + ServiceState.STOPPING: 5, + ServiceState.COMPLETE: 6, + } diff --git a/packages/models-library/src/models_library/services_resources.py b/packages/models-library/src/models_library/services_resources.py index 26e6d4fb183..b7e2b3315b9 100644 --- a/packages/models-library/src/models_library/services_resources.py +++ b/packages/models-library/src/models_library/services_resources.py @@ -1,9 +1,7 @@ import logging from enum import auto -from typing import Any, Final, TypeAlias +from typing import Any, ClassVar, Final, TypeAlias -from models_library.docker import DockerGenericTag -from models_library.utils.enums import StrAutoEnum from pydantic import ( BaseModel, ByteSize, @@ -14,9 +12,11 @@ root_validator, ) +from .docker import DockerGenericTag +from .utils.enums import StrAutoEnum from .utils.fastapi_encoders import jsonable_encoder -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) ResourceName = str @@ -90,7 +90,7 @@ def set_reservation_same_as_limit(self) -> None: resource.set_reservation_same_as_limit() class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "image": "simcore/service/dynamic/pretty-intense:1.0.0", "resources": { @@ -138,7 +138,7 @@ def create_jsonable( return output class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "examples": [ # no compose spec (majority of services) { diff --git a/packages/models-library/src/models_library/services_ui.py b/packages/models-library/src/models_library/services_ui.py index 2933c09a212..22196693334 100644 --- a/packages/models-library/src/models_library/services_ui.py +++ b/packages/models-library/src/models_library/services_ui.py @@ -1,5 +1,4 @@ from enum import Enum -from typing import Union from pydantic import BaseModel, Extra, Field from pydantic.types import PositiveInt @@ -20,7 +19,7 @@ class Config: class Structure(BaseModel): - key: Union[str, bool, float] + key: str | bool | float label: str class Config: @@ -38,7 +37,7 @@ class Widget(BaseModel): widget_type: WidgetType = Field( ..., alias="type", description="type of the property" ) - details: Union[TextArea, SelectBox] + details: TextArea | SelectBox class Config: extra = Extra.forbid diff --git a/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py b/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py index b51096b6bbe..95b074d34bb 100644 --- a/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py +++ b/packages/models-library/src/models_library/utils/_original_fastapi_encoders.py @@ -1,26 +1,26 @@ # pylint: disable-all -# nopycln: file # # wget https://raw.githubusercontent.com/tiangolo/fastapi/master/fastapi/encoders.py --output-document=_original_fastapi_encoders # import dataclasses from collections import defaultdict +from collections.abc import Callable from enum import Enum from pathlib import PurePath from types import GeneratorType -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any from pydantic import BaseModel from pydantic.json import ENCODERS_BY_TYPE -SetIntStr = Set[Union[int, str]] -DictIntStrAny = Dict[Union[int, str], Any] +SetIntStr = set[int | str] +DictIntStrAny = dict[int | str, Any] def generate_encoders_by_class_tuples( - type_encoder_map: Dict[Any, Callable[[Any], Any]] -) -> Dict[Callable[[Any], Any], Tuple[Any, ...]]: - encoders_by_class_tuples: Dict[Callable[[Any], Any], Tuple[Any, ...]] = defaultdict( + type_encoder_map: dict[Any, Callable[[Any], Any]] +) -> dict[Callable[[Any], Any], tuple[Any, ...]]: + encoders_by_class_tuples: dict[Callable[[Any], Any], tuple[Any, ...]] = defaultdict( tuple ) for type_, encoder in type_encoder_map.items(): @@ -33,13 +33,13 @@ def generate_encoders_by_class_tuples( def jsonable_encoder( obj: Any, - include: Optional[Union[SetIntStr, DictIntStrAny]] = None, - exclude: Optional[Union[SetIntStr, DictIntStrAny]] = None, + include: SetIntStr | DictIntStrAny | None = None, + exclude: SetIntStr | DictIntStrAny | None = None, by_alias: bool = True, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, - custom_encoder: Optional[Dict[Any, Callable[[Any], Any]]] = None, + custom_encoder: dict[Any, Callable[[Any], Any]] | None = None, sqlalchemy_safe: bool = True, ) -> Any: custom_encoder = custom_encoder or {} @@ -50,9 +50,9 @@ def jsonable_encoder( for encoder_type, encoder_instance in custom_encoder.items(): if isinstance(obj, encoder_type): return encoder_instance(obj) - if include is not None and not isinstance(include, (set, dict)): + if include is not None and not isinstance(include, set | dict): include = set(include) - if exclude is not None and not isinstance(exclude, (set, dict)): + if exclude is not None and not isinstance(exclude, set | dict): exclude = set(exclude) if isinstance(obj, BaseModel): encoder = getattr(obj.__config__, "json_encoders", {}) @@ -92,7 +92,7 @@ def jsonable_encoder( return obj.value if isinstance(obj, PurePath): return str(obj) - if isinstance(obj, (str, int, float, type(None))): + if isinstance(obj, str | int | float | type(None)): return obj if isinstance(obj, dict): encoded_dict = {} @@ -129,7 +129,7 @@ def jsonable_encoder( ) encoded_dict[encoded_key] = encoded_value return encoded_dict - if isinstance(obj, (list, set, frozenset, GeneratorType, tuple)): + if isinstance(obj, list | set | frozenset | GeneratorType | tuple): encoded_list = [] for item in obj: encoded_list.append( @@ -156,7 +156,7 @@ def jsonable_encoder( try: data = dict(obj) except Exception as e: - errors: List[Exception] = [] + errors: list[Exception] = [] errors.append(e) try: data = vars(obj) diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index 0c1439eedfd..09f3a241d38 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -14,16 +14,31 @@ class MyModel(BaseModel): SEE https://docs.pydantic.dev/usage/validators/#reuse-validators """ +import enum from typing import Any -def empty_str_to_none(value: Any): +def empty_str_to_none_pre_validator(value: Any): if isinstance(value, str) and value.strip() == "": return None return value -def none_to_empty_str(value: Any): +def none_to_empty_str_pre_validator(value: Any): if value is None: return "" return value + + +def create_enums_pre_validator(enum_cls: type[enum.Enum]): + """Enables parsing enums from equivalent enums + + SEE test__pydantic_models_and_enumps.py for more details + """ + + def _validator(value: Any): + if value and not isinstance(value, enum_cls) and isinstance(value, enum.Enum): + return value.value + return value + + return _validator diff --git a/packages/models-library/src/models_library/utils/database_models_factory.py b/packages/models-library/src/models_library/utils/database_models_factory.py index 372e70a2534..3f4a57a16ba 100644 --- a/packages/models-library/src/models_library/utils/database_models_factory.py +++ b/packages/models-library/src/models_library/utils/database_models_factory.py @@ -5,8 +5,9 @@ import json import warnings +from collections.abc import Callable, Container from datetime import datetime -from typing import Any, Callable, Container +from typing import Any from uuid import UUID import sqlalchemy as sa @@ -76,7 +77,7 @@ def _eval_defaults( elif issubclass(pydantic_type, datetime): assert isinstance( # nosec column.server_default.arg, # type: ignore - (type(null()), sqlalchemy.sql.functions.now), + type(null()) | sqlalchemy.sql.functions.now, ) default_factory = datetime.now return default, default_factory diff --git a/packages/models-library/src/models_library/utils/docker_compose.py b/packages/models-library/src/models_library/utils/docker_compose.py index d1b61f9adc5..cec056f02b3 100644 --- a/packages/models-library/src/models_library/utils/docker_compose.py +++ b/packages/models-library/src/models_library/utils/docker_compose.py @@ -1,6 +1,6 @@ import yaml -from models_library.service_settings_labels import ComposeSpecLabelDict +from ..service_settings_labels import ComposeSpecLabelDict from .string_substitution import SubstitutionsDict, TextTemplate # Notes on below env var names: diff --git a/packages/models-library/src/models_library/utils/enums.py b/packages/models-library/src/models_library/utils/enums.py index b8e624ea3be..21590c59dc8 100644 --- a/packages/models-library/src/models_library/utils/enums.py +++ b/packages/models-library/src/models_library/utils/enums.py @@ -1,8 +1,27 @@ +import inspect from enum import Enum, unique +from typing import Any @unique -class StrAutoEnum(str, Enum): +class StrAutoEnum(str, Enum): # noqa: SLOT000 @staticmethod def _generate_next_value_(name, start, count, last_values): return name.upper() + + +def enum_to_dict(enum_cls: type[Enum]) -> dict[str, Any]: + return {m.name: m.value for m in enum_cls} + + +def are_equivalent_enums(enum_cls1: type[Enum], enum_cls2: type[Enum]) -> bool: + assert inspect.isclass(enum_cls1) # nosec + assert issubclass(enum_cls1, Enum) # nosec + assert inspect.isclass(enum_cls2) # nosec + assert issubclass(enum_cls2, Enum) # nosec + + try: + return enum_to_dict(enum_cls1) == enum_to_dict(enum_cls2) + + except (AttributeError, TypeError): + return False diff --git a/packages/models-library/src/models_library/utils/json_schema.py b/packages/models-library/src/models_library/utils/json_schema.py index 9b6d76cf232..1c5afc4ca55 100644 --- a/packages/models-library/src/models_library/utils/json_schema.py +++ b/packages/models-library/src/models_library/utils/json_schema.py @@ -84,7 +84,7 @@ def jsonschema_validate_schema(schema: dict[str, Any]): def any_ref_key(obj): if isinstance(obj, dict): - return "$ref" in obj.keys() or any_ref_key(tuple(obj.values())) + return "$ref" in obj or any_ref_key(tuple(obj.values())) if isinstance(obj, Sequence) and not isinstance(obj, str): return any(any_ref_key(v) for v in obj) diff --git a/packages/models-library/src/models_library/utils/misc.py b/packages/models-library/src/models_library/utils/misc.py deleted file mode 100644 index c06dede6056..00000000000 --- a/packages/models-library/src/models_library/utils/misc.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Any, Dict, List, Type, Union - -from pydantic import BaseModel -from pydantic.config import SchemaExtraCallable - - -def extract_examples(model_cls: Type[BaseModel]) -> List[Dict[str, Any]]: - """Extracts examples from pydantic classes""" - - examples = [] - - schema_extra: Union[ - Dict[str, Any], SchemaExtraCallable - ] = model_cls.__config__.schema_extra - - if isinstance(schema_extra, dict): - # NOTE: Sometimes an example (singular) mistaken - # by exampleS. The assertions below should - # help catching this error while running tests - - examples = schema_extra.get("examples", []) - assert isinstance(examples, list) # nosec - - if example := schema_extra.get("example"): - assert not isinstance(example, list) # nosec - examples.append(example) - - # TODO: treat SchemaExtraCallable case (so far we only have one example) - # TODO: extract examples from single fields and compose model? - - return examples diff --git a/packages/models-library/src/models_library/utils/nodes.py b/packages/models-library/src/models_library/utils/nodes.py index e000be7bbc0..df2771e80ad 100644 --- a/packages/models-library/src/models_library/utils/nodes.py +++ b/packages/models-library/src/models_library/utils/nodes.py @@ -1,16 +1,16 @@ import hashlib import json import logging +from collections.abc import Callable, Coroutine from copy import deepcopy -from typing import Any, Callable, Coroutine +from typing import Any -from models_library.projects_nodes_io import UUIDStr from pydantic import BaseModel from ..projects import Project -from ..projects_nodes_io import NodeID, PortLink +from ..projects_nodes_io import NodeID, PortLink, UUIDStr -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def project_node_io_payload_cb( diff --git a/packages/models-library/src/models_library/utils/services_io.py b/packages/models-library/src/models_library/utils/services_io.py index 67225495e49..a323ad90b45 100644 --- a/packages/models-library/src/models_library/utils/services_io.py +++ b/packages/models-library/src/models_library/utils/services_io.py @@ -1,6 +1,6 @@ import mimetypes from copy import deepcopy -from typing import Any, Literal, Optional, Union +from typing import Any, Literal from pydantic import schema_of @@ -16,7 +16,7 @@ } -def guess_media_type(io: Union[ServiceInput, ServiceOutput]) -> str: +def guess_media_type(io: ServiceInput | ServiceOutput) -> str: # SEE https://docs.python.org/3/library/mimetypes.html # SEE https://www.iana.org/assignments/media-types/media-types.xhtml media_type = io.property_type.removeprefix("data:") @@ -28,7 +28,7 @@ def guess_media_type(io: Union[ServiceInput, ServiceOutput]) -> str: return media_type -def update_schema_doc(schema: dict[str, Any], port: Union[ServiceInput, ServiceOutput]): +def update_schema_doc(schema: dict[str, Any], port: ServiceInput | ServiceOutput): schema["title"] = port.label if port.label != port.description: schema["description"] = port.description @@ -36,8 +36,8 @@ def update_schema_doc(schema: dict[str, Any], port: Union[ServiceInput, ServiceO def get_service_io_json_schema( - port: Union[ServiceInput, ServiceOutput] -) -> Optional[JsonSchemaDict]: + port: ServiceInput | ServiceOutput, +) -> JsonSchemaDict | None: """Get json-schema for a i/o service For legacy metadata with property_type = integer, etc ... , it applies a conversion diff --git a/packages/models-library/src/models_library/utils/string_substitution.py b/packages/models-library/src/models_library/utils/string_substitution.py index 6ce86bc55eb..cb2d4be2905 100644 --- a/packages/models-library/src/models_library/utils/string_substitution.py +++ b/packages/models-library/src/models_library/utils/string_substitution.py @@ -70,9 +70,8 @@ def is_valid(self): ): # If all the groups are None, there must be # another group we're not expecting - raise ValueError( - "Unrecognized named group in pattern", self.pattern - ) + msg = "Unrecognized named group in pattern" + raise ValueError(msg, self.pattern) return True def get_identifiers(self): @@ -89,9 +88,8 @@ def get_identifiers(self): ): # If all the groups are None, there must be # another group we're not expecting - raise ValueError( - "Unrecognized named group in pattern", self.pattern - ) + msg = "Unrecognized named group in pattern" + raise ValueError(msg, self.pattern) return ids @@ -110,4 +108,4 @@ def __getitem__(self, key) -> Any: @property def unused(self): - return {key for key in self.keys() if key not in self.used} + return {key for key in self if key not in self.used} diff --git a/packages/models-library/tests/test__models_examples.py b/packages/models-library/tests/test__models_examples.py index 7c0adedd935..12809db713b 100644 --- a/packages/models-library/tests/test__models_examples.py +++ b/packages/models-library/tests/test__models_examples.py @@ -1,59 +1,19 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - - import json -from contextlib import suppress -from importlib import import_module -from inspect import getmembers, isclass -from pathlib import Path -from typing import Any, Iterable, Optional, Set, Tuple, Type +from typing import Any import models_library import pytest -from models_library.utils.misc import extract_examples -from pydantic import BaseModel, NonNegativeInt -from pydantic.json import pydantic_encoder - - -def iter_model_cls_examples( - exclude: Optional[Set] = None, -) -> Iterable[Tuple[str, Type[BaseModel], NonNegativeInt, Any]]: - def _is_model_cls(cls) -> bool: - with suppress(TypeError): - # NOTE: issubclass( dict[models_library.services.ConstrainedStrValue, models_library.services.ServiceInput] ) raises TypeError - return cls is not BaseModel and isclass(cls) and issubclass(cls, BaseModel) - return False - - exclude = exclude or set() - - for filepath in Path(models_library.__file__).resolve().parent.glob("*.py"): - if not filepath.name.startswith("_"): - mod = import_module(f"models_library.{filepath.stem}") - for name, model_cls in getmembers(mod, _is_model_cls): - if name in exclude: - continue - # NOTE: this is part of utils.misc and is tested here - examples = extract_examples(model_cls) - for index, example in enumerate(examples): - yield (name, model_cls, index, example) +from pydantic import BaseModel +from pytest_simcore.pydantic_models import walk_model_examples_in_package @pytest.mark.parametrize( - "class_name, model_cls, example_index, test_example", iter_model_cls_examples() + "model_cls, example_name, example_data", + walk_model_examples_in_package(models_library), ) -def test_all_module_model_examples( - class_name: str, - model_cls: Type[BaseModel], - example_index: NonNegativeInt, - test_example: Any, +def test_all_models_library_models_config_examples( + model_cls: type[BaseModel], example_name: int, example_data: Any ): - """Automatically collects all BaseModel subclasses having examples and tests them against schemas""" - print( - f"test {example_index=} for {class_name=}:\n", - json.dumps(test_example, default=pydantic_encoder, indent=2), - "---", - ) - model_instance = model_cls.parse_obj(test_example) - assert isinstance(model_instance, model_cls) + assert model_cls.parse_obj( + example_data + ), f"Failed {example_name} : {json.dumps(example_data)}" diff --git a/packages/models-library/tests/test__models_fit_schemas.py b/packages/models-library/tests/test__models_fit_schemas.py index 483d75c2ce4..b909a3c2890 100644 --- a/packages/models-library/tests/test__models_fit_schemas.py +++ b/packages/models-library/tests/test__models_fit_schemas.py @@ -3,7 +3,7 @@ # pylint:disable=redefined-outer-name # pylint:disable=protected-access import json -from typing import Callable +from collections.abc import Callable import pytest from models_library.projects import Project diff --git a/packages/models-library/tests/test__pydantic_models.py b/packages/models-library/tests/test__pydantic_models.py index 38e47bf3c76..716cf9f7906 100644 --- a/packages/models-library/tests/test__pydantic_models.py +++ b/packages/models-library/tests/test__pydantic_models.py @@ -123,7 +123,7 @@ class Func(BaseModel): print(model.json(indent=1)) assert model.input == 0.5 - assert model.output == False + assert model.output is False # (undefined) json string vs string ------------------------ model = Func.parse_obj( diff --git a/packages/models-library/tests/test__pydantic_models_and_enums.py b/packages/models-library/tests/test__pydantic_models_and_enums.py new file mode 100644 index 00000000000..832ec221c2e --- /dev/null +++ b/packages/models-library/tests/test__pydantic_models_and_enums.py @@ -0,0 +1,124 @@ +from enum import Enum, unique + +import pytest +from models_library.utils.enums import are_equivalent_enums, enum_to_dict +from pydantic import BaseModel, ValidationError, parse_obj_as + + +# +# Enum Color1 is **equivalent** to enum Color2 but not equal +# +@unique +class Color1(Enum): + RED = "RED" + + +@unique +class Color2(Enum): + RED = "RED" + + +def test_equivalent_enums_are_not_strictly_equal(): + assert Color1 != Color2 + + assert enum_to_dict(Color1) == enum_to_dict(Color2) + + assert are_equivalent_enums(Color1, Color2) + assert are_equivalent_enums(Color1, Color1) + + +# +# Here two equivalent enum BUT of type str-enum +# +# SEE from models_library.utils.enums.AutoStrEnum +# SEE https://docs.pydantic.dev/dev-v2/usage/types/enums/ +# + + +@unique +class ColorStrAndEnum1(str, Enum): + RED = "RED" + + +@unique +class ColorStrAndEnum2(str, Enum): + RED = "RED" + + +def test_enums_vs_strenums(): + # here are the differences + assert f"{Color1.RED}" == "Color1.RED" + assert f"{ColorStrAndEnum1.RED}" == "RED" + + assert Color1.RED != "RED" + assert ColorStrAndEnum1.RED == "RED" + + assert Color1.RED != ColorStrAndEnum1.RED + + # here are the analogies + assert Color1.RED.name == "RED" + assert ColorStrAndEnum1.RED.name == "RED" + + assert Color1.RED.value == "RED" + assert ColorStrAndEnum1.RED.value == "RED" + + +def test_enums_and_strenums_are_equivalent(): + + assert are_equivalent_enums(Color1, ColorStrAndEnum1) + assert are_equivalent_enums(Color2, ColorStrAndEnum2) + assert are_equivalent_enums(Color1, ColorStrAndEnum2) + + +class Model(BaseModel): + color: Color1 + + +def test_parsing_enums_in_pydantic(): + + model = parse_obj_as(Model, {"color": Color1.RED}) + assert model.color == Color1.RED + + # Can parse from STRING + model = parse_obj_as(Model, {"color": "RED"}) + assert model.color == Color1.RED + + # Can **NOT** parse from equilalent enum + with pytest.raises(ValidationError): + parse_obj_as(Model, {"color": Color2.RED}) + + +class ModelStrAndEnum(BaseModel): + color: ColorStrAndEnum1 + + +def test_parsing_strenum_in_pydantic(): + assert are_equivalent_enums(Color1, ColorStrAndEnum1) + + model = parse_obj_as(ModelStrAndEnum, {"color": ColorStrAndEnum1.RED}) + assert model.color == ColorStrAndEnum1.RED + + # Can parse from string + model = parse_obj_as(ModelStrAndEnum, {"color": "RED"}) + assert model.color == ColorStrAndEnum1.RED + + # **CAN** parse other equivalent str-enum + # Using str-enums allow you to parse from equivalent enums! + parse_obj_as(ModelStrAndEnum, {"color": ColorStrAndEnum2.RED}) + + +def test_parsing_str_and_enum_in_pydantic(): + + # Can still NOT parse equilalent enum(-only) + with pytest.raises(ValidationError): + parse_obj_as(ModelStrAndEnum, {"color": Color1.RED}) + + # And the opposite? NO!!! + with pytest.raises(ValidationError): + parse_obj_as(Color1, {"color": ColorStrAndEnum1.RED}) + + with pytest.raises(ValidationError): + parse_obj_as(Color1, {"color": ColorStrAndEnum2.RED}) + + # CONCLUSION: we need a validator to pre-process inputs ! + # SEE models_library.utils.common_validators diff --git a/services/catalog/tests/unit/test_models_schemas.py b/packages/models-library/tests/test_api_schemas_catalog.py similarity index 74% rename from services/catalog/tests/unit/test_models_schemas.py rename to packages/models-library/tests/test_api_schemas_catalog.py index adb7e226e99..0c815d7bd0c 100644 --- a/services/catalog/tests/unit/test_models_schemas.py +++ b/packages/models-library/tests/test_api_schemas_catalog.py @@ -3,32 +3,8 @@ # pylint: disable=unused-variable -from pprint import pformat - -import pytest +from models_library.api_schemas_catalog.services_ports import ServicePortGet from models_library.services import ServiceInput -from simcore_service_catalog.models.schemas.services import ( - ServiceGet, - ServiceItem, - ServiceUpdate, -) -from simcore_service_catalog.models.schemas.services_ports import ServicePortGet - - -@pytest.mark.parametrize( - "model_cls", - ( - ServiceGet, - ServiceUpdate, - ServiceItem, - ServicePortGet, - ), -) -def test_service_api_models_examples(model_cls, model_cls_examples): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" def test_service_port_with_file(): diff --git a/packages/models-library/tests/test_api_schemas_storage.py b/packages/models-library/tests/test_api_schemas_storage.py deleted file mode 100644 index 07349d39f92..00000000000 --- a/packages/models-library/tests/test_api_schemas_storage.py +++ /dev/null @@ -1,23 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -import pytest -from models_library.api_schemas_storage import ( - DatasetMetaDataGet, - FileLocation, - FileMetaDataGet, -) - - -@pytest.mark.parametrize( - "model_cls", (FileLocation, FileMetaDataGet, DatasetMetaDataGet) -) -def test_storage_api_models_examples(model_cls): - examples = model_cls.Config.schema_extra["examples"] - - for index, example in enumerate(examples): - print(f"{index:-^10}:\n", example) - - model_instance = model_cls(**example) - assert model_instance diff --git a/packages/models-library/tests/test_basic_regex.py b/packages/models-library/tests/test_basic_regex.py index 0b9de3f7e6f..252ca8fa953 100644 --- a/packages/models-library/tests/test_basic_regex.py +++ b/packages/models-library/tests/test_basic_regex.py @@ -6,8 +6,10 @@ import keyword import re +from collections.abc import Sequence from datetime import datetime -from typing import Any, Pattern, Sequence +from re import Pattern +from typing import Any import pytest from models_library.basic_regex import ( diff --git a/packages/models-library/tests/test_clusters.py b/packages/models-library/tests/test_clusters.py index d4d9b4b4c13..4aff7b8fd95 100644 --- a/packages/models-library/tests/test_clusters.py +++ b/packages/models-library/tests/test_clusters.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Any, Dict, Type +from typing import Any import pytest from faker import Faker @@ -18,7 +18,7 @@ (Cluster,), ) def test_cluster_access_rights_correctly_created_when_owner_access_rights_not_present( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] + model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] ): for example in model_cls_examples.values(): modified_example = deepcopy(example) @@ -38,8 +38,8 @@ def test_cluster_access_rights_correctly_created_when_owner_access_rights_not_pr (Cluster,), ) def test_cluster_fails_when_owner_has_no_admin_rights_unless_default_cluster( - model_cls: Type[BaseModel], - model_cls_examples: Dict[str, Dict[str, Any]], + model_cls: type[BaseModel], + model_cls_examples: dict[str, dict[str, Any]], faker: Faker, ): for example in model_cls_examples.values(): @@ -64,8 +64,8 @@ def test_cluster_fails_when_owner_has_no_admin_rights_unless_default_cluster( (Cluster,), ) def test_cluster_fails_when_owner_has_no_user_rights_if_default_cluster( - model_cls: Type[BaseModel], - model_cls_examples: Dict[str, Dict[str, Any]], + model_cls: type[BaseModel], + model_cls_examples: dict[str, dict[str, Any]], ): for example in model_cls_examples.values(): modified_example = deepcopy(example) diff --git a/packages/models-library/tests/test_project_networks.py b/packages/models-library/tests/test_project_networks.py index 85465159c15..c91f0503a8e 100644 --- a/packages/models-library/tests/test_project_networks.py +++ b/packages/models-library/tests/test_project_networks.py @@ -1,7 +1,4 @@ # pylint: disable=redefined-outer-name -import json -from pprint import pformat -from typing import Any, Dict, Type from uuid import UUID import pytest @@ -9,27 +6,8 @@ DockerNetworkAlias, DockerNetworkName, NetworksWithAliases, - ProjectsNetworks, ) -from pydantic import BaseModel, ValidationError, parse_obj_as - - -@pytest.mark.parametrize( - "model_cls", - ( - ProjectsNetworks, - NetworksWithAliases, - ), -) -def test_service_settings_model_examples( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]] -) -> None: - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - - model_instance = model_cls.parse_obj(example) - assert json.loads(model_instance.json()) == example - assert model_instance.json() == json.dumps(example) +from pydantic import ValidationError, parse_obj_as @pytest.mark.parametrize( @@ -40,7 +18,7 @@ def test_service_settings_model_examples( {"shr-ntwrk_5c743ad2-8fdb-11ec-bb3a-02420a000008_default": {}}, ], ) -def test_networks_with_aliases_ok(valid_example: Dict) -> None: +def test_networks_with_aliases_ok(valid_example: dict) -> None: assert NetworksWithAliases.parse_obj(valid_example) @@ -59,7 +37,7 @@ def test_networks_with_aliases_ok(valid_example: Dict) -> None: {"i_am_ok": {"5057e2c1-d392-4d31-b5c8-19f3db780390": "1_I_AM_INVALID"}}, ], ) -def test_networks_with_aliases_fail(invalid_example: Dict) -> None: +def test_networks_with_aliases_fail(invalid_example: dict) -> None: with pytest.raises(ValidationError): assert NetworksWithAliases.parse_obj(invalid_example) diff --git a/packages/models-library/tests/test_project_nodes.py b/packages/models-library/tests/test_project_nodes.py index bbdb2079dee..2edefd1533d 100644 --- a/packages/models-library/tests/test_project_nodes.py +++ b/packages/models-library/tests/test_project_nodes.py @@ -2,7 +2,7 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -from typing import Any, Dict +from typing import Any import pytest from models_library.projects_nodes import Node @@ -10,15 +10,15 @@ @pytest.fixture() -def minimal_node_data_sample() -> Dict[str, Any]: - return dict( - key="simcore/services/dynamic/3dviewer", - version="1.3.0-alpha", - label="3D viewer human message", - ) +def minimal_node_data_sample() -> dict[str, Any]: + return { + "key": "simcore/services/dynamic/3dviewer", + "version": "1.3.0-alpha", + "label": "3D viewer human message", + } -def test_create_minimal_node(minimal_node_data_sample: Dict[str, Any]): +def test_create_minimal_node(minimal_node_data_sample: dict[str, Any]): node = Node(**minimal_node_data_sample) # a nice way to see how the simplest node looks like @@ -35,7 +35,7 @@ def test_create_minimal_node(minimal_node_data_sample: Dict[str, Any]): def test_create_minimal_node_with_new_data_type( - minimal_node_data_sample: Dict[str, Any] + minimal_node_data_sample: dict[str, Any] ): old_node_data = minimal_node_data_sample # found some old data with this aspect @@ -57,7 +57,7 @@ def test_create_minimal_node_with_new_data_type( assert node.state.dependencies == set() -def test_backwards_compatibility_node_data(minimal_node_data_sample: Dict[str, Any]): +def test_backwards_compatibility_node_data(minimal_node_data_sample: dict[str, Any]): old_node_data = minimal_node_data_sample # found some old data with this aspect old_node_data.update({"thumbnail": "", "state": "FAILURE"}) diff --git a/packages/models-library/tests/test_project_nodes_io.py b/packages/models-library/tests/test_project_nodes_io.py index 13723a6f53f..992c4d1f604 100644 --- a/packages/models-library/tests/test_project_nodes_io.py +++ b/packages/models-library/tests/test_project_nodes_io.py @@ -2,7 +2,6 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -from pprint import pformat from typing import Any import pytest @@ -41,17 +40,6 @@ def test_simcore_file_link_with_label(minimal_simcore_file_link: dict[str, Any]) assert simcore_file_link.e_tag is None -@pytest.mark.parametrize("model_cls", [SimCoreFileLink, DatCoreFileLink]) -def test_project_nodes_io_model_examples(model_cls, model_cls_examples): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - - model_instance = model_cls(**example) - - assert model_instance, f"Failed with {name}" - print(name, ":", model_instance) - - def test_store_discriminator(): workbench = { "89f95b67-a2a3-4215-a794-2356684deb61": { diff --git a/packages/models-library/tests/test_projects.py b/packages/models-library/tests/test_projects.py index a8384257090..ecd6533ea83 100644 --- a/packages/models-library/tests/test_projects.py +++ b/packages/models-library/tests/test_projects.py @@ -3,7 +3,7 @@ # pylint:disable=redefined-outer-name from copy import deepcopy -from typing import Any, Dict +from typing import Any import pytest from faker import Faker @@ -11,7 +11,7 @@ @pytest.fixture() -def minimal_project(faker: Faker) -> Dict[str, Any]: +def minimal_project(faker: Faker) -> dict[str, Any]: # API request body payload return { "uuid": faker.uuid4(), @@ -26,20 +26,20 @@ def minimal_project(faker: Faker) -> Dict[str, Any]: } -def test_project_minimal_model(minimal_project: Dict[str, Any]): +def test_project_minimal_model(minimal_project: dict[str, Any]): project = Project.parse_obj(minimal_project) assert project - assert project.thumbnail == None + assert project.thumbnail is None -def test_project_with_thumbnail_as_empty_string(minimal_project: Dict[str, Any]): +def test_project_with_thumbnail_as_empty_string(minimal_project: dict[str, Any]): thumbnail_empty_string = deepcopy(minimal_project) thumbnail_empty_string.update({"thumbnail": ""}) project = Project.parse_obj(thumbnail_empty_string) assert project - assert project.thumbnail == None + assert project.thumbnail is None def test_project_type_in_models_package_same_as_in_postgres_database_package(): diff --git a/packages/models-library/tests/test_projects_pipeline.py b/packages/models-library/tests/test_projects_pipeline.py deleted file mode 100644 index 0cbf054eaca..00000000000 --- a/packages/models-library/tests/test_projects_pipeline.py +++ /dev/null @@ -1,23 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from pprint import pformat -from typing import Dict, Type - -import pytest -from models_library.projects_pipeline import ComputationTask -from pydantic import BaseModel - - -@pytest.mark.parametrize( - "model_cls", - (ComputationTask,), -) -def test_computation_task_model_examples( - model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict] -): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" diff --git a/packages/models-library/tests/test_projects_state.py b/packages/models-library/tests/test_projects_state.py index d4b73b689a1..ecfc7e97730 100644 --- a/packages/models-library/tests/test_projects_state.py +++ b/packages/models-library/tests/test_projects_state.py @@ -1,20 +1,7 @@ -from pprint import pformat - import pytest from models_library.projects_state import ProjectLocked, ProjectStatus -@pytest.mark.parametrize( - "model_cls", - (ProjectLocked,), -) -def test_projects_state_model_examples(model_cls, model_cls_examples): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" - - def test_project_locked_with_missing_owner_raises(): with pytest.raises(ValueError): ProjectLocked(**{"value": True, "status": ProjectStatus.OPENED}) diff --git a/packages/models-library/tests/test_service_resources.py b/packages/models-library/tests/test_service_resources.py index 34300d6b277..c119a33e898 100644 --- a/packages/models-library/tests/test_service_resources.py +++ b/packages/models-library/tests/test_service_resources.py @@ -81,7 +81,7 @@ def _assert_service_resources_dict( assert type(service_resources_dict) == dict print(service_resources_dict) - for _, image_resources in service_resources_dict.items(): + for image_resources in service_resources_dict.values(): _ensure_resource_value_is_an_object(image_resources.resources) service_resources_dict: ServiceResourcesDict = parse_obj_as( diff --git a/packages/models-library/tests/test_service_settings_labels.py b/packages/models-library/tests/test_service_settings_labels.py index 6bafd236eb7..3320c86101b 100644 --- a/packages/models-library/tests/test_service_settings_labels.py +++ b/packages/models-library/tests/test_service_settings_labels.py @@ -80,23 +80,6 @@ def test_service_settings(): service_setting._destination_containers = ["random_value1", "random_value2"] -@pytest.mark.parametrize( - "model_cls", - ( - SimcoreServiceSettingLabelEntry, - SimcoreServiceSettingsLabel, - SimcoreServiceLabels, - ), -) -def test_service_settings_model_examples( - model_cls: type[BaseModel], model_cls_examples: dict[str, dict[str, Any]] -): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" - - @pytest.mark.parametrize( "model_cls", (SimcoreServiceLabels,), @@ -544,7 +527,6 @@ def test_can_parse_labels_with_osparc_identifiers( def test_resolving_some_service_labels_at_load_time( vendor_environments: dict[str, Any], service_labels: dict[str, str] ): - print(json.dumps(service_labels, indent=1)) # Resolving at load-time (some of them are possible) diff --git a/packages/models-library/tests/test_services.py b/packages/models-library/tests/test_services.py index ad4bf8d6426..687edadbdf6 100644 --- a/packages/models-library/tests/test_services.py +++ b/packages/models-library/tests/test_services.py @@ -4,9 +4,9 @@ import re import urllib.parse +from collections.abc import Callable from copy import deepcopy -from pprint import pformat -from typing import Any, Callable +from typing import Any import pytest from models_library.basic_regex import VERSION_RE @@ -17,20 +17,16 @@ SERVICE_KEY_RE, BootOption, ServiceDockerData, - ServiceInput, - ServiceMetaData, - ServiceOutput, _BaseServiceCommonDataModel, ) -from models_library.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB @pytest.fixture() def minimal_service_common_data() -> dict[str, Any]: - return dict( - name="this is a nice sample service", - description="this is the description of the service", - ) + return { + "name": "this is a nice sample service", + "description": "this is the description of the service", + } def test_create_minimal_service_common_data( @@ -40,7 +36,7 @@ def test_create_minimal_service_common_data( assert service.name == minimal_service_common_data["name"] assert service.description == minimal_service_common_data["description"] - assert service.thumbnail == None + assert service.thumbnail is None def test_node_with_empty_thumbnail(minimal_service_common_data: dict[str, Any]): @@ -51,7 +47,7 @@ def test_node_with_empty_thumbnail(minimal_service_common_data: dict[str, Any]): assert service.name == minimal_service_common_data["name"] assert service.description == minimal_service_common_data["description"] - assert service.thumbnail == None + assert service.thumbnail is None def test_node_with_thumbnail(minimal_service_common_data: dict[str, Any]): @@ -72,21 +68,6 @@ def test_node_with_thumbnail(minimal_service_common_data: dict[str, Any]): ) -@pytest.mark.parametrize( - "model_cls", - ( - ServiceInput, - ServiceOutput, - BootOption, - ), -) -def test_service_models_examples(model_cls, model_cls_examples): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" - - @pytest.mark.parametrize("pattern", (SERVICE_KEY_RE, SERVICE_ENCODED_KEY_RE)) @pytest.mark.parametrize( "service_key", @@ -168,17 +149,6 @@ def test_SERVICE_KEY_RE(service_key: str, pattern: re.Pattern): assert new_match.groups() == match.groups() -@pytest.mark.parametrize( - "model_cls", - (ServiceAccessRightsAtDB, ServiceMetaDataAtDB, ServiceMetaData, ServiceDockerData), -) -def test_services_model_examples(model_cls, model_cls_examples): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" - - @pytest.mark.skip(reason="will be disabled by PC") @pytest.mark.parametrize( "python_regex_pattern, json_schema_file_name, json_schema_entry_paths", diff --git a/packages/models-library/tests/test_utils_common_validators.py b/packages/models-library/tests/test_utils_common_validators.py new file mode 100644 index 00000000000..d185b16f7b0 --- /dev/null +++ b/packages/models-library/tests/test_utils_common_validators.py @@ -0,0 +1,61 @@ +from enum import Enum + +import pytest +from models_library.utils.common_validators import ( + create_enums_pre_validator, + empty_str_to_none_pre_validator, + none_to_empty_str_pre_validator, +) +from pydantic import BaseModel, ValidationError, validator + + +def test_enums_pre_validator(): + class Enum1(Enum): + RED = "RED" + + class Model(BaseModel): + color: Enum1 + + class ModelWithPreValidator(BaseModel): + color: Enum1 + + _from_equivalent_enums = validator("color", allow_reuse=True, pre=True)( + create_enums_pre_validator(Enum1) + ) + + # with Enum1 + model = Model(color=Enum1.RED) + assert ModelWithPreValidator(color=Enum1.RED) == model + + # with Enum2 + class Enum2(Enum): + RED = "RED" + + with pytest.raises(ValidationError): + Model(color=Enum2.RED) + + assert ModelWithPreValidator(color=Enum2.RED) == model + + +def test_empty_str_to_none_pre_validator(): + class Model(BaseModel): + nullable_message: str | None + + _empty_is_none = validator("nullable_message", allow_reuse=True, pre=True)( + empty_str_to_none_pre_validator + ) + + model = Model.parse_obj({"nullable_message": None}) + assert model == Model.parse_obj({"nullable_message": ""}) + + +def test_none_to_empty_str_pre_validator(): + class Model(BaseModel): + message: str + + _none_is_empty = validator("message", allow_reuse=True, pre=True)( + none_to_empty_str_pre_validator + ) + + model = Model.parse_obj({"message": ""}) + assert model == Model.parse_obj({"message": None}) diff --git a/packages/models-library/tests/test_utils_enums.py b/packages/models-library/tests/test_utils_enums.py index c0d5ed6be8c..5eece557e05 100644 --- a/packages/models-library/tests/test_utils_enums.py +++ b/packages/models-library/tests/test_utils_enums.py @@ -11,4 +11,4 @@ class _Ordinal(StrAutoEnum): def test_strautoenum(): - assert list(f"{n}" for n in _Ordinal) == ["NORTH", "EAST", "SOUTH", "WEST"] + assert [f"{n}" for n in _Ordinal] == ["NORTH", "EAST", "SOUTH", "WEST"] diff --git a/packages/models-library/tests/test_utils_nodes.py b/packages/models-library/tests/test_utils_nodes.py index 736dc8bab6f..47465ce236d 100644 --- a/packages/models-library/tests/test_utils_nodes.py +++ b/packages/models-library/tests/test_utils_nodes.py @@ -2,7 +2,7 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -from typing import Any, Dict +from typing import Any from uuid import uuid4 import pytest @@ -77,9 +77,9 @@ def node_id() -> NodeID: ], ) async def test_compute_node_hash( - node_id: NodeID, node_payload: Dict[str, Any], expected_hash: str + node_id: NodeID, node_payload: dict[str, Any], expected_hash: str ): - async def get_node_io_payload_cb(some_node_id: NodeID) -> Dict[str, Any]: + async def get_node_io_payload_cb(some_node_id: NodeID) -> dict[str, Any]: assert some_node_id in [node_id, ANOTHER_NODE_ID] return node_payload if some_node_id == node_id else ANOTHER_NODE_PAYLOAD diff --git a/packages/models-library/tests/test_utils_string_substitution.py b/packages/models-library/tests/test_utils_string_substitution.py index 965f292323a..59aade68d2b 100644 --- a/packages/models-library/tests/test_utils_string_substitution.py +++ b/packages/models-library/tests/test_utils_string_substitution.py @@ -157,7 +157,7 @@ def test_substitution_with_new_and_legacy_identifiers(): } -@pytest.mark.diagnostics +@pytest.mark.diagnostics() @pytest.mark.parametrize( "metadata_path", TEST_DATA_FOLDER.rglob("metadata*.json"), @@ -226,7 +226,7 @@ def test_template_substitution_on_jsondumps(): json_dumps_template = json.dumps(json_template) # LIKE image labels! # NOTE: that here we are enforcing the values to be strings! - assert '{"x": "$VALUE1", "y": "$VALUE2"}' == json_dumps_template + assert json_dumps_template == '{"x": "$VALUE1", "y": "$VALUE2"}' template = TextTemplate(json_dumps_template) assert set(template.get_identifiers()) == {"VALUE1", "VALUE2"} diff --git a/packages/service-library/src/servicelib/fastapi/openapi.py b/packages/service-library/src/servicelib/fastapi/openapi.py index 63c5bfd375c..48e69594500 100644 --- a/packages/service-library/src/servicelib/fastapi/openapi.py +++ b/packages/service-library/src/servicelib/fastapi/openapi.py @@ -84,7 +84,7 @@ def _remove_named_groups(regex: str) -> str: def _patch_node_properties(key: str, node: dict): # Validation for URL is broken in the context of the license entry # this helps to bypass validation and then replace with the correct value - if key.startswith("__PLACEHOLDER___KEY_"): + if isinstance(key, str) and key.startswith("__PLACEHOLDER___KEY_"): new_key = key.replace("__PLACEHOLDER___KEY_", "") node[new_key] = node[key] node.pop(key) diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index db3f6421073..f8335bbeed2 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -1,5 +1,6 @@ import urllib.parse from functools import cached_property +from typing import Any, ClassVar from pydantic import Field, PostgresDsn, SecretStr, validator @@ -42,9 +43,8 @@ class PostgresSettings(BaseCustomSettings): @classmethod def _check_size(cls, v, values): if not (values["POSTGRES_MINSIZE"] <= v): - raise ValueError( - f"assert POSTGRES_MINSIZE={values['POSTGRES_MINSIZE']} <= POSTGRES_MAXSIZE={v}" - ) + msg = f"assert POSTGRES_MINSIZE={values['POSTGRES_MINSIZE']} <= POSTGRES_MAXSIZE={v}" + raise ValueError(msg) return v @cached_property @@ -82,7 +82,7 @@ def dsn_with_query(self) -> str: return dsn class Config(BaseCustomSettings.Config): - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] "examples": [ # minimal required { diff --git a/services/catalog/Makefile b/services/catalog/Makefile index 3981b930704..8fc2fe2a01d 100644 --- a/services/catalog/Makefile +++ b/services/catalog/Makefile @@ -54,11 +54,20 @@ run-prod: .env up-extra # BUILD ##################### + +# specification of the used openapi-generator-cli (see also https://github.com/ITISFoundation/openapi-generator) +OPENAPI_GENERATOR_NAME := itisfoundation/openapi-generator-cli-openapi-generator-v4.2.3 +OPENAPI_GENERATOR_TAG := v0 +OPENAPI_GENERATOR_IMAGE := $(OPENAPI_GENERATOR_NAME):$(OPENAPI_GENERATOR_TAG) + + .PHONY: openapi-specs openapi.json openapi-specs: openapi.json openapi.json: .env # generating openapi specs file python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ + # validates OAS file: $@ - @cd $(CURDIR); \ - $(SCRIPTS_DIR)/openapi-generator-cli.bash validate --input-spec /local/$@ + docker run --rm \ + --volume "$(CURDIR):/local" \ + $(OPENAPI_GENERATOR_IMAGE) validate --input-spec /local/$@ diff --git a/services/catalog/VERSION b/services/catalog/VERSION index 60a2d3e96c8..79a2734bbf3 100644 --- a/services/catalog/VERSION +++ b/services/catalog/VERSION @@ -1 +1 @@ -0.4.0 \ No newline at end of file +0.5.0 \ No newline at end of file diff --git a/services/catalog/openapi.json b/services/catalog/openapi.json index 0f1af3567d2..8e9a2d8034b 100644 --- a/services/catalog/openapi.json +++ b/services/catalog/openapi.json @@ -27,330 +27,6 @@ } } }, - "/v0/dags": { - "get": { - "tags": [ - "DAG" - ], - "summary": "List Dags", - "operationId": "list_dags_v0_dags_get", - "parameters": [ - { - "description": "Requests a specific page of the list results", - "required": false, - "schema": { - "title": "Page Token", - "type": "string", - "description": "Requests a specific page of the list results" - }, - "name": "page_token", - "in": "query" - }, - { - "description": "Maximum number of results to be returned by the server", - "required": false, - "schema": { - "title": "Page Size", - "minimum": 0, - "type": "integer", - "description": "Maximum number of results to be returned by the server", - "default": 0 - }, - "name": "page_size", - "in": "query" - }, - { - "description": "Sorts in ascending order comma-separated fields", - "required": false, - "schema": { - "title": "Order By", - "type": "string", - "description": "Sorts in ascending order comma-separated fields" - }, - "name": "order_by", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "title": "Response List Dags V0 Dags Get", - "type": "array", - "items": { - "$ref": "#/components/schemas/DAGOut" - } - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "post": { - "tags": [ - "DAG" - ], - "summary": "Create Dag", - "operationId": "create_dag_v0_dags_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DAGIn" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "Successfully created", - "content": { - "application/json": { - "schema": { - "title": "Response Create Dag V0 Dags Post", - "type": "integer" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v0/dags:batchGet": { - "get": { - "tags": [ - "DAG" - ], - "summary": "Batch Get Dags", - "operationId": "batch_get_dags_v0_dags_batchGet_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, - "/v0/dags:search": { - "get": { - "tags": [ - "DAG" - ], - "summary": "Search Dags", - "operationId": "search_dags_v0_dags_search_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, - "/v0/dags/{dag_id}": { - "get": { - "tags": [ - "DAG" - ], - "summary": "Get Dag", - "operationId": "get_dag_v0_dags__dag_id__get", - "parameters": [ - { - "required": true, - "schema": { - "title": "Dag Id", - "type": "integer" - }, - "name": "dag_id", - "in": "path" - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DAGOut" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "put": { - "tags": [ - "DAG" - ], - "summary": "Replace Dag", - "operationId": "replace_dag_v0_dags__dag_id__put", - "parameters": [ - { - "required": true, - "schema": { - "title": "Dag Id", - "type": "integer" - }, - "name": "dag_id", - "in": "path" - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DAGIn" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DAGOut" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "delete": { - "tags": [ - "DAG" - ], - "summary": "Delete Dag", - "operationId": "delete_dag_v0_dags__dag_id__delete", - "parameters": [ - { - "required": true, - "schema": { - "title": "Dag Id", - "type": "integer" - }, - "name": "dag_id", - "in": "path" - } - ], - "responses": { - "204": { - "description": "Successfully deleted" - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "patch": { - "tags": [ - "DAG" - ], - "summary": "Udpate Dag", - "operationId": "udpate_dag_v0_dags__dag_id__patch", - "parameters": [ - { - "required": true, - "schema": { - "title": "Dag Id", - "type": "integer" - }, - "name": "dag_id", - "in": "path" - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DAGIn" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DAGOut" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/services/{service_key}/{service_version}/resources": { "get": { "tags": [ @@ -862,16 +538,6 @@ }, "components": { "schemas": { - "AccessEnum": { - "title": "AccessEnum", - "enum": [ - "ReadAndWrite", - "Invisible", - "ReadOnly" - ], - "type": "string", - "description": "An enumeration." - }, "Author": { "title": "Author", "required": [ @@ -1243,129 +909,38 @@ }, "Ulimits": { "title": "Ulimits", - "type": "array", - "items": { - "$ref": "#/components/schemas/Ulimit1" - }, - "description": "A list of resource limits to set in the container. For example: `{\"Name\": \"nofile\", \"Soft\": 1024, \"Hard\": 2048}`\"\n" - } - }, - "description": " Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." - }, - "CredentialSpec": { - "title": "CredentialSpec", - "type": "object", - "properties": { - "Config": { - "title": "Config", - "type": "string", - "description": "Load credential spec from a Swarm Config with the given ID.\nThe specified config must also be present in the Configs\nfield with the Runtime property set.\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", - "example": "0bt9dmxjvjiqermk6xrop3ekq" - }, - "File": { - "title": "File", - "type": "string", - "description": "Load credential spec from this file. The file is read by\nthe daemon, and must be present in the `CredentialSpecs`\nsubdirectory in the docker data directory, which defaults\nto `C:\\ProgramData\\Docker\\` on Windows.\n\nFor example, specifying `spec.json` loads\n`C:\\ProgramData\\Docker\\CredentialSpecs\\spec.json`.\n\n


\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", - "example": "spec.json" - }, - "Registry": { - "title": "Registry", - "type": "string", - "description": "Load credential spec from this value in the Windows\nregistry. The specified registry value must be located in:\n\n`HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Virtualization\\Containers\\CredentialSpecs`\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n" - } - }, - "description": "CredentialSpec for managed service account (Windows only)" - }, - "DAGIn": { - "title": "DAGIn", - "required": [ - "key", - "version", - "name" - ], - "type": "object", - "properties": { - "key": { - "title": "Key", - "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", - "type": "string", - "example": "simcore/services/frontend/nodes-group/macros/1" - }, - "version": { - "title": "Version", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", - "type": "string", - "example": "1.0.0" - }, - "name": { - "title": "Name", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "contact": { - "title": "Contact", - "type": "string", - "format": "email" - }, - "workbench": { - "title": "Workbench", - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/Node" - } + "type": "array", + "items": { + "$ref": "#/components/schemas/Ulimit1" + }, + "description": "A list of resource limits to set in the container. For example: `{\"Name\": \"nofile\", \"Soft\": 1024, \"Hard\": 2048}`\"\n" } - } + }, + "description": " Container spec for the service.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." }, - "DAGOut": { - "title": "DAGOut", - "required": [ - "key", - "version", - "name", - "id" - ], + "CredentialSpec": { + "title": "CredentialSpec", "type": "object", "properties": { - "key": { - "title": "Key", - "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "Config": { + "title": "Config", "type": "string", - "example": "simcore/services/frontend/nodes-group/macros/1" + "description": "Load credential spec from a Swarm Config with the given ID.\nThe specified config must also be present in the Configs\nfield with the Runtime property set.\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", + "example": "0bt9dmxjvjiqermk6xrop3ekq" }, - "version": { - "title": "Version", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", + "File": { + "title": "File", "type": "string", - "example": "1.0.0" - }, - "name": { - "title": "Name", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" + "description": "Load credential spec from this file. The file is read by\nthe daemon, and must be present in the `CredentialSpecs`\nsubdirectory in the docker data directory, which defaults\nto `C:\\ProgramData\\Docker\\` on Windows.\n\nFor example, specifying `spec.json` loads\n`C:\\ProgramData\\Docker\\CredentialSpecs\\spec.json`.\n\n


\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n", + "example": "spec.json" }, - "contact": { - "title": "Contact", + "Registry": { + "title": "Registry", "type": "string", - "format": "email" - }, - "id": { - "title": "Id", - "type": "integer" - }, - "workbench": { - "title": "Workbench", - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/Node" - } + "description": "Load credential spec from this value in the Windows\nregistry. The specified registry value must be located in:\n\n`HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Virtualization\\Containers\\CredentialSpecs`\n\n


\n\n\n> **Note**: `CredentialSpec.File`, `CredentialSpec.Registry`,\n> and `CredentialSpec.Config` are mutually exclusive.\n" } - } + }, + "description": "CredentialSpec for managed service account (Windows only)" }, "DNSConfig": { "title": "DNSConfig", @@ -1398,54 +973,6 @@ }, "description": " Specification for DNS related configurations in resolver configuration\nfile (`resolv.conf`)." }, - "DatCoreFileLink": { - "title": "DatCoreFileLink", - "required": [ - "store", - "path", - "label", - "dataset" - ], - "type": "object", - "properties": { - "store": { - "title": "Store", - "type": "integer", - "description": "The store identifier: 0 for simcore S3, 1 for datcore" - }, - "path": { - "title": "Path", - "anyOf": [ - { - "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$", - "type": "string" - }, - { - "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$", - "type": "string" - } - ], - "description": "The path to the file in the storage provider domain" - }, - "label": { - "title": "Label", - "type": "string", - "description": "The real file name" - }, - "eTag": { - "title": "Etag", - "type": "string", - "description": "Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box)." - }, - "dataset": { - "title": "Dataset", - "type": "string", - "description": "Unique identifier to access the dataset on datcore (REQUIRED for datcore)" - } - }, - "additionalProperties": false, - "description": "I/O port type to hold a link to a file in DATCORE storage" - }, "DiscreteResourceSpec": { "title": "DiscreteResourceSpec", "type": "object", @@ -1460,29 +987,6 @@ } } }, - "DownloadLink": { - "title": "DownloadLink", - "required": [ - "downloadLink" - ], - "type": "object", - "properties": { - "downloadLink": { - "title": "Downloadlink", - "maxLength": 65536, - "minLength": 1, - "type": "string", - "format": "uri" - }, - "label": { - "title": "Label", - "type": "string", - "description": "Display name" - } - }, - "additionalProperties": false, - "description": "I/O port type to hold a generic download link to a file (e.g. S3 pre-signed link, etc)" - }, "DriverConfig": { "title": "DriverConfig", "type": "object", @@ -1845,14 +1349,14 @@ }, "version": { "title": "Version", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z]+)*)?$", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "type": "string" }, "released": { "title": "Released", "type": "object", "additionalProperties": { - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z]+)*)?$", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "type": "string" }, "description": "Maps every route's path tag with a released version" @@ -2019,182 +1523,6 @@ }, "description": " Read-only spec type for non-swarm containers attached to swarm overlay\nnetworks.\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." }, - "Node": { - "title": "Node", - "required": [ - "key", - "version", - "label" - ], - "type": "object", - "properties": { - "key": { - "title": "Key", - "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", - "type": "string", - "description": "distinctive name for the node based on the docker registry path" - }, - "version": { - "title": "Version", - "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", - "type": "string", - "description": "semantic version number of the node" - }, - "label": { - "title": "Label", - "type": "string", - "description": "The short name of the node" - }, - "progress": { - "title": "Progress", - "maximum": 100.0, - "minimum": 0.0, - "type": "number", - "description": "the node progress value", - "deprecated": true - }, - "thumbnail": { - "title": "Thumbnail", - "maxLength": 2083, - "minLength": 0, - "type": "string", - "description": "url of the latest screenshot of the node", - "format": "uri" - }, - "runHash": { - "anyOf": [ - { - "type": "null" - }, - { - "title": "Runhash", - "type": "string", - "description": "the hex digest of the resolved inputs +outputs hash at the time when the last outputs were generated" - } - ] - }, - "inputs": { - "title": "Inputs", - "type": "object", - "description": "values of input properties" - }, - "inputsUnits": { - "title": "Inputsunits", - "type": "object", - "description": "Overrides default unit (if any) defined in the service for each port" - }, - "inputAccess": { - "type": "object", - "description": "map with key - access level pairs" - }, - "inputNodes": { - "title": "Inputnodes", - "type": "array", - "items": { - "type": "string", - "format": "uuid" - }, - "description": "node IDs of where the node is connected to" - }, - "outputs": { - "title": "Outputs", - "type": "object", - "description": "values of output properties" - }, - "outputNode": { - "title": "Outputnode", - "type": "boolean", - "deprecated": true - }, - "outputNodes": { - "title": "Outputnodes", - "type": "array", - "items": { - "type": "string", - "format": "uuid" - }, - "description": "Used in group-nodes. Node IDs of those connected to the output" - }, - "parent": { - "anyOf": [ - { - "type": "null" - }, - { - "title": "Parent", - "type": "string", - "description": "Parent's (group-nodes') node ID s. Used to group", - "format": "uuid" - } - ] - }, - "position": { - "title": "Position", - "allOf": [ - { - "$ref": "#/components/schemas/Position" - } - ], - "description": "Use projects_ui.WorkbenchUI.position instead", - "deprecated": true - }, - "state": { - "title": "State", - "allOf": [ - { - "$ref": "#/components/schemas/NodeState" - } - ], - "description": "The node's state object" - }, - "bootOptions": { - "title": "Bootoptions", - "type": "object", - "description": "Some services provide alternative parameters to be injected at boot time. The user selection should be stored here, and it will overwrite the services's defaults." - } - }, - "additionalProperties": false - }, - "NodeState": { - "title": "NodeState", - "type": "object", - "properties": { - "modified": { - "title": "Modified", - "type": "boolean", - "description": "true if the node's outputs need to be re-computed", - "default": true - }, - "dependencies": { - "title": "Dependencies", - "uniqueItems": true, - "type": "array", - "items": { - "type": "string", - "format": "uuid" - }, - "description": "contains the node inputs dependencies if they need to be computed first" - }, - "currentStatus": { - "allOf": [ - { - "$ref": "#/components/schemas/RunningState" - } - ], - "description": "the node's current state", - "default": "NOT_STARTED" - }, - "progress": { - "title": "Progress", - "maximum": 1.0, - "minimum": 0.0, - "type": "number", - "description": "current progress of the task if available (None if not started or not a computational task)", - "default": 0 - } - }, - "additionalProperties": false - }, "Order": { "title": "Order", "enum": [ @@ -2333,57 +1661,6 @@ }, "description": " Plugin spec for the service. *(Experimental release only.)*\n\n


\n\n> **Note**: ContainerSpec, NetworkAttachmentSpec, and PluginSpec are\n> mutually exclusive. PluginSpec is only used when the Runtime field\n> is set to `plugin`. NetworkAttachmentSpec is used when the Runtime\n> field is set to `attachment`." }, - "PortLink": { - "title": "PortLink", - "required": [ - "nodeUuid", - "output" - ], - "type": "object", - "properties": { - "nodeUuid": { - "title": "Nodeuuid", - "type": "string", - "description": "The node to get the port output from", - "format": "uuid" - }, - "output": { - "title": "Output", - "pattern": "^[-_a-zA-Z0-9]+$", - "type": "string", - "description": "The port key in the node given by nodeUuid" - } - }, - "additionalProperties": false, - "description": "I/O port type to reference to an output port of another node in the same project" - }, - "Position": { - "title": "Position", - "required": [ - "x", - "y" - ], - "type": "object", - "properties": { - "x": { - "title": "X", - "type": "integer", - "description": "The x position", - "example": [ - "12" - ] - }, - "y": { - "title": "Y", - "type": "integer", - "description": "The y position", - "example": [ - "15" - ] - } - }, - "additionalProperties": false - }, "Preference": { "title": "Preference", "type": "object", @@ -2627,22 +1904,6 @@ }, "description": "Specification for the rollback strategy of the service." }, - "RunningState": { - "title": "RunningState", - "enum": [ - "UNKNOWN", - "PUBLISHED", - "NOT_STARTED", - "PENDING", - "STARTED", - "RETRY", - "SUCCESS", - "FAILED", - "ABORTED" - ], - "type": "string", - "description": "State of execution of a project's computational workflow\n\nSEE StateType for task state" - }, "SELinuxContext": { "title": "SELinuxContext", "type": "object", @@ -2886,7 +2147,7 @@ "description": "Static metadata for a service injected in the image labels\n\nThis is one to one with node-meta-v0.0.1.json", "example": { "name": "File Picker", - "description": "File Picker", + "description": "description", "classifiers": [], "quality": {}, "accessRights": { @@ -3342,52 +2603,6 @@ } } }, - "SimCoreFileLink": { - "title": "SimCoreFileLink", - "required": [ - "store", - "path" - ], - "type": "object", - "properties": { - "store": { - "title": "Store", - "type": "integer", - "description": "The store identifier: 0 for simcore S3, 1 for datcore" - }, - "path": { - "title": "Path", - "anyOf": [ - { - "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$", - "type": "string" - }, - { - "pattern": "^N:package:[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$", - "type": "string" - } - ], - "description": "The path to the file in the storage provider domain" - }, - "label": { - "title": "Label", - "type": "string", - "description": "The real file name" - }, - "eTag": { - "title": "Etag", - "type": "string", - "description": "Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box)." - }, - "dataset": { - "title": "Dataset", - "type": "string", - "deprecated": true - } - }, - "additionalProperties": false, - "description": "I/O port type to hold a link to a file in simcore S3 storage" - }, "Spread": { "title": "Spread", "type": "object", diff --git a/services/catalog/setup.cfg b/services/catalog/setup.cfg index e17423c5d32..32bc36fdac7 100644 --- a/services/catalog/setup.cfg +++ b/services/catalog/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.0 +current_version = 0.5.0 commit = True message = services/catalog version: {current_version} → {new_version} tag = False @@ -9,5 +9,5 @@ commit_args = --no-verify [tool:pytest] asyncio_mode = auto -markers = +markers = testit: "marks test to run during development" diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index fe9f5162528..ad51a5324bf 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -5,6 +5,10 @@ from fastapi import Depends, Header, HTTPException, status from fastapi.requests import Request +from models_library.api_schemas_catalog.services import ServiceGet +from models_library.api_schemas_catalog.services_specifications import ( + ServiceSpecifications, +) from models_library.services import ServiceKey, ServiceVersion from models_library.services_resources import ResourcesDict from pydantic import ValidationError @@ -12,8 +16,6 @@ from ...core.settings import ApplicationSettings from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository -from ...models.schemas.services import ServiceGet -from ...models.schemas.services_specifications import ServiceSpecifications from ...services.director import DirectorApi from ...services.function_services import get_function_service, is_function_service from .database import get_repository diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py b/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py index be13c8a58ac..277ad40f3cb 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py @@ -1,15 +1,14 @@ -from typing import Optional - from fastapi import Depends, Query +from models_library.groups import GroupAtDB from models_library.users import UserID from ...db.repositories.groups import GroupsRepository -from ...models.domain.group import GroupAtDB from .database import get_repository async def list_user_groups( - user_id: Optional[UserID] = Query( + user_id: UserID + | None = Query( default=None, description="if passed, and that user has custom resources, " "they will be merged with default resources and returned.", diff --git a/services/catalog/src/simcore_service_catalog/api/root.py b/services/catalog/src/simcore_service_catalog/api/root.py index d5e34da5e78..4efd75f9167 100644 --- a/services/catalog/src/simcore_service_catalog/api/root.py +++ b/services/catalog/src/simcore_service_catalog/api/root.py @@ -1,7 +1,6 @@ from fastapi import APIRouter from .routes import ( - dags, health, meta, services, @@ -16,7 +15,6 @@ # API router.include_router(meta.router, tags=["meta"], prefix="/meta") -router.include_router(dags.router, tags=["DAG"], prefix="/dags") SERVICE_PREFIX = "/services" SERVICE_TAGS = [ diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/constants.py b/services/catalog/src/simcore_service_catalog/api/routes/_constants.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/models/schemas/constants.py rename to services/catalog/src/simcore_service_catalog/api/routes/_constants.py diff --git a/services/catalog/src/simcore_service_catalog/api/routes/dags.py b/services/catalog/src/simcore_service_catalog/api/routes/dags.py deleted file mode 100644 index cf22a9d9c81..00000000000 --- a/services/catalog/src/simcore_service_catalog/api/routes/dags.py +++ /dev/null @@ -1,133 +0,0 @@ -import logging -from typing import Optional - -from fastapi import APIRouter, Body, Depends, HTTPException, Query -from starlette.status import ( - HTTP_201_CREATED, - HTTP_204_NO_CONTENT, - HTTP_409_CONFLICT, - HTTP_501_NOT_IMPLEMENTED, -) - -from ...db.repositories.dags import DAGsRepository -from ...models.schemas.dag import DAGIn, DAGOut -from ..dependencies.database import get_repository - -router = APIRouter() -log = logging.getLogger(__name__) - - -@router.get("", response_model=list[DAGOut]) -async def list_dags( - page_token: Optional[str] = Query( - None, description="Requests a specific page of the list results" - ), - page_size: int = Query( - 0, ge=0, description="Maximum number of results to be returned by the server" - ), - order_by: Optional[str] = Query( - None, description="Sorts in ascending order comma-separated fields" - ), - dags_repo: DAGsRepository = Depends(get_repository(DAGsRepository)), -): - - # List is suited to data from a single collection that is bounded in size and not cached - - # Applicable common patterns - # SEE pagination: https://cloud.google.com/apis/design/design_patterns#list_pagination - # SEE sorting https://cloud.google.com/apis/design/design_patterns#sorting_order - - # Applicable naming conventions - # TODO: filter: https://cloud.google.com/apis/design/naming_convention#list_filter_field - # SEE response: https://cloud.google.com/apis/design/naming_convention#list_response - log.debug("%s %s %s", page_token, page_size, order_by) - dags = await dags_repo.list_dags() - return dags - - -@router.get(":batchGet") -async def batch_get_dags(): - raise HTTPException( - status_code=HTTP_501_NOT_IMPLEMENTED, detail="Still not implemented" - ) - - -@router.get(":search") -async def search_dags(): - # A method that takes multiple resource IDs and returns an object for each of those IDs - # Alternative to List for fetching data that does not adhere to List semantics, such as services.search. - # https://cloud.google.com/apis/design/standard_methods#list - raise HTTPException( - status_code=HTTP_501_NOT_IMPLEMENTED, detail="Still not implemented" - ) - - -@router.get("/{dag_id}", response_model=DAGOut) -async def get_dag( - dag_id: int, - dags_repo: DAGsRepository = Depends(get_repository(DAGsRepository)), -): - dag = await dags_repo.get_dag(dag_id) - return dag - - -@router.post( - "", - response_model=int, - status_code=HTTP_201_CREATED, - response_description="Successfully created", -) -async def create_dag( - dag: DAGIn = Body(...), - dags_repo: DAGsRepository = Depends(get_repository(DAGsRepository)), -): - assert dag # nosec - - if dag.version == "0.0.0" and dag.key == "foo": - # client-assigned resouce name - raise HTTPException( - status_code=HTTP_409_CONFLICT, - detail=f"DAG {dag.key}:{dag.version} already exists", - ) - - # FIXME: conversion DAG (issue with workbench being json in orm and dict in schema) - dag_id = await dags_repo.create_dag(dag) - # TODO: no need to return since there is not extra info?, perhaps return - return dag_id - - -@router.patch("/{dag_id}", response_model=DAGOut) -async def udpate_dag( - dag_id: int, - dag: DAGIn = Body(None), - dags_repo: DAGsRepository = Depends(get_repository(DAGsRepository)), -): - async with dags_repo.db_engine.begin(): - await dags_repo.update_dag(dag_id, dag) - updated_dag = await dags_repo.get_dag(dag_id) - - return updated_dag - - -@router.put("/{dag_id}", response_model=Optional[DAGOut]) -async def replace_dag( - dag_id: int, - dag: DAGIn = Body(...), - dags_repo: DAGsRepository = Depends(get_repository(DAGsRepository)), -): - await dags_repo.replace_dag(dag_id, dag) - - -@router.delete( - "/{dag_id}", - status_code=HTTP_204_NO_CONTENT, - response_description="Successfully deleted", -) -async def delete_dag( - dag_id: int, - dags_repo: DAGsRepository = Depends(get_repository(DAGsRepository)), -): - # If the Delete method immediately removes the resource, it should return an empty response. - # If the Delete method initiates a long-running operation, it should return the long-running operation. - # If the Delete method only marks the resource as being deleted, it should return the updated resource. - await dags_repo.delete_dag(dag_id) diff --git a/services/catalog/src/simcore_service_catalog/api/routes/meta.py b/services/catalog/src/simcore_service_catalog/api/routes/meta.py index 2def46ab51b..6cfe6898ee4 100644 --- a/services/catalog/src/simcore_service_catalog/api/routes/meta.py +++ b/services/catalog/src/simcore_service_catalog/api/routes/meta.py @@ -1,8 +1,8 @@ from fastapi import APIRouter +from models_library.api_schemas_catalog.meta import Meta, VersionStr from pydantic import parse_obj_as from ..._meta import API_VERSION, API_VTAG -from ...models.schemas.meta import Meta, VersionStr router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/routes/services.py b/services/catalog/src/simcore_service_catalog/api/routes/services.py index 400444203cf..095c09a7252 100644 --- a/services/catalog/src/simcore_service_catalog/api/routes/services.py +++ b/services/catalog/src/simcore_service_catalog/api/routes/services.py @@ -3,10 +3,11 @@ import asyncio import logging import urllib.parse -from typing import Any, cast +from typing import Any, TypeAlias, cast from aiocache import cached from fastapi import APIRouter, Depends, Header, HTTPException, status +from models_library.api_schemas_catalog.services import ServiceGet, ServiceUpdate from models_library.services import ServiceKey, ServiceType, ServiceVersion from models_library.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB from pydantic import ValidationError @@ -15,22 +16,21 @@ from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository -from ...models.schemas.constants import ( - DIRECTOR_CACHING_TTL, - LIST_SERVICES_CACHING_TTL, - RESPONSE_MODEL_POLICY, -) -from ...models.schemas.services import ServiceGet, ServiceUpdate from ...services.director import DirectorApi from ...services.function_services import is_function_service from ...utils.requests_decorators import cancellable_request from ..dependencies.database import get_repository from ..dependencies.director import get_director_api from ..dependencies.services import get_service_from_registry +from ._constants import ( + DIRECTOR_CACHING_TTL, + LIST_SERVICES_CACHING_TTL, + RESPONSE_MODEL_POLICY, +) -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) -ServicesSelection = set[tuple[str, str]] +ServicesSelection: TypeAlias = set[tuple[str, str]] def _prepare_service_details( @@ -52,7 +52,7 @@ def _prepare_service_details( try: validated_service = ServiceGet(**composed_service) except ValidationError as exc: - logger.warning( + _logger.warning( "could not validate service [%s:%s]: %s", composed_service.get("key"), composed_service.get("version"), diff --git a/services/catalog/src/simcore_service_catalog/api/routes/services_access_rights.py b/services/catalog/src/simcore_service_catalog/api/routes/services_access_rights.py index 527fb8bc24b..644b019ff93 100644 --- a/services/catalog/src/simcore_service_catalog/api/routes/services_access_rights.py +++ b/services/catalog/src/simcore_service_catalog/api/routes/services_access_rights.py @@ -1,14 +1,16 @@ import logging from fastapi import APIRouter, Depends, Header -from models_library.api_schemas_catalog import ServiceAccessRightsGet +from models_library.api_schemas_catalog.service_access_rights import ( + ServiceAccessRightsGet, +) from models_library.services import ServiceKey, ServiceVersion from models_library.services_db import ServiceAccessRightsAtDB from ...db.repositories.services import ServicesRepository -from ...models.schemas.constants import RESPONSE_MODEL_POLICY from ..dependencies.database import get_repository from ..dependencies.services import AccessInfo, check_service_read_access +from ._constants import RESPONSE_MODEL_POLICY # # Routes ----------------------------------------------------------------------------------------------- diff --git a/services/catalog/src/simcore_service_catalog/api/routes/services_ports.py b/services/catalog/src/simcore_service_catalog/api/routes/services_ports.py index de6148dc5e6..24e0386b1c0 100644 --- a/services/catalog/src/simcore_service_catalog/api/routes/services_ports.py +++ b/services/catalog/src/simcore_service_catalog/api/routes/services_ports.py @@ -1,17 +1,17 @@ import logging from fastapi import APIRouter, Depends +from models_library.api_schemas_catalog.services import ServiceGet +from models_library.api_schemas_catalog.services_ports import ServicePortGet -from ...models.schemas.constants import RESPONSE_MODEL_POLICY -from ...models.schemas.services import ServiceGet -from ...models.schemas.services_ports import ServicePortGet from ..dependencies.services import ( AccessInfo, check_service_read_access, get_service_from_registry, ) +from ._constants import RESPONSE_MODEL_POLICY -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) # diff --git a/services/catalog/src/simcore_service_catalog/api/routes/services_resources.py b/services/catalog/src/simcore_service_catalog/api/routes/services_resources.py index 2e8843a7f88..b3848fa7b9b 100644 --- a/services/catalog/src/simcore_service_catalog/api/routes/services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/routes/services_resources.py @@ -6,6 +6,7 @@ import yaml from fastapi import APIRouter, Depends, HTTPException, status from models_library.docker import DockerGenericTag +from models_library.groups import GroupAtDB from models_library.service_settings_labels import ( ComposeSpecLabelDict, SimcoreServiceSettingLabelEntry, @@ -22,11 +23,6 @@ from pydantic import parse_obj_as, parse_raw_as from ...db.repositories.services import ServicesRepository -from ...models.domain.group import GroupAtDB -from ...models.schemas.constants import ( - RESPONSE_MODEL_POLICY, - SIMCORE_SERVICE_SETTINGS_LABELS, -) from ...services.director import DirectorApi from ...services.function_services import is_function_service from ...utils.service_resources import ( @@ -37,6 +33,7 @@ from ..dependencies.director import get_director_api from ..dependencies.services import get_default_service_resources from ..dependencies.user_groups import list_user_groups +from ._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS router = APIRouter() logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/routes/services_specifications.py b/services/catalog/src/simcore_service_catalog/api/routes/services_specifications.py index 9c04c8402c6..7b2461d050c 100644 --- a/services/catalog/src/simcore_service_catalog/api/routes/services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/routes/services_specifications.py @@ -1,22 +1,22 @@ import logging from fastapi import APIRouter, Depends, HTTPException, Query, status +from models_library.api_schemas_catalog.services_specifications import ( + ServiceSpecifications, + ServiceSpecificationsGet, +) from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository -from ...models.schemas.constants import RESPONSE_MODEL_POLICY -from ...models.schemas.services_specifications import ( - ServiceSpecifications, - ServiceSpecificationsGet, -) from ...services.function_services import is_function_service from ..dependencies.database import get_repository from ..dependencies.services import get_default_service_specifications +from ._constants import RESPONSE_MODEL_POLICY router = APIRouter() -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) @router.get( @@ -38,7 +38,7 @@ async def get_service_specifications( get_default_service_specifications ), ): - logger.debug("getting specifications for '%s:%s'", service_key, service_version) + _logger.debug("getting specifications for '%s:%s'", service_key, service_version) if is_function_service(service_key): # There is no specification for these, return empty specs @@ -64,5 +64,5 @@ async def get_service_specifications( # nothing found, let's return the default then service_specs = default_service_specifications.copy() - logger.debug("returning %s", f"{service_specs=}") + _logger.debug("returning %s", f"{service_specs=}") return service_specs diff --git a/services/catalog/src/simcore_service_catalog/cli.py b/services/catalog/src/simcore_service_catalog/cli.py index 613ae543caf..a0ab31715e2 100644 --- a/services/catalog/src/simcore_service_catalog/cli.py +++ b/services/catalog/src/simcore_service_catalog/cli.py @@ -6,12 +6,14 @@ from ._meta import PROJECT_NAME from .core.settings import ApplicationSettings -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) # NOTE: 'main' variable is referred in the setup's entrypoint! main = typer.Typer(name=PROJECT_NAME) -main.command()(create_settings_command(settings_cls=ApplicationSettings, logger=log)) +main.command()( + create_settings_command(settings_cls=ApplicationSettings, logger=_logger) +) @main.command() diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index e271bf3ebf2..236f903bd8b 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -2,6 +2,9 @@ from functools import cached_property from typing import Final +from models_library.api_schemas_catalog.services_specifications import ( + ServiceSpecifications, +) from models_library.basic_types import BootModeEnum, BuildTargetEnum, LogLevel from models_library.services_resources import ResourcesDict from pydantic import ByteSize, Field, PositiveInt, parse_obj_as @@ -10,8 +13,6 @@ from settings_library.postgres import PostgresSettings from settings_library.utils_logging import MixinLoggingSettings -from ..models.schemas.services_specifications import ServiceSpecifications - logger = logging.getLogger(__name__) @@ -51,12 +52,12 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): env=["CATALOG_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], ) CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - False, + default=False, env=["CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) CATALOG_DEV_FEATURES_ENABLED: bool = Field( - False, + default=False, description="Enables development features. WARNING: make sure it is disabled in production .env file!", ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/dags.py b/services/catalog/src/simcore_service_catalog/db/repositories/dags.py deleted file mode 100644 index d381d0b25fe..00000000000 --- a/services/catalog/src/simcore_service_catalog/db/repositories/dags.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -from typing import Optional - -import sqlalchemy as sa - -from ...models.domain.dag import DAGAtDB -from ...models.schemas.dag import DAGIn -from ..tables import dags -from ._base import BaseRepository - - -class DAGsRepository(BaseRepository): - async def list_dags(self) -> list[DAGAtDB]: - dagraphs = [] - async with self.db_engine.connect() as conn: - async for row in await conn.stream(dags.select()): - dagraphs.append(DAGAtDB.parse_obj(row)) - return dagraphs - - async def get_dag(self, dag_id: int) -> Optional[DAGAtDB]: - async with self.db_engine.connect() as conn: - result = await conn.execute(dags.select().where(dags.c.id == dag_id)) - row = result.first() - if row: - return DAGAtDB.from_orm(row) - return None - - async def create_dag(self, dag: DAGIn) -> int: - async with self.db_engine.begin() as conn: - new_id: int = await conn.scalar( - dags.insert().values( - workbench=dag.json(include={"workbench"}), - **dag.dict(exclude={"workbench"}) - ) - ) - - return new_id - - async def replace_dag(self, dag_id: int, dag: DAGIn) -> None: - async with self.db_engine.begin() as conn: - await conn.execute( - dags.update() - .values( - workbench=dag.json(include={"workbench"}), - **dag.dict(exclude={"workbench"}) - ) - .where(dags.c.id == dag_id) - ) - - async def update_dag(self, dag_id: int, dag: DAGIn) -> None: - patch = dag.dict(exclude_unset=True, exclude={"workbench"}) - if "workbench" in dag.__fields_set__: - patch["workbench"] = json.dumps(patch["workbench"]) - async with self.db_engine.begin() as conn: - await conn.execute( - sa.update(dags).values(**patch).where(dags.c.id == dag_id) - ) - - async def delete_dag(self, dag_id: int) -> None: - async with self.db_engine.begin() as conn: - await conn.execute(sa.delete(dags).where(dags.c.id == dag_id)) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py b/services/catalog/src/simcore_service_catalog/db/repositories/groups.py index c3f4780bd74..c1c293425e4 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/groups.py @@ -2,9 +2,9 @@ import sqlalchemy as sa from models_library.emails import LowerCaseEmailStr +from models_library.groups import GroupAtDB from pydantic.types import PositiveInt -from ...models.domain.group import GroupAtDB from ..errors import RepositoryError from ..tables import GroupType, groups, user_to_groups, users from ._base import BaseRepository diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index d6e898d6b40..05f1d3cda72 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -1,16 +1,20 @@ import logging from collections import defaultdict +from collections.abc import Iterable from itertools import chain -from typing import Any, Iterable, cast +from typing import Any, cast import packaging.version import sqlalchemy as sa +from models_library.api_schemas_catalog.services_specifications import ( + ServiceSpecifications, +) +from models_library.groups import GroupAtDB, GroupTypeInModel from models_library.services import ServiceKey, ServiceVersion from models_library.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB from models_library.users import GroupID from psycopg2.errors import ForeignKeyViolation from pydantic import ValidationError -from simcore_postgres_database.models.groups import GroupType from simcore_postgres_database.utils_services import create_select_latest_services_query from sqlalchemy import literal_column from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -18,9 +22,7 @@ from sqlalchemy.sql.expression import tuple_ from sqlalchemy.sql.selectable import Select -from ...models.domain.group import GroupAtDB -from ...models.domain.service_specifications import ServiceSpecificationsAtDB -from ...models.schemas.services_specifications import ServiceSpecifications +from ...models.service_specifications import ServiceSpecificationsAtDB from ..tables import services_access_rights, services_meta_data, services_specifications from ._base import BaseRepository @@ -37,11 +39,7 @@ def _make_list_services_query( query = sa.select(services_meta_data) if gids or execute_access or write_access: logic_operator = and_ if combine_access_with_and else or_ - default = ( - True # pylint: disable=simplifiable-if-expression - if combine_access_with_and - else False - ) + default = bool(combine_access_with_and) access_query_part = logic_operator( services_access_rights.c.execute_access if execute_access else default, services_access_rights.c.write_access if write_access else default, @@ -111,7 +109,8 @@ async def list_service_releases( limit_count limits returned value. None or non-positive values returns all matches """ if minor is not None and major is None: - raise ValueError("Expected only major.*.* or major.minor.*") + msg = "Expected only major.*.* or major.minor.*" + raise ValueError(msg) search_condition = services_meta_data.c.key == key if major is not None: @@ -142,8 +141,7 @@ async def list_service_releases( def _by_version(x: ServiceMetaDataAtDB) -> packaging.version.Version: return cast(packaging.version.Version, packaging.version.parse(x.version)) - releases_sorted = sorted(releases, key=_by_version, reverse=True) - return releases_sorted + return sorted(releases, key=_by_version, reverse=True) async def get_latest_release(self, key: str) -> ServiceMetaDataAtDB | None: """Returns last release or None if service was never released""" @@ -219,9 +217,8 @@ async def create_service( access_rights.key != new_service.key or access_rights.version != new_service.version ): - raise ValueError( - f"{access_rights} does not correspond to service {new_service.key}:{new_service.version}" - ) + msg = f"{access_rights} does not correspond to service {new_service.key}:{new_service.version}" + raise ValueError(msg) async with self.db_engine.begin() as conn: # NOTE: this ensure proper rollback in case of issue result = await conn.execute( @@ -258,8 +255,7 @@ async def update_service( ) row = result.first() assert row # nosec - updated_service = ServiceMetaDataAtDB.from_orm(row) - return updated_service + return ServiceMetaDataAtDB.from_orm(row) async def get_service_access_rights( self, @@ -410,16 +406,16 @@ async def get_service_specifications( continue # filter by group type group = gid_to_group_map[row.gid] - if (group.group_type == GroupType.STANDARD) and _is_newer( + if (group.group_type == GroupTypeInModel.STANDARD) and _is_newer( teams_specs.get(db_service_spec.gid), db_service_spec, ): teams_specs[db_service_spec.gid] = db_service_spec - elif (group.group_type == GroupType.EVERYONE) and _is_newer( + elif (group.group_type == GroupTypeInModel.EVERYONE) and _is_newer( everyone_specs, db_service_spec ): everyone_specs = db_service_spec - elif (group.group_type == GroupType.PRIMARY) and _is_newer( + elif (group.group_type == GroupTypeInModel.PRIMARY) and _is_newer( primary_specs, db_service_spec ): primary_specs = db_service_spec diff --git a/services/catalog/src/simcore_service_catalog/models/domain/dag.py b/services/catalog/src/simcore_service_catalog/models/domain/dag.py deleted file mode 100644 index f82c2755c68..00000000000 --- a/services/catalog/src/simcore_service_catalog/models/domain/dag.py +++ /dev/null @@ -1,30 +0,0 @@ -from models_library.basic_regex import VERSION_RE -from models_library.emails import LowerCaseEmailStr -from models_library.projects_nodes import Node -from models_library.services import SERVICE_KEY_RE -from pydantic import BaseModel, Field, Json - - -class DAGBase(BaseModel): - key: str = Field( - ..., - regex=SERVICE_KEY_RE.pattern, - example="simcore/services/frontend/nodes-group/macros/1", - ) - version: str = Field(..., regex=VERSION_RE, example="1.0.0") - name: str - description: str | None - contact: LowerCaseEmailStr | None - - -class DAGAtDB(DAGBase): - id: int - # pylint: disable=unsubscriptable-object - workbench: Json[dict[str, Node]] - - class Config: - orm_mode = True - - -class DAGData(DAGAtDB): - workbench: dict[str, Node] | None diff --git a/services/catalog/src/simcore_service_catalog/models/domain/group.py b/services/catalog/src/simcore_service_catalog/models/domain/group.py deleted file mode 100644 index b8eb35b34c0..00000000000 --- a/services/catalog/src/simcore_service_catalog/models/domain/group.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel, Field -from pydantic.types import PositiveInt - -from ...db.tables import GroupType - - -class Group(BaseModel): - gid: PositiveInt - name: str - description: str - group_type: GroupType = Field(..., alias="type") - thumbnail: Optional[str] - - -class GroupAtDB(Group): - class Config: - orm_mode = True - - schema_extra = { - "example": { - "gid": 218, - "name": "Friends group", - "description": "Joey, Ross, Rachel, Monica, Phoeby and Chandler", - "type": GroupType.STANDARD, - "thumbnail": "https://image.flaticon.com/icons/png/512/23/23374.png", - } - } diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/__init__.py b/services/catalog/src/simcore_service_catalog/models/schemas/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/dag.py b/services/catalog/src/simcore_service_catalog/models/schemas/dag.py deleted file mode 100644 index 1ebc66db648..00000000000 --- a/services/catalog/src/simcore_service_catalog/models/schemas/dag.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import Dict, Optional - -from models_library.projects_nodes import Node - -from ..domain.dag import DAGBase, DAGData - - -class DAGIn(DAGBase): - workbench: Optional[Dict[str, Node]] - - -class DAGInPath(DAGBase): - version: str - name: str - description: Optional[str] - contact: Optional[str] - workbench: Optional[Dict[str, Node]] - - -class DAGOut(DAGData): - pass diff --git a/services/catalog/src/simcore_service_catalog/models/schemas/meta.py b/services/catalog/src/simcore_service_catalog/models/schemas/meta.py deleted file mode 100644 index 3cb7da165d4..00000000000 --- a/services/catalog/src/simcore_service_catalog/models/schemas/meta.py +++ /dev/null @@ -1,31 +0,0 @@ -import re -from typing import Optional - -from pydantic import BaseModel, ConstrainedStr, Field - -# TODO: review this RE -# use https://www.python.org/dev/peps/pep-0440/#version-scheme -# or https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions -# -VERSION_RE = r"^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z]+)*)?$" - - -class VersionStr(ConstrainedStr): - regex = re.compile(VERSION_RE) - - -class Meta(BaseModel): - name: str - version: VersionStr - released: Optional[dict[str, VersionStr]] = Field( - None, description="Maps every route's path tag with a released version" - ) - - class Config: - schema_extra = { - "example": { - "name": "simcore_service_foo", - "version": "2.4.45", - "released": {"v1": "1.3.4", "v2": "2.4.45"}, - } - } diff --git a/services/catalog/src/simcore_service_catalog/models/domain/service_specifications.py b/services/catalog/src/simcore_service_catalog/models/service_specifications.py similarity index 75% rename from services/catalog/src/simcore_service_catalog/models/domain/service_specifications.py rename to services/catalog/src/simcore_service_catalog/models/service_specifications.py index d0966f0d0aa..ce40b492f07 100644 --- a/services/catalog/src/simcore_service_catalog/models/domain/service_specifications.py +++ b/services/catalog/src/simcore_service_catalog/models/service_specifications.py @@ -1,8 +1,9 @@ +from models_library.api_schemas_catalog.services_specifications import ( + ServiceSpecifications, +) from models_library.services import ServiceKey, ServiceVersion from models_library.users import GroupID -from ..schemas.services_specifications import ServiceSpecifications - class ServiceSpecificationsAtDB(ServiceSpecifications): service_key: ServiceKey diff --git a/services/catalog/tests/unit/conftest.py b/services/catalog/tests/unit/conftest.py index b7e70cbc85e..100d313cb84 100644 --- a/services/catalog/tests/unit/conftest.py +++ b/services/catalog/tests/unit/conftest.py @@ -3,12 +3,10 @@ # pylint:disable=redefined-outer-name import sys -from copy import deepcopy from pathlib import Path import pytest import simcore_service_catalog -from pytest import MonkeyPatch from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import load_dotenv, setenvs_from_envfile @@ -74,79 +72,7 @@ def testing_environ_vars( @pytest.fixture def service_test_environ( - service_env_file: Path, monkeypatch: MonkeyPatch + service_env_file: Path, monkeypatch: pytest.MonkeyPatch ) -> EnvVarsDict: # environs seen by app are defined by the service env-file! - app_envs = setenvs_from_envfile(monkeypatch, service_env_file, verbose=True) - return app_envs - - -# FAKE DATA ------ - - -@pytest.fixture() -def fake_data_dag_in() -> dict: - DAG_DATA_IN_DICT = { - "key": "simcore/services/frontend/nodes-group/macros/1", - "version": "1.0.0", - "name": "string", - "description": "string", - "contact": "user@example.com", - "workbench": { - "additionalProp1": { - "key": "simcore/services/comp/sleeper", - "version": "6.2.0", - "label": "string", - "progress": 0, - "thumbnail": "https://string.com", - "inputs": {}, - "inputAccess": { - "additionalProp1": "ReadAndWrite", - "additionalProp2": "ReadAndWrite", - "additionalProp3": "ReadAndWrite", - }, - "inputNodes": ["ba8e4558-1088-49b1-8fe6-f591634089e5"], - "outputs": {}, - "outputNodes": ["ba8e4558-1088-49b1-8fe6-f591634089e5"], - "parent": "ba8e4558-1088-49b1-8fe6-f591634089e5", - "position": {"x": 0, "y": 0}, - }, - "additionalProp2": { - "key": "simcore/services/comp/sleeper", - "version": "6.2.0", - "label": "string", - "progress": 0, - "thumbnail": "https://string.com", - "inputs": {}, - "inputAccess": { - "additionalProp1": "ReadAndWrite", - "additionalProp2": "ReadAndWrite", - "additionalProp3": "ReadAndWrite", - }, - "inputNodes": ["ba8e4558-1088-49b1-8fe6-f591634089e5"], - "outputs": {}, - "outputNodes": ["ba8e4558-1088-49b1-8fe6-f591634089e5"], - "parent": "ba8e4558-1088-49b1-8fe6-f591634089e5", - "position": {"x": 0, "y": 0}, - }, - "additionalProp3": { - "key": "simcore/services/comp/sleeper", - "version": "6.2.0", - "label": "string", - "progress": 0, - "thumbnail": "https://string.com", - "inputs": {}, - "inputAccess": { - "additionalProp1": "ReadAndWrite", - "additionalProp2": "ReadOnly", - "additionalProp3": "ReadAndWrite", - }, - "inputNodes": [], - "outputs": {}, - "outputNodes": [], - "parent": None, - "position": {"x": 0, "y": 0}, - }, - }, - } - return deepcopy(DAG_DATA_IN_DICT) + return setenvs_from_envfile(monkeypatch, service_env_file, verbose=True) diff --git a/services/catalog/tests/unit/test_models_domain_groups.py b/services/catalog/tests/unit/test_models_domain_groups.py deleted file mode 100644 index 84ef885f13c..00000000000 --- a/services/catalog/tests/unit/test_models_domain_groups.py +++ /dev/null @@ -1,17 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - - -from pprint import pformat - -import pytest -from simcore_service_catalog.models.domain.group import GroupAtDB - - -@pytest.mark.parametrize("model_cls", (GroupAtDB,)) -def test_service_api_models_examples(model_cls, model_cls_examples): - for name, example in model_cls_examples.items(): - print(name, ":", pformat(example)) - model_instance = model_cls(**example) - assert model_instance, f"Failed with {name}" diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_services_director.py index 2ec9f9e8682..d633f785ea6 100644 --- a/services/catalog/tests/unit/test_services_director.py +++ b/services/catalog/tests/unit/test_services_director.py @@ -11,7 +11,6 @@ import respx from fastapi import FastAPI from fastapi.testclient import TestClient -from pytest import MonkeyPatch from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api @@ -21,7 +20,7 @@ @pytest.fixture def minimal_app( - monkeypatch: MonkeyPatch, service_test_environ: EnvVarsDict + monkeypatch: pytest.MonkeyPatch, service_test_environ: EnvVarsDict ) -> Iterator[FastAPI]: # disable a couple of subsystems monkeypatch.setenv("CATALOG_POSTGRES", "null") diff --git a/services/catalog/tests/unit/test_services_function_services.py b/services/catalog/tests/unit/test_services_function_services.py index 798e414c1d6..69c8744d5d6 100644 --- a/services/catalog/tests/unit/test_services_function_services.py +++ b/services/catalog/tests/unit/test_services_function_services.py @@ -4,7 +4,7 @@ # pylint:disable=protected-access import pytest -from simcore_service_catalog.models.schemas.services import ServiceDockerData +from models_library.api_schemas_catalog.services import ServiceDockerData from simcore_service_catalog.services.function_services import ( is_function_service, iter_service_docker_data, diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index 3908d99b55c..bc42ee1b357 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -17,7 +17,6 @@ from fastapi import FastAPI from models_library.services import ServiceDockerData from models_library.users import UserID -from pytest import MonkeyPatch from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_postgres import PostgresTestConfig @@ -66,7 +65,7 @@ async def products_names( @pytest.fixture def app( - monkeypatch: MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, service_test_environ: EnvVarsDict, postgres_db: sa.engine.Engine, diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_dags.py b/services/catalog/tests/unit/with_dbs/test_api_routes_dags.py deleted file mode 100644 index 928502905df..00000000000 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_dags.py +++ /dev/null @@ -1,80 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from typing import Any - -import pytest -from fastapi import FastAPI -from respx.router import MockRouter -from simcore_service_catalog._meta import API_VERSION -from simcore_service_catalog.models.schemas.meta import Meta -from starlette.testclient import TestClient - -pytest_simcore_core_services_selection = [ - "postgres", -] -pytest_simcore_ops_services_selection = [ - "adminer", -] - - -def test_read_healthcheck(director_mockup: MockRouter, client: TestClient): - response = client.get("/") - assert response.status_code == 200 - assert response.text - - -def test_read_meta(director_mockup: MockRouter, app: FastAPI, client: TestClient): - response = client.get("/v0/meta") - assert response.status_code == 200 - meta = Meta(**response.json()) - assert meta.version == API_VERSION - assert meta.name == "simcore_service_catalog" - - -def test_list_dags(director_mockup: MockRouter, app: FastAPI, client: TestClient): - response = client.get("/v0/dags") - assert response.status_code == 200 - assert response.json() == [] - - # inject three dagin - response = client.get("/v0/dags") - assert response.status_code == 200 - # TODO: assert i can list them as dagouts - - # TODO: assert dagout have identifiers now - - -@pytest.mark.skip(reason="does not work") -def test_standard_operations_on_resource( - director_mockup: MockRouter, - app: FastAPI, - client: TestClient, - fake_data_dag_in: dict[str, Any], -): - - response = client.post("/v0/dags", json=fake_data_dag_in) - assert response.status_code == 201 - assert response.json() == 1 - - # list - response = client.get("/v0/dags") - assert response.status_code == 200 - got = response.json() - - assert isinstance(got, list) - assert len(got) == 1 - - # TODO: data_in is not the same as data_out?? - data_out = got[0] - assert data_out["id"] == 1 # extra key, once in db - - # get - response = client.get("/v0/dags/1") - assert response.status_code == 200 - assert response.json() == data_out - - # delete - response = client.delete("/v0/dags/1") - assert response.status_code == 204 diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py b/services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py index 53c6eaa8a58..47a54aa5164 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py +++ b/services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py @@ -6,14 +6,14 @@ # pylint: disable=unused-variable import re +from collections.abc import Callable from datetime import datetime, timedelta -from typing import Callable import pytest +from models_library.api_schemas_catalog.services import ServiceGet from models_library.services import ServiceDockerData from pydantic import parse_obj_as from respx.router import MockRouter -from simcore_service_catalog.models.schemas.services import ServiceGet from starlette import status from starlette.testclient import TestClient from yarl import URL diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py index 8359b8571a7..6869442702c 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py @@ -9,7 +9,9 @@ import random from typing import Any, Callable -from models_library.api_schemas_catalog import ServiceAccessRightsGet +from models_library.api_schemas_catalog.service_access_rights import ( + ServiceAccessRightsGet, +) from pydantic import parse_obj_as from respx.router import MockRouter from starlette.testclient import TestClient diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py b/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py index d6c63e6e503..3b7051d2a54 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py +++ b/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py @@ -13,6 +13,10 @@ from faker import Faker from fastapi import FastAPI from fastapi.encoders import jsonable_encoder +from models_library.api_schemas_catalog.services_specifications import ( + ServiceSpecifications, + ServiceSpecificationsGet, +) from models_library.generated_models.docker_rest_api import ( DiscreteResourceSpec, GenericResource, @@ -30,13 +34,9 @@ from simcore_postgres_database.models.services_specifications import ( services_specifications, ) -from simcore_service_catalog.models.domain.service_specifications import ( +from simcore_service_catalog.models.service_specifications import ( ServiceSpecificationsAtDB, ) -from simcore_service_catalog.models.schemas.services_specifications import ( - ServiceSpecifications, - ServiceSpecificationsGet, -) from sqlalchemy.ext.asyncio import AsyncEngine from starlette import status from starlette.testclient import TestClient diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py index b97ed64446d..d7e663da066 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py @@ -5,11 +5,11 @@ from typing import Callable from fastapi import FastAPI +from models_library.groups import GroupAtDB from models_library.services import ServiceDockerData, ServiceVersion from models_library.services_db import ServiceAccessRightsAtDB from pydantic import parse_obj_as from simcore_service_catalog.db.repositories.services import ServicesRepository -from simcore_service_catalog.models.domain.group import GroupAtDB from simcore_service_catalog.services.access_rights import ( evaluate_auto_upgrade_policy, evaluate_default_policy, diff --git a/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py b/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py index 9c3bff363c9..58d875dd1ab 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py +++ b/services/director-v2/src/simcore_service_director_v2/models/schemas/clusters.py @@ -8,7 +8,7 @@ Cluster, ClusterAccessRights, ClusterAuthentication, - ClusterType, + ClusterTypeInModel, ExternalClusterAuthentication, ) from models_library.generics import DictModel @@ -123,8 +123,8 @@ def set_default_thumbnail_if_empty(cls, v, values): if v is None: cluster_type = values["type"] default_thumbnails = { - ClusterType.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", - ClusterType.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", + ClusterTypeInModel.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", + ClusterTypeInModel.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", } return default_thumbnails[cluster_type] return v @@ -134,7 +134,7 @@ class Config(BaseCluster.Config): "examples": [ { "name": "My awesome cluster", - "type": ClusterType.ON_PREMISE, + "type": ClusterTypeInModel.ON_PREMISE, "endpoint": "https://registry.osparc-development.fake.dev", "authentication": { "type": "simple", @@ -145,7 +145,7 @@ class Config(BaseCluster.Config): { "name": "My AWS cluster", "description": "a AWS cluster administered by me", - "type": ClusterType.AWS, + "type": ClusterTypeInModel.AWS, "owner": 154, "endpoint": "https://registry.osparc-development.fake.dev", "authentication": { @@ -166,7 +166,7 @@ class Config(BaseCluster.Config): class ClusterPatch(BaseCluster): name: str | None description: str | None - type: ClusterType | None + type: ClusterTypeInModel | None owner: GroupID | None thumbnail: HttpUrl | None endpoint: AnyUrl | None diff --git a/services/web/server/src/simcore_service_webserver/catalog/client.py b/services/web/server/src/simcore_service_webserver/catalog/client.py index b464585585c..28de254ac3f 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/client.py +++ b/services/web/server/src/simcore_service_webserver/catalog/client.py @@ -11,7 +11,9 @@ ClientResponseError, InvalidURL, ) -from models_library.api_schemas_catalog import ServiceAccessRightsGet +from models_library.api_schemas_catalog.service_access_rights import ( + ServiceAccessRightsGet, +) from models_library.services_resources import ServiceResourcesDict from models_library.users import UserID from pydantic import parse_obj_as @@ -65,7 +67,6 @@ async def get_services_for_user_in_product( ) with handle_client_exceptions(app) as session: - async with session.get( url, headers={X_PRODUCT_NAME_HEADER: product_name}, diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_models.py b/services/web/server/src/simcore_service_webserver/director_v2/_models.py index 3cdf03f6827..d3c6b73c3a4 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_models.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_models.py @@ -4,6 +4,7 @@ CLUSTER_USER_RIGHTS, BaseCluster, ClusterAccessRights, + ClusterTypeInModel, ExternalClusterAuthentication, ) from models_library.users import GroupID @@ -17,6 +18,12 @@ class ClusterPing(BaseModel): authentication: ExternalClusterAuthentication +_DEFAULT_THUMBNAILS = { + f"{ClusterTypeInModel.AWS}": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", + f"{ClusterTypeInModel.ON_PREMISE}": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", +} + + class ClusterCreate(BaseCluster): owner: GroupID | None authentication: ExternalClusterAuthentication @@ -27,13 +34,10 @@ class ClusterCreate(BaseCluster): @validator("thumbnail", always=True, pre=True) @classmethod def set_default_thumbnail_if_empty(cls, v, values): - if v is None: - cluster_type = values["type"] - default_thumbnails = { - ClusterType.AWS.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/250px-Amazon_Web_Services_Logo.svg.png", - ClusterType.ON_PREMISE.value: "https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Crystal_Clear_app_network_local.png/120px-Crystal_Clear_app_network_local.png", - } - return default_thumbnails[cluster_type] + if v is None and ( + cluster_type := values.get("type", f"{ClusterTypeInModel.ON_PREMISE}") + ): + return _DEFAULT_THUMBNAILS[f"{cluster_type}"] return v class Config(BaseCluster.Config): @@ -41,7 +45,7 @@ class Config(BaseCluster.Config): "examples": [ { "name": "My awesome cluster", - "type": ClusterType.ON_PREMISE, + "type": ClusterType.ON_PREMISE, # can use also values from equivalent enum "endpoint": "https://registry.osparc-development.fake.dev", "authentication": { "type": "simple", diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 0bd35344547..e3e274719f0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -9,7 +9,9 @@ from typing import Any from aiohttp import web -from models_library.api_schemas_catalog import ServiceAccessRightsGet +from models_library.api_schemas_catalog.service_access_rights import ( + ServiceAccessRightsGet, +) from models_library.groups import EVERYONE_GROUP_ID from models_library.projects import Project, ProjectID from models_library.projects_nodes import NodeID diff --git a/services/web/server/tests/unit/isolated/test_groups_models.py b/services/web/server/tests/unit/isolated/test_groups_models.py index 1d3b085f175..3f0ca4f8954 100644 --- a/services/web/server/tests/unit/isolated/test_groups_models.py +++ b/services/web/server/tests/unit/isolated/test_groups_models.py @@ -1,8 +1,11 @@ from pprint import pformat from typing import Any +import models_library.groups import pytest +import simcore_postgres_database.models.groups from models_library.generics import Envelope +from models_library.utils.enums import enum_to_dict from pydantic import BaseModel from simcore_service_webserver.groups.schemas import ( AllUsersGroups, @@ -12,6 +15,15 @@ ) +def test_models_library_and_postgress_database_enums_are_equivalent(): + # For the moment these two libraries they do not have a common library to share these + # basic types so we test here that they are in sync + + assert enum_to_dict( + simcore_postgres_database.models.groups.GroupType + ) == enum_to_dict(models_library.groups.GroupTypeInModel) + + @pytest.mark.parametrize( "model_cls", ( diff --git a/services/web/server/tests/unit/isolated/test_projects_utils.py b/services/web/server/tests/unit/isolated/test_projects_utils.py index a3d905b0792..20fcc350f84 100644 --- a/services/web/server/tests/unit/isolated/test_projects_utils.py +++ b/services/web/server/tests/unit/isolated/test_projects_utils.py @@ -57,7 +57,7 @@ def test_clone_project_document( try: jsonschema.validate(instance=clone, schema=project_jsonschema) except ValidationError as err: - pytest.fail(f"Invalid clone of '{test_data_file_name}': {err.message}") + pytest.fail(f"Invalid clone of '{test_data_file_name}': {err}") @pytest.mark.parametrize( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py index 3eadb681ff1..5aaa8309e96 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py @@ -10,7 +10,9 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient -from models_library.api_schemas_catalog import ServiceAccessRightsGet +from models_library.api_schemas_catalog.service_access_rights import ( + ServiceAccessRightsGet, +) from models_library.projects_nodes import Node, NodeID from pydantic import parse_obj_as from pytest_mock import MockerFixture