Merge remote-tracking branch 'upstream/master' into pr-osparc-redirec…

…t-remain-dv2-calls-via-dynamic-scheduler
ITISFoundation · Dec 5, 2024 · 65ee01c · 65ee01c
2 parents 7edad2b + 3ade7ef
commit 65ee01c
Show file tree

Hide file tree

Showing 139 changed files with 6,034 additions and 4,466 deletions.
diff --git a/.env-devel b/.env-devel
@@ -126,6 +126,7 @@ DIRECTOR_V2_TRACING={}
 # DYNAMIC_SCHEDULER ----
 DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
+DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
 DYNAMIC_SCHEDULER_TRACING={}
 DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin

diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt
@@ -97,7 +97,7 @@ email-validator==2.2.0
     # via pydantic
 fast-depends==2.4.12
     # via faststream
-faststream==0.5.28
+faststream==0.5.31
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 frozenlist==1.4.1
     # via
@@ -218,7 +218,7 @@ protobuf==4.25.5
     #   opentelemetry-proto
 psutil==6.0.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
-pydantic==2.9.2
+pydantic==2.10.3
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -247,7 +247,7 @@ pydantic==2.9.2
     #   fast-depends
     #   pydantic-extra-types
     #   pydantic-settings
-pydantic-core==2.23.4
+pydantic-core==2.27.1
     # via pydantic
 pydantic-extra-types==2.9.0
     # via
@@ -307,7 +307,6 @@ redis==5.0.4
     #   -r requirements/../../../packages/service-library/requirements/_base.in
 referencing==0.29.3
     # via
-    #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
     #   jsonschema
     #   jsonschema-specifications
 repro-zipfile==0.3.1
@@ -393,6 +392,7 @@ wrapt==1.16.0
     #   opentelemetry-instrumentation-redis
 yarl==1.12.1
     # via
+    #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   aio-pika
     #   aiohttp
     #   aiormq

diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt
@@ -158,12 +158,12 @@ py-partiql-parser==0.5.6
     # via moto
 pycparser==2.22
     # via cffi
-pydantic==2.9.2
+pydantic==2.10.3
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   aws-sam-translator
-pydantic-core==2.23.4
+pydantic-core==2.27.1
     # via
     #   -c requirements/_base.txt
     #   pydantic

diff --git a/packages/common-library/requirements/_base.txt b/packages/common-library/requirements/_base.txt
@@ -4,12 +4,12 @@ orjson==3.10.10
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
-pydantic==2.9.2
+pydantic==2.10.3
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
     #   pydantic-extra-types
-pydantic-core==2.23.4
+pydantic-core==2.27.1
     # via pydantic
 pydantic-extra-types==2.9.0
     # via -r requirements/_base.in

diff --git a/packages/common-library/requirements/_test.txt b/packages/common-library/requirements/_test.txt
@@ -20,12 +20,12 @@ pluggy==1.5.0
     # via pytest
 pprintpp==0.4.0
     # via pytest-icdiff
-pydantic==2.9.2
+pydantic==2.10.3
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   pydantic-settings
-pydantic-core==2.23.4
+pydantic-core==2.27.1
     # via
     #   -c requirements/_base.txt
     #   pydantic

diff --git a/packages/common-library/src/common_library/basic_types.py b/packages/common-library/src/common_library/basic_types.py
@@ -1,4 +1,13 @@
 from enum import StrEnum
+from typing import Any
+
+from pydantic_core import PydanticUndefined
+
+# SEE https://github.com/fastapi/fastapi/blob/master/fastapi/_compat.py#L75-L78
+Undefined = PydanticUndefined
+DEFAULT_FACTORY: Any = Undefined
+# Use `UNSET` as default when default_factory
+# SEE https://github.com/ITISFoundation/osparc-simcore/pull/6882
 
 
 class LogLevel(StrEnum):

diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt
@@ -76,7 +76,7 @@ partd==1.4.2
     # via dask
 psutil==6.0.0
     # via distributed
-pydantic==2.9.2
+pydantic==2.10.3
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -92,7 +92,7 @@ pydantic==2.9.2
     #   -r requirements/_base.in
     #   pydantic-extra-types
     #   pydantic-settings
-pydantic-core==2.23.4
+pydantic-core==2.27.1
     # via pydantic
 pydantic-extra-types==2.9.0
     # via

diff --git a/packages/models-library/requirements/_base.txt b/packages/models-library/requirements/_base.txt
@@ -22,15 +22,15 @@ orjson==3.10.7
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/common-library/requirements/_base.in
     #   -r requirements/_base.in
-pydantic==2.9.2
+pydantic==2.10.3
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/common-library/requirements/_base.in
     #   -r requirements/_base.in
     #   pydantic-extra-types
     #   pydantic-settings
-pydantic-core==2.23.4
+pydantic-core==2.27.1
     # via pydantic
 pydantic-extra-types==2.9.0
     # via

diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py b/packages/models-library/src/models_library/api_schemas_directorv2/clusters.py
@@ -1,4 +1,4 @@
-from typing import Any, TypeAlias
+from typing import Annotated, Any, TypeAlias
 
 from pydantic import (
     BaseModel,
@@ -55,7 +55,7 @@ class Worker(BaseModel):
 
 class Scheduler(BaseModel):
     status: str = Field(..., description="The running status of the scheduler")
-    workers: WorkersDict | None = Field(default_factory=dict)
+    workers: Annotated[WorkersDict | None, Field(default_factory=dict)]
 
     @field_validator("workers", mode="before")
     @classmethod

diff --git a/packages/models-library/src/models_library/api_schemas_webserver/activity.py b/packages/models-library/src/models_library/api_schemas_webserver/activity.py
@@ -1,6 +1,6 @@
 from typing import TypeAlias
 
-from pydantic import BaseModel, Field, PositiveFloat
+from pydantic import BaseModel, PositiveFloat
 
 from ..projects_nodes_io import NodeID
 
@@ -18,7 +18,7 @@ class Limits(BaseModel):
 class Activity(BaseModel):
     stats: Stats
     limits: Limits
-    queued: bool = Field(None)  # TODO: review since it in NOT filled
+    queued: bool | None = None  # TODO: review since it in NOT filled
 
 
 ActivityStatusDict: TypeAlias = dict[NodeID, Activity]
diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py
@@ -88,9 +88,9 @@ class ProjectGet(OutputSchema):
     )
     state: ProjectState | None = None
     ui: EmptyModel | StudyUI | None = None
-    quality: dict[str, Any] = Field(
-        default_factory=dict, json_schema_extra={"default": {}}
-    )
+    quality: Annotated[
+        dict[str, Any], Field(default_factory=dict, json_schema_extra={"default": {}})
+    ]
     dev: dict | None
     permalink: ProjectPermalink | None = None
     workspace_id: WorkspaceID | None
@@ -122,16 +122,20 @@ class ProjectReplace(InputSchema):
     last_change_date: DateTimeStr
     workbench: NodesDict
     access_rights: dict[GroupIDStr, AccessRights]
-    tags: list[int] | None = Field(
-        default_factory=list, json_schema_extra={"default": []}
-    )
-    classifiers: list[ClassifierID] | None = Field(
-        default_factory=list, json_schema_extra={"default": []}
-    )
+    tags: Annotated[
+        list[int] | None, Field(default_factory=list, json_schema_extra={"default": []})
+    ]
+
+    classifiers: Annotated[
+        list[ClassifierID] | None,
+        Field(default_factory=list, json_schema_extra={"default": []}),
+    ]
+
     ui: StudyUI | None = None
-    quality: dict[str, Any] = Field(
-        default_factory=dict, json_schema_extra={"default": {}}
-    )
+
+    quality: Annotated[
+        dict[str, Any], Field(default_factory=dict, json_schema_extra={"default": {}})
+    ]
 
 
 class ProjectPatch(InputSchema):
@@ -141,7 +145,7 @@ class ProjectPatch(InputSchema):
         HttpUrl | None,
         BeforeValidator(empty_str_to_none_pre_validator),
         PlainSerializer(lambda x: str(x) if x is not None else None),
-    ] = Field(default=None)
+    ] = None
     access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None)
     classifiers: list[ClassifierID] | None = Field(default=None)
     dev: dict | None = Field(default=None)

diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py
@@ -1,5 +1,5 @@
 # mypy: disable-error-code=truthy-function
-from typing import Any, Literal, TypeAlias
+from typing import Annotated, Any, Literal, TypeAlias
 
 from pydantic import ConfigDict, Field
 
@@ -29,16 +29,23 @@ class NodePatch(InputSchemaWithoutCamelCase):
     service_key: ServiceKey | None = Field(default=None, alias="key")
     service_version: ServiceVersion | None = Field(default=None, alias="version")
     label: str | None = Field(default=None)
-    inputs: InputsDict = Field(default=None)
+    inputs: Annotated[
+        InputsDict, Field(default_factory=dict, json_schema_extra={"default": {}})
+    ]
     inputs_required: list[InputID] | None = Field(default=None, alias="inputsRequired")
     input_nodes: list[NodeID] | None = Field(default=None, alias="inputNodes")
-    progress: float | None = Field(
-        default=None, ge=0, le=100
-    )  # NOTE: it is used by frontend for File Picker progress
-    boot_options: BootOptions | None = Field(default=None, alias="bootOptions")
-    outputs: dict[str, Any] | None = Field(
-        default=None
-    )  # NOTE: it is used by frontend for File Picker
+    progress: Annotated[
+        float | None,
+        Field(
+            # NOTE: it is used by frontend for File Picker progress
+            ge=0,
+            le=100,
+        ),
+    ] = None
+    boot_options: Annotated[BootOptions | None, Field(alias="bootOptions")] = None
+    outputs: dict[
+        str, Any
+    ] | None = None  # NOTE: it is used by frontend for File Picker
 
 
 class NodeCreated(OutputSchema):

diff --git a/packages/models-library/src/models_library/app_diagnostics.py b/packages/models-library/src/models_library/app_diagnostics.py
@@ -1,28 +1,38 @@
-from typing import Any
+from typing import Annotated, Any
 
+from common_library.basic_types import DEFAULT_FACTORY
 from pydantic import AnyUrl, BaseModel, Field
 
 
 class AppStatusCheck(BaseModel):
     app_name: str = Field(..., description="Application name")
     version: str = Field(..., description="Application's version")
-    services: dict[str, Any] = Field(
-        default_factory=dict,
-        description="Other backend services connected from this service",
-        json_schema_extra={"default": {}},
-    )
+    services: Annotated[
+        dict[str, Any],
+        Field(
+            default_factory=dict,
+            description="Other backend services connected from this service",
+            json_schema_extra={"default": {}},
+        ),
+    ] = DEFAULT_FACTORY
 
-    sessions: dict[str, Any] | None = Field(
-        default_factory=dict,
-        description="Client sessions info. If single session per app, then is denoted as main",
-        json_schema_extra={"default": {}},
-    )
+    sessions: Annotated[
+        dict[str, Any] | None,
+        Field(
+            default_factory=dict,
+            description="Client sessions info. If single session per app, then is denoted as main",
+            json_schema_extra={"default": {}},
+        ),
+    ] = DEFAULT_FACTORY
 
     url: AnyUrl | None = Field(
         default=None,
         description="Link to current resource",
     )
-    diagnostics_url: AnyUrl | None = Field(
-        default=None,
-        description="Link to diagnostics report sub-resource. This MIGHT take some time to compute",
-    )
+
+    diagnostics_url: Annotated[
+        AnyUrl | None,
+        Field(
+            description="Link to diagnostics report sub-resource. This MIGHT take some time to compute",
+        ),
+    ] = None
diff --git a/packages/models-library/src/models_library/callbacks_mapping.py b/packages/models-library/src/models_library/callbacks_mapping.py
@@ -1,5 +1,5 @@
 from collections.abc import Sequence
-from typing import Final
+from typing import Annotated, Final
 
 from pydantic import BaseModel, ConfigDict, Field, NonNegativeFloat, field_validator
 
@@ -31,14 +31,17 @@ class CallbacksMapping(BaseModel):
         None,
         description="command to recover prometheus metrics from a specific user service",
     )
-    before_shutdown: list[UserServiceCommand] = Field(
-        default_factory=list,
-        description=(
-            "commands to run before shutting down the user services"
-            "commands get executed first to last, multiple commands for the same"
-            "user services are allowed"
+    before_shutdown: Annotated[
+        list[UserServiceCommand],
+        Field(
+            default_factory=list,
+            description=(
+                "commands to run before shutting down the user services"
+                "commands get executed first to last, multiple commands for the same"
+                "user services are allowed"
+            ),
         ),
-    )
+    ]
     inactivity: UserServiceCommand | None = Field(
         None,
         description=(