Adding fixes for dynamic-scheduler

packages/common-library/requirements/_test.in

@@ -10,6 +10,7 @@
 
 coverage
 faker
+pydantic-settings
 pytest
 pytest-asyncio
 pytest-cov

packages/common-library/requirements/_test.txt

@@ -1,8 +1,12 @@
+annotated-types==0.7.0
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 coverage==7.6.1
     # via
     #   -r requirements/_test.in
     #   pytest-cov
-faker==30.1.0
+faker==30.3.0
     # via -r requirements/_test.in
 icdiff==2.0.7
     # via pytest-icdiff
@@ -16,6 +20,17 @@ pluggy==1.5.0
     # via pytest
 pprintpp==0.4.0
     # via pytest-icdiff
+pydantic==2.9.2
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
+pydantic-settings==2.5.2
+    # via -r requirements/_test.in
 pytest==8.3.3
     # via
     #   -r requirements/_test.in
@@ -44,7 +59,9 @@ pytest-sugar==1.0.0
 python-dateutil==2.9.0.post0
     # via faker
 python-dotenv==1.0.1
-    # via -r requirements/_test.in
+    # via
+    #   -r requirements/_test.in
+    #   pydantic-settings
 six==1.16.0
     # via python-dateutil
 termcolor==2.5.0
@@ -53,3 +70,5 @@ typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   faker
+    #   pydantic
+    #   pydantic-core

packages/common-library/requirements/_tools.txt

@@ -1,6 +1,6 @@
 astroid==3.3.5
     # via pylint
-black==24.8.0
+black==24.10.0
     # via -r requirements/../../../requirements/devenv.txt
 build==1.2.2.post1
     # via pip-tools

packages/common-library/src/common_library/pydantic_networks_extension.py

@@ -1,5 +1,12 @@
 from typing import Annotated, TypeAlias
+
 from pydantic import AfterValidator, AnyHttpUrl
 
 
-AnyHttpUrlLegacy: TypeAlias = Annotated[str, AnyHttpUrl, AfterValidator(lambda u: u.rstrip("/"))]
+def _strip_last_slash(url: str) -> str:
+    return url.rstrip("/")
+
+
+AnyHttpUrlLegacy: TypeAlias = Annotated[
+    str, AnyHttpUrl, AfterValidator(_strip_last_slash)
+]

packages/common-library/src/common_library/pydantic_settings_validators.py

@@ -0,0 +1,20 @@
+import datetime
+
+from pydantic import field_validator
+
+
+def _get_float_string_as_seconds(
+    v: datetime.timedelta | str | float,
+) -> datetime.timedelta | float | str:
+    if isinstance(v, str):
+        try:
+            return float(v)
+        except ValueError:
+            # returns format like "1:00:00"
+            return v
+    return v
+
+
+def validate_timedelta_in_legacy_mode(field: str):
+    """Transforms a float/int number into a valid datetime as it used to work in the past"""
+    return field_validator(field, mode="before")(_get_float_string_as_seconds)

packages/common-library/src/common_library/serialization.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 from typing import Any
 
 from common_library.pydantic_fields_extension import get_type
@@ -15,6 +16,9 @@ def model_dump_with_secrets(
 
         field_data = data[field_name]
 
+        if isinstance(field_data, timedelta):
+            data[field_name] = field_data.total_seconds()
+
         if isinstance(field_data, SecretStr):
             if show_secrets:
                 data[field_name] = field_data.get_secret_value()

packages/common-library/tests/test_errors_classes.py

@@ -38,8 +38,7 @@ class B12(B1, ValueError):
 
 def test_error_codes_and_msg_template():
     class MyBaseError(OsparcErrorMixin, Exception):
-        def __init__(self, **ctx: Any) -> None:
-            super().__init__(**ctx)  # Do not forget this for base exceptions!
+        pass
 
     class MyValueError(MyBaseError, ValueError):
         msg_template = "Wrong value {value}"

packages/common-library/tests/test_pydantic_settings_validators.py

@@ -0,0 +1,42 @@
+from datetime import timedelta
+
+import pytest
+from common_library.pydantic_settings_validators import (
+    validate_timedelta_in_legacy_mode,
+)
+from faker import Faker
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
+
+
+def test_validate_timedelta_in_legacy_mode(
+    monkeypatch: pytest.MonkeyPatch, faker: Faker
+):
+    class Settings(BaseSettings):
+        APP_NAME: str
+        REQUEST_TIMEOUT: timedelta = Field(default=timedelta(seconds=40))
+
+        _legacy_parsing_request_timeout = validate_timedelta_in_legacy_mode(
+            "REQUEST_TIMEOUT"
+        )
+
+        model_config = SettingsConfigDict()
+
+    app_name = faker.pystr()
+    env_vars: dict[str, str | bool] = {"APP_NAME": app_name}
+
+    # without timedelta
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(seconds=40) == settings.REQUEST_TIMEOUT
+
+    # with timedelta in seconds
+    env_vars["REQUEST_TIMEOUT"] = "5555"
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(seconds=5555) == settings.REQUEST_TIMEOUT

packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py

@@ -33,7 +33,7 @@ class CommonServiceDetails(BaseModel):
 
 
 class ServiceDetails(CommonServiceDetails):
-    basepath: Path = Field(
+    basepath: Path | None = Field(
         default=None,
         description="predefined path where the dynamic service should be served. If empty, the service shall use the root endpoint.",
         alias="service_basepath",
@@ -68,7 +68,7 @@ class RunningDynamicServiceDetails(ServiceDetails):
     internal_port: PortInt = Field(
         ..., description="the service swarm internal port", alias="service_port"
     )
-    published_port: PortInt = Field(
+    published_port: PortInt | None = Field(
         default=None,
         description="the service swarm published port if any",
         deprecated=True,

packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py

@@ -103,7 +103,7 @@ class NodeGet(OutputSchema):
                 "service_basepath": "/x/E1O2E-LAH",
                 "service_state": "pending",
                 "service_message": "no suitable node (insufficient resources on 1 node)",
-                "user_id": 123,
+                "user_id": "123",
             }
         }
     )

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/errors.py

@@ -1,7 +1,7 @@
-from pydantic.errors import PydanticErrorMixin
+from common_library.errors_classes import OsparcErrorMixin
 
 
-class BaseDynamicSchedulerRPCError(PydanticErrorMixin, Exception):
+class BaseDynamicSchedulerRPCError(OsparcErrorMixin, Exception):
     ...
 
 

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/errors.py

@@ -1,4 +1,4 @@
-from models_library.errors_classes import OsparcErrorMixin
+from common_library.errors_classes import OsparcErrorMixin
 
 
 class ResourceUsageTrackerRuntimeError(OsparcErrorMixin, RuntimeError):

packages/service-library/tests/deferred_tasks/test_deferred_tasks.py

@@ -16,8 +16,8 @@
 import psutil
 import pytest
 from aiohttp.test_utils import unused_port
+from common_library.serialization import model_dump_with_secrets
 from models_library.utils.json_serialization import json_dumps
-from models_library.utils.serialization import model_dump_with_secrets
 from pydantic import NonNegativeFloat, NonNegativeInt
 from pytest_mock import MockerFixture
 from servicelib import redis as servicelib_redis

packages/settings-library/src/settings_library/base.py

@@ -2,11 +2,7 @@
 from functools import cached_property
 from typing import Any, Final, get_origin
 
-from common_library.utils.pydantic_fields_extension import (
-    get_type,
-    is_literal,
-    is_nullable,
-)
+from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable
 from pydantic import ValidationInfo, field_validator
 from pydantic.fields import FieldInfo
 from pydantic_core import PydanticUndefined, ValidationError
@@ -44,7 +40,7 @@ def _default_factory():
                     field_name,
                 )
                 return None
-
+            _logger.warning("Validation errors=%s", err.errors())
             raise DefaultFromEnvFactoryError(errors=err.errors()) from err
 
     return _default_factory

packages/settings-library/src/settings_library/utils_cli.py

@@ -7,7 +7,7 @@
 
 import rich
 import typer
-from models_library.utils.serialization import model_dump_with_secrets
+from common_library.serialization import model_dump_with_secrets
 from pydantic import ValidationError
 from pydantic_settings import BaseSettings
 

packages/settings-library/tests/test__pydantic_settings.py

@@ -12,10 +12,10 @@
 
 """
 
+from common_library.pydantic_fields_extension import is_nullable
 from pydantic import ValidationInfo, field_validator
 from pydantic.fields import PydanticUndefined
 from pydantic_settings import BaseSettings
-from common_library.utils.pydantic_fields_extension import is_nullable
 
 
 def assert_field_specs(

services/api-server/src/simcore_service_api_server/exceptions/_base.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from models_library.errors_classes import OsparcErrorMixin
+from common_library.errors_classes import OsparcErrorMixin
 
 
 class ApiServerBaseError(OsparcErrorMixin, Exception):

services/autoscaling/src/simcore_service_autoscaling/core/errors.py

@@ -1,4 +1,4 @@
-from models_library.errors_classes import OsparcErrorMixin
+from common_library.errors_classes import OsparcErrorMixin
 
 
 class AutoscalingRuntimeError(OsparcErrorMixin, RuntimeError):

services/catalog/src/simcore_service_catalog/exceptions/errors.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from models_library.errors_classes import OsparcErrorMixin
+from common_library.errors_classes import OsparcErrorMixin
 
 
 class CatalogBaseError(OsparcErrorMixin, Exception):

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py

@@ -1,8 +1,6 @@
 import logging
-import os
 
 import typer
-from settings_library.rabbit import RabbitSettings
 from settings_library.utils_cli import (
     create_settings_command,
     create_version_callback,
@@ -33,26 +31,7 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True):
     """
     assert ctx  # nosec
 
-    # NOTE: we normally DO NOT USE `os.environ` to capture env vars but this is a special case
-    # The idea here is to have a command that can generate a **valid** `.env` file that can be used
-    # to initialized the app. For that reason we fill required fields of the `ApplicationSettings` with
-    # "fake" but valid values (e.g. generating a password or adding tags as `replace-with-api-key).
-    # Nonetheless, if the caller of this CLI has already some **valid** env vars in the environment we want to use them ...
-    # and that is why we use `os.environ`.
-
-    settings = ApplicationSettings.create_from_envs(
-        DYNAMIC_SCHEDULER_RABBITMQ=os.environ.get(
-            "DYNAMIC_SCHEDULER_RABBITMQ",
-            RabbitSettings.create_from_envs(
-                RABBIT_HOST=os.environ.get("RABBIT_HOST", "replace-with-rabbit-host"),
-                RABBIT_SECURE=os.environ.get("RABBIT_SECURE", "0"),
-                RABBIT_USER=os.environ.get("RABBIT_USER", "replace-with-rabbit-user"),
-                RABBIT_PASSWORD=os.environ.get(
-                    "RABBIT_PASSWORD", "replace-with-rabbit-user"
-                ),
-            ),
-        ),
-    )
+    settings = ApplicationSettings.create_from_envs()
 
     print_as_envfile(
         settings,

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py

@@ -1,6 +1,9 @@
 import datetime
 from functools import cached_property
 
+from common_library.pydantic_settings_validators import (
+    validate_timedelta_in_legacy_mode,
+)
 from pydantic import Field, parse_obj_as, validator
 from settings_library.application import BaseApplicationSettings
 from settings_library.basic_types import LogLevel, VersionTag
@@ -43,6 +46,10 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings):
         ),
     )
 
+    _legacy_parsing_dynamic_scheduler_stop_service_timeout = (
+        validate_timedelta_in_legacy_mode("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT")
+    )
+
     @cached_property
     def LOG_LEVEL(self):  # noqa: N802
         return self.DYNAMIC_SCHEDULER__LOGLEVEL