♻️ Maintenance: cleanup catalog

packages/models-library/src/models_library/basic_regex.py

@@ -71,5 +71,5 @@
 DOCKER_IMAGE_KEY_RE = r"[\w/-]+"
 DOCKER_IMAGE_VERSION_RE = r"[\w/.]+"
 DOCKER_GENERIC_TAG_KEY_RE: Final[re.Pattern] = re.compile(
-    r"^(?P<registry_host>(?:(?:(?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9-]+(?::\d+)?)|[a-zA-Z0-9-]+:\d+))?(?:/)?(?P<docker_image>(?:[a-z0-9][a-z0-9_.-]*/)*[a-z0-9-_]+[a-z0-9])(?::(?P<docker_tag>[\w][\w.-]{0,126}[\w]))?$"
+    r"^(?P<registry_host>(?:(?:(?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9-]+(?::\d+)?)|[a-zA-Z0-9-]+:\d+))?(?:/)?(?P<docker_image>(?:[a-z0-9][a-z0-9_.-]*/)*[a-z0-9-_]+[a-z0-9])(?::(?P<docker_tag>[\w][\w.-]{0,126}[\w]))?(?P<docker_digest>\@sha256:[a-fA-F0-9]{64})?$"
 )

packages/models-library/src/models_library/docker.py

@@ -5,17 +5,9 @@
 from models_library.projects import ProjectID
 from models_library.projects_nodes import NodeID
 from models_library.users import UserID
-from pydantic import BaseModel, ConstrainedStr, Field, constr
+from pydantic import BaseModel, ConstrainedStr, Field
 
-from .basic_regex import (
-    DOCKER_GENERIC_TAG_KEY_RE,
-    DOCKER_IMAGE_KEY_RE,
-    DOCKER_IMAGE_VERSION_RE,
-    DOCKER_LABEL_KEY_REGEX,
-)
-
-DockerImageKey = constr(regex=DOCKER_IMAGE_KEY_RE)
-DockerImageVersion = constr(regex=DOCKER_IMAGE_VERSION_RE)
+from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX
 
 
 class DockerLabelKey(ConstrainedStr):

packages/models-library/src/models_library/projects_nodes.py

@@ -18,7 +18,6 @@
     validator,
 )
 
-from .basic_regex import VERSION_RE
 from .basic_types import EnvVarKey
 from .projects_access import AccessEnum
 from .projects_nodes_io import (
@@ -30,7 +29,7 @@
 )
 from .projects_nodes_ui import Position
 from .projects_state import RunningState
-from .services import PROPERTY_KEY_RE, SERVICE_KEY_RE
+from .services import PROPERTY_KEY_RE, ServiceKey, ServiceVersion
 
 # NOTE: WARNING the order here matters
 
@@ -100,20 +99,18 @@ class Config:
 
 
 class Node(BaseModel):
-    key: str = Field(
+    key: ServiceKey = Field(
         ...,
         description="distinctive name for the node based on the docker registry path",
-        regex=SERVICE_KEY_RE,
         examples=[
             "simcore/services/comp/itis/sleeper",
             "simcore/services/dynamic/3dviewer",
             "simcore/services/frontend/file-picker",
         ],
     )
-    version: str = Field(
+    version: ServiceVersion = Field(
         ...,
         description="semantic version number of the node",
-        regex=VERSION_RE,
         examples=["1.0.0", "0.0.1"],
     )
     label: str = Field(

packages/models-library/src/models_library/services.py

@@ -4,13 +4,15 @@
     python -c "from models_library.services import ServiceDockerData as cls; print(cls.schema_json(indent=2))" > services-schema.json
 """
 
+import re
 from datetime import datetime
 from enum import Enum
 from typing import Any, Optional, Union
 from uuid import UUID
 
 from pydantic import (
     BaseModel,
+    ConstrainedStr,
     EmailStr,
     Extra,
     Field,
@@ -54,8 +56,14 @@
 ServicePortKey = constr(regex=PROPERTY_KEY_RE)
 FileName = constr(regex=FILENAME_RE)
 
-ServiceKey = constr(regex=KEY_RE)
-ServiceVersion = constr(regex=VERSION_RE)
+
+class ServiceKey(ConstrainedStr):
+    regex = re.compile(SERVICE_KEY_RE)
+
+
+class ServiceVersion(ConstrainedStr):
+    regex = re.compile(VERSION_RE)
+
 
 RunID = UUID
 

packages/models-library/src/models_library/services_resources.py

@@ -1,13 +1,13 @@
 import logging
 from typing import Any, Final, Union
 
+from models_library.docker import DockerGenericTag
 from pydantic import (
     BaseModel,
     ByteSize,
     Field,
     StrictFloat,
     StrictInt,
-    constr,
     parse_obj_as,
     root_validator,
 )
@@ -16,13 +16,14 @@
 
 logger = logging.getLogger(__name__)
 
-DockerImage = constr(regex=r"[\w/-]+:[\w.@]+")
-DockerComposeServiceName = constr(regex=r"^[a-zA-Z0-9._-]+$")
+
 ResourceName = str
 
 # NOTE: replace hard coded `container` with function which can
 # extract the name from the `service_key` or `registry_address/service_key`
-DEFAULT_SINGLE_SERVICE_NAME: Final[DockerComposeServiceName] = "container"
+DEFAULT_SINGLE_SERVICE_NAME: Final[DockerGenericTag] = parse_obj_as(
+    DockerGenericTag, "container"
+)
 
 MEMORY_50MB: Final[int] = parse_obj_as(ByteSize, "50mib")
 MEMORY_250MB: Final[int] = parse_obj_as(ByteSize, "250mib")
@@ -58,7 +59,7 @@ class Config:
 
 
 class ImageResources(BaseModel):
-    image: DockerImage = Field(
+    image: DockerGenericTag = Field(
         ...,
         description=(
             "Used by the frontend to provide a context for the users."
@@ -87,23 +88,29 @@ class Config:
         }
 
 
-ServiceResourcesDict = dict[DockerComposeServiceName, ImageResources]
+ServiceResourcesDict = dict[DockerGenericTag, ImageResources]
 
 
 class ServiceResourcesDictHelpers:
     @staticmethod
     def create_from_single_service(
-        image: DockerComposeServiceName, resources: ResourcesDict
+        image: DockerGenericTag,
+        resources: ResourcesDict,
     ) -> ServiceResourcesDict:
         return parse_obj_as(
             ServiceResourcesDict,
-            {DEFAULT_SINGLE_SERVICE_NAME: {"image": image, "resources": resources}},
+            {
+                DEFAULT_SINGLE_SERVICE_NAME: {
+                    "image": image,
+                    "resources": resources,
+                }
+            },
         )
 
     @staticmethod
     def create_jsonable(
         service_resources: ServiceResourcesDict,
-    ) -> dict[DockerComposeServiceName, Any]:
+    ) -> dict[DockerGenericTag, Any]:
         return jsonable_encoder(service_resources)
 
     class Config:

packages/models-library/tests/test_service_resources.py

@@ -5,9 +5,8 @@
 from typing import Any
 
 import pytest
+from models_library.docker import DockerGenericTag
 from models_library.services_resources import (
-    DockerComposeServiceName,
-    DockerImage,
     ImageResources,
     ResourcesDict,
     ResourceValue,
@@ -24,11 +23,11 @@
         "simcore/services/dynamic/nice-service:v1.0.0",
         "a/docker-hub/image:1.0.0",
         "traefik:v1.0.0",
-        "traefik:v1.0.0@somehash",
+        "traefik:v1.0.0@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d",
     ),
 )
 def test_compose_image(example: str) -> None:
-    parse_obj_as(DockerImage, example)
+    parse_obj_as(DockerGenericTag, example)
 
 
 @pytest.fixture
@@ -39,8 +38,8 @@ def resources_dict() -> ResourcesDict:
 
 
 @pytest.fixture
-def compose_image() -> DockerImage:
-    return parse_obj_as(DockerImage, "image:latest")
+def compose_image() -> DockerGenericTag:
+    return parse_obj_as(DockerGenericTag, "image:latest")
 
 
 def _ensure_resource_value_is_an_object(data: ResourcesDict) -> None:
@@ -74,7 +73,7 @@ def test_image_resources_parsed_as_expected() -> None:
     "example", ServiceResourcesDictHelpers.Config.schema_extra["examples"]
 )
 def test_service_resource_parsed_as_expected(
-    example: dict[DockerComposeServiceName, Any], compose_image: DockerImage
+    example: dict[DockerGenericTag, Any], compose_image: DockerGenericTag
 ) -> None:
     def _assert_service_resources_dict(
         service_resources_dict: ServiceResourcesDict,
@@ -103,7 +102,7 @@ def _assert_service_resources_dict(
 @pytest.mark.parametrize(
     "example", ServiceResourcesDictHelpers.Config.schema_extra["examples"]
 )
-def test_create_jsonable_dict(example: dict[DockerComposeServiceName, Any]) -> None:
+def test_create_jsonable_dict(example: dict[DockerGenericTag, Any]) -> None:
     service_resources_dict: ServiceResourcesDict = parse_obj_as(
         ServiceResourcesDict, example
     )

packages/postgres-database/src/simcore_postgres_database/utils_products.py

@@ -2,7 +2,7 @@
 
 """
 
-from typing import Optional, Protocol
+from typing import Any, Optional, Protocol
 
 import sqlalchemy as sa
 
@@ -23,7 +23,7 @@ async def scalar(self, *args, **kwargs):
 
 class _AiopgConnection(Protocol):
     # Prototype to account for aiopg-only (this protocol avoids import <-> installation)
-    async def scalar(self, *args, **kwargs):
+    async def scalar(self, *args, **kwargs) -> Any:
         ...
 
     async def execute(self, *args, **kwargs):

services/catalog/requirements/_base.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile --output-file=requirements/_base.txt --resolver=backtracking --strip-extras requirements/_base.in
 #
@@ -148,7 +148,9 @@ pyyaml==5.4.1
     #   fastapi
     #   uvicorn
 redis==4.4.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
+    # via
+    #   -c requirements/../../../packages/service-library/requirements/./_base.in
+    #   -r requirements/../../../packages/service-library/requirements/_base.in
 requests==2.27.1
     # via fastapi
 rfc3986==1.4.0
@@ -166,7 +168,6 @@ sniffio==1.2.0
 sqlalchemy==1.4.37
     # via
     #   -r requirements/../../../packages/postgres-database/requirements/_base.in
-    #   -r requirements/_base.in
     #   alembic
 starlette==0.20.4
     # via fastapi

services/catalog/requirements/_test.in

@@ -25,3 +25,4 @@ pytest-docker
 pytest-mock
 pytest-runner
 respx
+sqlalchemy[asyncio, mypy]

services/catalog/requirements/_test.txt

@@ -101,6 +101,10 @@ multidict==6.0.2
     #   -c requirements/_base.txt
     #   aiohttp
     #   yarl
+mypy==1.1.1
+    # via sqlalchemy
+mypy-extensions==1.0.0
+    # via mypy
 packaging==21.3
     # via
     #   -c requirements/_base.txt
@@ -170,12 +174,20 @@ sniffio==1.2.0
     #   httpx
 sqlalchemy==1.4.37
     # via
-    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
     #   alembic
+sqlalchemy2-stubs==0.0.2a32
+    # via sqlalchemy
 tomli==2.0.1
     # via
     #   coverage
+    #   mypy
     #   pytest
+typing-extensions==4.3.0
+    # via
+    #   -c requirements/_base.txt
+    #   mypy
+    #   sqlalchemy2-stubs
 urllib3==1.26.9
     # via
     #   -c requirements/_base.txt

services/catalog/requirements/_tools.txt

@@ -36,7 +36,9 @@ lazy-object-proxy==1.9.0
 mccabe==0.7.0
     # via pylint
 mypy-extensions==1.0.0
-    # via black
+    # via
+    #   -c requirements/_test.txt
+    #   black
 nodeenv==1.7.0
     # via pre-commit
 packaging==21.3
@@ -78,7 +80,7 @@ tomlkit==0.11.6
     # via pylint
 typing-extensions==4.3.0
     # via
-    #   -c requirements/_base.txt
+    #   -c requirements/_test.txt
     #   astroid
     #   black
     #   pylint

services/catalog/src/simcore_service_catalog/api/dependencies/database.py

@@ -1,11 +1,11 @@
 import logging
-from typing import AsyncGenerator, Callable, Type
+from typing import AsyncGenerator, Callable
 
 from fastapi import Depends
 from fastapi.requests import Request
 from sqlalchemy.ext.asyncio import AsyncEngine
 
-from ...db.repositories import BaseRepository
+from ...db.repositories._base import BaseRepository
 
 logger = logging.getLogger(__name__)
 
@@ -14,7 +14,7 @@ def _get_db_engine(request: Request) -> AsyncEngine:
     return request.app.state.engine
 
 
-def get_repository(repo_type: Type[BaseRepository]) -> Callable:
+def get_repository(repo_type: type[BaseRepository]) -> Callable:
     async def _get_repo(
         engine: AsyncEngine = Depends(_get_db_engine),
     ) -> AsyncGenerator[BaseRepository, None]:
@@ -23,10 +23,6 @@ async def _get_repo(
         # 2nd one was acquiring a connection per request which works but blocks the director-v2 responsiveness once
         # the max amount of connections is reached
         # now the current solution is to connect connection when needed.
-        logger.info(
-            "%s",
-            f"current pool connections {engine.pool.checkedin()=},{engine.pool.checkedout()=}",
-        )
         yield repo_type(db_engine=engine)
 
     return _get_repo

services/catalog/src/simcore_service_catalog/api/dependencies/director.py

@@ -11,4 +11,5 @@ def _get_app(request: Request) -> FastAPI:
 def get_director_api(
     app: FastAPI = Depends(_get_app),
 ) -> DirectorApi:
-    return app.state.director_api
+    director: DirectorApi = app.state.director_api
+    return director

services/catalog/src/simcore_service_catalog/api/dependencies/services.py

@@ -17,7 +17,7 @@
 from ...services.director import DirectorApi
 from ...services.function_services import get_function_service, is_function_service
 from .database import get_repository
-from .director import DirectorApi, get_director_api
+from .director import get_director_api
 
 
 def get_default_service_resources(request: Request) -> ResourcesDict: