From abb8e728d55156aa38b736e303b30866cea09bc0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:14:18 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=E2=9C=85=20catalog:=20service-layer?= =?UTF-8?q?=20for=20registry=20and=20increased=20test=20coverage=20(part?= =?UTF-8?q?=204)=20(#6050)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- services/catalog/openapi.json | 56 +-- .../{api/rest => }/_constants.py | 0 .../api/dependencies/services.py | 35 +- .../api/rest/_services.py | 55 ++- .../api/rest/_services_access_rights.py | 2 +- .../api/rest/_services_ports.py | 8 +- .../api/rest/_services_resources.py | 2 +- .../api/rest/_services_specifications.py | 2 +- .../core/background_tasks.py | 76 +--- .../simcore_service_catalog/core/events.py | 32 +- .../services/director.py | 37 +- .../services/function_services.py | 17 +- .../services/manifest.py | 97 ++++ services/catalog/tests/unit/conftest.py | 428 ++++++++++++++---- services/catalog/tests/unit/test_api_rest.py | 37 ++ .../tests/unit/test_services_director.py | 62 ++- .../tests/unit/test_services_manifest.py | 67 +++ .../catalog/tests/unit/with_dbs/conftest.py | 37 +- .../with_dbs/test_api_rest_services__get.py | 96 ++++ ...ist.py => test_api_rest_services__list.py} | 32 +- ...> test_api_rest_services_access_rights.py} | 8 +- ...rts.py => test_api_rest_services_ports.py} | 8 +- ...py => test_api_rest_services_resources.py} | 28 +- ... test_api_rest_services_specifications.py} | 37 +- .../test_core_background_task__sync.py | 80 ++++ .../unit/with_dbs/test_db_repositories.py | 2 +- 27 files changed, 968 insertions(+), 375 deletions(-) rename services/catalog/src/simcore_service_catalog/{api/rest => }/_constants.py (100%) create mode 100644 services/catalog/src/simcore_service_catalog/services/manifest.py create mode 100644 services/catalog/tests/unit/test_api_rest.py create mode 100644 services/catalog/tests/unit/test_services_manifest.py create mode 100644 services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py rename services/catalog/tests/unit/with_dbs/{test_api_routes_services__list.py => test_api_rest_services__list.py} (91%) rename services/catalog/tests/unit/with_dbs/{test_api_routes_services_access_rights.py => test_api_rest_services_access_rights.py} (95%) rename services/catalog/tests/unit/with_dbs/{test_api_routes_services_ports.py => test_api_rest_services_ports.py} (96%) rename services/catalog/tests/unit/with_dbs/{test_api_routes_services_resources.py => test_api_rest_services_resources.py} (95%) rename services/catalog/tests/unit/with_dbs/{test_api_routes_services_specifications.py => test_api_rest_services_specifications.py} (94%) create mode 100644 services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py diff --git a/Makefile b/Makefile index 2326de6d67d..251779b2992 100644 --- a/Makefile +++ b/Makefile @@ -316,7 +316,7 @@ printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres& printf "$$rows" "Portainer" "http://$(get_my_ip).nip.io:9000" admin adminadmin;\ printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\ printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\ -printf "$$rows" "Docker Registry" "$${REGISTRY_URL}" $${REGISTRY_USER} $${REGISTRY_PW};\ +printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\ printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\ printf "$$rows" "Payments" "http://$(get_my_ip).nip.io:8011/dev/doc" $${PAYMENTS_USERNAME} $${PAYMENTS_PASSWORD};\ printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\ diff --git a/services/catalog/openapi.json b/services/catalog/openapi.json index bf8f8aee6b6..353ba9b0155 100644 --- a/services/catalog/openapi.json +++ b/services/catalog/openapi.json @@ -581,8 +581,7 @@ "name": { "type": "string", "title": "Name", - "description": "Name of the author", - "example": "Jim Knopf" + "description": "Name of the author" }, "email": { "type": "string", @@ -592,8 +591,7 @@ }, "affiliation": { "type": "string", - "title": "Affiliation", - "description": "Affiliation of the author" + "title": "Affiliation" } }, "type": "object", @@ -633,7 +631,12 @@ "image", "url" ], - "title": "Badge" + "title": "Badge", + "example": { + "name": "osparc.io", + "image": "https://img.shields.io/website-up-down-green-red/https/itisfoundation.github.io.svg?label=documentation", + "url": "https://itisfoundation.github.io/" + } }, "BaseMeta": { "properties": { @@ -2186,6 +2189,11 @@ "title": "Progress Regexp", "description": "regexp pattern for detecting computational service's progress" }, + "image_digest": { + "type": "string", + "title": "Image Digest", + "description": "Image manifest digest. Note that this is NOT injected as an image label" + }, "owner": { "type": "string", "format": "email", @@ -2205,43 +2213,7 @@ "outputs" ], "title": "ServiceGet", - "description": "Service metadata at publication time\n\n- read-only (can only be changed overwriting the image labels in the registry)\n- base metaddata\n- injected in the image labels\n\nNOTE: This model is serialized in .osparc/metadata.yml and in the labels of the docker image", - "example": { - "name": "File Picker", - "description": "description", - "classifiers": [], - "quality": {}, - "accessRights": { - "1": { - "execute_access": true, - "write_access": false - }, - "4": { - "execute_access": true, - "write_access": true - } - }, - "key": "simcore/services/frontend/file-picker", - "version": "1.0.0", - "type": "dynamic", - "authors": [ - { - "name": "Red Pandas", - "email": "redpandas@wonderland.com" - } - ], - "contact": "redpandas@wonderland.com", - "inputs": {}, - "outputs": { - "outFile": { - "displayOrder": 0, - "label": "File", - "description": "Chosen File", - "type": "data:*/*" - } - }, - "owner": "redpandas@wonderland.com" - } + "description": "Service metadata at publication time\n\n- read-only (can only be changed overwriting the image labels in the registry)\n- base metaddata\n- injected in the image labels\n\nNOTE: This model is serialized in .osparc/metadata.yml and in the labels of the docker image" }, "ServiceGroupAccessRights": { "properties": { diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_constants.py b/services/catalog/src/simcore_service_catalog/_constants.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/rest/_constants.py rename to services/catalog/src/simcore_service_catalog/_constants.py diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index dfea3b84b53..7855bc97ad5 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -1,23 +1,22 @@ import logging -import urllib.parse from dataclasses import dataclass -from typing import Annotated, Any, cast +from typing import Annotated from fastapi import Depends, FastAPI, Header, HTTPException, status -from models_library.api_schemas_catalog.services import ServiceGet from models_library.api_schemas_catalog.services_specifications import ( ServiceSpecifications, ) -from models_library.services import ServiceKey, ServiceVersion +from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_resources import ResourcesDict +from models_library.services_types import ServiceKey, ServiceVersion from pydantic import ValidationError from servicelib.fastapi.dependencies import get_app from ...core.settings import ApplicationSettings from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository +from ...services import manifest from ...services.director import DirectorApi -from ...services.function_services import get_function_service, is_function_service from .database import get_repository from .director import get_director_api @@ -84,32 +83,20 @@ async def check_service_read_access( ) -async def get_service_from_registry( +async def get_service_from_manifest( service_key: ServiceKey, service_version: ServiceVersion, director_client: Annotated[DirectorApi, Depends(get_director_api)], -) -> ServiceGet: +) -> ServiceMetaDataPublished: """ Retrieves service metadata from the docker registry via the director """ try: - if is_function_service(service_key): - frontend_service: dict[str, Any] = get_function_service( - key=service_key, version=service_version - ) - _service_data = frontend_service - else: - # NOTE: raises HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE) on ANY failure - services_in_registry = cast( - list[Any], - await director_client.get( - f"/services/{urllib.parse.quote_plus(service_key)}/{service_version}" - ), - ) - _service_data = services_in_registry[0] - - service: ServiceGet = ServiceGet.parse_obj(_service_data) - return service + return await manifest.get_service( + service_key=service_key, + service_version=service_version, + director_client=director_client, + ) except ValidationError as exc: _logger.warning( diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index abea57b24a0..5c98e535734 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -9,11 +9,17 @@ from fastapi import APIRouter, Depends, Header, HTTPException, status from models_library.api_schemas_catalog.services import ServiceGet, ServiceUpdate from models_library.services import ServiceKey, ServiceType, ServiceVersion +from models_library.services_metadata_published import ServiceMetaDataPublished from pydantic import ValidationError from pydantic.types import PositiveInt from servicelib.fastapi.requests_decorators import cancel_on_disconnect from starlette.requests import Request +from ..._constants import ( + DIRECTOR_CACHING_TTL, + LIST_SERVICES_CACHING_TTL, + RESPONSE_MODEL_POLICY, +) from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB @@ -21,12 +27,7 @@ from ...services.function_services import is_function_service from ..dependencies.database import get_repository from ..dependencies.director import get_director_api -from ..dependencies.services import get_service_from_registry -from ._constants import ( - DIRECTOR_CACHING_TTL, - LIST_SERVICES_CACHING_TTL, - RESPONSE_MODEL_POLICY, -) +from ..dependencies.services import get_service_from_manifest _logger = logging.getLogger(__name__) @@ -82,6 +83,7 @@ def _build_cache_key(fct, *_, **kwargs): ) async def list_services( request: Request, # pylint:disable=unused-argument + *, user_id: PositiveInt, director_client: Annotated[DirectorApi, Depends(get_director_api)], groups_repository: Annotated[ @@ -91,7 +93,7 @@ async def list_services( ServicesRepository, Depends(get_repository(ServicesRepository)) ], x_simcore_products_name: Annotated[str, Header(...)], - details: bool | None = True, # noqa: FBT002 + details: bool = True, ): # Access layer user_groups = await groups_repository.list_user_groups(user_id) @@ -116,7 +118,6 @@ async def list_services( # Non-detailed views from the services_repo database if not details: # only return a stripped down version - # FIXME: add name, ddescription, type, etc... # NOTE: here validation is not necessary since key,version were already validated # in terms of time, this takes the most return [ @@ -189,7 +190,9 @@ async def cached_registry_services() -> dict[str, Any]: ) async def get_service( user_id: int, - service: Annotated[ServiceGet, Depends(get_service_from_registry)], + service_in_manifest: Annotated[ + ServiceMetaDataPublished, Depends(get_service_from_manifest) + ], groups_repository: Annotated[ GroupsRepository, Depends(get_repository(GroupsRepository)) ], @@ -198,6 +201,8 @@ async def get_service( ], x_simcore_products_name: str = Header(None), ): + service_data: dict[str, Any] = {} + # get the user groups user_groups = await groups_repository.list_user_groups(user_id) if not user_groups: @@ -208,8 +213,8 @@ async def get_service( ) # check the user has access to this service and to which extent service_in_db = await services_repo.get_service( - service.key, - service.version, + service_in_manifest.key, + service_in_manifest.version, gids=[group.gid for group in user_groups], write_access=True, product_name=x_simcore_products_name, @@ -219,14 +224,18 @@ async def get_service( service_access_rights: list[ ServiceAccessRightsAtDB ] = await services_repo.get_service_access_rights( - service.key, service.version, product_name=x_simcore_products_name + service_in_manifest.key, + service_in_manifest.version, + product_name=x_simcore_products_name, ) - service.access_rights = {rights.gid: rights for rights in service_access_rights} + service_data["access_rights"] = { + rights.gid: rights for rights in service_access_rights + } else: # check if we have executable rights service_in_db = await services_repo.get_service( - service.key, - service.version, + service_in_manifest.key, + service_in_manifest.version, gids=[group.gid for group in user_groups], execute_access=True, product_name=x_simcore_products_name, @@ -237,17 +246,19 @@ async def get_service( status_code=status.HTTP_403_FORBIDDEN, detail="You have insufficient rights to access the service", ) - # access is allowed, override some of the values with what is in the db - service = service.copy( - update=service_in_db.dict(exclude_unset=True, exclude={"owner"}) - ) + # the owner shall be converted to an email address if service_in_db.owner: - service.owner = await groups_repository.get_user_email_from_gid( + service_data["owner"] = await groups_repository.get_user_email_from_gid( service_in_db.owner ) - return service + # access is allowed, override some of the values with what is in the db + service_in_manifest = service_in_manifest.copy( + update=service_in_db.dict(exclude_unset=True, exclude={"owner"}) + ) + service_data.update(service_in_manifest.dict(exclude_unset=True, by_alias=True)) + return service_data @router.patch( @@ -357,7 +368,7 @@ async def update_service( # now return the service return await get_service( user_id=user_id, - service=await get_service_from_registry( + service_in_manifest=await get_service_from_manifest( service_key, service_version, director_client ), groups_repository=groups_repository, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py index 46e0b337152..6abaf48dd04 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py @@ -7,11 +7,11 @@ ) from models_library.services import ServiceKey, ServiceVersion +from ..._constants import RESPONSE_MODEL_POLICY from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB from ..dependencies.database import get_repository from ..dependencies.services import AccessInfo, check_service_read_access -from ._constants import RESPONSE_MODEL_POLICY _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py index b58699866a7..84797e4832b 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py @@ -2,15 +2,15 @@ from typing import Annotated from fastapi import APIRouter, Depends -from models_library.api_schemas_catalog.services import ServiceGet from models_library.api_schemas_catalog.services_ports import ServicePortGet +from models_library.services_metadata_published import ServiceMetaDataPublished +from ..._constants import RESPONSE_MODEL_POLICY from ..dependencies.services import ( AccessInfo, check_service_read_access, - get_service_from_registry, + get_service_from_manifest, ) -from ._constants import RESPONSE_MODEL_POLICY _logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ ) async def list_service_ports( _user: Annotated[AccessInfo, Depends(check_service_read_access)], - service: Annotated[ServiceGet, Depends(get_service_from_registry)], + service: Annotated[ServiceMetaDataPublished, Depends(get_service_from_manifest)], ): ports: list[ServicePortGet] = [] diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 141f85c98b2..88e2ab4d958 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -22,6 +22,7 @@ from models_library.utils.docker_compose import replace_env_vars_in_compose_spec from pydantic import parse_obj_as, parse_raw_as +from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS from ...db.repositories.services import ServicesRepository from ...services.director import DirectorApi from ...services.function_services import is_function_service @@ -33,7 +34,6 @@ from ..dependencies.director import get_director_api from ..dependencies.services import get_default_service_resources from ..dependencies.user_groups import list_user_groups -from ._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS router = APIRouter() _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index bd398f25308..d5481b3cefd 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -9,12 +9,12 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID +from ..._constants import RESPONSE_MODEL_POLICY from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository from ...services.function_services import is_function_service from ..dependencies.database import get_repository from ..dependencies.services import get_default_service_specifications -from ._constants import RESPONSE_MODEL_POLICY router = APIRouter() _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index ffea02efeb4..89079b7561d 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -13,16 +13,16 @@ import logging from contextlib import suppress from pprint import pformat -from typing import Any, Final, NewType, TypeAlias, cast +from typing import Final from fastapi import FastAPI -from models_library.function_services_catalog.api import iter_service_docker_data from models_library.services import ServiceMetaDataPublished +from models_library.services_types import ServiceKey, ServiceVersion from packaging.version import Version -from pydantic import ValidationError +from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.services import manifest from sqlalchemy.ext.asyncio import AsyncEngine -from ..api.dependencies.director import get_director_api from ..db.repositories.groups import GroupsRepository from ..db.repositories.projects import ProjectsRepository from ..db.repositories.services import ServicesRepository @@ -31,49 +31,10 @@ _logger = logging.getLogger(__name__) -# NOTE: by PC I tried to unify with models_library.services but there are other inconsistencies so I leave if for another time! -ServiceKey = NewType("ServiceKey", str) -ServiceVersion = NewType("ServiceVersion", str) -ServiceDockerDataMap: TypeAlias = dict[ - tuple[ServiceKey, ServiceVersion], ServiceMetaDataPublished -] - -_error_already_logged: set[tuple[str, str]] = set() - - -async def _list_services_in_registry( - app: FastAPI, -) -> ServiceDockerDataMap: - client = get_director_api(app) - registry_services = cast(list[dict[str, Any]], await client.get("/services")) - - services: ServiceDockerDataMap = { - # functional-service: services w/o associated image - (s.key, s.version): s - for s in iter_service_docker_data() - } - for service in registry_services: - try: - service_data = ServiceMetaDataPublished.parse_obj(service) - services[(service_data.key, service_data.version)] = service_data - - except ValidationError: # noqa: PERF203 - errored_service = service.get("key"), service.get("version") - if errored_service not in _error_already_logged: - _logger.warning( - "Skipping '%s:%s' from the catalog of services! So far %s invalid services in registry.", - *errored_service, - len(_error_already_logged) + 1, - exc_info=True, - ) - _error_already_logged.add(errored_service) - - return services - async def _list_services_in_database( db_engine: AsyncEngine, -) -> set[tuple[ServiceKey, ServiceVersion]]: +): services_repo = ServicesRepository(db_engine=db_engine) return { (service.key, service.version) @@ -134,16 +95,15 @@ async def _ensure_registry_and_database_are_synced(app: FastAPI) -> None: Notice that a services here refers to a 2-tuple (key, version) """ - services_in_registry: dict[ - tuple[ServiceKey, ServiceVersion], ServiceMetaDataPublished - ] = await _list_services_in_registry(app) + director_api = get_director_api(app) + services_in_manifest_map = await manifest.get_services_map(director_api) services_in_db: set[ tuple[ServiceKey, ServiceVersion] ] = await _list_services_in_database(app.state.engine) # check that the db has all the services at least once - missing_services_in_db = set(services_in_registry.keys()) - services_in_db + missing_services_in_db = set(services_in_manifest_map.keys()) - services_in_db if missing_services_in_db: _logger.debug( "Missing services in db: %s", @@ -152,7 +112,7 @@ async def _ensure_registry_and_database_are_synced(app: FastAPI) -> None: # update db await _create_services_in_database( - app, missing_services_in_db, services_in_registry + app, missing_services_in_db, services_in_manifest_map ) @@ -197,20 +157,24 @@ async def _ensure_published_templates_accessible( await services_repo.upsert_service_access_rights(missing_services_access_rights) -async def _sync_services_task(app: FastAPI) -> None: +async def _run_sync_services(app: FastAPI): default_product: Final[str] = app.state.default_product_name engine: AsyncEngine = app.state.engine + # check that the list of services is in sync with the registry + await _ensure_registry_and_database_are_synced(app) + + # check that the published services are available to everyone + # (templates are published to GUESTs, so their services must be also accessible) + await _ensure_published_templates_accessible(engine, default_product) + + +async def _sync_services_task(app: FastAPI) -> None: while app.state.registry_syncer_running: try: _logger.debug("Syncing services between registry and database...") - # check that the list of services is in sync with the registry - await _ensure_registry_and_database_are_synced(app) - - # check that the published services are available to everyone - # (templates are published to GUESTs, so their services must be also accessible) - await _ensure_published_templates_accessible(engine, default_product) + await _run_sync_services(app) await asyncio.sleep(app.state.settings.CATALOG_BACKGROUND_TASK_REST_TIME) diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py index 8b668610f88..fb2329019b5 100644 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ b/services/catalog/src/simcore_service_catalog/core/events.py @@ -4,6 +4,7 @@ from fastapi import FastAPI from servicelib.db_async_engine import close_db_connection, connect_to_db +from servicelib.logging_utils import log_context from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..db.events import setup_default_product @@ -16,9 +17,18 @@ EventCallable: TypeAlias = Callable[[], Awaitable[None]] +def _flush_started_banner() -> None: + # WARNING: this function is spied in the tests + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + + +def _flush_finished_banner() -> None: + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + def create_on_startup(app: FastAPI) -> EventCallable: async def _() -> None: - print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + _flush_started_banner() # setup connection to pg db if app.state.settings.CATALOG_POSTGRES: @@ -40,16 +50,16 @@ async def _() -> None: def create_on_shutdown(app: FastAPI) -> EventCallable: async def _() -> None: - _logger.info("Application stopping") - if app.state.settings.CATALOG_DIRECTOR: - try: - await stop_registry_sync_task(app) - await close_director(app) - await close_db_connection(app) - except Exception: # pylint: disable=broad-except - _logger.exception("Unexpected error while closing application") - - print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + with log_context(_logger, logging.INFO, "Application shutdown"): + if app.state.settings.CATALOG_DIRECTOR: + try: + await stop_registry_sync_task(app) + await close_director(app) + await close_db_connection(app) + except Exception: # pylint: disable=broad-except + _logger.exception("Unexpected error while closing application") + + _flush_finished_banner() return _ diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/services/director.py index ae1ad14e48f..1f40b8ea3ec 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/services/director.py @@ -1,13 +1,17 @@ import asyncio import functools import logging +import urllib.parse from collections.abc import Awaitable, Callable from contextlib import suppress from typing import Any import httpx from fastapi import FastAPI, HTTPException +from models_library.services_metadata_published import ServiceMetaDataPublished +from models_library.services_types import ServiceKey, ServiceVersion from models_library.utils.json_serialization import json_dumps +from pydantic import parse_obj_as from servicelib.logging_utils import log_context from starlette import status from tenacity._asyncio import AsyncRetrying @@ -32,7 +36,7 @@ } -def _safe_request( +def _return_data_or_raise_error( request_func: Callable[..., Awaitable[httpx.Response]] ) -> Callable[..., Awaitable[list[Any] | dict[str, Any]]]: """ @@ -113,18 +117,18 @@ def __init__(self, base_url: str, app: FastAPI): async def close(self): await self.client.aclose() - # OPERATIONS - # TODO: policy to retry if NetworkError/timeout? - # TODO: add ping to healthcheck + # + # Low level API + # - @_safe_request + @_return_data_or_raise_error async def get(self, path: str) -> httpx.Response: # temp solution: default timeout increased to 20" return await self.client.get(path, timeout=20.0) - @_safe_request - async def put(self, path: str, body: dict) -> httpx.Response: - return await self.client.put(path, json=body) + # + # High level API + # async def is_responsive(self) -> bool: try: @@ -136,6 +140,23 @@ async def is_responsive(self) -> bool: except (httpx.HTTPStatusError, httpx.RequestError, httpx.TimeoutException): return False + async def list_all_services(self) -> list[ServiceMetaDataPublished]: + # WARNING: this function probably raise ValidationError since director does NOT offer guarantees. + # SEE list_registered_services + data = await self.get("/services") + return parse_obj_as(list[ServiceMetaDataPublished], data) + + async def get_service( + self, service_key: ServiceKey, service_version: ServiceVersion + ) -> ServiceMetaDataPublished: + data = await self.get( + f"/services/{urllib.parse.quote_plus(service_key)}/{service_version}" + ) + # NOTE: the fact that it returns a list of one element is a defect of the director API + assert isinstance(data, list) # nosec + assert len(data) == 1 # nosec + return ServiceMetaDataPublished.parse_obj(data[0]) + async def setup_director(app: FastAPI) -> None: if settings := app.state.settings.CATALOG_DIRECTOR: diff --git a/services/catalog/src/simcore_service_catalog/services/function_services.py b/services/catalog/src/simcore_service_catalog/services/function_services.py index ae6b7224a69..e56ef218bdc 100644 --- a/services/catalog/src/simcore_service_catalog/services/function_services.py +++ b/services/catalog/src/simcore_service_catalog/services/function_services.py @@ -1,7 +1,3 @@ -""" - Catalog of i/o metadata for functions implemented in the front-end -""" - from typing import Any, cast from fastapi import status @@ -17,19 +13,16 @@ def _as_dict(model_instance: ServiceMetaDataPublished) -> dict[str, Any]: - # FIXME: In order to convert to ServiceOut, now we have to convert back to front-end service because of alias - # FIXME: set the same policy for f/e and director datasets! return cast(dict[str, Any], model_instance.dict(by_alias=True, exclude_unset=True)) -def get_function_service(key, version) -> dict[str, Any]: +def get_function_service(key, version) -> ServiceMetaDataPublished: try: - found = next( + return next( s for s in iter_service_docker_data() if s.key == key and s.version == version ) - return _as_dict(found) except StopIteration as err: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -38,12 +31,6 @@ def get_function_service(key, version) -> dict[str, Any]: def setup_function_services(app: FastAPI): - """ - Setup entrypoint for this app module. - - Used in core.application.init_app - """ - def _on_startup() -> None: catalog = [_as_dict(metadata) for metadata in iter_service_docker_data()] app.state.frontend_services_catalog = catalog diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py new file mode 100644 index 00000000000..33ab9bfe657 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -0,0 +1,97 @@ +""" Services Manifest API Documentation + +The `services.manifest` module provides a read-only API to access the services catalog. The term "Manifest" refers to a detailed, finalized list, +traditionally used to denote items that are recorded as part of an official inventory or log, emphasizing the immutable nature of the data. + +### Service Registration +Services are registered within the manifest in two distinct methods: + +1. **Docker Registry Integration:** + - Services can be registered by pushing a Docker image, complete with appropriate labels and tags, to a Docker registry. + - These are generally services registered through the Docker registry method, catering primarily to end-user functionalities. + - Example services include user-oriented applications like `sleeper`. + +2. **Function Service Definition:** + - Services can also be directly defined in the codebase as function services, which typically support framework operations. + - These services are usually defined programmatically within the code and are integral to the framework's infrastructure. + - Examples include utility services like `FilePicker`. + + +### Usage +This API is designed for read-only interactions, allowing users to retrieve information about registered services but not to modify the registry. +This ensures data integrity and consistency across the system. + + +""" + +import logging +from typing import Any, TypeAlias, cast + +from models_library.function_services_catalog.api import iter_service_docker_data +from models_library.services_metadata_published import ServiceMetaDataPublished +from models_library.services_types import ServiceKey, ServiceVersion +from pydantic import ValidationError + +from .director import DirectorApi +from .function_services import get_function_service, is_function_service + +_logger = logging.getLogger(__name__) + + +ServiceMetaDataPublishedDict: TypeAlias = dict[ + tuple[ServiceKey, ServiceVersion], ServiceMetaDataPublished +] + + +_error_already_logged: set[tuple[str | None, str | None]] = set() + + +async def get_services_map( + director_client: DirectorApi, +) -> ServiceMetaDataPublishedDict: + + # NOTE: using Low-level API to avoid validation + services_in_registry = cast( + list[dict[str, Any]], await director_client.get("/services") + ) + + # NOTE: functional-services are services w/o associated image + services: ServiceMetaDataPublishedDict = { + (s.key, s.version): s for s in iter_service_docker_data() + } + for service in services_in_registry: + try: + service_data = ServiceMetaDataPublished.parse_obj(service) + services[(service_data.key, service_data.version)] = service_data + + except ValidationError: # noqa: PERF203 + # NOTE: this is necessary since registry DOES NOT provides any guarantee of the meta-data + # in the labels, i.e. it is not validated + errored_service = (service.get("key"), service.get("version")) + if errored_service not in _error_already_logged: + _logger.warning( + "Skipping '%s:%s' from the catalog of services! So far %s invalid services in registry.", + *errored_service, + len(_error_already_logged) + 1, + exc_info=True, + ) + _error_already_logged.add(errored_service) + + return services + + +async def get_service( + service_key: ServiceKey, + service_version: ServiceVersion, + director_client: DirectorApi, +) -> ServiceMetaDataPublished: + """ + Retrieves service metadata from the docker registry via the director and accounting + """ + if is_function_service(service_key): + service = get_function_service(key=service_key, version=service_version) + else: + service = await director_client.get_service( + service_key=service_key, service_version=service_version + ) + return service diff --git a/services/catalog/tests/unit/conftest.py b/services/catalog/tests/unit/conftest.py index cb5d8504660..6069514b085 100644 --- a/services/catalog/tests/unit/conftest.py +++ b/services/catalog/tests/unit/conftest.py @@ -8,20 +8,22 @@ import hashlib from collections.abc import AsyncIterator, Awaitable, Callable, Iterator -from copy import deepcopy from pathlib import Path -from typing import Any +from typing import Any, NamedTuple import httpx import pytest import respx import simcore_service_catalog +import simcore_service_catalog.core.events import yaml from asgi_lifespan import LifespanManager from faker import Faker from fastapi import FastAPI, status +from fastapi.testclient import TestClient from packaging.version import Version -from pytest_mock import MockerFixture +from pydantic import EmailStr +from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient @@ -98,18 +100,108 @@ def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: return ApplicationSettings.create_from_envs() +class AppLifeSpanSpyTargets(NamedTuple): + on_startup: MockType + on_shutdown: MockType + + +@pytest.fixture +def spy_app(mocker: MockerFixture) -> AppLifeSpanSpyTargets: + # Used to ensure startup/teardown workflows using different fixtures + # work as expected + return AppLifeSpanSpyTargets( + on_startup=mocker.spy( + simcore_service_catalog.core.events, + "_flush_started_banner", + ), + on_shutdown=mocker.spy( + simcore_service_catalog.core.events, + "_flush_finished_banner", + ), + ) + + @pytest.fixture async def app( - app_settings: ApplicationSettings, is_pdb_enabled: bool + app_settings: ApplicationSettings, + is_pdb_enabled: bool, + spy_app: AppLifeSpanSpyTargets, ) -> AsyncIterator[FastAPI]: + """ + NOTE that this app was started when the fixture is setup + and shutdown when the fixture is tear-down + """ + + # create instance assert app_environment - the_test_app = create_app(settings=app_settings) + app_under_test = create_app(settings=app_settings) + + assert spy_app.on_startup.call_count == 0 + assert spy_app.on_shutdown.call_count == 0 + async with LifespanManager( - the_test_app, + app_under_test, startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP, shutdown_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_SHUTDOWN, ): - yield the_test_app + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 0 + + yield app_under_test + + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 1 + + +@pytest.fixture +def client( + app_settings: ApplicationSettings, spy_app: AppLifeSpanSpyTargets +) -> Iterator[TestClient]: + # NOTE: DO NOT add `app` as a dependency since it is already initialized + + # create instance + assert app_environment + app_under_test = create_app(settings=app_settings) + + assert ( + spy_app.on_startup.call_count == 0 + ), "TIP: Remove dependencies from `app` fixture and get it via `client.app`" + assert spy_app.on_shutdown.call_count == 0 + + with TestClient(app_under_test) as cli: + + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 0 + + yield cli + + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 1 + + +@pytest.fixture +async def aclient( + app: FastAPI, spy_app: AppLifeSpanSpyTargets +) -> AsyncIterator[httpx.AsyncClient]: + # NOTE: Avoids TestClient since `app` fixture already runs LifespanManager + # Otherwise `with TestClient` will call twice start/shutdown events + + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 0 + + async with httpx.AsyncClient( + base_url="http://catalog.testserver.io", + headers={"Content-Type": "application/json"}, + transport=httpx.ASGITransport(app=app), + ) as acli: + assert isinstance(acli._transport, httpx.ASGITransport) + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 0 + + yield acli + + assert spy_app.on_startup.call_count == 1 + assert spy_app.on_shutdown.call_count == 0 @pytest.fixture @@ -122,13 +214,37 @@ def postgres_setup_disabled(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("CATALOG_POSTGRES", "null") +@pytest.fixture +def background_tasks_setup_disabled(mocker: MockerFixture) -> None: + """patch the setup of the background task so we can call it manually""" + + def _factory(name): + async def _side_effect(app: FastAPI): + assert app + print( + "TEST", + background_tasks_setup_disabled.__name__, + "Disabled background tasks. Skipping execution of", + name, + ) + + return _side_effect + + for name in ("start_registry_sync_task", "stop_registry_sync_task"): + mocker.patch( + f"simcore_service_catalog.core.events.{name}", + side_effect=_factory(name), + autospec=True, + ) + + # # rabbit-MQ # @pytest.fixture -def setup_rabbitmq_and_rpc_disabled(mocker: MockerFixture): +def rabbitmq_and_rpc_setup_disabled(mocker: MockerFixture): # The following services are affected if rabbitmq is not in place mocker.patch("simcore_service_catalog.core.application.setup_rabbitmq") mocker.patch("simcore_service_catalog.core.application.setup_rpc_api_routes") @@ -168,24 +284,98 @@ def director_service_openapi_specs( @pytest.fixture -def mocked_director_service_api( +def expected_director_list_services( + user_email: EmailStr, user_first_name: str, user_last_name: str +) -> list[dict[str, Any]]: + """This fixture has at least TWO purposes: + + 1. can be used as a reference to check the results at the other end + 2. can be used to change responses of the director API downstream (override fixture) + + """ + return [ + { + "image_digest": hashlib.sha256( + f"simcore/services/comp/ans-model:{major}".encode() + ).hexdigest(), + "authors": [ + { + "name": f"{user_first_name} {user_last_name}", + "email": user_email, + "affiliation": "ACME", + } + ], + "contact": user_email, + "description": "Autonomous Nervous System Network model", + "inputs": { + "input_1": { + "displayOrder": 1, + "label": "Simulation time", + "description": "Duration of the simulation", + "type": "ref_contentSchema", + "contentSchema": { + "type": "number", + "x_unit": "milli-second", + }, + "defaultValue": 2, + } + }, + "integration-version": "1.0.0", + "key": "simcore/services/comp/ans-model", + "name": "Autonomous Nervous System Network model", + "outputs": { + "output_1": { + "displayOrder": 1, + "label": "ANS output", + "description": "Output of simulation of Autonomous Nervous System Network model", + "type": "data:*/*", + "fileToKeyMap": {"ANS_output.txt": "output_1"}, + }, + "output_2": { + "displayOrder": 2, + "label": "Stimulation parameters", + "description": "stim_param.txt file containing the input provided in the inputs port", + "type": "data:*/*", + "fileToKeyMap": {"ANS_stim_param.txt": "output_2"}, + }, + }, + "thumbnail": "https://www.statnews.com/wp-content/uploads/2020/05/3D-rat-heart.-iScience--768x432.png", + "type": "computational", + "version": f"{major}.0.0", + } + for major in range(1, 4) + ] + + +@pytest.fixture +def mocked_director_service_api_base( app_settings: ApplicationSettings, director_service_openapi_specs: dict[str, Any], ) -> Iterator[respx.MockRouter]: - assert app_settings.CATALOG_DIRECTOR + """ + BASIC fixture to mock director service API + + Use `mocked_director_service_api_base` to customize the mocks + + """ + assert ( + app_settings.CATALOG_DIRECTOR + ), "Check dependency on fixture `director_setup_disabled`" + + # NOTE: this MUST be in sync with services/director/src/simcore_service_director/api/v0/openapi.yaml + openapi = director_service_openapi_specs + assert Version(openapi["info"]["version"]) == Version("0.1.0") + with respx.mock( - base_url=app_settings.CATALOG_DIRECTOR.base_url, + base_url=app_settings.CATALOG_DIRECTOR.base_url, # NOTE: it include v0/ assert_all_called=False, assert_all_mocked=True, ) as respx_mock: - # NOTE: this MUST be in sync with services/director/src/simcore_service_director/api/v0/openapi.yaml - openapi = deepcopy(director_service_openapi_specs) - assert Version(openapi["info"]["version"]) == Version("0.1.0") - - # Validate responses against OAS + # HEATHCHECK + assert openapi["paths"].get("/") respx_mock.head("/", name="healthcheck").respond( - 200, + status.HTTP_200_OK, json={ "data": { "name": "simcore-service-director", @@ -196,85 +386,141 @@ def mocked_director_service_api( }, ) - _services = [ - { - "image_digest": hashlib.sha256( - f"simcore/services/comp/ans-model:{major}".encode() - ).hexdigest(), - "authors": [ - { - "name": "John Smith", - "email": "smith@acme.com", - "affiliation": "ACME", - } - ], - "contact": "smith@acme.com", - "description": "Autonomous Nervous System Network model", - "inputs": { - "input_1": { - "displayOrder": 1, - "label": "Simulation time", - "description": "Duration of the simulation", - "type": "ref_contentSchema", - "contentSchema": { - "type": "number", - "x_unit": "milli-second", - }, - "defaultValue": 2, - } - }, - "integration-version": "1.0.0", - "key": "simcore/services/comp/ans-model", - "name": "Autonomous Nervous System Network model", - "outputs": { - "output_1": { - "displayOrder": 1, - "label": "ANS output", - "description": "Output of simulation of Autonomous Nervous System Network model", - "type": "data:*/*", - "fileToKeyMap": {"ANS_output.txt": "output_1"}, - }, - "output_2": { - "displayOrder": 2, - "label": "Stimulation parameters", - "description": "stim_param.txt file containing the input provided in the inputs port", - "type": "data:*/*", - "fileToKeyMap": {"ANS_stim_param.txt": "output_2"}, - }, - }, - "thumbnail": "https://www.statnews.com/wp-content/uploads/2020/05/3D-rat-heart.-iScience--768x432.png", - "type": "computational", - "version": f"{major}.0.0", - } - for major in range(1, 4) - ] - - respx_mock.get("/services", name="list_services").respond( - status.HTTP_200_OK, - json={"data": _services}, - ) + yield respx_mock + + +@pytest.fixture +def mocked_director_service_api( + mocked_director_service_api_base: respx.MockRouter, + director_service_openapi_specs: dict[str, Any], + expected_director_list_services: list[dict[str, Any]], +) -> respx.MockRouter: + """ + STANDARD fixture to mock director service API + + To customize the mock responses use `mocked_director_service_api_base` instead + """ + # alias + openapi = director_service_openapi_specs + respx_mock = mocked_director_service_api_base + + def _search(service_key, service_version): + try: + return next( + s + for s in expected_director_list_services + if (s["key"] == service_key and s["version"] == service_version) + ) + except StopIteration: + return None + + # LIST + assert openapi["paths"].get("/services") + + respx_mock.get(path__regex=r"/services$", name="list_services").respond( + status.HTTP_200_OK, json={"data": expected_director_list_services} + ) + + # GET + assert openapi["paths"].get("/services/{service_key}/{service_version}") - @respx_mock.get(r"/services/(?P\w+)/(?P\w+)") - def get_service(request): - key = request.path_params["services_key"] - version = request.path_params["service_version"] - for service in _services: - if service["key"] == key and service["version"] == version: - return httpx.Response(status.HTTP_200_OK, json={"data": service}) + @respx_mock.get( + path__regex=r"^/services/(?P[/\w-]+)/(?P[0-9.]+)$", + name="get_service", + ) + def _get_service(request, service_key, service_version): + if found := _search(service_key, service_version): + # NOTE: this is a defect in director's API + single_service_list = [found] return httpx.Response( - status.HTTP_404_NOT_FOUND, json={"error": "Service not found"} + status.HTTP_200_OK, json={"data": single_service_list} ) + return httpx.Response( + status.HTTP_404_NOT_FOUND, + json={ + "data": { + "status": status.HTTP_404_NOT_FOUND, + "message": f"The service {service_key}:{service_version} does not exist", + } + }, + ) + + # GET LABELS + assert openapi["paths"].get("/services/{service_key}/{service_version}/labels") - @respx_mock.get( - r"/services/(?P\w+)/(?P\w+)/labels" + @respx_mock.get( + path__regex=r"^/services/(?P[/\w-]+)/(?P[0-9\.]+)/labels$", + name="get_service_labels", + ) + def _get_service_labels(request, service_key, service_version): + if found := _search(service_key, service_version): + return httpx.Response( + status_code=status.HTTP_200_OK, + json={ + "data": { + "io.simcore.authors": '{"authors": [{"name": "John Smith", "email": "john@acme.com", "affiliation": "ACME\'IS Foundation"}]}', + "io.simcore.contact": '{"contact": "john@acme.com"}', + "io.simcore.description": '{"description": "Autonomous Nervous System Network model"}', + "io.simcore.inputs": '{"inputs": {"input_1": {"displayOrder": 1.0, "label": "Simulation time", "description": "Duration of the simulation", "type": "ref_contentSchema", "contentSchema": {"type": "number", "x_unit": "milli-second"}, "defaultValue": 2.0}}}', + "io.simcore.integration-version": '{"integration-version": "1.0.0"}', + "io.simcore.key": '{"key": "xxxxx"}'.replace( + "xxxxx", found["key"] + ), + "io.simcore.name": '{"name": "Autonomous Nervous System Network model"}', + "io.simcore.outputs": '{"outputs": {"output_1": {"displayOrder": 1.0, "label": "ANS output", "description": "Output of simulation of Autonomous Nervous System Network model", "type": "data:*/*", "fileToKeyMap": {"ANS_output.txt": "output_1"}}, "output_2": {"displayOrder": 2.0, "label": "Stimulation parameters", "description": "stim_param.txt file containing the input provided in the inputs port", "type": "data:*/*", "fileToKeyMap": {"ANS_stim_param.txt": "output_2"}}}}', + "io.simcore.thumbnail": '{"thumbnail": "https://www.statnews.com/wp-content/uploads/2020/05/3D-rat-heart.-iScience--768x432.png"}', + "io.simcore.type": '{"type": "computational"}', + "io.simcore.version": '{"version": "xxxxx"}'.replace( + "xxxxx", found["version"] + ), + "maintainer": "iavarone", + "org.label-schema.build-date": "2023-04-17T08:04:15Z", + "org.label-schema.schema-version": "1.0", + "org.label-schema.vcs-ref": "", + "org.label-schema.vcs-url": "", + "simcore.service.restart-policy": "no-restart", + "simcore.service.settings": '[{"name": "Resources", "type": "Resources", "value": {"Limits": {"NanoCPUs": 4000000000, "MemoryBytes": 2147483648}, "Reservations": {"NanoCPUs": 4000000000, "MemoryBytes": 2147483648}}}]', + } + }, + ) + return httpx.Response( + status.HTTP_404_NOT_FOUND, + json={ + "data": { + "status": status.HTTP_404_NOT_FOUND, + "message": f"The service {service_key}:{service_version} does not exist", + } + }, ) - def get_service_labels(request): - raise NotImplementedError - @respx_mock.get( - r"/services_extras/(?P\w+)/(?P\w+)" + # GET EXTRAS + assert openapi["paths"].get("/service_extras/{service_key}/{service_version}") + + @respx_mock.get( + path__regex=r"^/service_extras/(?P[/\w-]+)/(?P[0-9\.]+)$", + name="get_service_extras", + ) + def _get_service_extras(request, service_key, service_version): + if _search(service_key, service_version): + return httpx.Response( + status.HTTP_200_OK, + json={ + "data": { + "node_requirements": {"CPU": 4, "RAM": 2147483648}, + "build_date": "2023-04-17T08:04:15Z", + "vcs_ref": "", + "vcs_url": "", + } + }, + ) + return httpx.Response( + status.HTTP_404_NOT_FOUND, + json={ + "data": { + "status": status.HTTP_404_NOT_FOUND, + "message": f"The service {service_key}:{service_version} does not exist", + } + }, ) - def get_service_extras(request): - raise NotImplementedError - yield respx_mock + return respx_mock diff --git a/services/catalog/tests/unit/test_api_rest.py b/services/catalog/tests/unit/test_api_rest.py new file mode 100644 index 00000000000..393c65abbc1 --- /dev/null +++ b/services/catalog/tests/unit/test_api_rest.py @@ -0,0 +1,37 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +import httpx +from fastapi import status +from fastapi.testclient import TestClient + + +def test_sync_client( + postgres_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + background_tasks_setup_disabled: None, + director_setup_disabled: None, + client: TestClient, +): + + response = client.get("/v0/") + assert response.status_code == status.HTTP_200_OK + + response = client.get("/v0/meta") + assert response.status_code == status.HTTP_200_OK + + +async def test_async_client( + postgres_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + background_tasks_setup_disabled: None, + director_setup_disabled: None, + aclient: httpx.AsyncClient, +): + response = await aclient.get("/v0/") + assert response.status_code == status.HTTP_200_OK + + response = await aclient.get("/v0/meta") + assert response.status_code == status.HTTP_200_OK diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_services_director.py index 61b8ddb854b..b809e7a8315 100644 --- a/services/catalog/tests/unit/test_services_director.py +++ b/services/catalog/tests/unit/test_services_director.py @@ -6,10 +6,11 @@ # pylint: disable=unused-variable +import urllib.parse +from typing import Any + import pytest from fastapi import FastAPI -from models_library.services_metadata_published import ServiceMetaDataPublished -from pydantic import parse_obj_as from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter @@ -31,8 +32,9 @@ def app_environment( ) -async def test_director_client_setup( - setup_rabbitmq_and_rpc_disabled: None, +async def test_director_client_high_level_api( + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_api: MockRouter, app: FastAPI, ): @@ -42,15 +44,51 @@ async def test_director_client_setup( assert app.state.director_api == director_api assert isinstance(director_api, DirectorApi) - # use it - data = await director_api.get("/services") + # PING + assert await director_api.is_responsive() - # director entry-point has hit + # LIST + all_services = await director_api.list_all_services() assert mocked_director_service_api["list_services"].called - # returns un-enveloped response - got_services = parse_obj_as(list[ServiceMetaDataPublished], data) - - services_image_digest = {service.image_digest for service in got_services} + services_image_digest = {service.image_digest for service in all_services} assert None not in services_image_digest - assert len(services_image_digest) == len(got_services) + assert len(services_image_digest) == len(all_services) + + # GET + expected_service = all_services[0] + assert ( + await director_api.get_service(expected_service.key, expected_service.version) + == expected_service + ) + + +async def test_director_client_low_level_api( + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: MockRouter, + expected_director_list_services: list[dict[str, Any]], + app: FastAPI, +): + director_api = get_director_api(app) + + expected_service = expected_director_list_services[0] + key = expected_service["key"] + version = expected_service["version"] + + service_labels = await director_api.get( + f"/services/{urllib.parse.quote_plus(key)}/{version}/labels" + ) + + assert service_labels + + service_extras = await director_api.get( + f"/service_extras/{urllib.parse.quote_plus(key)}/{version}" + ) + + assert service_extras + + service = await director_api.get( + f"/services/{urllib.parse.quote_plus(key)}/{version}" + ) + assert service diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py new file mode 100644 index 00000000000..4a6fcbdd025 --- /dev/null +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -0,0 +1,67 @@ +# pylint: disable=not-context-manager +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest +from fastapi import FastAPI +from models_library.function_services_catalog.api import is_function_service +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from respx.router import MockRouter +from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.services import manifest +from simcore_service_catalog.services.director import DirectorApi + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + "CATALOG_POSTGRES": "null", # disable postgres + "SC_BOOT_MODE": "local-development", + }, + ) + + +async def test_services_manifest_api( + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: MockRouter, + app: FastAPI, +): + director_api = get_director_api(app) + + assert app.state.director_api == director_api + assert isinstance(director_api, DirectorApi) + + # LIST + all_services_map = await manifest.get_services_map(director_api) + assert mocked_director_service_api["list_services"].called + + for service in all_services_map.values(): + if is_function_service(service.key): + assert service.image_digest is None + else: + assert service.image_digest is not None + + services_image_digest = { + s.image_digest for s in all_services_map.values() if s.image_digest + } + assert len(services_image_digest) < len(all_services_map) + + # GET + for expected_service in all_services_map.values(): + service = await manifest.get_service( + expected_service.key, expected_service.version, director_api + ) + + assert service == expected_service + if not is_function_service(service.key): + assert mocked_director_service_api["get_service"].called diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index c49b95b6793..f720e89bcdc 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -1,11 +1,12 @@ # pylint: disable=not-context-manager +# pylint: disable=protected-access # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable import itertools import random -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable from copy import deepcopy from datetime import datetime from typing import Any @@ -13,13 +14,10 @@ import pytest import sqlalchemy as sa from faker import Faker -from fastapi import FastAPI -from fastapi.testclient import TestClient from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished from models_library.users import UserID from pydantic import parse_obj_as -from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.postgres_tools import ( PostgresTestConfig, @@ -78,14 +76,6 @@ async def app_settings( # starts postgres service before app starts return app_settings -@pytest.fixture -def client(app: FastAPI) -> Iterator[TestClient]: - # NOTE: sync client since we use benchmarch fixture! - with TestClient(app) as cli: - # Note: this way we ensure the events are run in the application - yield cli - - # DATABASE tables fixtures ----------------------------------- # # These are the tables accessible by the catalog service: @@ -206,19 +196,19 @@ async def services_db_tables_injector( """Returns a helper function to init services_meta_data and services_access_rights tables - Can use service_catalog_faker to generate inputs + Can use `create_fake_service_data` to generate inputs Example: await services_db_tables_injector( [ - service_catalog_faker( + create_fake_service_data( "simcore/services/dynamic/jupyterlab", "0.0.1", team_access=None, everyone_access=None, product=target_product, ), - service_catalog_faker( + create_fake_service_data( "simcore/services/dynamic/jupyterlab", "0.0.7", team_access=None, @@ -371,7 +361,7 @@ async def create_fake_service_data( Example: - fake_service, *fake_access_rights = service_catalog_faker( + fake_service, *fake_access_rights = create_fake_service_data( "simcore/services/dynamic/jupyterlab", "0.0.1", team_access=None, @@ -457,18 +447,3 @@ def _fake_factory( return tuple(fakes) return _fake_factory - - -@pytest.fixture -def mocked_catalog_background_task(mocker: MockerFixture) -> None: - """patch the setup of the background task so we can call it manually""" - mocker.patch( - "simcore_service_catalog.core.events.start_registry_sync_task", - return_value=None, - autospec=True, - ) - mocker.patch( - "simcore_service_catalog.core.events.stop_registry_sync_task", - return_value=None, - autospec=True, - ) diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py new file mode 100644 index 00000000000..aa6e569fff8 --- /dev/null +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py @@ -0,0 +1,96 @@ +# pylint: disable=no-value-for-parameter +# pylint: disable=not-an-iterable +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import urllib.parse +from collections.abc import Callable +from typing import Any + +import pytest +import respx +from fastapi.testclient import TestClient +from models_library.api_schemas_catalog.services import ServiceGet +from models_library.products import ProductName +from models_library.users import UserID +from yarl import URL + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +async def expected_service( + expected_director_list_services: list[dict[str, Any]], + user: dict[str, Any], + services_db_tables_injector: Callable, + target_product: ProductName, +) -> dict[str, Any]: + # Just selected one of the list provided by the director (i.e. emulated from registry) + service = expected_director_list_services[-1] + + # Emulates sync of registry with db and injects the expected response model + # of the director (i.e. coming from the registry) in the database + await services_db_tables_injector( + [ + ( # service + { + "key": service["key"], + "version": service["version"], + "owner": user["primary_gid"], + "name": service["name"], + "description": service["description"], + "thumbnail": service["thumbnail"], + }, + # owner_access, + { + "key": service["key"], + "version": service["version"], + "gid": user["primary_gid"], + "execute_access": True, + "write_access": True, + "product_name": target_product, + } + # team_access, everyone_access [optional] + ) + ] + ) + return service + + +def test_get_service_with_details( + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: respx.MockRouter, + user_id: UserID, + expected_service: dict[str, Any], + target_product: ProductName, + client: TestClient, +): + service_key = expected_service["key"] + service_version = expected_service["version"] + + url = URL( + f"/v0/services/{urllib.parse.quote_plus(service_key)}/{service_version}" + ).with_query({"user_id": user_id}) + + response = client.get( + f"{url}", + headers={ + "x-simcore-products-name": target_product, + }, + ) + + assert response.status_code == 200 + + got = ServiceGet.parse_obj(response.json()) + assert got.key == service_key + assert got.version == service_version + + assert mocked_director_service_api["get_service"].called diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py similarity index 91% rename from services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py rename to services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py index dbdc78d4f1f..4b0bd5dceb6 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services__list.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__list.py @@ -28,9 +28,9 @@ async def test_list_services_with_details( - mocked_catalog_background_task: None, - setup_rabbitmq_and_rpc_disabled: None, - mocked_director_service_api: MockRouter, + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api_base: MockRouter, user_id: UserID, target_product: ProductName, create_fake_service_data: Callable, @@ -60,7 +60,7 @@ async def test_list_services_with_details( "examples" ][0] - mocked_director_service_api.get("/services", name="list_services").respond( + mocked_director_service_api_base.get("/services", name="list_services").respond( 200, json={ "data": [ @@ -84,9 +84,9 @@ async def test_list_services_with_details( async def test_list_services_without_details( - mocked_catalog_background_task: None, + background_tasks_setup_disabled: None, mocked_director_service_api: MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, user_id: int, target_product: ProductName, create_fake_service_data: Callable, @@ -128,9 +128,9 @@ async def test_list_services_without_details( async def test_list_services_without_details_with_wrong_user_id_returns_403( service_caching_disabled, - mocked_catalog_background_task: None, + background_tasks_setup_disabled: None, mocked_director_service_api: MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, user_id: int, target_product: ProductName, create_fake_service_data: Callable, @@ -160,9 +160,9 @@ async def test_list_services_without_details_with_wrong_user_id_returns_403( async def test_list_services_without_details_with_another_product_returns_other_services( service_caching_disabled: None, - mocked_catalog_background_task: None, + background_tasks_setup_disabled: None, mocked_director_service_api: MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, user_id: int, target_product: ProductName, other_product: ProductName, @@ -193,9 +193,9 @@ async def test_list_services_without_details_with_another_product_returns_other_ async def test_list_services_without_details_with_wrong_product_returns_0_service( service_caching_disabled, - mocked_catalog_background_task, + background_tasks_setup_disabled, mocked_director_service_api: MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, user_id: int, target_product: ProductName, create_fake_service_data: Callable, @@ -229,9 +229,9 @@ async def test_list_services_without_details_with_wrong_product_returns_0_servic async def test_list_services_that_are_deprecated( service_caching_disabled, - mocked_catalog_background_task, - setup_rabbitmq_and_rpc_disabled: None, - mocked_director_service_api: MockRouter, + background_tasks_setup_disabled, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api_base: MockRouter, user_id: int, target_product: ProductName, create_fake_service_data: Callable, @@ -265,7 +265,7 @@ async def test_list_services_that_are_deprecated( fake_registry_service_data = ServiceMetaDataPublished.Config.schema_extra[ "examples" ][0] - mocked_director_service_api.get("/services", name="list_services").respond( + mocked_director_service_api_base.get("/services", name="list_services").respond( 200, json={ "data": [ diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py similarity index 95% rename from services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py rename to services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py index ca0df29f837..87ac133a0df 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_access_rights.py @@ -28,9 +28,9 @@ async def test_get_service_access_rights( - mocked_catalog_background_task: None, + background_tasks_setup_disabled: None, mocked_director_service_api: MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, user: dict[str, Any], target_product: ProductName, create_fake_service_data: Callable, @@ -75,9 +75,9 @@ async def test_get_service_access_rights( async def test_get_service_access_rights_with_more_gids( - mocked_catalog_background_task: None, + background_tasks_setup_disabled: None, mocked_director_service_api: MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, user: dict[str, Any], other_product: ProductName, create_fake_service_data: Callable, diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_ports.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_ports.py similarity index 96% rename from services/catalog/tests/unit/with_dbs/test_api_routes_services_ports.py rename to services/catalog/tests/unit/with_dbs/test_api_rest_services_ports.py index cd4e8bead8b..1ba249dfc01 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_ports.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_ports.py @@ -59,13 +59,13 @@ async def mocked_check_service_read_access( @pytest.fixture async def mocked_director_service_api( - mocked_director_service_api: MockRouter, + mocked_director_service_api_base: MockRouter, service_key: str, service_version: str, service_metadata: dict[str, Any], ): # SEE services/director/src/simcore_service_director/api/v0/openapi.yaml - mocked_director_service_api.get( + mocked_director_service_api_base.get( f"/services/{urllib.parse.quote_plus(service_key)}/{service_version}", name="services_by_key_version_get", ).respond( @@ -80,10 +80,10 @@ async def mocked_director_service_api( async def test_list_service_ports( service_caching_disabled: None, - mocked_catalog_background_task: None, + background_tasks_setup_disabled: None, mocked_check_service_read_access: None, mocked_director_service_api: None, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, client: TestClient, product_name: str, user_id: int, diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_resources.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py similarity index 95% rename from services/catalog/tests/unit/with_dbs/test_api_routes_services_resources.py rename to services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py index 24371a5a54a..1ea7e40f18f 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_resources.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_resources.py @@ -13,7 +13,6 @@ import pytest import respx from faker import Faker -from fastapi import FastAPI from models_library.docker import DockerGenericTag from models_library.services_resources import ( BootMode, @@ -38,10 +37,13 @@ @pytest.fixture def mocked_director_service_labels( - mocked_director_service_api: respx.MockRouter, app: FastAPI + mocked_director_service_api_base: respx.MockRouter, ) -> Route: + """ + Customizes mock for labels entrypoints at the director service's API + """ slash = urllib.parse.quote_plus("/") - return mocked_director_service_api.get( + return mocked_director_service_api_base.get( url__regex=rf"v0/services/simcore{slash}services{slash}(comp|dynamic|frontend)({slash}[\w{slash}-]+)+/[0-9]+.[0-9]+.[0-9]+/labels", name="get_service_labels", ).respond(200, json={"data": {}}) @@ -56,7 +58,7 @@ def creator(): @pytest.fixture -def service_key(faker: Faker) -> str: +def service_key() -> str: return f"simcore/services/{choice(['comp', 'dynamic','frontend'])}/jupyter-math" @@ -66,7 +68,7 @@ def service_version() -> str: @pytest.fixture -def mock_service_labels(faker: Faker) -> dict[str, Any]: +def mock_service_labels() -> dict[str, Any]: return { "simcore.service.settings": '[ {"name": "ports", "type": "int", "value": 8888}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}, {"name": "Resources", "type": "Resources", "value": { "Limits": { "NanoCPUs": 4000000000, "MemoryBytes": 17179869184 } } } ]', } @@ -182,8 +184,8 @@ class _ServiceResourceParams: ], ) async def test_get_service_resources( - mocked_catalog_background_task, - setup_rabbitmq_and_rpc_disabled: None, + background_tasks_setup_disabled, + rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_labels: Route, client: TestClient, params: _ServiceResourceParams, @@ -209,7 +211,7 @@ async def test_get_service_resources( @pytest.fixture def create_mock_director_service_labels( - mocked_director_service_api: respx.MockRouter, app: FastAPI + mocked_director_service_api_base: respx.MockRouter, ) -> Callable: def factory(services_labels: dict[str, dict[str, Any]]) -> None: for service_name, data in services_labels.items(): @@ -217,7 +219,7 @@ def factory(services_labels: dict[str, dict[str, Any]]) -> None: f"simcore/services/dynamic/{service_name}" ) for k, mock_key in enumerate((encoded_key, service_name)): - mocked_director_service_api.get( + mocked_director_service_api_base.get( url__regex=rf"v0/services/{mock_key}/[\w/.]+/labels", name=f"get_service_labels_for_{service_name}_{k}", ).respond(200, json={"data": data}) @@ -287,8 +289,8 @@ def factory(services_labels: dict[str, dict[str, Any]]) -> None: ], ) async def test_get_service_resources_sim4life_case( - mocked_catalog_background_task, - setup_rabbitmq_and_rpc_disabled: None, + background_tasks_setup_disabled, + rabbitmq_and_rpc_setup_disabled: None, create_mock_director_service_labels: Callable, client: TestClient, mapped_services_labels: dict[str, dict[str, Any]], @@ -308,8 +310,8 @@ async def test_get_service_resources_sim4life_case( async def test_get_service_resources_raises_errors( - mocked_catalog_background_task, - setup_rabbitmq_and_rpc_disabled: None, + background_tasks_setup_disabled, + rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_labels: Route, client: TestClient, ) -> None: diff --git a/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py similarity index 94% rename from services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py rename to services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py index af9aef432ad..f8515b57298 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_routes_services_specifications.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services_specifications.py @@ -12,7 +12,6 @@ import pytest import respx from faker import Faker -from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from models_library.api_schemas_catalog.services_specifications import ( ServiceSpecifications, @@ -123,9 +122,9 @@ def _creator(service_key, service_version, gid) -> ServiceSpecificationsAtDB: async def test_get_service_specifications_returns_403_if_user_does_not_exist( - mocked_catalog_background_task, + background_tasks_setup_disabled, mocked_director_service_api: respx.MockRouter, - setup_rabbitmq_and_rpc_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, client: TestClient, user_id: UserID, ): @@ -139,10 +138,9 @@ async def test_get_service_specifications_returns_403_if_user_does_not_exist( async def test_get_service_specifications_of_unknown_service_returns_default_specs( - mocked_catalog_background_task, + background_tasks_setup_disabled, mocked_director_service_api: respx.MockRouter, - setup_rabbitmq_and_rpc_disabled: None, - app: FastAPI, + rabbitmq_and_rpc_setup_disabled: None, client: TestClient, user_id: UserID, user: dict[str, Any], @@ -160,14 +158,16 @@ async def test_get_service_specifications_of_unknown_service_returns_default_spe service_specs = ServiceSpecificationsGet.parse_obj(response.json()) assert service_specs - assert service_specs == app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + assert ( + service_specs + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + ) async def test_get_service_specifications( - mocked_catalog_background_task, + background_tasks_setup_disabled, mocked_director_service_api: respx.MockRouter, - setup_rabbitmq_and_rpc_disabled: None, - app: FastAPI, + rabbitmq_and_rpc_setup_disabled: None, client: TestClient, user_id: UserID, user: dict[str, Any], @@ -202,7 +202,10 @@ async def test_get_service_specifications( assert response.status_code == status.HTTP_200_OK service_specs = ServiceSpecificationsGet.parse_obj(response.json()) assert service_specs - assert service_specs == app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + assert ( + service_specs + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + ) everyone_gid, user_gid, team_gid = user_groups_ids # let's inject some rights for everyone group @@ -257,10 +260,9 @@ async def test_get_service_specifications( async def test_get_service_specifications_are_passed_to_newer_versions_of_service( - mocked_catalog_background_task, + background_tasks_setup_disabled, mocked_director_service_api: respx.MockRouter, - setup_rabbitmq_and_rpc_disabled: None, - app: FastAPI, + rabbitmq_and_rpc_setup_disabled: None, client: TestClient, user_id: UserID, user: dict[str, Any], @@ -328,7 +330,8 @@ async def test_get_service_specifications_are_passed_to_newer_versions_of_servic service_specs = ServiceSpecificationsGet.parse_obj(response.json()) assert service_specs assert ( - service_specs == app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + service_specs + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS ) # check version between first index and second all return the specs of the first @@ -371,10 +374,10 @@ async def test_get_service_specifications_are_passed_to_newer_versions_of_servic if version in versions_with_specs: assert ( service_specs - != app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + != client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS ) else: assert ( service_specs - == app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS + == client.app.state.settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS ) diff --git a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py new file mode 100644 index 00000000000..0093c5dd70c --- /dev/null +++ b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py @@ -0,0 +1,80 @@ +# pylint: disable=no-value-for-parameter +# pylint: disable=not-an-iterable +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from typing import Any + +import pytest +from fastapi import FastAPI +from respx.router import MockRouter +from simcore_postgres_database.models.services import services_meta_data +from simcore_service_catalog.core.background_tasks import _run_sync_services +from simcore_service_catalog.db.repositories.services import ServicesRepository +from sqlalchemy.ext.asyncio.engine import AsyncEngine + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +def services_repo(app: FastAPI) -> ServicesRepository: + # depends on client so the app has a state ready + assert len(app.state._state) > 0 # noqa: SLF001 + return ServicesRepository(app.state.engine) + + +@pytest.fixture +async def cleanup_service_meta_data_db_content(sqlalchemy_async_engine: AsyncEngine): + # NOTE: necessary because _run_sync_services fills tables + yield + + async with sqlalchemy_async_engine.begin() as conn: + await conn.execute(services_meta_data.delete()) + + +async def test_registry_sync_task( + background_tasks_setup_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: MockRouter, + expected_director_list_services: list[dict[str, Any]], + user: dict[str, Any], + app: FastAPI, + services_repo: ServicesRepository, + cleanup_service_meta_data_db_content: None, +): + + assert app.state + + service_key = expected_director_list_services[0]["key"] + service_version = expected_director_list_services[0]["version"] + + # in registry but NOT in db + got_from_db = await services_repo.get_service_with_history( + product_name="osparc", + user_id=user["id"], + key=service_key, + version=service_version, + ) + assert not got_from_db + + # let's sync + await _run_sync_services(app) + + # after sync, it should be in db as well + got_from_db = await services_repo.get_service_with_history( + product_name="osparc", + user_id=user["id"], + key=service_key, + version=service_version, + ) + assert got_from_db + assert got_from_db.key == service_key + assert got_from_db.version == service_version diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_db_repositories.py index e09f334f720..3efa24c4dd0 100644 --- a/services/catalog/tests/unit/with_dbs/test_db_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_db_repositories.py @@ -28,7 +28,7 @@ @pytest.fixture -def services_repo(sqlalchemy_async_engine: AsyncEngine): +def services_repo(sqlalchemy_async_engine: AsyncEngine) -> ServicesRepository: return ServicesRepository(sqlalchemy_async_engine)