From 81c8ea001d15885b374c95f0694114d2d1614e8d Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:03:17 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8Adding=20dynamic=20services=20monitori?= =?UTF-8?q?ng=20dashboard=20(=E2=9A=A0=EF=B8=8Fdevops)=20(#6784)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .env-devel | 1 + Makefile | 1 + .../tests/test_service_settings_labels.py | 11 +- .../tests/test_utils_fastapi_encoders.py | 21 +-- services/docker-compose.yml | 1 + services/dynamic-scheduler/Dockerfile | 2 +- .../dynamic-scheduler/requirements/_base.in | 1 + .../dynamic-scheduler/requirements/_base.txt | 66 ++++++- .../dynamic-scheduler/requirements/_test.in | 2 + .../dynamic-scheduler/requirements/_test.txt | 25 +++ .../api/frontend/__init__.py | 3 + .../api/frontend/_setup.py | 19 ++ .../api/frontend/_utils.py | 11 ++ .../api/frontend/routes/__init__.py | 10 ++ .../api/frontend/routes/_index.py | 169 ++++++++++++++++++ .../api/frontend/routes/_render_utils.py | 23 +++ .../api/frontend/routes/_service.py | 146 +++++++++++++++ .../api/rest/_health.py | 2 +- .../simcore_service_dynamic_scheduler/cli.py | 4 + .../core/application.py | 2 + .../core/settings.py | 10 +- .../services/rabbitmq.py | 10 ++ .../services/service_tracker/__init__.py | 4 +- .../services/service_tracker/_api.py | 2 +- .../status_monitor/_deferred_get_status.py | 2 +- .../services/status_monitor/_monitor.py | 6 +- .../tests/unit/api_frontend/conftest.py | 122 +++++++++++++ .../tests/unit/api_frontend/helpers.py | 104 +++++++++++ .../test_api_frontend_routes_index.py | 125 +++++++++++++ .../test_api_frontend_routes_service.py | 121 +++++++++++++ .../unit/api_rest/test_api_rest__health.py | 4 +- .../unit/service_tracker/test__tracker.py | 2 +- .../test_services_status_monitor__monitor.py | 4 +- 33 files changed, 990 insertions(+), 46 deletions(-) create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/__init__.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_setup.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_utils.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/__init__.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_index.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_render_utils.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_service.py create mode 100644 services/dynamic-scheduler/tests/unit/api_frontend/conftest.py create mode 100644 services/dynamic-scheduler/tests/unit/api_frontend/helpers.py create mode 100644 services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py create mode 100644 services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py diff --git a/.env-devel b/.env-devel index df3ea3bb4a7..6a32129a920 100644 --- a/.env-devel +++ b/.env-devel @@ -128,6 +128,7 @@ DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG DYNAMIC_SCHEDULER_PROFILING=1 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00 DYNAMIC_SCHEDULER_TRACING={} +DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' diff --git a/Makefile b/Makefile index e6176c55136..564e353ee58 100644 --- a/Makefile +++ b/Makefile @@ -322,6 +322,7 @@ printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\ printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\ printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\ printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\ +printf "$$rows" "Dy-scheduler Dashboard" "http://$(get_my_ip).nip.io:8012";\ printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\ printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\ printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\ diff --git a/packages/models-library/tests/test_service_settings_labels.py b/packages/models-library/tests/test_service_settings_labels.py index 0c582905d22..775facf96a4 100644 --- a/packages/models-library/tests/test_service_settings_labels.py +++ b/packages/models-library/tests/test_service_settings_labels.py @@ -8,8 +8,8 @@ from pprint import pformat from typing import Any, Final, NamedTuple -import pydantic_core import pytest +from common_library.json_serialization import json_dumps from models_library.basic_types import PortInt from models_library.osparc_variable_identifier import ( OsparcVariableIdentifier, @@ -558,11 +558,6 @@ def test_can_parse_labels_with_osparc_identifiers( assert "$" not in service_meta_str -def servicelib__json_serialization__json_dumps(obj: Any, **kwargs): - # Analogous to 'models_library.utils.json_serialization.json_dumps' - return json.dumps(obj, default=pydantic_core.to_jsonable_python, **kwargs) - - def test_resolving_some_service_labels_at_load_time( vendor_environments: dict[str, Any], service_labels: dict[str, str] ): @@ -579,9 +574,7 @@ def test_resolving_some_service_labels_at_load_time( ("settings", SimcoreServiceSettingsLabel), ): to_serialize = getattr(service_meta, attribute_name) - template = TextTemplate( - servicelib__json_serialization__json_dumps(to_serialize) - ) + template = TextTemplate(json_dumps(to_serialize)) assert template.is_valid() resolved_label: str = template.safe_substitute(vendor_environments) to_restore = TypeAdapter(pydantic_model).validate_json(resolved_label) diff --git a/packages/models-library/tests/test_utils_fastapi_encoders.py b/packages/models-library/tests/test_utils_fastapi_encoders.py index 6ee05a56e57..ecd046af24e 100644 --- a/packages/models-library/tests/test_utils_fastapi_encoders.py +++ b/packages/models-library/tests/test_utils_fastapi_encoders.py @@ -4,36 +4,25 @@ # pylint: disable=too-many-arguments import json -from typing import Any from uuid import uuid4 -import pytest +from common_library.json_serialization import json_dumps from faker import Faker from models_library.utils.fastapi_encoders import servicelib_jsonable_encoder -from pydantic.json import pydantic_encoder - - -def servicelib__json_serialization__json_dumps(obj: Any, **kwargs): - # Analogous to 'models_library.utils.json_serialization.json_dumps' - return json.dumps(obj, default=pydantic_encoder, **kwargs) def test_using_uuids_as_keys(faker: Faker): uuid_key = uuid4() - with pytest.raises(TypeError): - # IMPORTANT NOTE: we cannot serialize UUID objects as keys. - # We have to convert them to strings but then the class information is lost upon deserialization i.e. it is not reversable! - # NOTE: This could potentially be solved using 'orjson' !! - # - servicelib__json_serialization__json_dumps({uuid_key: "value"}, indent=1) + # this was previously failing + assert json_dumps({uuid_key: "value"}, indent=1) - # use encoder + # uuid keys now serialize without raising to the expected format string data = servicelib_jsonable_encoder({uuid_key: "value"}) assert data == {f"{uuid_key}": "value"} # serialize w/o raising - dumped_data = servicelib__json_serialization__json_dumps(data, indent=1) + dumped_data = json_dumps(data, indent=1) # deserialize w/o raising loaded_data = json.loads(dumped_data) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 691e544b0c0..5da1a28ba0d 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -566,6 +566,7 @@ services: DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING} DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING} + DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET} TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} static-webserver: diff --git a/services/dynamic-scheduler/Dockerfile b/services/dynamic-scheduler/Dockerfile index b3e9119c898..bffb3808bdd 100644 --- a/services/dynamic-scheduler/Dockerfile +++ b/services/dynamic-scheduler/Dockerfile @@ -146,7 +146,7 @@ HEALTHCHECK --interval=30s \ --timeout=20s \ --start-period=30s \ --retries=3 \ - CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/"] + CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/health"] ENTRYPOINT [ "/bin/sh", "services/dynamic-scheduler/docker/entrypoint.sh" ] CMD ["/bin/sh", "services/dynamic-scheduler/docker/boot.sh"] diff --git a/services/dynamic-scheduler/requirements/_base.in b/services/dynamic-scheduler/requirements/_base.in index fa6e19b5a14..a5926615337 100644 --- a/services/dynamic-scheduler/requirements/_base.in +++ b/services/dynamic-scheduler/requirements/_base.in @@ -18,6 +18,7 @@ arrow fastapi httpx +nicegui packaging python-socketio typer[all] diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 7fbf832f7df..6cf4dc07c90 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -7,7 +7,9 @@ aiodebug==2.3.0 aiodocker==0.24.0 # via -r requirements/../../../packages/service-library/requirements/_base.in aiofiles==24.1.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # nicegui aiohappyeyeballs==2.4.3 # via aiohttp aiohttp==3.11.7 @@ -27,6 +29,8 @@ aiohttp==3.11.7 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiodocker + # nicegui + # python-socketio aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 @@ -77,6 +81,7 @@ certifi==2024.8.30 # -c requirements/../../../requirements/constraints.txt # httpcore # httpx + # nicegui # requests charset-normalizer==3.4.0 # via requests @@ -92,6 +97,8 @@ deprecated==1.2.15 # opentelemetry-semantic-conventions dnspython==2.7.0 # via email-validator +docutils==0.21.2 + # via nicegui email-validator==2.2.0 # via pydantic exceptiongroup==1.2.2 @@ -102,6 +109,7 @@ fastapi==0.115.5 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in + # nicegui faststream==0.5.30 # via -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.5.0 @@ -143,6 +151,7 @@ httpx==0.27.2 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in + # nicegui idna==3.10 # via # anyio @@ -150,8 +159,29 @@ idna==3.10 # httpx # requests # yarl +ifaddr==0.2.0 + # via nicegui importlib-metadata==8.5.0 # via opentelemetry-api +itsdangerous==2.2.0 + # via nicegui +jinja2==3.1.4 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # nicegui jsonschema==4.23.0 # via # -r requirements/../../../packages/models-library/requirements/_base.in @@ -177,14 +207,20 @@ mako==1.3.6 # alembic markdown-it-py==3.0.0 # via rich +markdown2==2.5.1 + # via nicegui markupsafe==3.0.2 - # via mako + # via + # jinja2 + # mako mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 # via # aiohttp # yarl +nicegui==2.7.0 + # via -r requirements/_base.in opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in @@ -280,6 +316,7 @@ orjson==3.10.12 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # nicegui packaging==24.2 # via # -r requirements/_base.in @@ -300,6 +337,8 @@ protobuf==5.28.3 # via # googleapis-common-protos # opentelemetry-proto +pscript==0.7.7 + # via vbuild psutil==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in psycopg2-binary==2.9.10 @@ -357,7 +396,9 @@ pydantic-settings==2.6.1 # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 - # via rich + # via + # nicegui + # rich pyinstrument==5.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 @@ -368,8 +409,12 @@ python-dotenv==1.0.1 # uvicorn python-engineio==4.10.1 # via python-socketio +python-multipart==0.0.17 + # via nicegui python-socketio==5.11.4 - # via -r requirements/_base.in + # via + # -r requirements/_base.in + # nicegui pyyaml==6.0.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -412,7 +457,9 @@ referencing==0.35.1 repro-zipfile==0.3.1 # via -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 - # via opentelemetry-exporter-otlp-proto-http + # via + # nicegui + # opentelemetry-exporter-otlp-proto-http rich==13.9.4 # via # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -487,6 +534,7 @@ typing-extensions==4.12.2 # alembic # fastapi # faststream + # nicegui # opentelemetry-sdk # pydantic # pydantic-core @@ -510,15 +558,21 @@ urllib3==2.2.3 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # nicegui # requests uvicorn==0.32.1 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in + # nicegui uvloop==0.21.0 # via uvicorn +vbuild==0.8.2 + # via nicegui watchfiles==1.0.0 - # via uvicorn + # via + # nicegui + # uvicorn websockets==14.1 # via uvicorn wrapt==1.17.0 diff --git a/services/dynamic-scheduler/requirements/_test.in b/services/dynamic-scheduler/requirements/_test.in index 455f92720bb..1bc0580e049 100644 --- a/services/dynamic-scheduler/requirements/_test.in +++ b/services/dynamic-scheduler/requirements/_test.in @@ -15,6 +15,8 @@ asgi_lifespan coverage docker faker +hypercorn +playwright pytest pytest-asyncio pytest-cov diff --git a/services/dynamic-scheduler/requirements/_test.txt b/services/dynamic-scheduler/requirements/_test.txt index 2aeab660bbb..d951c31a63c 100644 --- a/services/dynamic-scheduler/requirements/_test.txt +++ b/services/dynamic-scheduler/requirements/_test.txt @@ -23,10 +23,20 @@ docker==7.1.0 # via -r requirements/_test.in faker==33.0.0 # via -r requirements/_test.in +greenlet==3.1.1 + # via + # -c requirements/_base.txt + # playwright h11==0.14.0 # via # -c requirements/_base.txt # httpcore + # hypercorn + # wsproto +h2==4.1.0 + # via hypercorn +hpack==4.0.0 + # via h2 httpcore==1.0.7 # via # -c requirements/_base.txt @@ -36,6 +46,10 @@ httpx==0.27.2 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # respx +hypercorn==0.17.3 + # via -r requirements/_test.in +hyperframe==6.0.1 + # via h2 icdiff==2.0.7 # via pytest-icdiff idna==3.10 @@ -51,10 +65,16 @@ packaging==24.2 # -c requirements/_base.txt # pytest # pytest-sugar +playwright==1.49.0 + # via -r requirements/_test.in pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff +priority==2.0.0 + # via hypercorn +pyee==12.0.0 + # via playwright pytest==8.3.3 # via # -r requirements/_test.in @@ -107,9 +127,14 @@ typing-extensions==4.12.2 # via # -c requirements/_base.txt # faker + # pyee urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # docker # requests +wsproto==1.2.0 + # via + # -c requirements/_base.txt + # hypercorn diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/__init__.py new file mode 100644 index 00000000000..7f991346a4b --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/__init__.py @@ -0,0 +1,3 @@ +from ._setup import setup_frontend + +__all__: tuple[str, ...] = ("setup_frontend",) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_setup.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_setup.py new file mode 100644 index 00000000000..9e689c86023 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_setup.py @@ -0,0 +1,19 @@ +import nicegui +from fastapi import FastAPI + +from ...core.settings import ApplicationSettings +from ._utils import set_parent_app +from .routes import router + + +def setup_frontend(app: FastAPI) -> None: + settings: ApplicationSettings = app.state.settings + + nicegui.app.include_router(router) + + nicegui.ui.run_with( + app, + mount_path="/", + storage_secret=settings.DYNAMIC_SCHEDULER_UI_STORAGE_SECRET.get_secret_value(), + ) + set_parent_app(app) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_utils.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_utils.py new file mode 100644 index 00000000000..6e890b8b8fe --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_utils.py @@ -0,0 +1,11 @@ +import nicegui +from fastapi import FastAPI + + +def set_parent_app(parent_app: FastAPI) -> None: + nicegui.app.state.parent_app = parent_app + + +def get_parent_app(app: FastAPI) -> FastAPI: + parent_app: FastAPI = app.state.parent_app + return parent_app diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/__init__.py new file mode 100644 index 00000000000..098f68217be --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/__init__.py @@ -0,0 +1,10 @@ +from nicegui import APIRouter + +from . import _index, _service + +router = APIRouter() + +router.include_router(_index.router) +router.include_router(_service.router) + +__all__: tuple[str, ...] = ("router",) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_index.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_index.py new file mode 100644 index 00000000000..5c864651427 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_index.py @@ -0,0 +1,169 @@ +import json + +import httpx +from fastapi import FastAPI +from models_library.projects_nodes_io import NodeID +from nicegui import APIRouter, app, ui +from nicegui.element import Element +from nicegui.elements.label import Label +from settings_library.utils_service import DEFAULT_FASTAPI_PORT + +from ....services.service_tracker import TrackedServiceModel, get_all_tracked_services +from ....services.service_tracker._models import SchedulerServiceState +from .._utils import get_parent_app +from ._render_utils import base_page, get_iso_formatted_date + +router = APIRouter() + + +def _render_service_details(node_id: NodeID, service: TrackedServiceModel) -> None: + dict_to_render: dict[str, tuple[str, str]] = { + "NodeID": ("copy", f"{node_id}"), + "Display State": ("label", service.current_state), + "Last State Change": ( + "label", + get_iso_formatted_date(service.last_state_change), + ), + "UserID": ("copy", f"{service.user_id}"), + "ProjectID": ("copy", f"{service.project_id}"), + "User Requested": ("label", service.requested_state), + } + + if service.dynamic_service_start: + dict_to_render["Service"] = ( + "label", + f"{service.dynamic_service_start.key}:{service.dynamic_service_start.version}", + ) + dict_to_render["Product"] = ( + "label", + service.dynamic_service_start.product_name, + ) + service_status = ( + json.loads(service.service_status) if service.service_status else {} + ) + dict_to_render["Service State"] = ( + "label", + service_status.get( + "state" if "boot_type" in service_status else "service_state", "N/A" + ), + ) + + with ui.column().classes("gap-0"): + for key, (widget, value) in dict_to_render.items(): + with ui.row(align_items="baseline"): + ui.label(key).classes("font-bold") + match widget: + case "copy": + ui.label(value).classes("border bg-slate-200 px-1") + case "label": + ui.label(value) + case _: + ui.label(value) + + +def _render_buttons(node_id: NodeID, service: TrackedServiceModel) -> None: + + with ui.dialog() as confirm_dialog, ui.card(): + ui.markdown(f"Stop service **{node_id}**?") + ui.label("The service will be stopped and its data will be saved.") + with ui.row(): + + async def _stop_service() -> None: + confirm_dialog.close() + await httpx.AsyncClient(timeout=10).get( + f"http://localhost:{DEFAULT_FASTAPI_PORT}/service/{node_id}:stop" + ) + + ui.notify( + f"Submitted stop request for {node_id}. Please give the service some time to stop!" + ) + + ui.button("Stop Now", color="red", on_click=_stop_service) + ui.button("Cancel", on_click=confirm_dialog.close) + + with ui.button_group(): + ui.button( + "Details", + icon="source", + on_click=lambda: ui.navigate.to(f"/service/{node_id}:details"), + ).tooltip("Display more information about what the scheduler is tracking") + + if service.current_state != SchedulerServiceState.RUNNING: + return + + ui.button( + "Stop Service", + icon="stop", + color="orange", + on_click=confirm_dialog.open, + ).tooltip("Stops the service and saves the data") + + +def _render_card( + card_container: Element, node_id: NodeID, service: TrackedServiceModel +) -> None: + with card_container: # noqa: SIM117 + with ui.column().classes("border p-1"): + _render_service_details(node_id, service) + _render_buttons(node_id, service) + + +def _get_clean_hashable(model: TrackedServiceModel) -> dict: + """removes items which trigger frequent updates and are not interesting to the user""" + data = model.model_dump(mode="json") + data.pop("check_status_after") + data.pop("last_status_notification") + data.pop("service_status_task_uid") + return data + + +def _get_hash(items: list[tuple[NodeID, TrackedServiceModel]]) -> int: + return hash( + json.dumps([(f"{key}", _get_clean_hashable(model)) for key, model in items]) + ) + + +class CardUpdater: + def __init__( + self, parent_app: FastAPI, container: Element, services_count_label: Label + ) -> None: + self.parent_app = parent_app + self.container = container + self.services_count_label = services_count_label + self.last_hash: int = _get_hash([]) + + async def update(self) -> None: + tracked_services = await get_all_tracked_services(self.parent_app) + tracked_items: list[tuple[NodeID, TrackedServiceModel]] = sorted( + tracked_services.items(), reverse=True + ) + + current_hash = _get_hash(tracked_items) + + if self.last_hash != current_hash: + self.services_count_label.set_text(f"{len(tracked_services)}") + # Clear the current cards + self.container.clear() + for node_id, service in tracked_items: + _render_card(self.container, node_id, service) + + self.last_hash = current_hash + + +@router.page("/") +async def index(): + with base_page(): + with ui.row().classes("gap-0"): + ui.label("Total tracked services:") + ui.label("").classes("w-1") + with ui.label("0") as services_count_label: + pass + + card_container: Element = ui.row() + + updater = CardUpdater(get_parent_app(app), card_container, services_count_label) + + # render cards when page is loaded + await updater.update() + # update card at a set interval + ui.timer(1, updater.update) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_render_utils.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_render_utils.py new file mode 100644 index 00000000000..c3a315be2d7 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_render_utils.py @@ -0,0 +1,23 @@ +from collections.abc import Iterator +from contextlib import contextmanager + +import arrow +from nicegui import ui + + +@contextmanager +def base_page(*, title: str | None = None) -> Iterator[None]: + display_title = ( + "Dynamic Scheduler" if title is None else f"Dynamic Scheduler - {title}" + ) + ui.page_title(display_title) + + with ui.header(elevated=True).classes("items-center"): + ui.button(icon="o_home", on_click=lambda: ui.navigate.to("/")) + ui.label(display_title) + + yield None + + +def get_iso_formatted_date(timestamp: float) -> str: + return arrow.get(timestamp).isoformat() diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_service.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_service.py new file mode 100644 index 00000000000..b4d9327df0f --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/_service.py @@ -0,0 +1,146 @@ +import json + +import httpx +from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( + DynamicServiceStop, +) +from models_library.projects_nodes_io import NodeID +from nicegui import APIRouter, app, ui +from servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services import ( + stop_dynamic_service, +) +from settings_library.utils_service import DEFAULT_FASTAPI_PORT +from simcore_service_dynamic_scheduler.services.rabbitmq import get_rabbitmq_rpc_client + +from ....core.settings import ApplicationSettings +from ....services.service_tracker import get_tracked_service, remove_tracked_service +from .._utils import get_parent_app +from ._render_utils import base_page + +router = APIRouter() + + +def _render_remove_from_tracking(node_id): + with ui.dialog() as confirm_dialog, ui.card(): + + async def remove_from_tracking(): + confirm_dialog.close() + await httpx.AsyncClient(timeout=10).get( + f"http://localhost:{DEFAULT_FASTAPI_PORT}/service/{node_id}/tracker:remove" + ) + + ui.notify(f"Service {node_id} removed from tracking") + ui.navigate.to("/") + + ui.markdown(f"Remove the service **{node_id}** form the tracker?") + ui.label( + "This action will result in the removal of the service form the internal tracker. " + "This action should be used whn you are facing issues and the service is not " + "automatically removed." + ) + ui.label( + "NOTE 1: the system normally cleans up services but it might take a few minutes. " + "Only use this option when you have observed enough time passing without any change." + ).classes("text-red-600") + ui.label( + "NOTE 2: This will break the fronted for the user! If the user has the service opened, " + "it will no longer receive an status updates." + ).classes("text-red-600") + + with ui.row(): + ui.button("Remove service", color="red", on_click=remove_from_tracking) + ui.button("Cancel", on_click=confirm_dialog.close) + + ui.button( + "Remove from tracking", + icon="remove_circle", + color="red", + on_click=confirm_dialog.open, + ).tooltip("Removes the service form the dynamic-scheduler's internal tracking") + + +def _render_danger_zone(node_id: NodeID) -> None: + ui.separator() + + ui.markdown("**Danger Zone, beware!**").classes("text-2xl text-red-700") + ui.label( + "Do not use these actions if you do not know what they are doing." + ).classes("text-red-700") + + ui.label( + "They are reserved as means of recovering the system form a failing state." + ).classes("text-red-700") + + _render_remove_from_tracking(node_id) + + +@router.page("/service/{node_id}:details") +async def service_details(node_id: NodeID): + with base_page(title=f"{node_id} details"): + service_model = await get_tracked_service(get_parent_app(app), node_id) + + if not service_model: + ui.markdown( + f"Sorry could not find any details for **node_id={node_id}**. " + "Please make sure the **node_id** is correct. " + "Also make sure you have not provided a **product_id**." + ) + return + + scheduler_internals = service_model.model_dump(mode="json") + service_status = scheduler_internals.pop("service_status", "{}") + service_status = json.loads("{}" if service_status == "" else service_status) + dynamic_service_start = scheduler_internals.pop("dynamic_service_start") + + ui.markdown("**Service Status**") + ui.code(json.dumps(service_status, indent=2), language="json") + + ui.markdown("**Scheduler Internals**") + ui.code(json.dumps(scheduler_internals, indent=2), language="json") + + ui.markdown("**Start Parameters**") + ui.code(json.dumps(dynamic_service_start, indent=2), language="json") + + ui.markdown("**Raw serialized data (the one used to render the above**") + ui.code(service_model.model_dump_json(indent=2), language="json") + + _render_danger_zone(node_id) + + +@router.page("/service/{node_id}:stop") +async def service_stop(node_id: NodeID): + parent_app = get_parent_app(app) + + service_model = await get_tracked_service(parent_app, node_id) + if not service_model: + ui.notify(f"Could not stop service {node_id}. Was not abel to find it") + return + + settings: ApplicationSettings = parent_app.state.settings + + assert service_model.user_id # nosec + assert service_model.project_id # nosec + + await stop_dynamic_service( + get_rabbitmq_rpc_client(get_parent_app(app)), + dynamic_service_stop=DynamicServiceStop( + user_id=service_model.user_id, + project_id=service_model.project_id, + node_id=node_id, + simcore_user_agent="", + save_state=True, + ), + timeout_s=int(settings.DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT.total_seconds()), + ) + + +@router.page("/service/{node_id}/tracker:remove") +async def remove_service_from_tracking(node_id: NodeID): + parent_app = get_parent_app(app) + + service_model = await get_tracked_service(parent_app, node_id) + if not service_model: + ui.notify(f"Could not remove service {node_id}. Was not abel to find it") + return + + await remove_tracked_service(parent_app, node_id) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py index 7e87c57fd06..ff5fe204132 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py @@ -24,7 +24,7 @@ class HealthCheckError(RuntimeError): """Failed a health check""" -@router.get("/", response_class=PlainTextResponse) +@router.get("/health", response_class=PlainTextResponse) async def healthcheck( rabbit_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client_from_request)], rabbit_rpc_server: Annotated[ diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py index e06b8f25129..0b7d56fccda 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py @@ -52,6 +52,10 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True): ), ), ), + DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=os.environ.get( + "DYNAMIC_SCHEDULER_UI_STORAGE_SECRET", + "replace-with-ui-storage-secret", + ), ) print_as_envfile( diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py index e6ba2bbb53f..971b82888be 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py @@ -15,6 +15,7 @@ PROJECT_NAME, SUMMARY, ) +from ..api.frontend import setup_frontend from ..api.rest.routes import setup_rest_api from ..api.rpc.routes import setup_rpc_api_routes from ..services.deferred_manager import setup_deferred_manager @@ -74,6 +75,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_status_monitor(app) setup_rest_api(app) + setup_frontend(app) # ERROR HANDLERS # ... add here ... diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index e577a806712..94acb6eaac4 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,7 +1,7 @@ import datetime from typing import Annotated -from pydantic import AliasChoices, Field, TypeAdapter, field_validator +from pydantic import AliasChoices, Field, SecretStr, TypeAdapter, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -80,6 +80,14 @@ class ApplicationSettings(_BaseApplicationSettings): These settings includes extra configuration for the http-API """ + DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: SecretStr = Field( + ..., + description=( + "secret required to enabled browser-based storage for the UI. " + "Enables the full set of features to be used for NiceUI" + ), + ) + DYNAMIC_SCHEDULER_RABBITMQ: RabbitSettings = Field( json_schema_extra={"auto_default_from_env": True}, description="settings for service/rabbitmq", diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/rabbitmq.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/rabbitmq.py index ff3a6ef9b94..b7b3d30425c 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/rabbitmq.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/rabbitmq.py @@ -21,6 +21,9 @@ async def _on_startup() -> None: app.state.rabbitmq_client = RabbitMQClient( client_name="dynamic_scheduler", settings=settings ) + app.state.rabbitmq_rpc_client = await RabbitMQRPCClient.create( + client_name="dynamic_scheduler_rpc_client", settings=settings + ) app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( client_name="dynamic_scheduler_rpc_server", settings=settings ) @@ -28,6 +31,8 @@ async def _on_startup() -> None: async def _on_shutdown() -> None: if app.state.rabbitmq_client: await app.state.rabbitmq_client.close() + if app.state.rabbitmq_rpc_client: + await app.state.rabbitmq_rpc_client.close() if app.state.rabbitmq_rpc_server: await app.state.rabbitmq_rpc_server.close() @@ -40,6 +45,11 @@ def get_rabbitmq_client(app: FastAPI) -> RabbitMQClient: return cast(RabbitMQClient, app.state.rabbitmq_client) +def get_rabbitmq_rpc_client(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_client + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_client) + + def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: assert app.state.rabbitmq_rpc_server # nosec return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/__init__.py index abf543d1bef..e4cf7e50705 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/__init__.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/__init__.py @@ -4,7 +4,7 @@ get_tracked_service, get_user_id_for_service, remove_tracked_service, - set_frontned_notified_for_service, + set_frontend_notified_for_service, set_if_status_changed_for_service, set_request_as_running, set_request_as_stopped, @@ -21,7 +21,7 @@ "get_user_id_for_service", "NORMAL_RATE_POLL_INTERVAL", "remove_tracked_service", - "set_frontned_notified_for_service", + "set_frontend_notified_for_service", "set_if_status_changed_for_service", "set_request_as_running", "set_request_as_stopped", diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py index 99215c69123..09e4c3b965f 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/service_tracker/_api.py @@ -184,7 +184,7 @@ async def should_notify_frontend_for_service( ) -async def set_frontned_notified_for_service(app: FastAPI, node_id: NodeID) -> None: +async def set_frontend_notified_for_service(app: FastAPI, node_id: NodeID) -> None: tracker = get_tracker(app) model: TrackedServiceModel | None = await tracker.load(node_id) if model is None: diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_deferred_get_status.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_deferred_get_status.py index f710204504c..4cd8209d1ae 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_deferred_get_status.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_deferred_get_status.py @@ -76,7 +76,7 @@ async def on_result( ) if user_id: await notify_service_status_change(app, user_id, result) - await service_tracker.set_frontned_notified_for_service(app, node_id) + await service_tracker.set_frontend_notified_for_service(app, node_id) else: _logger.info( "Did not find a user for '%s', skipping status delivery of: %s", diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_monitor.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_monitor.py index 8ba70997a93..432cf8896d8 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_monitor.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_monitor.py @@ -70,9 +70,9 @@ def __init__(self, app: FastAPI, status_worker_interval: timedelta) -> None: def status_worker_interval_seconds(self) -> NonNegativeFloat: return self.status_worker_interval.total_seconds() - async def _worker_start_get_status_requests(self) -> None: + async def _worker_check_services_require_status_update(self) -> None: """ - Check if a service requires it's status to be polled. + Check if any service requires it's status to be polled. Note that the interval at which the status is polled can vary. This is a relatively low resource check. """ @@ -136,7 +136,7 @@ async def _worker_start_get_status_requests(self) -> None: async def setup(self) -> None: self.app.state.status_monitor_background_task = start_exclusive_periodic_task( get_redis_client(self.app, RedisDatabase.LOCKS), - self._worker_start_get_status_requests, + self._worker_check_services_require_status_update, task_period=_INTERVAL_BETWEEN_CHECKS, retry_after=_INTERVAL_BETWEEN_CHECKS, task_name="periodic_service_status_update", diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py b/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py new file mode 100644 index 00000000000..9d131549faf --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py @@ -0,0 +1,122 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +import asyncio +import subprocess +from collections.abc import AsyncIterable +from contextlib import suppress +from typing import Final +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI, status +from httpx import AsyncClient +from hypercorn.asyncio import serve +from hypercorn.config import Config +from playwright.async_api import Page, async_playwright +from pytest_mock import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings +from settings_library.utils_service import DEFAULT_FASTAPI_PORT +from simcore_service_dynamic_scheduler.core.application import create_app +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +_MODULE: Final["str"] = "simcore_service_dynamic_scheduler" + + +@pytest.fixture +def disable_status_monitor_background_task(mocker: MockerFixture) -> None: + mocker.patch( + f"{_MODULE}.services.status_monitor._monitor.Monitor._worker_check_services_require_status_update" + ) + + +@pytest.fixture +def mock_stop_dynamic_service(mocker: MockerFixture) -> AsyncMock: + async_mock = AsyncMock() + mocker.patch( + f"{_MODULE}.api.frontend.routes._service.stop_dynamic_service", async_mock + ) + return async_mock + + +@pytest.fixture +def mock_remove_tracked_service(mocker: MockerFixture) -> AsyncMock: + async_mock = AsyncMock() + mocker.patch( + f"{_MODULE}.api.frontend.routes._service.remove_tracked_service", async_mock + ) + return async_mock + + +@pytest.fixture +def app_environment( + app_environment: EnvVarsDict, + disable_status_monitor_background_task: None, + rabbit_service: RabbitSettings, + redis_service: RedisSettings, + remove_redis_data: None, +) -> EnvVarsDict: + return app_environment + + +@pytest.fixture +def server_host_port() -> str: + return f"127.0.0.1:{DEFAULT_FASTAPI_PORT}" + + +@pytest.fixture +def not_initialized_app(app_environment: EnvVarsDict) -> FastAPI: + return create_app() + + +@pytest.fixture +async def app_runner( + not_initialized_app: FastAPI, server_host_port: str +) -> AsyncIterable[None]: + + shutdown_event = asyncio.Event() + + async def _wait_for_shutdown_event(): + await shutdown_event.wait() + + async def _run_server() -> None: + config = Config() + config.bind = [server_host_port] + + with suppress(asyncio.CancelledError): + await serve( + not_initialized_app, config, shutdown_trigger=_wait_for_shutdown_event + ) + + server_task = asyncio.create_task(_run_server()) + + async for attempt in AsyncRetrying( + reraise=True, wait=wait_fixed(0.1), stop=stop_after_delay(2) + ): + with attempt: + async with AsyncClient(timeout=1) as client: + result = await client.get(f"http://{server_host_port}") + assert result.status_code == status.HTTP_200_OK + + yield + + shutdown_event.set() + await server_task + + +@pytest.fixture +def download_playwright_browser() -> None: + subprocess.run( # noqa: S603 + ["playwright", "install", "chromium"], check=True # noqa: S607 + ) + + +@pytest.fixture +async def async_page(download_playwright_browser: None) -> AsyncIterable[Page]: + async with async_playwright() as p: + browser = await p.chromium.launch() + page = await browser.new_page() + yield page + await browser.close() diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/helpers.py b/services/dynamic-scheduler/tests/unit/api_frontend/helpers.py new file mode 100644 index 00000000000..91c2058c869 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/api_frontend/helpers.py @@ -0,0 +1,104 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Final +from uuid import uuid4 + +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_webserver.projects_nodes import NodeGet +from playwright.async_api import Locator, Page +from pydantic import NonNegativeFloat, NonNegativeInt, TypeAdapter +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +_HERE: Final[Path] = ( + Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +) +_DEFAULT_TIMEOUT: Final[NonNegativeFloat] = 10 + + +@asynccontextmanager +async def take_screenshot_on_error( + async_page: Page, +) -> AsyncIterator[None]: + try: + yield + # allows to also capture exceptions form `with pytest.raise(...)`` + except BaseException: + path = _HERE / f"{uuid4()}.ignore.png" + await async_page.screenshot(path=path) + print(f"Please check :{path}") + + raise + + +async def _get_locator( + async_page: Page, + text: str, + instances: NonNegativeInt | None, + timeout: float, # noqa: ASYNC109 +) -> Locator: + async with take_screenshot_on_error(async_page): + async for attempt in AsyncRetrying( + reraise=True, wait=wait_fixed(0.1), stop=stop_after_delay(timeout) + ): + with attempt: + locator = async_page.get_by_text(text) + count = await locator.count() + if instances is None: + assert count > 0, f"cold not find text='{text}'" + else: + assert ( + count == instances + ), f"found {count} instances of text='{text}'. Expected {instances}" + return locator + + +async def assert_contains_text( + async_page: Page, + text: str, + instances: NonNegativeInt | None = None, + timeout: float = _DEFAULT_TIMEOUT, # noqa: ASYNC109 +) -> None: + await _get_locator(async_page, text, instances=instances, timeout=timeout) + + +async def click_on_text( + async_page: Page, + text: str, + instances: NonNegativeInt | None = None, + timeout: float = _DEFAULT_TIMEOUT, # noqa: ASYNC109 +) -> None: + locator = await _get_locator(async_page, text, instances=instances, timeout=timeout) + await locator.click() + + +async def assert_not_contains_text( + async_page: Page, + text: str, + timeout: float = _DEFAULT_TIMEOUT, # noqa: ASYNC109 +) -> None: + async with take_screenshot_on_error(async_page): + async for attempt in AsyncRetrying( + reraise=True, wait=wait_fixed(0.1), stop=stop_after_delay(timeout) + ): + with attempt: + locator = async_page.get_by_text(text) + assert await locator.count() < 1, f"found text='{text}' in body" + + +def get_new_style_service_status(state: str) -> DynamicServiceGet: + return TypeAdapter(DynamicServiceGet).validate_python( + DynamicServiceGet.model_config["json_schema_extra"]["examples"][0] + | {"state": state} + ) + + +def get_legacy_service_status(state: str) -> NodeGet: + return TypeAdapter(NodeGet).validate_python( + NodeGet.model_config["json_schema_extra"]["examples"][0] + | {"service_state": state} + ) diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py new file mode 100644 index 00000000000..1cdb66ba587 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py @@ -0,0 +1,125 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from collections.abc import Callable +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from helpers import ( + assert_contains_text, + assert_not_contains_text, + click_on_text, + get_legacy_service_status, + get_new_style_service_status, +) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( + DynamicServiceStart, + DynamicServiceStop, +) +from models_library.api_schemas_webserver.projects_nodes import NodeGet +from models_library.projects_nodes_io import NodeID +from playwright.async_api import Page +from simcore_service_dynamic_scheduler.services.service_tracker import ( + set_if_status_changed_for_service, + set_request_as_running, + set_request_as_stopped, +) +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +pytest_simcore_core_services_selection = [ + "rabbit", + "redis", +] + +pytest_simcore_ops_services_selection = [ + # "redis-commander", +] + + +async def test_index_with_elements( + app_runner: None, + async_page: Page, + server_host_port: str, + not_initialized_app: FastAPI, + get_dynamic_service_start: Callable[[NodeID], DynamicServiceStart], + get_dynamic_service_stop: Callable[[NodeID], DynamicServiceStop], +): + await async_page.goto(server_host_port) + + # 1. no content + await assert_contains_text(async_page, "Total tracked services:") + await assert_contains_text(async_page, "0") + await assert_not_contains_text(async_page, "Details") + + # 2. add elements and check + await set_request_as_running( + not_initialized_app, get_dynamic_service_start(uuid4()) + ) + await set_request_as_stopped(not_initialized_app, get_dynamic_service_stop(uuid4())) + + await assert_contains_text(async_page, "2") + await assert_contains_text(async_page, "Details", instances=2) + + +@pytest.mark.parametrize( + "service_status", + [ + get_new_style_service_status("running"), + get_legacy_service_status("running"), + ], +) +async def test_main_page( + app_runner: None, + async_page: Page, + server_host_port: str, + node_id: NodeID, + service_status: NodeGet | DynamicServiceGet, + not_initialized_app: FastAPI, + get_dynamic_service_start: Callable[[NodeID], DynamicServiceStart], + mock_stop_dynamic_service: AsyncMock, +): + await async_page.goto(server_host_port) + + # 1. no content + await assert_contains_text(async_page, "Total tracked services:") + await assert_contains_text(async_page, "0") + await assert_not_contains_text(async_page, "Details") + + # 2. start a service shows content + await set_request_as_running( + not_initialized_app, get_dynamic_service_start(node_id) + ) + await set_if_status_changed_for_service( + not_initialized_app, node_id, service_status + ) + + await assert_contains_text(async_page, "1") + await assert_contains_text(async_page, "Details") + + # 3. click on stop and then cancel + await click_on_text(async_page, "Stop Service") + await assert_contains_text( + async_page, "The service will be stopped and its data will be saved" + ) + await click_on_text(async_page, "Cancel") + + # 4. click on stop then confirm + + await assert_not_contains_text( + async_page, "The service will be stopped and its data will be saved" + ) + await click_on_text(async_page, "Stop Service") + await assert_contains_text( + async_page, "The service will be stopped and its data will be saved" + ) + + mock_stop_dynamic_service.assert_not_awaited() + await click_on_text(async_page, "Stop Now") + async for attempt in AsyncRetrying( + reraise=True, wait=wait_fixed(0.1), stop=stop_after_delay(3) + ): + with attempt: + mock_stop_dynamic_service.assert_awaited_once() diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py new file mode 100644 index 00000000000..c37b7b0a4f1 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py @@ -0,0 +1,121 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from collections.abc import Callable +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI +from helpers import ( + assert_contains_text, + click_on_text, + get_legacy_service_status, + get_new_style_service_status, +) +from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet +from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( + DynamicServiceStart, +) +from models_library.api_schemas_webserver.projects_nodes import NodeGet +from models_library.projects_nodes_io import NodeID +from playwright.async_api import Page +from simcore_service_dynamic_scheduler.services.service_tracker import ( + set_if_status_changed_for_service, + set_request_as_running, +) +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +pytest_simcore_core_services_selection = [ + "rabbit", + "redis", +] + +pytest_simcore_ops_services_selection = [ + # "redis-commander", +] + + +async def test_service_details_no_status_present( + app_runner: None, + async_page: Page, + server_host_port: str, + node_id: NodeID, + not_initialized_app: FastAPI, + get_dynamic_service_start: Callable[[NodeID], DynamicServiceStart], +): + await set_request_as_running( + not_initialized_app, get_dynamic_service_start(node_id) + ) + + await async_page.goto(server_host_port) + + # 1. one service is tracked + await assert_contains_text(async_page, "Total tracked services:") + await assert_contains_text(async_page, "1") + await assert_contains_text(async_page, "Details", instances=1) + + # 2. open details page + await click_on_text(async_page, "Details") + # NOTE: if something is wrong with the page the bottom to remove from tracking + # will not be present + await assert_contains_text(async_page, "Remove from tracking", instances=1) + + +async def test_service_details_renders_friendly_404( + app_runner: None, async_page: Page, server_host_port: str, node_id: NodeID +): + # node was not started + await async_page.goto(f"{server_host_port}/service/{node_id}:details") + await assert_contains_text(async_page, "Sorry could not find any details for") + + +@pytest.mark.parametrize( + "service_status", + [ + get_new_style_service_status("running"), + get_legacy_service_status("running"), + ], +) +async def test_service_details( + app_runner: None, + async_page: Page, + server_host_port: str, + node_id: NodeID, + not_initialized_app: FastAPI, + get_dynamic_service_start: Callable[[NodeID], DynamicServiceStart], + mock_remove_tracked_service: AsyncMock, + service_status: NodeGet | DynamicServiceGet, +): + await set_request_as_running( + not_initialized_app, get_dynamic_service_start(node_id) + ) + await set_request_as_running( + not_initialized_app, get_dynamic_service_start(node_id) + ) + await set_if_status_changed_for_service( + not_initialized_app, node_id, service_status + ) + + await async_page.goto(server_host_port) + + # 1. one service is tracked + await assert_contains_text(async_page, "Total tracked services:") + await assert_contains_text(async_page, "1") + await assert_contains_text(async_page, "Details", instances=1) + + # 2. open details page + await click_on_text(async_page, "Details") + + # 3. click "Remove from tracking" -> cancel + await click_on_text(async_page, "Remove from tracking") + await click_on_text(async_page, "Cancel") + mock_remove_tracked_service.assert_not_awaited() + + # 4. click "Remove from tracking" -> confirm + await click_on_text(async_page, "Remove from tracking") + await click_on_text(async_page, "Remove service") + async for attempt in AsyncRetrying( + reraise=True, wait=wait_fixed(0.1), stop=stop_after_delay(3) + ): + with attempt: + mock_remove_tracked_service.assert_awaited_once() diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py index 9b5648e12b4..cb7939c5824 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py @@ -68,9 +68,9 @@ def app_environment( ) async def test_health(client: AsyncClient, is_ok: bool): if is_ok: - response = await client.get("/") + response = await client.get("/health") assert response.status_code == status.HTTP_200_OK assert datetime.fromisoformat(response.text.split("@")[1]) else: with pytest.raises(HealthCheckError): - await client.get("/") + await client.get("/health") diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py index 20293f343b5..f1c29a3d3f7 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py @@ -28,7 +28,7 @@ @pytest.fixture def disable_monitor_task(mocker: MockerFixture) -> None: mocker.patch( - "simcore_service_dynamic_scheduler.services.status_monitor._monitor.Monitor._worker_start_get_status_requests", + "simcore_service_dynamic_scheduler.services.status_monitor._monitor.Monitor._worker_check_services_require_status_update", autospec=True, ) diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index 2578114e541..5924f9dec84 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -404,7 +404,7 @@ async def test_expected_calls_to_notify_frontend( # pylint:disable=too-many-arg ): with attempt: # pylint:disable=protected-access - await monitor._worker_start_get_status_requests() # noqa: SLF001 + await monitor._worker_check_services_require_status_update() # noqa: SLF001 for method in ("start", "on_created", "on_result"): await _assert_call_to( deferred_status_spies, method=method, count=i + 1 @@ -428,7 +428,7 @@ async def test_expected_calls_to_notify_frontend( # pylint:disable=too-many-arg ): with attempt: # pylint:disable=protected-access - await monitor._worker_start_get_status_requests() # noqa: SLF001 + await monitor._worker_check_services_require_status_update() # noqa: SLF001 assert remove_tracked_spy.call_count == remove_tracked_count