Skip to content

Commit

Permalink
✨Adding dynamic services monitoring dashboard (⚠️devops) (ITISFoundat…
Browse files Browse the repository at this point in the history
…ion#6784)

Co-authored-by: Andrei Neagu <[email protected]>
  • Loading branch information
GitHK and Andrei Neagu authored Dec 4, 2024
1 parent b1fe598 commit 81c8ea0
Show file tree
Hide file tree
Showing 33 changed files with 990 additions and 46 deletions.
1 change: 1 addition & 0 deletions .env-devel
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
DYNAMIC_SCHEDULER_PROFILING=1
DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
DYNAMIC_SCHEDULER_TRACING={}
DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin

FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'

Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\
printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\
printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
printf "$$rows" "Dy-scheduler Dashboard" "http://$(get_my_ip).nip.io:8012";\
printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\
Expand Down
11 changes: 2 additions & 9 deletions packages/models-library/tests/test_service_settings_labels.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
from pprint import pformat
from typing import Any, Final, NamedTuple

import pydantic_core
import pytest
from common_library.json_serialization import json_dumps
from models_library.basic_types import PortInt
from models_library.osparc_variable_identifier import (
OsparcVariableIdentifier,
Expand Down Expand Up @@ -558,11 +558,6 @@ def test_can_parse_labels_with_osparc_identifiers(
assert "$" not in service_meta_str


def servicelib__json_serialization__json_dumps(obj: Any, **kwargs):
# Analogous to 'models_library.utils.json_serialization.json_dumps'
return json.dumps(obj, default=pydantic_core.to_jsonable_python, **kwargs)


def test_resolving_some_service_labels_at_load_time(
vendor_environments: dict[str, Any], service_labels: dict[str, str]
):
Expand All @@ -579,9 +574,7 @@ def test_resolving_some_service_labels_at_load_time(
("settings", SimcoreServiceSettingsLabel),
):
to_serialize = getattr(service_meta, attribute_name)
template = TextTemplate(
servicelib__json_serialization__json_dumps(to_serialize)
)
template = TextTemplate(json_dumps(to_serialize))
assert template.is_valid()
resolved_label: str = template.safe_substitute(vendor_environments)
to_restore = TypeAdapter(pydantic_model).validate_json(resolved_label)
Expand Down
21 changes: 5 additions & 16 deletions packages/models-library/tests/test_utils_fastapi_encoders.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,25 @@
# pylint: disable=too-many-arguments

import json
from typing import Any
from uuid import uuid4

import pytest
from common_library.json_serialization import json_dumps
from faker import Faker
from models_library.utils.fastapi_encoders import servicelib_jsonable_encoder
from pydantic.json import pydantic_encoder


def servicelib__json_serialization__json_dumps(obj: Any, **kwargs):
# Analogous to 'models_library.utils.json_serialization.json_dumps'
return json.dumps(obj, default=pydantic_encoder, **kwargs)


def test_using_uuids_as_keys(faker: Faker):
uuid_key = uuid4()

with pytest.raises(TypeError):
# IMPORTANT NOTE: we cannot serialize UUID objects as keys.
# We have to convert them to strings but then the class information is lost upon deserialization i.e. it is not reversable!
# NOTE: This could potentially be solved using 'orjson' !!
#
servicelib__json_serialization__json_dumps({uuid_key: "value"}, indent=1)
# this was previously failing
assert json_dumps({uuid_key: "value"}, indent=1)

# use encoder
# uuid keys now serialize without raising to the expected format string
data = servicelib_jsonable_encoder({uuid_key: "value"})
assert data == {f"{uuid_key}": "value"}

# serialize w/o raising
dumped_data = servicelib__json_serialization__json_dumps(data, indent=1)
dumped_data = json_dumps(data, indent=1)

# deserialize w/o raising
loaded_data = json.loads(dumped_data)
Expand Down
1 change: 1 addition & 0 deletions services/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,7 @@ services:
DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING}
DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING}
DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET}
TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}
TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
static-webserver:
Expand Down
2 changes: 1 addition & 1 deletion services/dynamic-scheduler/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ HEALTHCHECK --interval=30s \
--timeout=20s \
--start-period=30s \
--retries=3 \
CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/"]
CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/health"]

ENTRYPOINT [ "/bin/sh", "services/dynamic-scheduler/docker/entrypoint.sh" ]
CMD ["/bin/sh", "services/dynamic-scheduler/docker/boot.sh"]
Expand Down
1 change: 1 addition & 0 deletions services/dynamic-scheduler/requirements/_base.in
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
arrow
fastapi
httpx
nicegui
packaging
python-socketio
typer[all]
Expand Down
66 changes: 60 additions & 6 deletions services/dynamic-scheduler/requirements/_base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ aiodebug==2.3.0
aiodocker==0.24.0
# via -r requirements/../../../packages/service-library/requirements/_base.in
aiofiles==24.1.0
# via -r requirements/../../../packages/service-library/requirements/_base.in
# via
# -r requirements/../../../packages/service-library/requirements/_base.in
# nicegui
aiohappyeyeballs==2.4.3
# via aiohttp
aiohttp==3.11.7
Expand All @@ -27,6 +29,8 @@ aiohttp==3.11.7
# -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../requirements/constraints.txt
# aiodocker
# nicegui
# python-socketio
aiormq==6.8.1
# via aio-pika
aiosignal==1.3.1
Expand Down Expand Up @@ -77,6 +81,7 @@ certifi==2024.8.30
# -c requirements/../../../requirements/constraints.txt
# httpcore
# httpx
# nicegui
# requests
charset-normalizer==3.4.0
# via requests
Expand All @@ -92,6 +97,8 @@ deprecated==1.2.15
# opentelemetry-semantic-conventions
dnspython==2.7.0
# via email-validator
docutils==0.21.2
# via nicegui
email-validator==2.2.0
# via pydantic
exceptiongroup==1.2.2
Expand All @@ -102,6 +109,7 @@ fastapi==0.115.5
# via
# -r requirements/../../../packages/service-library/requirements/_fastapi.in
# -r requirements/_base.in
# nicegui
faststream==0.5.30
# via -r requirements/../../../packages/service-library/requirements/_base.in
frozenlist==1.5.0
Expand Down Expand Up @@ -143,15 +151,37 @@ httpx==0.27.2
# -c requirements/../../../requirements/constraints.txt
# -r requirements/../../../packages/service-library/requirements/_fastapi.in
# -r requirements/_base.in
# nicegui
idna==3.10
# via
# anyio
# email-validator
# httpx
# requests
# yarl
ifaddr==0.2.0
# via nicegui
importlib-metadata==8.5.0
# via opentelemetry-api
itsdangerous==2.2.0
# via nicegui
jinja2==3.1.4
# via
# -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../requirements/constraints.txt
# nicegui
jsonschema==4.23.0
# via
# -r requirements/../../../packages/models-library/requirements/_base.in
Expand All @@ -177,14 +207,20 @@ mako==1.3.6
# alembic
markdown-it-py==3.0.0
# via rich
markdown2==2.5.1
# via nicegui
markupsafe==3.0.2
# via mako
# via
# jinja2
# mako
mdurl==0.1.2
# via markdown-it-py
multidict==6.1.0
# via
# aiohttp
# yarl
nicegui==2.7.0
# via -r requirements/_base.in
opentelemetry-api==1.28.2
# via
# -r requirements/../../../packages/service-library/requirements/_base.in
Expand Down Expand Up @@ -280,6 +316,7 @@ orjson==3.10.12
# -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
# -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
# -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
# nicegui
packaging==24.2
# via
# -r requirements/_base.in
Expand All @@ -300,6 +337,8 @@ protobuf==5.28.3
# via
# googleapis-common-protos
# opentelemetry-proto
pscript==0.7.7
# via vbuild
psutil==6.1.0
# via -r requirements/../../../packages/service-library/requirements/_base.in
psycopg2-binary==2.9.10
Expand Down Expand Up @@ -357,7 +396,9 @@ pydantic-settings==2.6.1
# -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
# -r requirements/../../../packages/settings-library/requirements/_base.in
pygments==2.18.0
# via rich
# via
# nicegui
# rich
pyinstrument==5.0.0
# via -r requirements/../../../packages/service-library/requirements/_base.in
python-dateutil==2.9.0.post0
Expand All @@ -368,8 +409,12 @@ python-dotenv==1.0.1
# uvicorn
python-engineio==4.10.1
# via python-socketio
python-multipart==0.0.17
# via nicegui
python-socketio==5.11.4
# via -r requirements/_base.in
# via
# -r requirements/_base.in
# nicegui
pyyaml==6.0.2
# via
# -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
Expand Down Expand Up @@ -412,7 +457,9 @@ referencing==0.35.1
repro-zipfile==0.3.1
# via -r requirements/../../../packages/service-library/requirements/_base.in
requests==2.32.3
# via opentelemetry-exporter-otlp-proto-http
# via
# nicegui
# opentelemetry-exporter-otlp-proto-http
rich==13.9.4
# via
# -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
Expand Down Expand Up @@ -487,6 +534,7 @@ typing-extensions==4.12.2
# alembic
# fastapi
# faststream
# nicegui
# opentelemetry-sdk
# pydantic
# pydantic-core
Expand All @@ -510,15 +558,21 @@ urllib3==2.2.3
# -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../requirements/constraints.txt
# nicegui
# requests
uvicorn==0.32.1
# via
# -r requirements/../../../packages/service-library/requirements/_fastapi.in
# -r requirements/_base.in
# nicegui
uvloop==0.21.0
# via uvicorn
vbuild==0.8.2
# via nicegui
watchfiles==1.0.0
# via uvicorn
# via
# nicegui
# uvicorn
websockets==14.1
# via uvicorn
wrapt==1.17.0
Expand Down
2 changes: 2 additions & 0 deletions services/dynamic-scheduler/requirements/_test.in
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ asgi_lifespan
coverage
docker
faker
hypercorn
playwright
pytest
pytest-asyncio
pytest-cov
Expand Down
25 changes: 25 additions & 0 deletions services/dynamic-scheduler/requirements/_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,20 @@ docker==7.1.0
# via -r requirements/_test.in
faker==33.0.0
# via -r requirements/_test.in
greenlet==3.1.1
# via
# -c requirements/_base.txt
# playwright
h11==0.14.0
# via
# -c requirements/_base.txt
# httpcore
# hypercorn
# wsproto
h2==4.1.0
# via hypercorn
hpack==4.0.0
# via h2
httpcore==1.0.7
# via
# -c requirements/_base.txt
Expand All @@ -36,6 +46,10 @@ httpx==0.27.2
# -c requirements/../../../requirements/constraints.txt
# -c requirements/_base.txt
# respx
hypercorn==0.17.3
# via -r requirements/_test.in
hyperframe==6.0.1
# via h2
icdiff==2.0.7
# via pytest-icdiff
idna==3.10
Expand All @@ -51,10 +65,16 @@ packaging==24.2
# -c requirements/_base.txt
# pytest
# pytest-sugar
playwright==1.49.0
# via -r requirements/_test.in
pluggy==1.5.0
# via pytest
pprintpp==0.4.0
# via pytest-icdiff
priority==2.0.0
# via hypercorn
pyee==12.0.0
# via playwright
pytest==8.3.3
# via
# -r requirements/_test.in
Expand Down Expand Up @@ -107,9 +127,14 @@ typing-extensions==4.12.2
# via
# -c requirements/_base.txt
# faker
# pyee
urllib3==2.2.3
# via
# -c requirements/../../../requirements/constraints.txt
# -c requirements/_base.txt
# docker
# requests
wsproto==1.2.0
# via
# -c requirements/_base.txt
# hypercorn
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from ._setup import setup_frontend

__all__: tuple[str, ...] = ("setup_frontend",)
Loading

0 comments on commit 81c8ea0

Please sign in to comment.