From 88e23a6b017cb62c6445cb982b5c22f6cc8c394a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 <60785969+matusdrobuliak66@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:43:33 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20introducing=20parent=20ids=20to?= =?UTF-8?q?=20rut=20(=F0=9F=97=83=EF=B8=8F)=20(#5891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service_runs.py | 3 +- .../api_schemas_webserver/resource_usage.py | 3 +- .../src/models_library/rabbitmq_messages.py | 7 + .../481d5b472721_add_parent_fields_to_rut.py | 120 ++++++++++++++++++ .../models/resource_tracker_service_runs.py | 35 ++++- .../utils/rabbitmq.py | 5 + .../modules/resource_tracking/_core.py | 5 + .../models/resource_tracker_service_runs.py | 10 ++ .../db/repositories/resource_tracker.py | 10 ++ .../resource_tracker_process_messages.py | 5 + .../services/resource_tracker_service_runs.py | 3 +- .../tests/unit/with_dbs/conftest.py | 10 ++ .../api/v0/openapi.yaml | 13 +- .../resource_usage/_service_runs_handlers.py | 2 + .../test_usage_services__list.py | 12 +- 15 files changed, 224 insertions(+), 19 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/481d5b472721_add_parent_fields_to_rut.py diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py index 3c3c5905d58..8abb2da421e 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py @@ -22,10 +22,11 @@ class ServiceRunGet(BaseModel): project_name: str node_id: NodeID node_name: str + root_parent_project_id: ProjectID + root_parent_project_name: str service_key: ServiceKey service_version: ServiceVersion service_type: str - service_resources: dict started_at: datetime stopped_at: datetime | None service_run_status: ServiceRunStatus diff --git a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py index 4e514b8776e..fa150f9ffc6 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py @@ -35,10 +35,11 @@ class ServiceRunGet( project_name: str node_id: NodeID node_name: str + root_parent_project_id: ProjectID + root_parent_project_name: str service_key: ServiceKey service_version: ServiceVersion service_type: str - service_resources: dict started_at: datetime stopped_at: datetime | None service_run_status: ServiceRunStatus diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 42374baeb4c..902bac4d279 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -217,6 +217,13 @@ class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage): node_id: NodeID node_name: str + parent_project_id: ProjectID + root_parent_project_id: ProjectID + root_parent_project_name: str + + parent_node_id: NodeID + root_parent_node_id: NodeID + service_key: ServiceKey service_version: ServiceVersion service_type: ServiceType diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/481d5b472721_add_parent_fields_to_rut.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/481d5b472721_add_parent_fields_to_rut.py new file mode 100644 index 00000000000..8458cf73442 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/481d5b472721_add_parent_fields_to_rut.py @@ -0,0 +1,120 @@ +"""add_parent_fields_to_rut + +Revision ID: 481d5b472721 +Revises: 0d85bd35bdaa +Create Date: 2024-06-03 08:58:35.086686+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "481d5b472721" +down_revision = "0d85bd35bdaa" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "resource_tracker_service_runs", + sa.Column("parent_project_id", sa.String(), nullable=True), + ) + op.add_column( + "resource_tracker_service_runs", + sa.Column("root_parent_project_id", sa.String(), nullable=True), + ) + op.add_column( + "resource_tracker_service_runs", + sa.Column("root_parent_project_name", sa.String(), nullable=True), + ) + op.add_column( + "resource_tracker_service_runs", + sa.Column("parent_node_id", sa.String(), nullable=True), + ) + op.add_column( + "resource_tracker_service_runs", + sa.Column("root_parent_node_id", sa.String(), nullable=True), + ) + + # Populate new columns with values from the existing column + op.execute( + sa.DDL( + f""" + + UPDATE resource_tracker_service_runs + SET parent_project_id = project_id, + root_parent_project_id = project_id, + root_parent_project_name = project_name, + parent_node_id = node_id, + root_parent_node_id = node_id + """ + ) + ) + + # Make newly created column non-nullable + op.alter_column( + "resource_tracker_service_runs", + "parent_project_id", + nullable=False, + ) + op.alter_column( + "resource_tracker_service_runs", + "root_parent_project_id", + nullable=False, + ) + op.alter_column( + "resource_tracker_service_runs", + "root_parent_project_name", + nullable=False, + ) + op.alter_column( + "resource_tracker_service_runs", + "parent_node_id", + nullable=False, + ) + op.alter_column( + "resource_tracker_service_runs", + "root_parent_node_id", + nullable=False, + ) + + # Make already existing columns non-nullable + op.alter_column( + "resource_tracker_service_runs", + "project_name", + existing_type=sa.VARCHAR(), + nullable=False, + ) + op.alter_column( + "resource_tracker_service_runs", + "node_name", + existing_type=sa.VARCHAR(), + nullable=False, + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "resource_tracker_service_runs", + "node_name", + existing_type=sa.VARCHAR(), + nullable=True, + ) + op.alter_column( + "resource_tracker_service_runs", + "project_name", + existing_type=sa.VARCHAR(), + nullable=True, + ) + op.drop_column("resource_tracker_service_runs", "root_parent_node_id") + op.drop_column("resource_tracker_service_runs", "parent_node_id") + op.drop_column("resource_tracker_service_runs", "root_parent_project_name") + op.drop_column("resource_tracker_service_runs", "root_parent_project_id") + op.drop_column("resource_tracker_service_runs", "parent_project_id") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs.py index 012d0e6b27c..33eddcb9fc7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs.py @@ -104,7 +104,7 @@ class ResourceTrackerServiceRunStatus(str, enum.Enum): sa.Column( "project_name", sa.String, - nullable=True, + nullable=False, doc="we want to store the project name for tracking/billing purposes and be sure it stays there even when the project is deleted (that's also reason why we do not introduce foreign key)", ), # Node fields @@ -117,9 +117,40 @@ class ResourceTrackerServiceRunStatus(str, enum.Enum): sa.Column( "node_name", sa.String, - nullable=True, + nullable=False, doc="we want to store the node/service name/label for tracking/billing purposes and be sure it stays there even when the node is deleted.", ), + # Project/Node parent fields + sa.Column( + "parent_project_id", # UUID + sa.String, + nullable=False, + doc="If a user starts computational jobs via a dynamic service, a new project is created in the backend. This newly created project is considered a child project, and the project from which it was created is the parent project. We want to store the parent project ID for tracking and billing purposes, and ensure it remains even when the node is deleted. This is also the reason why we do not introduce a foreign key.", + ), + sa.Column( + "root_parent_project_id", # UUID + sa.String, + nullable=False, + doc="Similar to the parent project concept, we are flexible enough to allow multiple nested computational jobs, which create multiple nested projects. For this reason, we keep the parent project ID, so we know from which project the user started their computation.", + ), + sa.Column( + "root_parent_project_name", + sa.String, + nullable=False, + doc="We want to store the root parent project name for tracking/billing purposes.", + ), + sa.Column( + "parent_node_id", # UUID + sa.String, + nullable=False, + doc="Since each project can have multiple nodes, similar to the parent project concept, we also store the parent node..", + ), + sa.Column( + "root_parent_node_id", # UUID + sa.String, + nullable=False, + doc="Since each project can have multiple nodes, similar to the root parent project concept, we also store the root parent node.", + ), # Service fields sa.Column( "service_key", diff --git a/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py b/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py index 87b99c46071..e239d57b673 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py @@ -106,6 +106,11 @@ async def publish_service_resource_tracking_started( # pylint: disable=too-many project_name=project_name, node_id=node_id, node_name=node_name, + parent_project_id=project_id, # <-- SAN please modify + root_parent_project_id=project_id, # <-- SAN please modify + root_parent_project_name=project_name, # <-- SAN please modify + parent_node_id=node_id, # <-- SAN please modify + root_parent_node_id=node_id, # <-- SAN please modify service_key=service_key, service_version=service_version, service_type=service_type, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_core.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_core.py index 524f21c83e0..031b42ff324 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_core.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_core.py @@ -110,6 +110,11 @@ async def send_service_started( project_name=metrics_params.project_name, node_id=settings.DY_SIDECAR_NODE_ID, node_name=metrics_params.node_name, + parent_project_id=settings.DY_SIDECAR_PROJECT_ID, + root_parent_project_id=settings.DY_SIDECAR_PROJECT_ID, + root_parent_project_name=metrics_params.project_name, + parent_node_id=settings.DY_SIDECAR_NODE_ID, + root_parent_node_id=settings.DY_SIDECAR_NODE_ID, service_key=metrics_params.service_key, service_version=metrics_params.service_version, service_type=ServiceType.DYNAMIC, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py index eadab0831d4..f68e9ff2c5a 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py @@ -35,6 +35,11 @@ class ServiceRunCreate(BaseModel): project_name: str node_id: NodeID node_name: str + parent_project_id: ProjectID + root_parent_project_id: ProjectID + root_parent_project_name: str + parent_node_id: NodeID + root_parent_node_id: NodeID service_key: ServiceKey service_version: ServiceVersion service_type: ResourceTrackerServiceType @@ -72,6 +77,11 @@ class ServiceRunDB(BaseModel): project_name: str node_id: NodeID node_name: str + parent_project_id: ProjectID + root_parent_project_id: ProjectID + root_parent_project_name: str + parent_node_id: NodeID + root_parent_node_id: NodeID service_key: ServiceKey service_version: ServiceVersion service_type: ResourceTrackerServiceType diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py index 560138176d7..f24f4930344 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py @@ -104,6 +104,11 @@ async def create_service_run(self, data: ServiceRunCreate) -> ServiceRunId: project_name=data.project_name, node_id=f"{data.node_id}", node_name=data.node_name, + parent_project_id=f"{data.parent_project_id}", + root_parent_project_id=f"{data.root_parent_project_id}", + root_parent_project_name=data.root_parent_project_name, + parent_node_id=f"{data.parent_node_id}", + root_parent_node_id=f"{data.root_parent_node_id}", service_key=data.service_key, service_version=data.service_version, service_type=data.service_type, @@ -231,6 +236,11 @@ async def list_service_runs_by_product_and_user_and_wallet( resource_tracker_service_runs.c.project_name, resource_tracker_service_runs.c.node_id, resource_tracker_service_runs.c.node_name, + resource_tracker_service_runs.c.parent_project_id, + resource_tracker_service_runs.c.root_parent_project_id, + resource_tracker_service_runs.c.root_parent_project_name, + resource_tracker_service_runs.c.parent_node_id, + resource_tracker_service_runs.c.root_parent_node_id, resource_tracker_service_runs.c.service_key, resource_tracker_service_runs.c.service_version, resource_tracker_service_runs.c.service_type, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py index 591bbb98b47..9a1bc1b6170 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py @@ -109,6 +109,11 @@ async def _process_start_event( project_name=msg.project_name, node_id=msg.node_id, node_name=msg.node_name, + parent_project_id=msg.parent_project_id, + root_parent_project_id=msg.root_parent_project_id, + root_parent_project_name=msg.root_parent_project_name, + parent_node_id=msg.parent_node_id, + root_parent_node_id=msg.root_parent_node_id, service_key=msg.service_key, service_version=msg.service_version, service_type=service_type, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index 0618838c69b..536d7c9423f 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -118,12 +118,13 @@ async def list_service_runs( user_email=service.user_email, project_id=service.project_id, project_name=service.project_name, + root_parent_project_id=service.root_parent_project_id, + root_parent_project_name=service.root_parent_project_name, node_id=service.node_id, node_name=service.node_name, service_key=service.service_key, service_version=service.service_version, service_type=service.service_type, - service_resources=service.service_resources, started_at=service.started_at, stopped_at=service.stopped_at, service_run_status=service.service_run_status, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index fda8e0eea1a..3d476e01d6d 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -98,6 +98,11 @@ def _creator(**overrides) -> dict[str, Any]: "project_name": faker.word(), "node_id": faker.uuid4(), "node_name": faker.word(), + "parent_project_id": faker.uuid4(), + "root_parent_project_id": faker.uuid4(), + "root_parent_project_name": faker.pystr(), + "parent_node_id": faker.uuid4(), + "root_parent_node_id": faker.uuid4(), "service_key": "simcore/services/dynamic/jupyter-smash", "service_version": "3.0.7", "service_type": "DYNAMIC_SERVICE", @@ -238,6 +243,11 @@ def _creator(**kwargs: dict[str, Any]) -> RabbitResourceTrackingStartedMessage: "project_name": faker.pystr(), "node_id": faker.uuid4(), "node_name": faker.pystr(), + "parent_project_id": faker.uuid4(), + "root_parent_project_id": faker.uuid4(), + "root_parent_project_name": faker.pystr(), + "parent_node_id": faker.uuid4(), + "root_parent_node_id": faker.uuid4(), "service_key": "simcore/services/comp/itis/sleeper", "service_version": "2.1.6", "service_type": "computational", diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d930d3e42b8..621a50ff25a 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -10015,10 +10015,11 @@ components: - project_name - node_id - node_name + - root_parent_project_id + - root_parent_project_name - service_key - service_version - service_type - - service_resources - started_at - service_run_status type: object @@ -10053,6 +10054,13 @@ components: node_name: title: Node Name type: string + root_parent_project_id: + title: Root Parent Project Id + type: string + format: uuid + root_parent_project_name: + title: Root Parent Project Name + type: string service_key: title: Service Key pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$ @@ -10064,9 +10072,6 @@ components: service_type: title: Service Type type: string - service_resources: - title: Service Resources - type: object started_at: title: Started At type: string diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py index a605fe1f853..a5efdd515f2 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py @@ -89,6 +89,8 @@ def validate_order_by_field(cls, v): "project_name", "node_id", "node_name", + "root_parent_project_id", + "root_parent_project_name", "service_key", "service_version", "service_type", diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py index 0dea4c45928..e5fe848543b 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_usage_services__list.py @@ -37,19 +37,11 @@ "project_name": "osparc", "node_id": "3d2133f4-aba4-4364-9f7a-9377dea1221f", "node_name": "sleeper", + "root_parent_project_id": "5c2110be-441b-11ee-a0e8-02420a000040", + "root_parent_project_name": "osparc", "service_key": "simcore/services/comp/itis/sleeper", "service_version": "2.0.2", "service_type": "DYNAMIC_SERVICE", - "service_resources": { - "container": { - "image": "simcore/services/comp/itis/sleeper:2.0.2", - "resources": { - "CPU": {"limit": 0.1, "reservation": 0.1}, - "RAM": {"limit": 2147483648, "reservation": 2147483648}, - }, - "boot_modes": ["CPU"], - } - }, "started_at": "2023-08-26T14:18:17.600493+00:00", "stopped_at": "2023-08-26T14:18:19.358355+00:00", "service_run_status": "SUCCESS",