🎨 introducing parent ids to rut (🗃️) (ITISFoundation#5891)

packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py

@@ -22,10 +22,11 @@ class ServiceRunGet(BaseModel):
     project_name: str
     node_id: NodeID
     node_name: str
+    root_parent_project_id: ProjectID
+    root_parent_project_name: str
     service_key: ServiceKey
     service_version: ServiceVersion
     service_type: str
-    service_resources: dict
     started_at: datetime
     stopped_at: datetime | None
     service_run_status: ServiceRunStatus

packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py

@@ -35,10 +35,11 @@ class ServiceRunGet(
     project_name: str
     node_id: NodeID
     node_name: str
+    root_parent_project_id: ProjectID
+    root_parent_project_name: str
     service_key: ServiceKey
     service_version: ServiceVersion
     service_type: str
-    service_resources: dict
     started_at: datetime
     stopped_at: datetime | None
     service_run_status: ServiceRunStatus

packages/models-library/src/models_library/rabbitmq_messages.py

@@ -217,6 +217,13 @@ class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage):
     node_id: NodeID
     node_name: str
 
+    parent_project_id: ProjectID
+    root_parent_project_id: ProjectID
+    root_parent_project_name: str
+
+    parent_node_id: NodeID
+    root_parent_node_id: NodeID
+
     service_key: ServiceKey
     service_version: ServiceVersion
     service_type: ServiceType

packages/postgres-database/src/simcore_postgres_database/migration/versions/481d5b472721_add_parent_fields_to_rut.py

@@ -0,0 +1,120 @@
+"""add_parent_fields_to_rut
+
+Revision ID: 481d5b472721
+Revises: 0d85bd35bdaa
+Create Date: 2024-06-03 08:58:35.086686+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "481d5b472721"
+down_revision = "0d85bd35bdaa"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "resource_tracker_service_runs",
+        sa.Column("parent_project_id", sa.String(), nullable=True),
+    )
+    op.add_column(
+        "resource_tracker_service_runs",
+        sa.Column("root_parent_project_id", sa.String(), nullable=True),
+    )
+    op.add_column(
+        "resource_tracker_service_runs",
+        sa.Column("root_parent_project_name", sa.String(), nullable=True),
+    )
+    op.add_column(
+        "resource_tracker_service_runs",
+        sa.Column("parent_node_id", sa.String(), nullable=True),
+    )
+    op.add_column(
+        "resource_tracker_service_runs",
+        sa.Column("root_parent_node_id", sa.String(), nullable=True),
+    )
+
+    # Populate new columns with values from the existing column
+    op.execute(
+        sa.DDL(
+            f"""
+
+                UPDATE resource_tracker_service_runs
+                SET parent_project_id = project_id,
+                    root_parent_project_id = project_id,
+                    root_parent_project_name = project_name,
+                    parent_node_id = node_id,
+                    root_parent_node_id = node_id
+                """
+        )
+    )
+
+    # Make newly created column non-nullable
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "parent_project_id",
+        nullable=False,
+    )
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "root_parent_project_id",
+        nullable=False,
+    )
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "root_parent_project_name",
+        nullable=False,
+    )
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "parent_node_id",
+        nullable=False,
+    )
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "root_parent_node_id",
+        nullable=False,
+    )
+
+    # Make already existing columns non-nullable
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "project_name",
+        existing_type=sa.VARCHAR(),
+        nullable=False,
+    )
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "node_name",
+        existing_type=sa.VARCHAR(),
+        nullable=False,
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "node_name",
+        existing_type=sa.VARCHAR(),
+        nullable=True,
+    )
+    op.alter_column(
+        "resource_tracker_service_runs",
+        "project_name",
+        existing_type=sa.VARCHAR(),
+        nullable=True,
+    )
+    op.drop_column("resource_tracker_service_runs", "root_parent_node_id")
+    op.drop_column("resource_tracker_service_runs", "parent_node_id")
+    op.drop_column("resource_tracker_service_runs", "root_parent_project_name")
+    op.drop_column("resource_tracker_service_runs", "root_parent_project_id")
+    op.drop_column("resource_tracker_service_runs", "parent_project_id")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs.py

@@ -104,7 +104,7 @@ class ResourceTrackerServiceRunStatus(str, enum.Enum):
     sa.Column(
         "project_name",
         sa.String,
-        nullable=True,
+        nullable=False,
         doc="we want to store the project name for tracking/billing purposes and be sure it stays there even when the project is deleted (that's also reason why we do not introduce foreign key)",
     ),
     # Node fields
@@ -117,9 +117,40 @@ class ResourceTrackerServiceRunStatus(str, enum.Enum):
     sa.Column(
         "node_name",
         sa.String,
-        nullable=True,
+        nullable=False,
         doc="we want to store the node/service name/label for tracking/billing purposes and be sure it stays there even when the node is deleted.",
     ),
+    # Project/Node parent fields
+    sa.Column(
+        "parent_project_id",  # UUID
+        sa.String,
+        nullable=False,
+        doc="If a user starts computational jobs via a dynamic service, a new project is created in the backend. This newly created project is considered a child project, and the project from which it was created is the parent project. We want to store the parent project ID for tracking and billing purposes, and ensure it remains even when the node is deleted. This is also the reason why we do not introduce a foreign key.",
+    ),
+    sa.Column(
+        "root_parent_project_id",  # UUID
+        sa.String,
+        nullable=False,
+        doc="Similar to the parent project concept, we are flexible enough to allow multiple nested computational jobs, which create multiple nested projects. For this reason, we keep the parent project ID, so we know from which project the user started their computation.",
+    ),
+    sa.Column(
+        "root_parent_project_name",
+        sa.String,
+        nullable=False,
+        doc="We want to store the root parent project name for tracking/billing purposes.",
+    ),
+    sa.Column(
+        "parent_node_id",  # UUID
+        sa.String,
+        nullable=False,
+        doc="Since each project can have multiple nodes, similar to the parent project concept, we also store the parent node..",
+    ),
+    sa.Column(
+        "root_parent_node_id",  # UUID
+        sa.String,
+        nullable=False,
+        doc="Since each project can have multiple nodes, similar to the root parent project concept, we also store the root parent node.",
+    ),
     # Service fields
     sa.Column(
         "service_key",

services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py

@@ -106,6 +106,11 @@ async def publish_service_resource_tracking_started(  # pylint: disable=too-many
         project_name=project_name,
         node_id=node_id,
         node_name=node_name,
+        parent_project_id=project_id,  # <-- SAN please modify
+        root_parent_project_id=project_id,  # <-- SAN please modify
+        root_parent_project_name=project_name,  # <-- SAN please modify
+        parent_node_id=node_id,  # <-- SAN please modify
+        root_parent_node_id=node_id,  # <-- SAN please modify
         service_key=service_key,
         service_version=service_version,
         service_type=service_type,

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/resource_tracking/_core.py

@@ -110,6 +110,11 @@ async def send_service_started(
         project_name=metrics_params.project_name,
         node_id=settings.DY_SIDECAR_NODE_ID,
         node_name=metrics_params.node_name,
+        parent_project_id=settings.DY_SIDECAR_PROJECT_ID,
+        root_parent_project_id=settings.DY_SIDECAR_PROJECT_ID,
+        root_parent_project_name=metrics_params.project_name,
+        parent_node_id=settings.DY_SIDECAR_NODE_ID,
+        root_parent_node_id=settings.DY_SIDECAR_NODE_ID,
         service_key=metrics_params.service_key,
         service_version=metrics_params.service_version,
         service_type=ServiceType.DYNAMIC,

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py

@@ -35,6 +35,11 @@ class ServiceRunCreate(BaseModel):
     project_name: str
     node_id: NodeID
     node_name: str
+    parent_project_id: ProjectID
+    root_parent_project_id: ProjectID
+    root_parent_project_name: str
+    parent_node_id: NodeID
+    root_parent_node_id: NodeID
     service_key: ServiceKey
     service_version: ServiceVersion
     service_type: ResourceTrackerServiceType
@@ -72,6 +77,11 @@ class ServiceRunDB(BaseModel):
     project_name: str
     node_id: NodeID
     node_name: str
+    parent_project_id: ProjectID
+    root_parent_project_id: ProjectID
+    root_parent_project_name: str
+    parent_node_id: NodeID
+    root_parent_node_id: NodeID
     service_key: ServiceKey
     service_version: ServiceVersion
     service_type: ResourceTrackerServiceType

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py

@@ -104,6 +104,11 @@ async def create_service_run(self, data: ServiceRunCreate) -> ServiceRunId:
                     project_name=data.project_name,
                     node_id=f"{data.node_id}",
                     node_name=data.node_name,
+                    parent_project_id=f"{data.parent_project_id}",
+                    root_parent_project_id=f"{data.root_parent_project_id}",
+                    root_parent_project_name=data.root_parent_project_name,
+                    parent_node_id=f"{data.parent_node_id}",
+                    root_parent_node_id=f"{data.root_parent_node_id}",
                     service_key=data.service_key,
                     service_version=data.service_version,
                     service_type=data.service_type,
@@ -231,6 +236,11 @@ async def list_service_runs_by_product_and_user_and_wallet(
                     resource_tracker_service_runs.c.project_name,
                     resource_tracker_service_runs.c.node_id,
                     resource_tracker_service_runs.c.node_name,
+                    resource_tracker_service_runs.c.parent_project_id,
+                    resource_tracker_service_runs.c.root_parent_project_id,
+                    resource_tracker_service_runs.c.root_parent_project_name,
+                    resource_tracker_service_runs.c.parent_node_id,
+                    resource_tracker_service_runs.c.root_parent_node_id,
                     resource_tracker_service_runs.c.service_key,
                     resource_tracker_service_runs.c.service_version,
                     resource_tracker_service_runs.c.service_type,

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py

@@ -109,6 +109,11 @@ async def _process_start_event(
         project_name=msg.project_name,
         node_id=msg.node_id,
         node_name=msg.node_name,
+        parent_project_id=msg.parent_project_id,
+        root_parent_project_id=msg.root_parent_project_id,
+        root_parent_project_name=msg.root_parent_project_name,
+        parent_node_id=msg.parent_node_id,
+        root_parent_node_id=msg.root_parent_node_id,
         service_key=msg.service_key,
         service_version=msg.service_version,
         service_type=service_type,

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py

@@ -118,12 +118,13 @@ async def list_service_runs(
                 user_email=service.user_email,
                 project_id=service.project_id,
                 project_name=service.project_name,
+                root_parent_project_id=service.root_parent_project_id,
+                root_parent_project_name=service.root_parent_project_name,
                 node_id=service.node_id,
                 node_name=service.node_name,
                 service_key=service.service_key,
                 service_version=service.service_version,
                 service_type=service.service_type,
-                service_resources=service.service_resources,
                 started_at=service.started_at,
                 stopped_at=service.stopped_at,
                 service_run_status=service.service_run_status,

services/resource-usage-tracker/tests/unit/with_dbs/conftest.py

@@ -98,6 +98,11 @@ def _creator(**overrides) -> dict[str, Any]:
             "project_name": faker.word(),
             "node_id": faker.uuid4(),
             "node_name": faker.word(),
+            "parent_project_id": faker.uuid4(),
+            "root_parent_project_id": faker.uuid4(),
+            "root_parent_project_name": faker.pystr(),
+            "parent_node_id": faker.uuid4(),
+            "root_parent_node_id": faker.uuid4(),
             "service_key": "simcore/services/dynamic/jupyter-smash",
             "service_version": "3.0.7",
             "service_type": "DYNAMIC_SERVICE",
@@ -238,6 +243,11 @@ def _creator(**kwargs: dict[str, Any]) -> RabbitResourceTrackingStartedMessage:
             "project_name": faker.pystr(),
             "node_id": faker.uuid4(),
             "node_name": faker.pystr(),
+            "parent_project_id": faker.uuid4(),
+            "root_parent_project_id": faker.uuid4(),
+            "root_parent_project_name": faker.pystr(),
+            "parent_node_id": faker.uuid4(),
+            "root_parent_node_id": faker.uuid4(),
             "service_key": "simcore/services/comp/itis/sleeper",
             "service_version": "2.1.6",
             "service_type": "computational",

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -10015,10 +10015,11 @@ components:
       - project_name
       - node_id
       - node_name
+      - root_parent_project_id
+      - root_parent_project_name
       - service_key
       - service_version
       - service_type
-      - service_resources
       - started_at
       - service_run_status
       type: object
@@ -10053,6 +10054,13 @@ components:
         node_name:
           title: Node Name
           type: string
+        root_parent_project_id:
+          title: Root Parent Project Id
+          type: string
+          format: uuid
+        root_parent_project_name:
+          title: Root Parent Project Name
+          type: string
         service_key:
           title: Service Key
           pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$
@@ -10064,9 +10072,6 @@ components:
         service_type:
           title: Service Type
           type: string
-        service_resources:
-          title: Service Resources
-          type: object
         started_at:
           title: Started At
           type: string

services/web/server/src/simcore_service_webserver/resource_usage/_service_runs_handlers.py

@@ -89,6 +89,8 @@ def validate_order_by_field(cls, v):
             "project_name",
             "node_id",
             "node_name",
+            "root_parent_project_id",
+            "root_parent_project_name",
             "service_key",
             "service_version",
             "service_type",