🐛 Fix aiohttp server autoinstrumentation

.env-devel

@@ -104,6 +104,7 @@ DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKE
 DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
+DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600
 # DIRECTOR_V2 ----
 
 DYNAMIC_SCHEDULER_PROFILING=1
@@ -153,7 +154,7 @@ POSTGRES_PORT=5432
 POSTGRES_USER=scu
 
 POSTGRES_READONLY_PASSWORD=readonly
-POSTGRES_READONLY_USER=readonly
+POSTGRES_READONLY_USER=postgres_readonly
 
 
 RABBIT_HOST=rabbit
@@ -220,6 +221,13 @@ STORAGE_PROFILING=1
 
 SWARM_STACK_NAME=master-simcore
 
+## VENDOR DEVELOPMENT SERVICES ---
+VENDOR_DEV_MANUAL_IMAGE=containous/whoami
+VENDOR_DEV_MANUAL_REPLICAS=1
+VENDOR_DEV_MANUAL_SUBDOMAIN=manual
+
+## VENDOR DEVELOPMENT SERVICES ---
+
 WB_API_WEBSERVER_HOST=wb-api-server
 WB_API_WEBSERVER_PORT=8080
 

.gitignore

@@ -181,6 +181,3 @@ tests/public-api/osparc_python_wheels/*
 
 # osparc-config repo files
 repo.config
-
-# scripts resolved with .env s
-services/postgres/scripts/create-readonly-user.sql

.vscode/settings.template.json

@@ -9,11 +9,12 @@
   "files.associations": {
     ".*rc": "ini",
     ".env*": "ini",
+    "*.logs*": "log",
     "**/requirements/*.in": "pip-requirements",
     "**/requirements/*.txt": "pip-requirements",
     "*logs.txt": "log",
-    "*.logs*": "log",
     "*Makefile": "makefile",
+    "*sql.*": "sql",
     "docker-compose*.yml": "dockercompose",
     "Dockerfile*": "dockerfile"
   },

Makefile

@@ -269,6 +269,11 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 		services/docker-compose.local.yml \
 		> $@
 
+.stack-vendor-services.yml: .env $(docker-compose-configs)
+	# Creating config for vendors stack to $@
+	@scripts/docker/docker-stack-config.bash -e $< \
+		services/docker-compose-dev-vendors.yml \
+		> $@
 
 .stack-ops.yml: .env $(docker-compose-configs)
 	# Creating config for ops stack to $@
@@ -288,7 +293,11 @@ endif
 
 
 
-.PHONY: up-devel up-prod up-prod-ci up-version up-latest .deploy-ops
+.PHONY: up-devel up-prod up-prod-ci up-version up-latest .deploy-ops .deploy-vendors
+
+.deploy-vendors: .stack-vendor-services.yml
+	# Deploy stack 'vendors'
+	docker stack deploy --detach=true --with-registry-auth -c $< vendors
 
 .deploy-ops: .stack-ops.yml
 	# Deploy stack 'ops'
@@ -338,6 +347,7 @@ up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Dep
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME) [back-end]
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
 	@$(MAKE_C) services/static-webserver/client follow-dev-logs
@@ -348,6 +358,7 @@ up-devel-frontend: .stack-simcore-development-frontend.yml .init-swarm ## Every
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME)  [back-end]
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
 	@$(MAKE_C) services/static-webserver/client follow-dev-logs
@@ -358,6 +369,7 @@ ifeq ($(target),)
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME)
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 else
 	# deploys ONLY $(target) service
@@ -369,6 +381,7 @@ up-version: .stack-simcore-version.yml .init-swarm ## Deploys versioned stack '$
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME)
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
 

api/specs/web-server/_auth.py

@@ -155,6 +155,21 @@ async def logout(_body: LogoutBody):
     """user logout"""
 
 
+@router.get(
+    "/auth:check",
+    operation_id="check_authentication",
+    status_code=status.HTTP_204_NO_CONTENT,
+    responses={
+        status.HTTP_401_UNAUTHORIZED: {
+            "model": Envelope[Error],
+            "description": "unauthorized reset due to invalid token code",
+        }
+    },
+)
+async def check_auth():
+    """checks if user is authenticated in the platform"""
+
+
 @router.post(
     "/auth/reset-password",
     response_model=Envelope[Log],

api/specs/web-server/_projects_crud.py

@@ -142,10 +142,17 @@ async def clone_project(
 
 @router.get(
     "/projects:search",
-    response_model=Page[ProjectListItem],
+    response_model=Page[ProjectListFullSearchParams],
 )
 async def list_projects_full_search(
     _params: Annotated[ProjectListFullSearchParams, Depends()],
+    order_by: Annotated[
+        Json,
+        Query(
+            description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.",
+            example='{"field": "last_change_date", "direction": "desc"}',
+        ),
+    ] = ('{"field": "last_change_date", "direction": "desc"}',),
 ):
     ...
 

packages/aws-library/requirements/_base.txt

@@ -224,6 +224,8 @@ referencing==0.29.3
     #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
     #   jsonschema
     #   jsonschema-specifications
+repro-zipfile==0.3.1
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
 requests==2.32.3
     # via opentelemetry-exporter-otlp-proto-http
 rich==13.8.1

packages/models-library/src/models_library/api_schemas_directorv2/services.py

@@ -1,4 +1,4 @@
-from typing import Any, ClassVar
+from typing import Any, ClassVar, Final
 
 from pydantic import BaseModel, Field, validator
 from pydantic.types import ByteSize, NonNegativeInt
@@ -90,3 +90,6 @@ class Config:
                 for node_example in NodeRequirements.Config.schema_extra["examples"]
             ]
         }
+
+
+CHARS_IN_VOLUME_NAME_BEFORE_DIR_NAME: Final[NonNegativeInt] = 89

packages/models-library/src/models_library/rabbitmq_messages.py

@@ -194,6 +194,24 @@ def routing_key(self) -> str | None:
         return None
 
 
+class DynamicServiceRunningMessage(RabbitMessageBase):
+    channel_name: Literal["io.simcore.service.dynamic-service-running"] = Field(
+        default="io.simcore.service.dynamic-service-running", const=True
+    )
+
+    project_id: ProjectID
+    node_id: NodeID
+    user_id: UserID
+    product_name: ProductName | None
+    created_at: datetime.datetime = Field(
+        default_factory=lambda: arrow.utcnow().datetime,
+        description="message creation datetime",
+    )
+
+    def routing_key(self) -> str | None:
+        return None
+
+
 class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage):
     message_type: RabbitResourceTrackingMessageType = Field(
         default=RabbitResourceTrackingMessageType.TRACKING_STARTED, const=True

packages/postgres-database/src/simcore_postgres_database/migration/versions/8a742f3efdd9_new_tags_priority_column.py

@@ -0,0 +1,27 @@
+"""new tags priority column
+
+Revision ID: 8a742f3efdd9
+Revises: 10729e07000d
+Create Date: 2024-10-02 15:23:27.446241+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "8a742f3efdd9"
+down_revision = "10729e07000d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("tags", sa.Column("priority", sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("tags", "priority")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/migration/versions/ea3952fe5a0e_add_enable_efs_to_group_extra_properties.py

@@ -0,0 +1,32 @@
+"""add `enable_efs` to group extra properties
+
+Revision ID: ea3952fe5a0e
+Revises: 8a742f3efdd9
+Create Date: 2024-10-07 06:24:42.464942+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ea3952fe5a0e"
+down_revision = "8a742f3efdd9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "groups_extra_properties",
+        sa.Column(
+            "enable_efs", sa.Boolean(), server_default=sa.text("false"), nullable=False
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("groups_extra_properties", "enable_efs")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/groups_extra_properties.py

@@ -63,6 +63,13 @@
         server_default=sa.sql.expression.false(),
         doc="If true, will send telemetry for new style dynamic services to frontend",
     ),
+    sa.Column(
+        "enable_efs",
+        sa.Boolean(),
+        nullable=False,
+        server_default=sa.sql.expression.false(),
+        doc="If true, will mount efs distributed file system when dynamic services starts",
+    ),
     sa.UniqueConstraint(
         "group_id", "product_name", name="group_id_product_name_uniqueness"
     ),

packages/postgres-database/src/simcore_postgres_database/models/services_tags.py

@@ -14,20 +14,21 @@
         "service_key",
         sa.String,
         nullable=False,
-        doc="Service Key Identifier",
+        doc="Key name identifier for the service, without specifiying its versions",
     ),
     sa.Column(
         "service_version",
         sa.String,
         nullable=False,
-        doc="Service version",
+        doc="Version of the service. Combined with 'service_key', it forms a unique identifier for this service.",
     ),
     # Tag
     sa.Column(
         "tag_id",
         sa.BigInteger,
         sa.ForeignKey(tags.c.id, onupdate="CASCADE", ondelete="CASCADE"),
         nullable=False,
+        doc="Identifier of the tag assigned to this specific service (service_key, service_version).",
     ),
     # Constraints
     sa.ForeignKeyConstraint(

packages/postgres-database/src/simcore_postgres_database/models/tags.py

@@ -2,35 +2,42 @@
 
 from .base import metadata
 
-#
-# tags: a way to mark any entity (e.g. a project, ...)
-#       this can be used to perform operations as filter, select, compare, etc
-#
 tags = sa.Table(
+    #
+    # A way to mark any entity (e.g. a project, ...)
+    # this can be used to perform operations as filter, select, compare, etc
+    #
     "tags",
     metadata,
     sa.Column(
         "id",
         sa.BigInteger(),
         nullable=False,
         primary_key=True,
+        doc="Unique identifier for each tag.",
     ),
-    sa.Column(
-        "name",
-        sa.String(),
-        nullable=False,
-        doc="display name",
-    ),
+    sa.Column("name", sa.String(), nullable=False, doc="The display name of the tag."),
     sa.Column(
         "description",
         sa.String(),
         nullable=True,
-        doc="description displayed",
+        doc="A brief description displayed for the tag.",
     ),
     sa.Column(
         "color",
         sa.String(),
         nullable=False,
-        doc="Hex color (see https://www.color-hex.com/)",
+        doc="Hexadecimal color code representing the tag (e.g., #FF5733).",
+    ),
+    sa.Column(
+        "priority",
+        sa.Integer(),
+        nullable=True,
+        doc=(
+            "Explicit ordering priority when displaying tags. "
+            "Tags with a lower value are displayed first. "
+            "If NULL, tags are considered to have the lowest priority and "
+            "are displayed after non-NULL values, ordered by their ID (reflecting creation order)."
+        ),
     ),
 )

packages/postgres-database/src/simcore_postgres_database/models/tags_access_rights.py

@@ -22,7 +22,7 @@
             name="fk_tag_to_group_tag_id",
         ),
         nullable=False,
-        doc="Tag unique ID",
+        doc="References the unique identifier of the tag that these access rights apply to.",
     ),
     sa.Column(
         "group_id",
@@ -34,30 +34,32 @@
             name="fk_tag_to_group_group_id",
         ),
         nullable=False,
-        doc="Group unique ID",
+        doc="References the unique identifier of the group that has access rights to the tag.",
     ),
     # ACCESS RIGHTS ---
     sa.Column(
         "read",
         sa.Boolean(),
         nullable=False,
         server_default=sa.sql.expression.true(),
-        doc="If true, group can *read* a tag."
-        "This column can be used to set the tag invisible",
+        doc="Indicates whether the group has permission to view the tag. "
+        "A value of 'True' allows the group to access the tag's details.",
     ),
     sa.Column(
         "write",
         sa.Boolean(),
         nullable=False,
         server_default=sa.sql.expression.false(),
-        doc="If true, group can *create* and *update* a tag",
+        doc="Indicates whether the group has permission to modify the tag. "
+        "A value of 'True' grants write access to the group.",
     ),
     sa.Column(
         "delete",
         sa.Boolean(),
         nullable=False,
         server_default=sa.sql.expression.false(),
-        doc="If true, group can *delete* the tag",
+        doc="Indicates whether the group has permission to delete the tag. "
+        "A value of 'True' allows the group to remove the tag.",
     ),
     # TIME STAMPS ----
     column_created_datetime(timezone=False),