From e05d046b9804f6cc96aba75bbdfed558a73bdcb5 Mon Sep 17 00:00:00 2001 From: Matus Drobuliak <60785969+matusdrobuliak66@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:46:36 +0100 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=A8=20Introduce=20vip=20models=20pric?= =?UTF-8?q?ing=201=20of=202=20parts=20(#6897)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../web-server/_catalog_licensed_items.py | 66 ++++++ api/specs/web-server/openapi.py | 1 + .../api_schemas_webserver/licensed_items.py | 22 ++ .../src/models_library/licensed_items.py | 44 ++++ .../postgres-database/requirements/_base.in | 2 +- .../postgres-database/requirements/_base.txt | 2 +- .../requirements/_migration.txt | 2 +- .../postgres-database/requirements/_test.txt | 17 +- .../postgres-database/requirements/_tools.txt | 32 +-- .../e5555076ef50_add_license_db_tables.py | 171 ++++++++++++++++ .../models/licensed_items.py | 63 ++++++ ...source_tracker_licensed_items_purchases.py | 61 ++++++ .../resource_tracker_licensed_items_usage.py | 76 +++++++ .../simcore_postgres_database/utils_repos.py | 8 + .../api/v0/openapi.yaml | 190 +++++++++++++++++- .../application_settings.py | 1 + .../catalog/licenses/__init__.py | 0 .../catalog/licenses/_exceptions_handlers.py | 26 +++ .../catalog/licenses/_licensed_items_api.py | 77 +++++++ .../catalog/licenses/_licensed_items_db.py | 181 +++++++++++++++++ .../licenses/_licensed_items_handlers.py | 112 +++++++++++ .../catalog/licenses/_models.py | 54 +++++ .../catalog/licenses/api.py | 3 + .../catalog/licenses/errors.py | 9 + .../catalog/licenses/plugin.py | 26 +++ .../catalog/plugin.py | 3 + .../security/_authz_access_roles.py | 1 + .../unit/with_dbs/04/licenses/conftest.py | 44 ++++ .../04/licenses/test_licensed_items_db.py | 94 +++++++++ .../licenses/test_licensed_items_handlers.py | 66 ++++++ tests/swarm-deploy/requirements/_test.txt | 2 +- 31 files changed, 1424 insertions(+), 32 deletions(-) create mode 100644 api/specs/web-server/_catalog_licensed_items.py create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py create mode 100644 packages/models-library/src/models_library/licensed_items.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/__init__.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/api.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/conftest.py create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py diff --git a/api/specs/web-server/_catalog_licensed_items.py b/api/specs/web-server/_catalog_licensed_items.py new file mode 100644 index 00000000000..29b39853c95 --- /dev/null +++ b/api/specs/web-server/_catalog_licensed_items.py @@ -0,0 +1,66 @@ +""" Helper script to generate OAS automatically +""" + +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from typing import Annotated + +from _common import as_query +from fastapi import APIRouter, Depends, status +from models_library.api_schemas_webserver.licensed_items import LicensedItemGet +from models_library.generics import Envelope +from models_library.rest_error import EnvelopedError +from simcore_service_webserver._meta import API_VTAG +from simcore_service_webserver.catalog.licenses._exceptions_handlers import ( + _TO_HTTP_ERROR_MAP, +) +from simcore_service_webserver.catalog.licenses._models import ( + LicensedItemsBodyParams, + LicensedItemsListQueryParams, + LicensedItemsPathParams, +) + +router = APIRouter( + prefix=f"/{API_VTAG}", + tags=[ + "licenses", + "catalog", + ], + responses={ + i.status_code: {"model": EnvelopedError} for i in _TO_HTTP_ERROR_MAP.values() + }, +) + + +@router.get( + "/catalog/licensed-items", + response_model=Envelope[list[LicensedItemGet]], +) +async def list_licensed_items( + _query: Annotated[as_query(LicensedItemsListQueryParams), Depends()], +): + ... + + +@router.get( + "/catalog/licensed-items/{licensed_item_id}", + response_model=Envelope[LicensedItemGet], +) +async def get_licensed_item( + _path: Annotated[LicensedItemsPathParams, Depends()], +): + ... + + +@router.post( + "/catalog/licensed-items/{licensed_item_id}:purchase", + status_code=status.HTTP_204_NO_CONTENT, +) +async def purchase_licensed_item( + _path: Annotated[LicensedItemsPathParams, Depends()], + _body: LicensedItemsBodyParams, +): + ... diff --git a/api/specs/web-server/openapi.py b/api/specs/web-server/openapi.py index 8e6b562c96d..5a679b75713 100644 --- a/api/specs/web-server/openapi.py +++ b/api/specs/web-server/openapi.py @@ -31,6 +31,7 @@ "_announcements", "_catalog", "_catalog_tags", # MUST BE after _catalog + "_catalog_licensed_items", "_computations", "_exporter", "_folders", diff --git a/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py b/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py new file mode 100644 index 00000000000..5c170588856 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py @@ -0,0 +1,22 @@ +from datetime import datetime +from typing import NamedTuple + +from models_library.licensed_items import LicensedItemID, LicensedResourceType +from models_library.resource_tracker import PricingPlanId +from pydantic import PositiveInt + +from ._base import OutputSchema + + +class LicensedItemGet(OutputSchema): + licensed_item_id: LicensedItemID + name: str + licensed_resource_type: LicensedResourceType + pricing_plan_id: PricingPlanId + created_at: datetime + modified_at: datetime + + +class LicensedItemGetPage(NamedTuple): + items: list[LicensedItemGet] + total: PositiveInt diff --git a/packages/models-library/src/models_library/licensed_items.py b/packages/models-library/src/models_library/licensed_items.py new file mode 100644 index 00000000000..021cf214ce5 --- /dev/null +++ b/packages/models-library/src/models_library/licensed_items.py @@ -0,0 +1,44 @@ +from datetime import datetime +from enum import auto +from typing import TypeAlias +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from .products import ProductName +from .resource_tracker import PricingPlanId +from .utils.enums import StrAutoEnum + +LicensedItemID: TypeAlias = UUID + + +class LicensedResourceType(StrAutoEnum): + VIP_MODEL = auto() + + +# +# DB +# + + +class LicensedItemDB(BaseModel): + licensed_item_id: LicensedItemID + name: str + licensed_resource_type: LicensedResourceType + pricing_plan_id: PricingPlanId + product_name: ProductName + created: datetime = Field( + ..., + description="Timestamp on creation", + ) + modified: datetime = Field( + ..., + description="Timestamp of last modification", + ) + # ---- + model_config = ConfigDict(from_attributes=True) + + +class LicensedItemUpdateDB(BaseModel): + name: str | None = None + pricing_plan_id: PricingPlanId | None = None diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index 645b7aae0fb..c5aa128b710 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -6,7 +6,7 @@ --requirement ../../../packages/common-library/requirements/_base.in alembic +opentelemetry-instrumentation-asyncpg pydantic sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43 -opentelemetry-instrumentation-asyncpg yarl diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index e14ea9e8ab2..2a6f133725f 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -54,7 +54,7 @@ pydantic-core==2.27.1 # via pydantic pydantic-extra-types==2.10.0 # via -r requirements/../../../packages/common-library/requirements/_base.in -setuptools==75.2.0 +setuptools==75.6.0 # via opentelemetry-instrumentation sqlalchemy==1.4.54 # via diff --git a/packages/postgres-database/requirements/_migration.txt b/packages/postgres-database/requirements/_migration.txt index a0dd4d6577f..903b84d3681 100644 --- a/packages/postgres-database/requirements/_migration.txt +++ b/packages/postgres-database/requirements/_migration.txt @@ -6,7 +6,7 @@ certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via -r requirements/_migration.in diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt index d6059bacd37..49636a365c3 100644 --- a/packages/postgres-database/requirements/_test.txt +++ b/packages/postgres-database/requirements/_test.txt @@ -6,11 +6,11 @@ async-timeout==4.0.3 # aiopg attrs==24.2.0 # via pytest-docker -coverage==7.6.1 +coverage==7.6.8 # via # -r requirements/_test.in # pytest-cov -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in greenlet==3.1.1 # via @@ -19,11 +19,11 @@ greenlet==3.1.1 # sqlalchemy iniconfig==2.0.0 # via pytest -mypy==1.12.0 +mypy==1.13.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -packaging==24.1 +packaging==24.2 # via pytest pluggy==1.5.0 # via pytest @@ -32,7 +32,7 @@ psycopg2-binary==2.9.9 # -c requirements/_base.txt # aiopg # sqlalchemy -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -43,7 +43,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-docker==3.1.1 # via -r requirements/_test.in @@ -70,14 +70,15 @@ sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy types-docker==7.1.0.20240827 # via -r requirements/_test.in -types-psycopg2==2.9.21.20240819 +types-psycopg2==2.9.21.20241019 # via -r requirements/_test.in -types-requests==2.32.0.20240914 +types-requests==2.32.0.20241016 # via types-docker typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_migration.txt + # faker # mypy # sqlalchemy2-stubs urllib3==2.2.3 diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt index 61c9a3ec7e1..98fce79f69a 100644 --- a/packages/postgres-database/requirements/_tools.txt +++ b/packages/postgres-database/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -12,13 +12,13 @@ click==8.1.7 # via # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -26,7 +26,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via # -c requirements/_test.txt # -r requirements/../../../requirements/devenv.txt @@ -37,14 +37,14 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.1 +packaging==24.2 # via # -c requirements/_test.txt # black # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -53,11 +53,11 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -66,9 +66,9 @@ pyyaml==6.0.2 # -c requirements/../../../requirements/constraints.txt # -c requirements/_test.txt # pre-commit -ruff==0.6.7 +ruff==0.8.1 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.2.0 +setuptools==75.6.0 # via # -c requirements/_base.txt # pip-tools @@ -79,7 +79,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py new file mode 100644 index 00000000000..abdfafefdf7 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py @@ -0,0 +1,171 @@ +"""add license db tables + +Revision ID: e5555076ef50 +Revises: e05bdc5b3c7b +Create Date: 2024-12-05 10:57:16.867891+00:00 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "e5555076ef50" +down_revision = "e05bdc5b3c7b" +branch_labels = None +depends_on = None + + +def upgrade(): + # CREATE EXTENSION pgcrypto; + op.execute( + """ + DO + $$ + BEGIN + IF EXISTS(SELECT * FROM pg_available_extensions WHERE name = 'pgcrypto') THEN + -- Create the extension + CREATE EXTENSION if not exists pgcrypto; + END IF; + END + $$; + """ + ) + + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "resource_tracker_licensed_items_purchases", + sa.Column( + "licensed_item_purchase_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("licensed_item_id", sa.BigInteger(), nullable=False), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column( + "start_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "expire_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("purchased_by_user", sa.BigInteger(), nullable=False), + sa.Column( + "purchased_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("licensed_item_purchase_id"), + ) + op.create_table( + "resource_tracker_licensed_items_usage", + sa.Column( + "licensed_item_usage_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("licensed_item_id", sa.String(), nullable=True), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("user_email", sa.String(), nullable=True), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("service_run_id", sa.String(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("num_of_seats", sa.SmallInteger(), nullable=False), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["product_name", "service_run_id"], + [ + "resource_tracker_service_runs.product_name", + "resource_tracker_service_runs.service_run_id", + ], + name="resource_tracker_license_checkouts_service_run_id_fkey", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("licensed_item_usage_id"), + ) + op.create_index( + op.f("ix_resource_tracker_licensed_items_usage_wallet_id"), + "resource_tracker_licensed_items_usage", + ["wallet_id"], + unique=False, + ) + op.create_table( + "licensed_items", + sa.Column( + "licensed_item_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "licensed_resource_type", + sa.Enum("VIP_MODEL", name="licensedresourcetype"), + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("licensed_item_id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("licensed_items") + op.drop_index( + op.f("ix_resource_tracker_licensed_items_usage_wallet_id"), + table_name="resource_tracker_licensed_items_usage", + ) + op.drop_table("resource_tracker_licensed_items_usage") + op.drop_table("resource_tracker_licensed_items_purchases") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py new file mode 100644 index 00000000000..63301eb9c1d --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -0,0 +1,63 @@ +""" resource_tracker_service_runs table +""" + +import enum + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions, column_created_datetime, column_modified_datetime +from .base import metadata + + +class LicensedResourceType(str, enum.Enum): + VIP_MODEL = "VIP_MODEL" + + +licensed_items = sa.Table( + "licensed_items", + metadata, + sa.Column( + "licensed_item_id", + UUID(as_uuid=True), + nullable=False, + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column( + "name", + sa.String, + nullable=False, + ), + sa.Column( + "licensed_resource_type", + sa.Enum(LicensedResourceType), + nullable=False, + doc="Item type, ex. VIP_MODEL", + ), + sa.Column( + "pricing_plan_id", + sa.BigInteger, + sa.ForeignKey( + "resource_tracker_pricing_plans.pricing_plan_id", + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), + nullable=False, + ), + sa.Column( + "product_name", + sa.String, + sa.ForeignKey( + "products.name", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_resource_tracker_license_packages_product_name", + ), + nullable=False, + doc="Product name", + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py new file mode 100644 index 00000000000..2a13e3d718e --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py @@ -0,0 +1,61 @@ +""" resource_tracker_service_runs table +""" + + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import column_modified_datetime +from .base import metadata + +resource_tracker_licensed_items_purchases = sa.Table( + "resource_tracker_licensed_items_purchases", + metadata, + sa.Column( + "licensed_item_purchase_id", + UUID(as_uuid=True), + nullable=False, + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column( + "product_name", + sa.String, + nullable=False, + doc="Product name", + ), + sa.Column( + "licensed_item_id", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "wallet_id", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "start_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.sql.func.now(), + ), + sa.Column( + "expire_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.sql.func.now(), + ), + sa.Column( + "purchased_by_user", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "purchased_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.sql.func.now(), + ), + column_modified_datetime(timezone=True), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py new file mode 100644 index 00000000000..27d6afe8250 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py @@ -0,0 +1,76 @@ +""" resource_tracker_service_runs table +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions, column_modified_datetime +from .base import metadata + +resource_tracker_licensed_items_usage = sa.Table( + "resource_tracker_licensed_items_usage", + metadata, + sa.Column( + "licensed_item_usage_id", + UUID(as_uuid=True), + nullable=False, + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column( + "licensed_item_id", + sa.String, + nullable=True, + ), + sa.Column( + "wallet_id", + sa.BigInteger, + nullable=False, + index=True, + ), + sa.Column( + "user_id", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "user_email", + sa.String, + nullable=True, + ), + sa.Column("product_name", sa.String, nullable=False, doc="Product name"), + sa.Column( + "service_run_id", + sa.String, + nullable=True, + ), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + doc="Timestamp when the service was started", + ), + sa.Column( + "stopped_at", + sa.DateTime(timezone=True), + nullable=True, + doc="Timestamp when the service was stopped", + ), + sa.Column( + "num_of_seats", + sa.SmallInteger, + nullable=False, + ), + column_modified_datetime(timezone=True), + # --------------------------- + sa.ForeignKeyConstraint( + ["product_name", "service_run_id"], + [ + "resource_tracker_service_runs.product_name", + "resource_tracker_service_runs.service_run_id", + ], + name="resource_tracker_license_checkouts_service_run_id_fkey", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_repos.py b/packages/postgres-database/src/simcore_postgres_database/utils_repos.py index f2c96313ea9..e013a09b526 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_repos.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_repos.py @@ -11,6 +11,10 @@ async def pass_or_acquire_connection( engine: AsyncEngine, connection: AsyncConnection | None = None ) -> AsyncIterator[AsyncConnection]: + """ + When to use: For READ operations! + It ensures that a connection is available for use within the context, either by using an existing connection passed as a parameter or by acquiring a new one from the engine. The caller must manage the lifecycle of any connection explicitly passed in, but the function handles the cleanup for connections it creates itself. This function **does not open new transactions** and therefore is recommended only for read-only database operations. + """ # NOTE: When connection is passed, the engine is actually not needed # NOTE: Creator is responsible of closing connection is_connection_created = connection is None @@ -30,6 +34,10 @@ async def pass_or_acquire_connection( async def transaction_context( engine: AsyncEngine, connection: AsyncConnection | None = None ): + """ + When to use: For WRITE operations! + This function manages the database connection and ensures that a transaction context is established for write operations. It supports both outer and nested transactions, providing flexibility for scenarios where transactions may already exist in the calling context. + """ async with pass_or_acquire_connection(engine, connection) as conn: if conn.in_transaction(): async with conn.begin_nested(): # inner transaction (savepoint) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 29338d83996..829aa2be9c2 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2347,6 +2347,108 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_CatalogServiceGet_' + /v0/catalog/licensed-items: + get: + tags: + - licenses + - catalog + summary: List Licensed Items + operationId: list_licensed_items + parameters: + - name: order_by + in: query + required: false + schema: + type: string + contentMediaType: application/json + contentSchema: {} + default: '{"field":"modified","direction":"desc"}' + title: Order By + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_list_LicensedItemGet__' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + /v0/catalog/licensed-items/{licensed_item_id}: + get: + tags: + - licenses + - catalog + summary: Get Licensed Item + operationId: get_licensed_item + parameters: + - name: licensed_item_id + in: path + required: true + schema: + type: string + format: uuid + title: Licensed Item Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_LicensedItemGet_' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + /v0/catalog/licensed-items/{licensed_item_id}:purchase: + post: + tags: + - licenses + - catalog + summary: Purchase Licensed Item + operationId: purchase_licensed_item + parameters: + - name: licensed_item_id + in: path + required: true + schema: + type: string + format: uuid + title: Licensed Item Id + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LicensedItemsBodyParams' + responses: + '204': + description: Successful Response + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found /v0/computations/{project_id}: get: tags: @@ -7710,6 +7812,19 @@ components: title: Error type: object title: Envelope[InvitationInfo] + Envelope_LicensedItemGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/LicensedItemGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[LicensedItemGet] Envelope_Log_: properties: data: @@ -8483,6 +8598,22 @@ components: title: Error type: object title: Envelope[list[GroupUserGet]] + Envelope_list_LicensedItemGet__: + properties: + data: + anyOf: + - items: + $ref: '#/components/schemas/LicensedItemGet' + type: array + - type: 'null' + title: Data + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[list[LicensedItemGet]] Envelope_list_OsparcCreditsAggregatedByServiceGet__: properties: data: @@ -9878,6 +10009,61 @@ components: additionalProperties: false type: object title: InvitationInfo + LicensedItemGet: + properties: + licensedItemId: + type: string + format: uuid + title: Licenseditemid + name: + type: string + title: Name + licensedResourceType: + $ref: '#/components/schemas/LicensedResourceType' + pricingPlanId: + type: integer + exclusiveMinimum: true + title: Pricingplanid + minimum: 0 + createdAt: + type: string + format: date-time + title: Createdat + modifiedAt: + type: string + format: date-time + title: Modifiedat + type: object + required: + - licensedItemId + - name + - licensedResourceType + - pricingPlanId + - createdAt + - modifiedAt + title: LicensedItemGet + LicensedItemsBodyParams: + properties: + wallet_id: + type: integer + exclusiveMinimum: true + title: Wallet Id + minimum: 0 + num_of_seeds: + type: integer + title: Num Of Seeds + additionalProperties: false + type: object + required: + - wallet_id + - num_of_seeds + title: LicensedItemsBodyParams + LicensedResourceType: + type: string + enum: + - VIP_MODEL + const: VIP_MODEL + title: LicensedResourceType Limits: properties: cpus: @@ -14034,9 +14220,7 @@ components: required: - walletId - name - - description - owner - - thumbnail - status - created - modified @@ -14085,9 +14269,7 @@ components: required: - walletId - name - - description - owner - - thumbnail - status - created - modified diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index ed4e519141b..b15cd73e1c8 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -271,6 +271,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_DB_LISTENER: bool = True WEBSERVER_FOLDERS: bool = True WEBSERVER_GROUPS: bool = True + WEBSERVER_LICENSES: bool = True WEBSERVER_META_MODELING: bool = True WEBSERVER_NOTIFICATIONS: bool = Field(default=True) WEBSERVER_PRODUCTS: bool = True diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/__init__.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py new file mode 100644 index 00000000000..0abb7671b16 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py @@ -0,0 +1,26 @@ +import logging + +from servicelib.aiohttp import status + +from ...exception_handling import ( + ExceptionToHttpErrorMap, + HttpErrorInfo, + exception_handling_decorator, + to_exceptions_handlers_map, +) +from .errors import LicensedItemNotFoundError + +_logger = logging.getLogger(__name__) + + +_TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { + LicensedItemNotFoundError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, + "Market item {licensed_item_id} not found.", + ) +} + + +handle_plugin_requests_exceptions = exception_handling_decorator( + to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP) +) diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py new file mode 100644 index 00000000000..bb024b0423b --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py @@ -0,0 +1,77 @@ +# pylint: disable=unused-argument + +import logging + +from aiohttp import web +from models_library.api_schemas_webserver.licensed_items import ( + LicensedItemGet, + LicensedItemGetPage, +) +from models_library.licensed_items import LicensedItemID +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy +from models_library.users import UserID +from pydantic import NonNegativeInt + +from . import _licensed_items_db +from ._models import LicensedItemsBodyParams + +_logger = logging.getLogger(__name__) + + +async def get_licensed_item( + app: web.Application, + *, + licensed_item_id: LicensedItemID, + product_name: ProductName, +) -> LicensedItemGet: + + licensed_item_db = await _licensed_items_db.get( + app, licensed_item_id=licensed_item_id, product_name=product_name + ) + return LicensedItemGet( + licensed_item_id=licensed_item_db.licensed_item_id, + name=licensed_item_db.name, + licensed_resource_type=licensed_item_db.licensed_resource_type, + pricing_plan_id=licensed_item_db.pricing_plan_id, + created_at=licensed_item_db.created, + modified_at=licensed_item_db.modified, + ) + + +async def list_licensed_items( + app: web.Application, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: int, + order_by: OrderBy, +) -> LicensedItemGetPage: + total_count, licensed_item_db_list = await _licensed_items_db.list_( + app, product_name=product_name, offset=offset, limit=limit, order_by=order_by + ) + return LicensedItemGetPage( + items=[ + LicensedItemGet( + licensed_item_id=licensed_item_db.licensed_item_id, + name=licensed_item_db.name, + licensed_resource_type=licensed_item_db.licensed_resource_type, + pricing_plan_id=licensed_item_db.pricing_plan_id, + created_at=licensed_item_db.created, + modified_at=licensed_item_db.modified, + ) + for licensed_item_db in licensed_item_db_list + ], + total=total_count, + ) + + +async def purchase_licensed_item( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + licensed_item_id: LicensedItemID, + body_params: LicensedItemsBodyParams, +) -> None: + raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py new file mode 100644 index 00000000000..fc14221ff91 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py @@ -0,0 +1,181 @@ +""" Database API + + - Adds a layer to the postgres API with a focus on the projects comments + +""" + +import logging +from typing import cast + +from aiohttp import web +from models_library.licensed_items import ( + LicensedItemDB, + LicensedItemID, + LicensedItemUpdateDB, + LicensedResourceType, +) +from models_library.products import ProductName +from models_library.resource_tracker import PricingPlanId +from models_library.rest_ordering import OrderBy, OrderDirection +from pydantic import NonNegativeInt +from simcore_postgres_database.models.licensed_items import licensed_items +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy import asc, desc, func +from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.sql import select + +from ...db.plugin import get_asyncpg_engine +from .errors import LicensedItemNotFoundError + +_logger = logging.getLogger(__name__) + + +_SELECTION_ARGS = ( + licensed_items.c.licensed_item_id, + licensed_items.c.name, + licensed_items.c.licensed_resource_type, + licensed_items.c.pricing_plan_id, + licensed_items.c.product_name, + licensed_items.c.created, + licensed_items.c.modified, +) + +assert set(LicensedItemDB.model_fields) == {c.name for c in _SELECTION_ARGS} # nosec + + +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + name: str, + licensed_resource_type: LicensedResourceType, + pricing_plan_id: PricingPlanId, +) -> LicensedItemDB: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + licensed_items.insert() + .values( + name=name, + licensed_resource_type=licensed_resource_type, + pricing_plan_id=pricing_plan_id, + product_name=product_name, + created=func.now(), + modified=func.now(), + ) + .returning(*_SELECTION_ARGS) + ) + row = await result.first() + return LicensedItemDB.model_validate(row) + + +async def list_( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: NonNegativeInt, + order_by: OrderBy, +) -> tuple[int, list[LicensedItemDB]]: + base_query = ( + select(*_SELECTION_ARGS) + .select_from(licensed_items) + .where(licensed_items.c.product_name == product_name) + ) + + # Select total count from base_query + subquery = base_query.subquery() + count_query = select(func.count()).select_from(subquery) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by(asc(getattr(licensed_items.c, order_by.field))) + else: + list_query = base_query.order_by( + desc(getattr(licensed_items.c, order_by.field)) + ) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + items: list[LicensedItemDB] = [ + LicensedItemDB.model_validate(row) async for row in result + ] + + return cast(int, total_count), items + + +async def get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_item_id: LicensedItemID, + product_name: ProductName, +) -> LicensedItemDB: + base_query = ( + select(*_SELECTION_ARGS) + .select_from(licensed_items) + .where( + (licensed_items.c.licensed_item_id == licensed_item_id) + & (licensed_items.c.product_name == product_name) + ) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(base_query) + row = await result.first() + if row is None: + raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) + return LicensedItemDB.model_validate(row) + + +async def update( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + licensed_item_id: LicensedItemID, + updates: LicensedItemUpdateDB, +) -> LicensedItemDB: + # NOTE: at least 'touch' if updated_values is empty + _updates = { + **updates.dict(exclude_unset=True), + "modified": func.now(), + } + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + licensed_items.update() + .values(**_updates) + .where( + (licensed_items.c.licensed_item_id == licensed_item_id) + & (licensed_items.c.product_name == product_name) + ) + .returning(*_SELECTION_ARGS) + ) + row = await result.first() + if row is None: + raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) + return LicensedItemDB.model_validate(row) + + +async def delete( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_item_id: LicensedItemID, + product_name: ProductName, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + licensed_items.delete().where( + (licensed_items.c.licensed_item_id == licensed_item_id) + & (licensed_items.c.product_name == product_name) + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py new file mode 100644 index 00000000000..6ed227500e5 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py @@ -0,0 +1,112 @@ +import logging + +from aiohttp import web +from models_library.api_schemas_webserver.licensed_items import ( + LicensedItemGet, + LicensedItemGetPage, +) +from models_library.rest_ordering import OrderBy +from models_library.rest_pagination import Page +from models_library.rest_pagination_utils import paginate_data +from servicelib.aiohttp import status +from servicelib.aiohttp.requests_validation import ( + parse_request_body_as, + parse_request_path_parameters_as, + parse_request_query_parameters_as, +) +from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON +from servicelib.rest_constants import RESPONSE_MODEL_POLICY + +from ..._meta import API_VTAG as VTAG +from ...login.decorators import login_required +from ...security.decorators import permission_required +from ...utils_aiohttp import envelope_json_response +from . import _licensed_items_api +from ._exceptions_handlers import handle_plugin_requests_exceptions +from ._models import ( + LicensedItemsBodyParams, + LicensedItemsListQueryParams, + LicensedItemsPathParams, + LicensedItemsRequestContext, +) + +_logger = logging.getLogger(__name__) + + +routes = web.RouteTableDef() + + +@routes.get(f"/{VTAG}/catalog/licensed-items", name="list_licensed_items") +@login_required +@permission_required("catalog/licensed-items.*") +@handle_plugin_requests_exceptions +async def list_workspaces(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + query_params: LicensedItemsListQueryParams = parse_request_query_parameters_as( + LicensedItemsListQueryParams, request + ) + + licensed_item_get_page: LicensedItemGetPage = ( + await _licensed_items_api.list_licensed_items( + app=request.app, + product_name=req_ctx.product_name, + offset=query_params.offset, + limit=query_params.limit, + order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), + ) + ) + + page = Page[LicensedItemGet].model_validate( + paginate_data( + chunk=licensed_item_get_page.items, + request_url=request.url, + total=licensed_item_get_page.total, + limit=query_params.limit, + offset=query_params.offset, + ) + ) + return web.Response( + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), + content_type=MIMETYPE_APPLICATION_JSON, + ) + + +@routes.get( + f"/{VTAG}/catalog/licensed-items/{{licensed_item_id}}", name="get_licensed_item" +) +@login_required +@permission_required("catalog/licensed-items.*") +@handle_plugin_requests_exceptions +async def get_workspace(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(LicensedItemsPathParams, request) + + licensed_item_get: LicensedItemGet = await _licensed_items_api.get_licensed_item( + app=request.app, + licensed_item_id=path_params.licensed_item_id, + product_name=req_ctx.product_name, + ) + + return envelope_json_response(licensed_item_get) + + +@routes.post( + f"/{VTAG}/catalog/licensed-items/{{licensed_item_id}}:purchase", + name="purchase_licensed_item", +) +@login_required +@permission_required("catalog/licensed-items.*") +@handle_plugin_requests_exceptions +async def purchase_licensed_item(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(LicensedItemsPathParams, request) + body_params = await parse_request_body_as(LicensedItemsBodyParams, request) + + await _licensed_items_api.purchase_licensed_item( + app=request.app, + user_id=req_ctx.user_id, + licensed_item_id=path_params.licensed_item_id, + product_name=req_ctx.product_name, + body_params=body_params, + ) + return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py new file mode 100644 index 00000000000..40d287faa92 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py @@ -0,0 +1,54 @@ +import logging + +from models_library.basic_types import IDStr +from models_library.licensed_items import LicensedItemID +from models_library.rest_base import RequestParameters, StrictRequestParameters +from models_library.rest_ordering import ( + OrderBy, + OrderDirection, + create_ordering_query_model_class, +) +from models_library.rest_pagination import PageQueryParameters +from models_library.users import UserID +from models_library.wallets import WalletID +from pydantic import BaseModel, ConfigDict, Field +from servicelib.request_keys import RQT_USERID_KEY + +from ..._constants import RQ_PRODUCT_KEY + +_logger = logging.getLogger(__name__) + + +class LicensedItemsRequestContext(RequestParameters): + user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[literal-required] + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[literal-required] + + +class LicensedItemsPathParams(StrictRequestParameters): + licensed_item_id: LicensedItemID + + +_LicensedItemsListOrderQueryParams: type[ + RequestParameters +] = create_ordering_query_model_class( + ordering_fields={ + "modified_at", + "name", + }, + default=OrderBy(field=IDStr("modified_at"), direction=OrderDirection.DESC), + ordering_fields_api_to_column_map={"modified_at": "modified"}, +) + + +class LicensedItemsListQueryParams( + PageQueryParameters, + _LicensedItemsListOrderQueryParams, # type: ignore[misc, valid-type] +): + ... + + +class LicensedItemsBodyParams(BaseModel): + wallet_id: WalletID + num_of_seeds: int + + model_config = ConfigDict(extra="forbid") diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/api.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/api.py new file mode 100644 index 00000000000..07b8034ea85 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/api.py @@ -0,0 +1,3 @@ +# mypy: disable-error-code=truthy-function + +__all__: tuple[str, ...] = () diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py new file mode 100644 index 00000000000..0c8bae69b03 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py @@ -0,0 +1,9 @@ +from ...errors import WebServerBaseError + + +class LicensesValueError(WebServerBaseError, ValueError): + ... + + +class LicensedItemNotFoundError(LicensesValueError): + msg_template = "License good {licensed_item_id} not found" diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py new file mode 100644 index 00000000000..ef124c69fad --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py @@ -0,0 +1,26 @@ +""" tags management subsystem + +""" +import logging + +from aiohttp import web +from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY +from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup + +from . import _licensed_items_handlers + +_logger = logging.getLogger(__name__) + + +@app_module_setup( + __name__, + ModuleCategory.ADDON, + settings_name="WEBSERVER_LICENSES", + depends=["simcore_service_webserver.rest"], + logger=_logger, +) +def setup_licenses(app: web.Application): + assert app[APP_SETTINGS_KEY].WEBSERVER_LICENSES # nosec + + # routes + app.router.add_routes(_licensed_items_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index 2af8da917f0..74c36bcbcb4 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -9,6 +9,7 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from . import _handlers, _tags_handlers +from .licenses.plugin import setup_licenses _logger = logging.getLogger(__name__) @@ -27,6 +28,8 @@ def setup_catalog(app: web.Application): for route_def in _handlers.routes ) + setup_licenses(app) + app.add_routes(_handlers.routes) app.add_routes(_tags_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py index 919486962d3..da342e1b996 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py @@ -59,6 +59,7 @@ class PermissionDict(TypedDict, total=False): "folder.delete", "folder.access_rights.update", "groups.*", + "catalog/licensed-items.*", "product.price.read", "project.folders.*", "project.access_rights.update", diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py new file mode 100644 index 00000000000..5971ed9f168 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -0,0 +1,44 @@ +from collections.abc import AsyncIterator + +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +import pytest +from aiohttp.test_utils import TestClient +from simcore_postgres_database.models.licensed_items import licensed_items +from simcore_postgres_database.models.resource_tracker_pricing_plans import ( + resource_tracker_pricing_plans, +) +from simcore_postgres_database.utils_repos import transaction_context +from simcore_service_webserver.db.plugin import get_asyncpg_engine + + +@pytest.fixture +async def pricing_plan_id( + client: TestClient, + osparc_product_name: str, +) -> AsyncIterator[int]: + assert client.app + + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + result = await conn.execute( + resource_tracker_pricing_plans.insert() + .values( + product_name=osparc_product_name, + display_name="ISolve Thermal", + description="", + classification="TIER", + is_active=True, + pricing_plan_key="isolve-thermal", + ) + .returning(resource_tracker_pricing_plans.c.pricing_plan_id) + ) + row = result.first() + + assert row + + yield int(row[0]) + + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + await conn.execute(licensed_items.delete()) + await conn.execute(resource_tracker_pricing_plans.delete()) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py new file mode 100644 index 00000000000..5455c280cd7 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py @@ -0,0 +1,94 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements +from http import HTTPStatus + +import pytest +from aiohttp.test_utils import TestClient +from models_library.licensed_items import ( + LicensedItemDB, + LicensedItemUpdateDB, + LicensedResourceType, +) +from models_library.rest_ordering import OrderBy +from pytest_simcore.helpers.webserver_login import UserInfoDict +from servicelib.aiohttp import status +from simcore_service_webserver.catalog.licenses import _licensed_items_db +from simcore_service_webserver.catalog.licenses.errors import LicensedItemNotFoundError +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test_licensed_items_db_crud( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: str, + expected: HTTPStatus, + pricing_plan_id: int, +): + assert client.app + + output: tuple[int, list[LicensedItemDB]] = await _licensed_items_db.list_( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="modified"), + ) + assert output[0] == 0 + + licensed_item_db = await _licensed_items_db.create( + client.app, + product_name=osparc_product_name, + name="Model A", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + pricing_plan_id=pricing_plan_id, + ) + _licensed_item_id = licensed_item_db.licensed_item_id + + output: tuple[int, list[LicensedItemDB]] = await _licensed_items_db.list_( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="modified"), + ) + assert output[0] == 1 + + licensed_item_db = await _licensed_items_db.get( + client.app, + licensed_item_id=_licensed_item_id, + product_name=osparc_product_name, + ) + assert licensed_item_db.name == "Model A" + + await _licensed_items_db.update( + client.app, + licensed_item_id=_licensed_item_id, + product_name=osparc_product_name, + updates=LicensedItemUpdateDB(name="Model B"), + ) + + licensed_item_db = await _licensed_items_db.get( + client.app, + licensed_item_id=_licensed_item_id, + product_name=osparc_product_name, + ) + assert licensed_item_db.name == "Model B" + + licensed_item_db = await _licensed_items_db.delete( + client.app, + licensed_item_id=_licensed_item_id, + product_name=osparc_product_name, + ) + + with pytest.raises(LicensedItemNotFoundError): + await _licensed_items_db.get( + client.app, + licensed_item_id=_licensed_item_id, + product_name=osparc_product_name, + ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py new file mode 100644 index 00000000000..eb63d9bb75a --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py @@ -0,0 +1,66 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements +from http import HTTPStatus + +import pytest +from aiohttp.test_utils import TestClient +from models_library.api_schemas_webserver.licensed_items import LicensedItemGet +from models_library.licensed_items import LicensedResourceType +from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.webserver_login import UserInfoDict +from servicelib.aiohttp import status +from simcore_service_webserver.catalog.licenses import _licensed_items_db +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test_licensed_items_db_crud( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: str, + expected: HTTPStatus, + pricing_plan_id: int, +): + assert client.app + + # list + url = client.app.router["list_licensed_items"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert data == [] + + licensed_item_db = await _licensed_items_db.create( + client.app, + product_name=osparc_product_name, + name="Model A", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + pricing_plan_id=pricing_plan_id, + ) + _licensed_item_id = licensed_item_db.licensed_item_id + + # list + url = client.app.router["list_licensed_items"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + assert LicensedItemGet(**data[0]) + + # get + url = client.app.router["get_licensed_item"].url_for( + licensed_item_id=f"{_licensed_item_id}" + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert LicensedItemGet(**data) + + # purchase + url = client.app.router["purchase_licensed_item"].url_for( + licensed_item_id=f"{_licensed_item_id}" + ) + resp = await client.post(f"{url}", json={"wallet_id": 1, "num_of_seeds": 5}) + # NOTE: Not yet implemented diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index fd400882d27..ddd2910ff59 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -120,7 +120,7 @@ certifi==2024.8.30 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # requests From 1f9c110d66f6b79b1080236e616ee28869f42910 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:12:49 +0100 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=8E=A8=20Tracing:=20instrument=20logs?= =?UTF-8?q?=20and=20ensure=20traces=20are=20propagated=20through=20`asynci?= =?UTF-8?q?o.Queue`s=20(#6857)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/aws-library/requirements/_base.txt | 4 +++ .../service-library/requirements/_base.in | 1 + .../service-library/requirements/_base.txt | 4 +++ .../src/servicelib/async_utils.py | 24 +++++++------ .../src/servicelib/fastapi/tracing.py | 1 - .../src/servicelib/logging_utils.py | 26 +++++++++++++- .../service-library/src/servicelib/tracing.py | 35 +++++++++++++++++++ packages/simcore-sdk/requirements/_base.txt | 4 +++ services/agent/requirements/_base.txt | 6 +++- .../simcore_service_agent/core/application.py | 1 + services/api-server/requirements/_base.txt | 6 ++++ .../core/application.py | 1 + services/autoscaling/requirements/_base.txt | 6 ++++ .../src/simcore_service_autoscaling/main.py | 1 + services/catalog/requirements/_base.txt | 4 +++ .../src/simcore_service_catalog/main.py | 1 + .../clusters-keeper/requirements/_base.txt | 6 ++++ .../simcore_service_clusters_keeper/main.py | 1 + services/dask-sidecar/requirements/_base.txt | 4 +++ .../src/simcore_service_dask_sidecar/tasks.py | 1 + .../datcore-adapter/requirements/_base.txt | 4 +++ .../core/application.py | 1 + services/director-v2/requirements/_base.txt | 6 ++++ .../core/application.py | 1 + services/director/requirements/_base.txt | 4 +++ .../src/simcore_service_director/main.py | 1 + .../dynamic-scheduler/requirements/_base.txt | 6 +++- .../simcore_service_dynamic_scheduler/main.py | 1 + .../dynamic-sidecar/requirements/_base.txt | 8 ++++- .../core/application.py | 1 + services/efs-guardian/requirements/_base.txt | 6 ++++ .../src/simcore_service_efs_guardian/main.py | 1 + services/invitations/requirements/_base.txt | 6 +++- .../src/simcore_service_invitations/main.py | 1 + services/payments/requirements/_base.txt | 6 +++- .../src/simcore_service_payments/main.py | 1 + .../requirements/_base.txt | 6 ++++ .../main.py | 1 + services/storage/requirements/_base.txt | 6 ++++ .../src/simcore_service_storage/cli.py | 1 + services/web/server/requirements/_base.txt | 6 ++++ .../src/simcore_service_webserver/cli.py | 1 + .../src/simcore_service_webserver/log.py | 3 ++ .../with_dbs/03/meta_modeling/conftest.py | 5 ++- .../with_dbs/03/version_control/conftest.py | 5 ++- .../04/studies_dispatcher/conftest.py | 5 ++- tests/swarm-deploy/requirements/_test.txt | 6 ++++ 47 files changed, 216 insertions(+), 20 deletions(-) create mode 100644 packages/service-library/src/servicelib/tracing.py diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index f4ed9a597c8..ab1ba69c3e5 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -142,6 +142,7 @@ opentelemetry-api==1.27.0 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-botocore + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray @@ -160,10 +161,13 @@ opentelemetry-exporter-otlp-proto-http==1.27.0 opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-botocore + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-botocore==0.48b0 # via -r requirements/_base.in +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in index 5a51d883b74..7ac1ce0da04 100644 --- a/packages/service-library/requirements/_base.in +++ b/packages/service-library/requirements/_base.in @@ -20,6 +20,7 @@ opentelemetry-api opentelemetry-exporter-otlp opentelemetry-instrumentation-requests opentelemetry-instrumentation-redis +opentelemetry-instrumentation-logging opentelemetry-sdk psutil pydantic diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt index e65bfe29d46..9c66cf51cab 100644 --- a/packages/service-library/requirements/_base.txt +++ b/packages/service-library/requirements/_base.txt @@ -101,6 +101,7 @@ opentelemetry-api==1.27.0 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -117,8 +118,11 @@ opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp opentelemetry-instrumentation==0.48b0 # via + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 diff --git a/packages/service-library/src/servicelib/async_utils.py b/packages/service-library/src/servicelib/async_utils.py index 42ba951aafb..0b1a6b27ca5 100644 --- a/packages/service-library/src/servicelib/async_utils.py +++ b/packages/service-library/src/servicelib/async_utils.py @@ -6,6 +6,7 @@ from functools import wraps from typing import TYPE_CHECKING, Any, Awaitable, Callable, Deque +from . import tracing from .utils_profiling_middleware import dont_profile, is_profiling, profile_context logger = logging.getLogger(__name__) @@ -34,6 +35,7 @@ class Context: @dataclass class QueueElement: + tracing_context: tracing.TracingContext do_profile: bool = False input: Awaitable | None = None output: Any | None = None @@ -162,16 +164,17 @@ async def worker(in_q: Queue[QueueElement], out_q: Queue) -> None: while True: element = await in_q.get() in_q.task_done() - # check if requested to shutdown - try: - do_profile = element.do_profile - awaitable = element.input - if awaitable is None: - break - with profile_context(do_profile): - result = await awaitable - except Exception as e: # pylint: disable=broad-except - result = e + with tracing.use_tracing_context(element.tracing_context): + # check if requested to shutdown + try: + do_profile = element.do_profile + awaitable = element.input + if awaitable is None: + break + with profile_context(do_profile): + result = await awaitable + except Exception as e: # pylint: disable=broad-except + result = e await out_q.put(result) logging.info( @@ -189,6 +192,7 @@ async def worker(in_q: Queue[QueueElement], out_q: Queue) -> None: queue_input = QueueElement( input=decorated_function(*args, **kwargs), do_profile=is_profiling(), + tracing_context=tracing.get_context(), ) await context.in_queue.put(queue_input) wrapped_result = await context.out_queue.get() diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index ee0258d93f4..e2cce12c319 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -70,7 +70,6 @@ def setup_tracing( ): _logger.warning("Skipping opentelemetry tracing setup") return - # Set up the tracer provider resource = Resource(attributes={"service.name": service_name}) trace.set_tracer_provider(TracerProvider(resource=resource)) diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 1ee9494a84b..0db4b2febd3 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -16,7 +16,10 @@ from pathlib import Path from typing import Any, NotRequired, TypeAlias, TypedDict, TypeVar +from settings_library.tracing import TracingSettings + from .logging_utils_filtering import GeneralLogFilter, LoggerName, MessageSubstring +from .tracing import setup_log_tracing from .utils_secrets import mask_sensitive_data _logger = logging.getLogger(__name__) @@ -121,11 +124,11 @@ def config_all_loggers( *, log_format_local_dev_enabled: bool, logger_filter_mapping: dict[LoggerName, list[MessageSubstring]], + tracing_settings: TracingSettings | None, ) -> None: """ Applies common configuration to ALL registered loggers """ - fmt = DEFAULT_FORMATTING the_manager: logging.Manager = logging.Logger.manager root_logger = logging.getLogger() @@ -133,8 +136,29 @@ def config_all_loggers( logging.getLogger(name) for name in the_manager.loggerDict ] + fmt = DEFAULT_FORMATTING + if tracing_settings is not None: + fmt = ( + "log_level=%(levelname)s " + "| log_timestamp=%(asctime)s " + "| log_source=%(name)s:%(funcName)s(%(lineno)d) " + "| log_uid=%(log_uid)s " + "| log_oec=%(log_oec)s" + "| log_trace_id=%(otelTraceID)s " + "| log_span_id=%(otelSpanID)s " + "| log_resource.service.name=%(otelServiceName)s " + "| log_trace_sampled=%(otelTraceSampled)s] " + "| log_msg=%(message)s" + ) + setup_log_tracing(tracing_settings=tracing_settings) if log_format_local_dev_enabled: fmt = LOCAL_FORMATTING + if tracing_settings is not None: + fmt = ( + "%(levelname)s: [%(asctime)s/%(processName)s] " + "[log_trace_id=%(otelTraceID)s log_span_id=%(otelSpanID)s log_resource.service.name=%(otelServiceName)s log_trace_sampled=%(otelTraceSampled)s] " + "[%(name)s:%(funcName)s(%(lineno)d)] - %(message)s" + ) for logger in loggers: _set_logging_handler( diff --git a/packages/service-library/src/servicelib/tracing.py b/packages/service-library/src/servicelib/tracing.py new file mode 100644 index 00000000000..6635e824a46 --- /dev/null +++ b/packages/service-library/src/servicelib/tracing.py @@ -0,0 +1,35 @@ +from contextlib import contextmanager +from typing import TypeAlias + +from opentelemetry import context as otcontext +from opentelemetry import trace +from opentelemetry.instrumentation.logging import LoggingInstrumentor +from settings_library.tracing import TracingSettings + +TracingContext: TypeAlias = otcontext.Context | None + + +def _is_tracing() -> bool: + return trace.get_current_span().is_recording() + + +def get_context() -> TracingContext: + if not _is_tracing(): + return None + return otcontext.get_current() + + +@contextmanager +def use_tracing_context(context: TracingContext): + if context is not None: + otcontext.attach(context) + try: + yield + finally: + if context is not None: + otcontext.detach(context) + + +def setup_log_tracing(tracing_settings: TracingSettings): + _ = tracing_settings + LoggingInstrumentor().instrument(set_logging_format=False) diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 541037d54fe..b00b9f66957 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -165,6 +165,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -184,6 +185,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiopg==0.48b0 @@ -192,6 +194,8 @@ opentelemetry-instrumentation-asyncpg==0.48b0 # via -r requirements/../../../packages/postgres-database/requirements/_base.in opentelemetry-instrumentation-dbapi==0.48b0 # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 4fddb46bb50..95f8e56d4cb 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -157,6 +157,7 @@ opentelemetry-api==1.28.2 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -176,6 +177,7 @@ opentelemetry-instrumentation==0.49b2 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.49b2 @@ -184,7 +186,9 @@ opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-redis==0.49b2 +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/agent/src/simcore_service_agent/core/application.py b/services/agent/src/simcore_service_agent/core/application.py index 42cc067a211..fe226a33558 100644 --- a/services/agent/src/simcore_service_agent/core/application.py +++ b/services/agent/src/simcore_service_agent/core/application.py @@ -33,6 +33,7 @@ def _setup_logger(settings: ApplicationSettings): config_all_loggers( log_format_local_dev_enabled=settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.AGENT_VOLUMES_LOG_FILTER_MAPPING, + tracing_settings=settings.AGENT_TRACING, ) diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index bb064316789..48d12d8b832 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -357,6 +357,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -381,6 +382,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiopg==0.48b0 @@ -397,6 +399,10 @@ opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 8f9eed26ef3..714d7993b6b 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -54,6 +54,7 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: config_all_loggers( log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.API_SERVER_LOG_FILTER_MAPPING, + tracing_settings=settings.API_SERVER_TRACING, ) _logger.debug("App settings:\n%s", settings.model_dump_json(indent=2)) diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index 5b02f55cb96..dec4d478b86 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -297,6 +297,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray @@ -320,6 +321,7 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 @@ -330,6 +332,10 @@ opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/autoscaling/src/simcore_service_autoscaling/main.py b/services/autoscaling/src/simcore_service_autoscaling/main.py index c73f3bdf94c..102258cac70 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/main.py +++ b/services/autoscaling/src/simcore_service_autoscaling/main.py @@ -15,6 +15,7 @@ config_all_loggers( log_format_local_dev_enabled=the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=the_settings.AUTOSCALING_LOG_FILTER_MAPPING, + tracing_settings=the_settings.AUTOSCALING_TRACING, ) # SINGLETON FastAPI app diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 78fac9431bf..3c2384e6ebb 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -222,6 +222,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -242,6 +243,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.48b0 @@ -252,6 +254,8 @@ opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.48b0 diff --git a/services/catalog/src/simcore_service_catalog/main.py b/services/catalog/src/simcore_service_catalog/main.py index 53cd5da07c5..52bd949a542 100644 --- a/services/catalog/src/simcore_service_catalog/main.py +++ b/services/catalog/src/simcore_service_catalog/main.py @@ -16,6 +16,7 @@ config_all_loggers( log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_the_settings.CATALOG_LOG_FILTER_MAPPING, + tracing_settings=_the_settings.CATALOG_TRACING, ) diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 15a9a8f8114..b5e0a2e1b12 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -295,6 +295,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray @@ -318,6 +319,7 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 @@ -328,6 +330,10 @@ opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py index bcf872e807d..b2844bde6af 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py @@ -15,6 +15,7 @@ config_all_loggers( log_format_local_dev_enabled=the_settings.CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=the_settings.CLUSTERS_KEEPER_LOG_FILTER_MAPPING, + tracing_settings=the_settings.CLUSTERS_KEEPER_TRACING, ) # SINGLETON FastAPI app diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 15392d474dd..0096bb1b261 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -209,6 +209,7 @@ opentelemetry-api==1.26.0 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -225,8 +226,11 @@ opentelemetry-exporter-otlp-proto-http==1.26.0 # via opentelemetry-exporter-otlp opentelemetry-instrumentation==0.47b0 # via + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-logging==0.47b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py index c8611a500ed..cc061d6cd39 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py @@ -65,6 +65,7 @@ async def dask_setup(worker: distributed.Worker) -> None: config_all_loggers( log_format_local_dev_enabled=settings.DASK_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.DASK_LOG_FILTER_MAPPING, + tracing_settings=None, # no tracing for dask sidecar ) logger.info("Setting up worker...") diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index fca17558d12..7e0a7936e48 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -178,6 +178,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -197,6 +198,7 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 @@ -205,6 +207,8 @@ opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.47b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.47b0 diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py index ff2e7741164..d653dc6a090 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py @@ -42,6 +42,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: config_all_loggers( log_format_local_dev_enabled=settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.DATCORE_ADAPTER_LOG_FILTER_MAPPING, + tracing_settings=settings.DATCORE_ADAPTER_TRACING, ) # keep mostly quiet noisy loggers diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 20ff744f93c..0d514a1b886 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -406,6 +406,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -430,6 +431,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiopg==0.48b0 @@ -446,6 +448,10 @@ opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 4b62b4ce73c..be96e845307 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -108,6 +108,7 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: config_all_loggers( log_format_local_dev_enabled=settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.DIRECTOR_V2_LOG_FILTER_MAPPING, + tracing_settings=settings.DIRECTOR_V2_TRACING, ) _logger.debug(settings.model_dump_json(indent=2)) diff --git a/services/director/requirements/_base.txt b/services/director/requirements/_base.txt index 67e2c6e55f3..ced8f7909f0 100644 --- a/services/director/requirements/_base.txt +++ b/services/director/requirements/_base.txt @@ -192,6 +192,7 @@ opentelemetry-api==1.28.1 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -211,6 +212,7 @@ opentelemetry-instrumentation==0.49b1 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.49b1 @@ -219,6 +221,8 @@ opentelemetry-instrumentation-fastapi==0.49b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.49b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.49b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.49b1 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.49b1 diff --git a/services/director/src/simcore_service_director/main.py b/services/director/src/simcore_service_director/main.py index 02636e753bf..da0c480065f 100644 --- a/services/director/src/simcore_service_director/main.py +++ b/services/director/src/simcore_service_director/main.py @@ -17,6 +17,7 @@ config_all_loggers( log_format_local_dev_enabled=_the_settings.DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_the_settings.DIRECTOR_LOG_FILTER_MAPPING, + tracing_settings=_the_settings.DIRECTOR_TRACING, ) # SINGLETON FastAPI app diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index c6fcc166562..dd73276b15a 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -231,6 +231,7 @@ opentelemetry-api==1.28.2 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -251,6 +252,7 @@ opentelemetry-instrumentation==0.49b2 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.49b2 @@ -261,7 +263,9 @@ opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-redis==0.49b2 +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py index 4431038df10..ab726883237 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py @@ -15,6 +15,7 @@ config_all_loggers( log_format_local_dev_enabled=_the_settings.DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_the_settings.DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING, + tracing_settings=_the_settings.DYNAMIC_SCHEDULER_TRACING, ) # SINGLETON FastAPI app diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 10ad9581448..02aab36cf3b 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -286,6 +286,7 @@ opentelemetry-api==1.28.2 # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -310,6 +311,7 @@ opentelemetry-instrumentation==0.49b2 # opentelemetry-instrumentation-dbapi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiopg==0.49b2 @@ -326,7 +328,11 @@ opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-redis==0.49b2 +opentelemetry-instrumentation-logging==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 5fd6c90b26b..58c5495563f 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -119,6 +119,7 @@ def setup_logger(settings: ApplicationSettings): config_all_loggers( log_format_local_dev_enabled=settings.DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.DY_SIDECAR_LOG_FILTER_MAPPING, + tracing_settings=settings.DYNAMIC_SIDECAR_TRACING, ) diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index c5dbeb68306..558b6f5b43e 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -278,6 +278,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray @@ -302,6 +303,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.48b0 @@ -314,6 +316,10 @@ opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.48b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/main.py b/services/efs-guardian/src/simcore_service_efs_guardian/main.py index f155d24f0f8..711d32d83ee 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/main.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/main.py @@ -15,6 +15,7 @@ config_all_loggers( log_format_local_dev_enabled=the_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=the_settings.EFS_GUARDIAN_LOG_FILTER_MAPPING, + tracing_settings=the_settings.EFS_GUARDIAN_TRACING, ) # SINGLETON FastAPI app diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt index 38827d398db..4aad9be2472 100644 --- a/services/invitations/requirements/_base.txt +++ b/services/invitations/requirements/_base.txt @@ -175,6 +175,7 @@ opentelemetry-api==1.28.2 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -194,6 +195,7 @@ opentelemetry-instrumentation==0.49b2 # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.49b2 @@ -202,7 +204,9 @@ opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-redis==0.49b2 +opentelemetry-instrumentation-logging==0.47b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/invitations/src/simcore_service_invitations/main.py b/services/invitations/src/simcore_service_invitations/main.py index 407e239c1bd..4a21e994b31 100644 --- a/services/invitations/src/simcore_service_invitations/main.py +++ b/services/invitations/src/simcore_service_invitations/main.py @@ -17,6 +17,7 @@ config_all_loggers( log_format_local_dev_enabled=the_settings.INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=the_settings.INVITATIONS_LOG_FILTER_MAPPING, + tracing_settings=the_settings.INVITATIONS_TRACING, ) # SINGLETON FastAPI app diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index 4d758aea4d6..e021ae5074c 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -236,6 +236,7 @@ opentelemetry-api==1.28.2 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -256,6 +257,7 @@ opentelemetry-instrumentation==0.49b2 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.49b2 @@ -266,7 +268,9 @@ opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-redis==0.49b2 +opentelemetry-instrumentation-logging==0.48b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/payments/src/simcore_service_payments/main.py b/services/payments/src/simcore_service_payments/main.py index 6ff09676f1f..53e19bd22a1 100644 --- a/services/payments/src/simcore_service_payments/main.py +++ b/services/payments/src/simcore_service_payments/main.py @@ -17,6 +17,7 @@ config_all_loggers( log_format_local_dev_enabled=_the_settings.PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_the_settings.PAYMENTS_LOG_FILTER_MAPPING, + tracing_settings=_the_settings.PAYMENTS_TRACING, ) # SINGLETON FastAPI app diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index da9a82895d9..45848d96288 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -300,6 +300,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray @@ -324,6 +325,7 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.47b0 @@ -336,6 +338,10 @@ opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py index 05890bd7a51..079ba5cdf79 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py @@ -17,6 +17,7 @@ config_all_loggers( log_format_local_dev_enabled=the_settings.RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=the_settings.RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING, + tracing_settings=the_settings.RESOURCE_USAGE_TRACKER_TRACING, ) # SINGLETON FastAPI app diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index c5a74d9831e..203572eef60 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -276,6 +276,7 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-propagator-aws-xray @@ -301,6 +302,7 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiohttp-client==0.47b0 @@ -317,6 +319,10 @@ opentelemetry-instrumentation-botocore==0.47b0 # -r requirements/_base.in opentelemetry-instrumentation-dbapi==0.47b0 # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-logging==0.47b0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/storage/src/simcore_service_storage/cli.py b/services/storage/src/simcore_service_storage/cli.py index d50d540d5b4..bf37855e6c8 100644 --- a/services/storage/src/simcore_service_storage/cli.py +++ b/services/storage/src/simcore_service_storage/cli.py @@ -28,6 +28,7 @@ def run(): config_all_loggers( log_format_local_dev_enabled=settings_obj.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings_obj.STORAGE_LOG_FILTER_MAPPING, + tracing_settings=settings_obj.STORAGE_TRACING, ) # keep mostly quiet noisy loggers diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index cc870dec999..ada75776f9e 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -340,6 +340,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -363,6 +364,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiohttp-client==0.48b0 @@ -380,6 +382,10 @@ opentelemetry-instrumentation-asyncpg==0.48b0 # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in opentelemetry-instrumentation-dbapi==0.48b0 # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-logging==0.48b0 + # via + # -r requirements/../../../../packages/service-library/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via # -r requirements/../../../../packages/service-library/requirements/_base.in diff --git a/services/web/server/src/simcore_service_webserver/cli.py b/services/web/server/src/simcore_service_webserver/cli.py index b09f4eca0c4..feec4b29e0c 100644 --- a/services/web/server/src/simcore_service_webserver/cli.py +++ b/services/web/server/src/simcore_service_webserver/cli.py @@ -57,6 +57,7 @@ def _setup_app_from_settings( slow_duration=settings.AIODEBUG_SLOW_DURATION_SECS, log_format_local_dev_enabled=settings.WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.WEBSERVER_LOG_FILTER_MAPPING, + tracing_settings=settings.WEBSERVER_TRACING, ) app = create_application() diff --git a/services/web/server/src/simcore_service_webserver/log.py b/services/web/server/src/simcore_service_webserver/log.py index ee6957732f4..f1cbbada057 100644 --- a/services/web/server/src/simcore_service_webserver/log.py +++ b/services/web/server/src/simcore_service_webserver/log.py @@ -7,6 +7,7 @@ from aiodebug import log_slow_callbacks # type: ignore[import-untyped] from aiohttp.log import access_logger from servicelib.logging_utils import config_all_loggers +from settings_library.tracing import TracingSettings LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR NOISY_LOGGERS = ( @@ -29,6 +30,7 @@ def setup_logging( slow_duration: float | None = None, log_format_local_dev_enabled: bool, logger_filter_mapping: dict, + tracing_settings: TracingSettings | None ): # service log level logging.basicConfig(level=level) @@ -38,6 +40,7 @@ def setup_logging( config_all_loggers( log_format_local_dev_enabled=log_format_local_dev_enabled, logger_filter_mapping=logger_filter_mapping, + tracing_settings=tracing_settings, ) # Enforces same log-level to aiohttp & gunicorn access loggers diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py index 2a26d924622..3b965a09e46 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py @@ -68,7 +68,10 @@ def app_cfg(default_app_cfg, unused_tcp_port_factory, monkeypatch) -> dict[str, # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG setup_logging( - level=logging.DEBUG, log_format_local_dev_enabled=True, logger_filter_mapping={} + level=logging.DEBUG, + log_format_local_dev_enabled=True, + logger_filter_mapping={}, + tracing_settings=None, ) # Enforces smallest GC in the background task diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index 679091f6e85..6903e2e1ef5 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -122,7 +122,10 @@ def app_cfg( # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG setup_logging( - level=logging.DEBUG, log_format_local_dev_enabled=True, logger_filter_mapping={} + level=logging.DEBUG, + log_format_local_dev_enabled=True, + logger_filter_mapping={}, + tracing_settings=None, ) # Enforces smallest GC in the background task diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py index a03a5713e59..864dbbe7be0 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py @@ -54,7 +54,10 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG setup_logging( - level=logging.DEBUG, log_format_local_dev_enabled=True, logger_filter_mapping={} + level=logging.DEBUG, + log_format_local_dev_enabled=True, + logger_filter_mapping={}, + tracing_settings=None, ) plugin_settings = StudiesDispatcherSettings.create_from_envs() diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index ddd2910ff59..e18b8c99a1b 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -240,6 +240,7 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk @@ -261,6 +262,7 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aiopg==0.48b0 @@ -269,6 +271,10 @@ opentelemetry-instrumentation-asyncpg==0.48b0 # via -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in opentelemetry-instrumentation-dbapi==0.48b0 # via opentelemetry-instrumentation-aiopg +opentelemetry-instrumentation-logging==0.48b0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.48b0 # via # -r requirements/../../../packages/service-library/requirements/_base.in From 35f11dfc84076d12a75fe7292a07726eb918f6c9 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:22:36 +0100 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=90=9B=20ensure=20backwards=20compati?= =?UTF-8?q?bility=20of=20api=20server=20(#6866)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-testing-deploy.yml | 77 ++ Makefile | 20 +- api/specs/director/openapi.yaml | 492 -------- api/specs/web-server/_projects_states.py | 18 +- api/specs/web-server/openapi.py | 3 + ci/github/helpers/openapi-specs-diff.bash | 47 + services/api-server/openapi.json | 40 +- .../api/routes/credits.py | 4 +- .../api/routes/solvers.py | 4 +- .../api/routes/solvers_jobs_getters.py | 12 +- .../api/routes/wallets.py | 6 +- .../models/schemas/model_adapter.py | 123 ++ .../services/webserver.py | 34 +- .../tests/unit/_with_db/test_product.py | 8 +- .../tests/unit/test_api_solver_jobs.py | 12 +- .../api-server/tests/unit/test_api_wallets.py | 10 +- services/catalog/openapi.json | 16 +- services/director-v2/openapi.json | 37 +- .../src/simcore_service_director/api/Makefile | 6 - services/dynamic-scheduler/openapi.json | 32 +- services/payments/gateway/openapi.json | 1039 ----------------- services/resource-usage-tracker/openapi.json | 2 +- .../api/v0/openapi.yaml | 193 +-- 23 files changed, 452 insertions(+), 1783 deletions(-) delete mode 100644 api/specs/director/openapi.yaml create mode 100755 ci/github/helpers/openapi-specs-diff.bash create mode 100644 services/api-server/src/simcore_service_api_server/models/schemas/model_adapter.py delete mode 100644 services/payments/gateway/openapi.json diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 516a401f9e3..63623a2c521 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -2655,6 +2655,8 @@ jobs: system-test-environment-setup, system-test-public-api, system-test-swarm-deploy, + system-api-specs, + system-backwards-compatibility ] runs-on: ubuntu-latest steps: @@ -2719,3 +2721,78 @@ jobs: env: TAG_PREFIX: hotfix-staging-github run: ./ci/deploy/dockerhub-deploy.bash -n + + system-api-specs: + needs: [changes] + if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }} + timeout-minutes: 10 + name: "[sys] check api-specs are up to date" + runs-on: ubuntu-latest + steps: + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: install uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.4.x" + enable-cache: false + - name: checkout source branch + uses: actions/checkout@v4 + - name: Regenerate specs and check + run: | + uv venv .venv && source .venv/bin/activate + make openapi-specs + ./ci/github/helpers/openapi-specs-diff.bash diff \ + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/refs/heads/${{ github.event.pull_request.head.ref }} \ + . + + system-backwards-compatibility: + needs: [changes, system-api-specs] + if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }} + timeout-minutes: 10 + name: "[sys] api-server backwards compatibility" + runs-on: ubuntu-latest + steps: + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: install uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.4.x" + enable-cache: false + - name: checkout + uses: actions/checkout@v4 + - name: check api-server backwards compatibility + run: | + ./scripts/openapi-diff.bash breaking --fail-on ERR\ + https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }}/services/api-server/openapi.json \ + /specs/services/api-server/openapi.json + + api-spec-backwards-compatibility: + needs: [changes, system-api-specs] + if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }} + continue-on-error: true + timeout-minutes: 10 + name: "api-specs-backwards-compatibility" + runs-on: ubuntu-latest + steps: + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: install uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.4.x" + enable-cache: false + - name: checkout + uses: actions/checkout@v4 + - name: Check openapi-specs backwards compatibility + run: | + ./ci/github/helpers/openapi-specs-diff.bash breaking \ + https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }} \ + . diff --git a/Makefile b/Makefile index 564e353ee58..1ecccd09739 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ .DEFAULT_GOAL := help SHELL := /bin/bash - +.SHELLFLAGS := -o errexit -o pipefail -c MAKE_C := $(MAKE) --no-print-directory --directory # Operating system @@ -84,7 +84,7 @@ export SWARM_STACK_NAME_NO_HYPHEN = $(subst -,_,$(SWARM_STACK_NAME)) export DOCKER_IMAGE_TAG ?= latest export DOCKER_REGISTRY ?= itisfoundation - +MAKEFILES_WITH_OPENAPI_SPECS := $(shell find . -mindepth 2 -type f -name 'Makefile' -not -path '*/.*' -exec grep -l '^openapi-specs:' {} \; | xargs realpath) get_my_ip := $(shell (hostname --all-ip-addresses || hostname -i) 2>/dev/null | cut --delimiter=" " --fields=1) @@ -131,6 +131,12 @@ test_python_version: ## Check Python version, throw error if compilation would f @.venv/bin/python ./scripts/test_python_version.py +.PHONY: _check_venv_active +_check_venv_active: + # Checking whether virtual environment was activated + @python3 -c "import sys; assert sys.base_prefix!=sys.prefix" + + ## DOCKER BUILD ------------------------------- # # - all builds are immediatly tagged as 'local/{service}:${BUILD_TARGET}' where BUILD_TARGET='development', 'production', 'cache' @@ -573,9 +579,13 @@ new-service: .venv ## Bakes a new project from cookiecutter-simcore-pyservice an .PHONY: openapi-specs -openapi-specs: ## bundles and validates openapi specifications and schemas of ALL service's API - @$(MAKE_C) services/web/server $@ - @$(MAKE_C) services/storage $@ +openapi-specs: .env _check_venv_active ## generates and validates openapi specifications and schemas of ALL service's API + @for makefile in $(MAKEFILES_WITH_OPENAPI_SPECS); do \ + echo "Generating openapi-specs using $${makefile}"; \ + $(MAKE_C) $$(dirname $${makefile}) install-dev; \ + $(MAKE_C) $$(dirname $${makefile}) $@; \ + printf "%0.s=" {1..100} && printf "\n"; \ + done .PHONY: settings-schema.json diff --git a/api/specs/director/openapi.yaml b/api/specs/director/openapi.yaml deleted file mode 100644 index fe95950536e..00000000000 --- a/api/specs/director/openapi.yaml +++ /dev/null @@ -1,492 +0,0 @@ -openapi: "3.0.0" -info: - description: This is the oSparc's director API - version: 0.1.0 - title: Director API - contact: - name: IT'IS Foundation - email: support@simcore.com - license: - name: MIT - url: https://github.com/ITISFoundation/osparc-simcore/blob/master/LICENSE - -servers: - - description: Development server - url: http://{host}:{port}/{version} - variables: - host: - default: "localhost" - port: - default: "8080" - version: - default: "v0" - enum: - - "v0" - - description: Production server - url: http://director:{port}/{version} - variables: - port: - default: "8080" - version: - default: "v0" - enum: - - "v0" - -# tags are used for organizing operations -tags: - - name: admins - description: Secured Admin-only calls - - name: developers - description: Operations available to regular developers - - name: users - description: Operations available to regular users - -paths: - /: - get: - tags: - - users - summary: Service health-check endpoint - description: Some general information on the API and state of the service behind - operationId: root_get - responses: - "200": - description: Service information - content: - application/json: - schema: - $ref: "#/components/schemas/HealthCheckEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - - /services: - get: - tags: - - users - summary: Lists available services in the oSparc platform - description: Lists available services in the oSparc platform - operationId: services_get - parameters: - - $ref: "#/components/parameters/ServiceType" - responses: - "200": - description: Success, returns the list of available services - content: - application/json: - schema: - $ref: "#/components/schemas/ServicesEnveloped" - "401": - description: Unauthorized access - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - - /services/{service_key}/{service_version}: - get: - tags: - - users - summary: Returns details of the selected service if available in the oSparc platform - description: Returns details of the selected service if available in the oSparc platform - operationId: services_by_key_version_get - parameters: - - $ref: "#/components/parameters/ServiceKeyPath" - - $ref: "#/components/parameters/ServiceVersionPath" - responses: - "200": - description: Success, returns the details of the service - content: - application/json: - schema: - $ref: "#/components/schemas/ServicesEnveloped" - "401": - description: Unauthorized access - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "404": - description: Service not found - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - - /services/{service_key}/{service_version}/labels: - get: - tags: - - users - summary: Returns the list of tags attached to a service - operationId: get_service_labels - parameters: - - $ref: "#/components/parameters/ServiceKeyPath" - - $ref: "#/components/parameters/ServiceVersionPath" - responses: - "200": - description: Success, returns the details of the service - content: - application/json: - schema: - type: object - additionalProperties: - type: string - "401": - description: Unauthorized access - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "404": - description: Service not found - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - - /service_extras/{service_key}/{service_version}: - get: - tags: - - users - summary: Returns the service's details which should be hidden from the user defined as extras. - description: Currently returns the node_requirements an array of resoruces needed for scheduling. - operationId: service_extras_by_key_version_get - parameters: - - $ref: "#/components/parameters/ServiceKeyPath" - - $ref: "#/components/parameters/ServiceVersionPath" - responses: - "200": - description: Success, returns an object containing details hidden from the user - content: - application/json: - schema: - $ref: "#/components/schemas/ServiceExtrasEnveloped" - "401": - description: Unauthorized access - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "404": - description: Service not found - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - - /running_interactive_services: - get: - tags: - - users - summary: Returns a list of interactive services - operationId: running_interactive_services_list_get - parameters: - - in: query - name: user_id - required: false - schema: - type: string - - in: query - name: project_id - required: false - schema: - type: string - responses: - "200": - description: Returns the running services instances - content: - application/json: - schema: - $ref: "#/components/schemas/RunningServicesEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - post: - tags: - - users - summary: Starts an interactive service in the oSparc platform - operationId: running_interactive_services_post - parameters: - - $ref: "#/components/parameters/UserId" - - $ref: "#/components/parameters/ProjectId" - - $ref: "#/components/parameters/ServiceKey" - - $ref: "#/components/parameters/ServiceVersion" - - $ref: "#/components/parameters/AssignmentUuid" - - $ref: "#/components/parameters/ServiceBasePath" - responses: - "201": - description: Succesfully created the service in the oSparc platform. Returns the location where the service runs. - content: - application/json: - schema: - $ref: "#/components/schemas/RunningServiceEnveloped" - "400": - description: Malformed function call, missing field - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "401": - description: Unauthorized access - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "404": - description: Service not found - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "409": - description: A service with the same uuid already exists - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - - /running_interactive_services/{service_uuid}: - get: - tags: - - users - summary: Succesfully returns if a service with the defined uuid is up and running - description: Succesfully returns if a service with the defined uuid is up and running - operationId: running_interactive_services_get - parameters: - - $ref: "#/components/parameters/ServiceUuid" - responses: - "200": - description: OK service exists and runs. Returns service location. - content: - application/json: - schema: - $ref: "#/components/schemas/RunningServiceEnveloped" - "400": - description: Malformed function call, missing field - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "404": - description: Service not found - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - delete: - tags: - - users - summary: Stops and removes an interactive service from the oSparc platform - description: Stops and removes an interactive service from the oSparc platform - operationId: running_interactive_services_delete - parameters: - - $ref: "#/components/parameters/ServiceUuid" - - $ref: "#/components/parameters/SaveState" - responses: - "204": - description: Succesfully stopped and removed the service from the oSparc platform - "400": - description: Malformed function call, missing field - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - "404": - description: Service not found - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - default: - description: Unexpected error - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnveloped" - -components: - parameters: - UserId: - in: query - name: user_id - description: The ID of the user that starts the service - required: true - schema: - type: string - example: asdfgj233 - ProjectId: - in: query - name: project_id - description: The ID of the project in which the service starts - required: true - schema: - type: string - example: asdfgj233 - AssignmentUuid: - in: query - name: service_uuid - description: The uuid to assign the service with - required: true - schema: - type: string - # format: uuid - example: 123e4567-e89b-12d3-a456-426655440000 - - ServiceKeyPath: - in: path - name: service_key - description: The key (url) of the service - required: true - schema: - type: string - description: distinctive name for the node based on the docker registry path - pattern: '^(simcore)/(services)/(comp|dynamic)(/[\w/-]+)+$' - example: - - simcore/services/comp/itis/sleeper - - simcore/services/dynamic/3dviewer - - ServiceKey: - in: query - name: service_key - description: The key (url) of the service - required: true - schema: - type: string - description: distinctive name for the node based on the docker registry path - pattern: '^(simcore)/(services)/(comp|dynamic)(/[\w/-]+)+$' - example: - - simcore/services/comp/itis/sleeper - - simcore/services/dynamic/3dviewer - - ServiceType: - in: query - name: service_type - description: | - The service type: - * computational - a computational service - * interactive - an interactive service - required: false - schema: - type: string - enum: - - computational - - interactive - example: computational - - ServiceBasePath: - in: query - name: service_basepath - description: predefined basepath for the backend service otherwise uses root - required: false - schema: - type: string - example: "/x/EycCXbU0H/" - default: "" - - ServiceUuid: - in: path - name: service_uuid - description: The uuid of the service - required: true - schema: - type: string - # format: uuid - example: 123e4567-e89b-12d3-a456-426655440000 - - ServiceVersionPath: - in: path - name: service_version - description: The tag/version of the service - required: true - schema: - type: string - description: semantic version number - pattern: >- - ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ - example: - - 1.0.0 - - 0.0.1 - - ServiceVersion: - in: query - name: service_tag - description: The tag/version of the service - required: false - schema: - type: string - description: semantic version number - pattern: >- - ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ - example: - - 1.0.0 - - 0.0.1 - - SaveState: - in: query - name: save_state - description: Save the state prior to removing the service - required: false - schema: - type: boolean - default: true - - schemas: - ErrorEnveloped: - $ref: "./schemas/error.yaml#/components/schemas/ErrorEnveloped" - - RunningServiceEnveloped: - $ref: "./schemas/running_service.yaml#/components/schemas/RunningServiceEnveloped" - - RunningServicesEnveloped: - $ref: "./schemas/running_service.yaml#/components/schemas/RunningServicesEnveloped" - - ServicesEnveloped: - $ref: "./schemas/services.yaml#/components/schemas/ServicesEnveloped" - - ServiceExtrasEnveloped: - $ref: "./schemas/services.yaml#/components/schemas/ServiceExtrasEnveloped" - - HealthCheckEnveloped: - $ref: "./schemas/health_check.yaml#/components/schemas/HealthCheckEnveloped" diff --git a/api/specs/web-server/_projects_states.py b/api/specs/web-server/_projects_states.py index 0c737062285..1547e8f3b35 100644 --- a/api/specs/web-server/_projects_states.py +++ b/api/specs/web-server/_projects_states.py @@ -34,8 +34,8 @@ ) -def to_desc(exceptions: set[type[Exception]] | type[Exception]): - exc_classes = {exceptions} if not isinstance(exceptions, set) else exceptions +def to_desc(exceptions: list[type[Exception]] | type[Exception]): + exc_classes = [exceptions] if not isinstance(exceptions, list) else exceptions return ", ".join(f"{cls.__name__}" for cls in exc_classes) @@ -43,26 +43,26 @@ def to_desc(exceptions: set[type[Exception]] | type[Exception]): "/projects/{project_id}:open", response_model=Envelope[ProjectGet], responses={ - status.HTTP_400_BAD_REQUEST: {"description": to_desc({ValidationError})}, + status.HTTP_400_BAD_REQUEST: {"description": to_desc([ValidationError])}, status.HTTP_402_PAYMENT_REQUIRED: { - "description": to_desc({WalletNotEnoughCreditsError}) + "description": to_desc([WalletNotEnoughCreditsError]) }, status.HTTP_403_FORBIDDEN: { - "description": to_desc({ProjectInvalidRightsError}) + "description": to_desc([ProjectInvalidRightsError]) }, status.HTTP_404_NOT_FOUND: { "description": to_desc( - {ProjectNotFoundError, UserDefaultWalletNotFoundError} + [ProjectNotFoundError, UserDefaultWalletNotFoundError] ) }, status.HTTP_409_CONFLICT: { - "description": to_desc({ProjectTooManyProjectOpenedError}), + "description": to_desc([ProjectTooManyProjectOpenedError]), }, status.HTTP_422_UNPROCESSABLE_ENTITY: { - "description": to_desc({ValidationError}) + "description": to_desc([ValidationError]) }, status.HTTP_503_SERVICE_UNAVAILABLE: { - "description": to_desc({DirectorServiceError}) + "description": to_desc([DirectorServiceError]) }, }, ) diff --git a/api/specs/web-server/openapi.py b/api/specs/web-server/openapi.py index 5a679b75713..77e656efdaa 100644 --- a/api/specs/web-server/openapi.py +++ b/api/specs/web-server/openapi.py @@ -98,6 +98,9 @@ def main(): # .yaml oas_path = webserver_resources.get_path("api/v0/openapi.yaml").resolve() + if not oas_path.exists(): + oas_path.parent.mkdir(parents=True) + oas_path.write_text("") print(f"Writing {oas_path}...", end=None) with oas_path.open("wt") as fh: yaml.safe_dump(openapi, stream=fh, sort_keys=False) diff --git a/ci/github/helpers/openapi-specs-diff.bash b/ci/github/helpers/openapi-specs-diff.bash new file mode 100755 index 00000000000..b4409c174a8 --- /dev/null +++ b/ci/github/helpers/openapi-specs-diff.bash @@ -0,0 +1,47 @@ +#!/bin/bash + +# Recursively checks if all openapi specs within a local osparc-simcore revision are different/backwards compatible with a remote base +# Example: +# bash osparc-simcore/ci/github/helpers/openapi-specs-diff.bash diff \ +# https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master \ +# ./osparc-simcore/ +# or +# bash osparc-simcore/ci/github/helpers/openapi-specs-diff.bash breaking \ +# https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master \ +# ./osparc-simcore/ +# +# The script generates github error annotations for better being able to locate issues. + +operation=$1 +base_remote=$2 +revision_local=$3 + +repo_base_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")/../../..") +openapi_specs=$(find "${revision_local}" -type f \( -name 'openapi.json' -o -name 'openapi.yaml' \) -not -path '*/.*' -exec realpath --relative-to="${revision_local}" {} \;) + +cd "${revision_local}" || exit 1 # required to mount correct dir for diff tool + + +function run_diff_tool() { + exit_status=0 + for spec in ${openapi_specs}; do + echo "Comparing ${spec}" + if ! "${repo_base_dir}/scripts/openapi-diff.bash" "$@" "${base_remote}/${spec}" "/specs/${spec}"; then + echo "::error file=${spec}:: Error when checking ${spec}" + exit_status=$(("${exit_status}" + "1")) + fi + printf "%0.s=" {1..100} && printf "\n" + done + + exit "${exit_status}" +} + + +if [[ "${operation}" == "diff" ]]; then + run_diff_tool "diff" "--fail-on-diff" +elif [[ "${operation}" == "breaking" ]]; then + run_diff_tool "breaking" "--fail-on" "ERR" +else + echo "the operation '${operation}' is not supported" + exit 1 +fi diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 883b7364473..e04d4b41d32 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -1931,7 +1931,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ServicePricingPlanGet" + "$ref": "#/components/schemas/ServicePricingPlanGetLegacy" } } } @@ -2650,6 +2650,7 @@ "type": "null" } ], + "deprecated": true, "title": "Cluster Id" }, "deprecated": true @@ -3789,7 +3790,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WalletGetWithAvailableCredits" + "$ref": "#/components/schemas/WalletGetWithAvailableCreditsLegacy" } } } @@ -3928,7 +3929,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PricingUnitGet" + "$ref": "#/components/schemas/PricingUnitGetLegacy" } } } @@ -4628,6 +4629,7 @@ "type": "null" } ], + "deprecated": true, "title": "Cluster Id" }, "deprecated": true @@ -5119,7 +5121,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WalletGetWithAvailableCredits" + "$ref": "#/components/schemas/WalletGetWithAvailableCreditsLegacy" } } } @@ -5232,7 +5234,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/WalletGetWithAvailableCredits" + "$ref": "#/components/schemas/WalletGetWithAvailableCreditsLegacy" } } } @@ -5334,7 +5336,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GetCreditPrice" + "$ref": "#/components/schemas/GetCreditPriceLegacy" } } } @@ -5563,7 +5565,7 @@ ], "title": "FileUploadData" }, - "GetCreditPrice": { + "GetCreditPriceLegacy": { "properties": { "productName": { "type": "string", @@ -5602,7 +5604,7 @@ "usdPerCredit", "minPaymentAmountUsd" ], - "title": "GetCreditPrice" + "title": "GetCreditPriceLegacy" }, "Groups": { "properties": { @@ -6471,7 +6473,7 @@ ], "title": "PricingPlanClassification" }, - "PricingUnitGet": { + "PricingUnitGetLegacy": { "properties": { "pricingUnitId": { "type": "integer", @@ -6487,7 +6489,8 @@ "$ref": "#/components/schemas/UnitExtraInfo" }, "currentCostPerUnit": { - "type": "string", + "type": "number", + "minimum": 0.0, "title": "Currentcostperunit" }, "default": { @@ -6503,7 +6506,7 @@ "currentCostPerUnit", "default" ], - "title": "PricingUnitGet" + "title": "PricingUnitGetLegacy" }, "Profile": { "properties": { @@ -6645,7 +6648,7 @@ "title": "RunningState", "description": "State of execution of a project's computational workflow\n\nSEE StateType for task state" }, - "ServicePricingPlanGet": { + "ServicePricingPlanGetLegacy": { "properties": { "pricingPlanId": { "type": "integer", @@ -6675,7 +6678,7 @@ }, "pricingUnits": { "items": { - "$ref": "#/components/schemas/PricingUnitGet" + "$ref": "#/components/schemas/PricingUnitGetLegacy" }, "type": "array", "title": "Pricingunits" @@ -6691,7 +6694,7 @@ "pricingPlanKey", "pricingUnits" ], - "title": "ServicePricingPlanGet" + "title": "ServicePricingPlanGetLegacy" }, "Solver": { "properties": { @@ -7032,7 +7035,7 @@ ], "title": "ValidationError" }, - "WalletGetWithAvailableCredits": { + "WalletGetWithAvailableCreditsLegacy": { "properties": { "walletId": { "type": "integer", @@ -7088,7 +7091,8 @@ "title": "Modified" }, "availableCredits": { - "type": "string", + "type": "number", + "minimum": 0.0, "title": "Availablecredits" } }, @@ -7096,15 +7100,13 @@ "required": [ "walletId", "name", - "description", "owner", - "thumbnail", "status", "created", "modified", "availableCredits" ], - "title": "WalletGetWithAvailableCredits" + "title": "WalletGetWithAvailableCreditsLegacy" }, "WalletStatus": { "type": "string", diff --git a/services/api-server/src/simcore_service_api_server/api/routes/credits.py b/services/api-server/src/simcore_service_api_server/api/routes/credits.py index 5370e6dd72d..5b5258cfb01 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/credits.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/credits.py @@ -1,8 +1,8 @@ from typing import Annotated from fastapi import APIRouter, Depends, status -from models_library.api_schemas_webserver.product import GetCreditPrice +from ...models.schemas.model_adapter import GetCreditPriceLegacy from ..dependencies.webserver import AuthSession, get_webserver_session from ._constants import FMSG_CHANGELOG_NEW_IN_VERSION @@ -12,7 +12,7 @@ @router.get( "/price", status_code=status.HTTP_200_OK, - response_model=GetCreditPrice, + response_model=GetCreditPriceLegacy, description=FMSG_CHANGELOG_NEW_IN_VERSION.format("0.6.0"), ) async def get_credits_price( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index fca71667030..c85b8b39baf 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -5,13 +5,13 @@ from fastapi import APIRouter, Depends, HTTPException, status from httpx import HTTPStatusError -from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet from pydantic import ValidationError from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.basic_types import VersionStr from ...models.pagination import OnePage, Page, PaginationParams from ...models.schemas.errors import ErrorGet +from ...models.schemas.model_adapter import ServicePricingPlanGetLegacy from ...models.schemas.solvers import Solver, SolverKeyId, SolverPort from ...services.catalog import CatalogApi from ..dependencies.application import get_reverse_url_mapper @@ -262,7 +262,7 @@ async def list_solver_ports( @router.get( "/{solver_key:path}/releases/{version}/pricing_plan", - response_model=ServicePricingPlanGet, + response_model=ServicePricingPlanGetLegacy, description="Gets solver pricing plan\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7"), responses=_SOLVER_STATUS_CODES, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index 708b9871a68..4903d1cb815 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -11,8 +11,6 @@ from fastapi.responses import RedirectResponse from fastapi_pagination.api import create_page from models_library.api_schemas_webserver.projects import ProjectGet -from models_library.api_schemas_webserver.resource_usage import PricingUnitGet -from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from models_library.projects_nodes_io import BaseFileLink from models_library.users import UserID from models_library.wallets import ZERO_CREDITS @@ -35,6 +33,10 @@ JobMetadata, JobOutputs, ) +from ...models.schemas.model_adapter import ( + PricingUnitGetLegacy, + WalletGetWithAvailableCreditsLegacy, +) from ...models.schemas.solvers import SolverKeyId from ...services.catalog import CatalogApi from ...services.director_v2 import DirectorV2Api @@ -376,7 +378,7 @@ async def get_job_custom_metadata( @router.get( "/{solver_key:path}/releases/{version}/jobs/{job_id:uuid}/wallet", - response_model=WalletGetWithAvailableCredits, + response_model=WalletGetWithAvailableCreditsLegacy, responses=WALLET_STATUS_CODES, description=("Get job wallet\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7")), ) @@ -385,7 +387,7 @@ async def get_job_wallet( version: VersionStr, job_id: JobID, webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -) -> WalletGetWithAvailableCredits: +) -> WalletGetWithAvailableCreditsLegacy: job_name = _compose_job_resource_name(solver_key, version, job_id) _logger.debug("Getting wallet for job '%s'", job_name) @@ -396,7 +398,7 @@ async def get_job_wallet( @router.get( "/{solver_key:path}/releases/{version}/jobs/{job_id:uuid}/pricing_unit", - response_model=PricingUnitGet, + response_model=PricingUnitGetLegacy, responses=_PRICING_UNITS_STATUS_CODES, description=( "Get job pricing unit\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7") diff --git a/services/api-server/src/simcore_service_api_server/api/routes/wallets.py b/services/api-server/src/simcore_service_api_server/api/routes/wallets.py index 0b3df66b1d5..40e2233fce0 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/wallets.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/wallets.py @@ -2,10 +2,10 @@ from typing import Annotated, Any from fastapi import APIRouter, Depends, status -from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES from ...models.schemas.errors import ErrorGet +from ...models.schemas.model_adapter import WalletGetWithAvailableCreditsLegacy from ..dependencies.webserver import AuthSession, get_webserver_session from ._constants import FMSG_CHANGELOG_NEW_IN_VERSION @@ -29,7 +29,7 @@ @router.get( "/default", description="Get default wallet\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7"), - response_model=WalletGetWithAvailableCredits, + response_model=WalletGetWithAvailableCreditsLegacy, responses=WALLET_STATUS_CODES, ) async def get_default_wallet( @@ -40,7 +40,7 @@ async def get_default_wallet( @router.get( "/{wallet_id}", - response_model=WalletGetWithAvailableCredits, + response_model=WalletGetWithAvailableCreditsLegacy, responses=WALLET_STATUS_CODES, description="Get wallet\n\n" + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.7"), ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/model_adapter.py b/services/api-server/src/simcore_service_api_server/models/schemas/model_adapter.py new file mode 100644 index 00000000000..9cc8b768d45 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/model_adapter.py @@ -0,0 +1,123 @@ +# Models added here "cover" models from within the deployment in order to restore backwards compatibility + +from datetime import datetime +from decimal import Decimal +from typing import Annotated + +from models_library.api_schemas_api_server.pricing_plans import ( + ServicePricingPlanGet as _ServicePricingPlanGet, +) +from models_library.api_schemas_webserver.product import ( + GetCreditPrice as _GetCreditPrice, +) +from models_library.api_schemas_webserver.resource_usage import ( + PricingUnitGet as _PricingUnitGet, +) +from models_library.api_schemas_webserver.wallets import ( + WalletGetWithAvailableCredits as _WalletGetWithAvailableCredits, +) +from models_library.basic_types import IDStr, NonNegativeDecimal +from models_library.resource_tracker import ( + PricingPlanClassification, + PricingPlanId, + PricingUnitId, + UnitExtraInfo, +) +from models_library.users import GroupID +from models_library.wallets import WalletID, WalletStatus +from pydantic import ( + BaseModel, + ConfigDict, + Field, + NonNegativeFloat, + NonNegativeInt, + PlainSerializer, +) + + +class GetCreditPriceLegacy(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + product_name: str = Field(alias="productName") + usd_per_credit: ( + Annotated[ + NonNegativeDecimal, + PlainSerializer(float, return_type=NonNegativeFloat, when_used="json"), + ] + | None + ) = Field( + ..., + description="Price of a credit in USD. " + "If None, then this product's price is UNDEFINED", + alias="usdPerCredit", + ) + min_payment_amount_usd: NonNegativeInt | None = Field( + ..., + description="Minimum amount (included) in USD that can be paid for this product" + "Can be None if this product's price is UNDEFINED", + alias="minPaymentAmountUsd", + ) + + +assert set(GetCreditPriceLegacy.model_fields.keys()) == set( + _GetCreditPrice.model_fields.keys() +) + + +class PricingUnitGetLegacy(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + pricing_unit_id: PricingUnitId = Field(alias="pricingUnitId") + unit_name: str = Field(alias="unitName") + unit_extra_info: UnitExtraInfo = Field(alias="unitExtraInfo") + current_cost_per_unit: Annotated[ + Decimal, PlainSerializer(float, return_type=NonNegativeFloat, when_used="json") + ] = Field(alias="currentCostPerUnit") + default: bool + + +assert set(PricingUnitGetLegacy.model_fields.keys()) == set( + _PricingUnitGet.model_fields.keys() +) + + +class WalletGetWithAvailableCreditsLegacy(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + wallet_id: WalletID = Field(alias="walletId") + name: IDStr + description: str | None = None + owner: GroupID + thumbnail: str | None = None + status: WalletStatus + created: datetime + modified: datetime + available_credits: Annotated[ + Decimal, PlainSerializer(float, return_type=NonNegativeFloat, when_used="json") + ] = Field(alias="availableCredits") + + +assert set(WalletGetWithAvailableCreditsLegacy.model_fields.keys()) == set( + _WalletGetWithAvailableCredits.model_fields.keys() +) + + +class ServicePricingPlanGetLegacy(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + pricing_plan_id: PricingPlanId = Field(alias="pricingPlanId") + display_name: str = Field(alias="displayName") + description: str + classification: PricingPlanClassification + created_at: datetime = Field(alias="createdAt") + pricing_plan_key: str = Field(alias="pricingPlanKey") + pricing_units: list[PricingUnitGetLegacy] = Field(alias="pricingUnits") + + +assert set(ServicePricingPlanGetLegacy.model_fields.keys()) == set( + _ServicePricingPlanGet.model_fields.keys() +) diff --git a/services/api-server/src/simcore_service_api_server/services/webserver.py b/services/api-server/src/simcore_service_api_server/services/webserver.py index ec828687b77..cba31689654 100644 --- a/services/api-server/src/simcore_service_api_server/services/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services/webserver.py @@ -14,7 +14,6 @@ from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet from models_library.api_schemas_long_running_tasks.tasks import TaskGet from models_library.api_schemas_webserver.computations import ComputationStart -from models_library.api_schemas_webserver.product import GetCreditPrice from models_library.api_schemas_webserver.projects import ( ProjectCreateNew, ProjectGet, @@ -29,14 +28,8 @@ ProjectInputGet, ProjectInputUpdate, ) -from models_library.api_schemas_webserver.resource_usage import ( - PricingPlanGet, - PricingUnitGet, -) -from models_library.api_schemas_webserver.wallets import ( - WalletGet, - WalletGetWithAvailableCredits, -) +from models_library.api_schemas_webserver.resource_usage import PricingPlanGet +from models_library.api_schemas_webserver.wallets import WalletGet from models_library.generics import Envelope from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID @@ -64,6 +57,7 @@ SolverOutputNotFoundError, WalletNotFoundError, ) +from simcore_service_api_server.models.schemas.model_adapter import GetCreditPriceLegacy from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -79,6 +73,10 @@ from ..models.basic_types import VersionStr from ..models.pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE from ..models.schemas.jobs import MetaValueType +from ..models.schemas.model_adapter import ( + PricingUnitGetLegacy, + WalletGetWithAvailableCreditsLegacy, +) from ..models.schemas.profiles import Profile, ProfileUpdate from ..models.schemas.solvers import SolverKeyId from ..models.schemas.studies import StudyPort @@ -409,14 +407,14 @@ async def update_project_metadata( @_exception_mapper({status.HTTP_404_NOT_FOUND: PricingUnitNotFoundError}) async def get_project_node_pricing_unit( self, *, project_id: UUID, node_id: UUID - ) -> PricingUnitGet: + ) -> PricingUnitGetLegacy: response = await self.client.get( f"/projects/{project_id}/nodes/{node_id}/pricing-unit", cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[PricingUnitGet].model_validate_json(response.text).data + data = Envelope[PricingUnitGetLegacy].model_validate_json(response.text).data assert data is not None # nosec return data @@ -531,14 +529,14 @@ async def update_node_outputs( # WALLETS ------------------------------------------------- @_exception_mapper(_WALLET_STATUS_MAP) - async def get_default_wallet(self) -> WalletGetWithAvailableCredits: + async def get_default_wallet(self) -> WalletGetWithAvailableCreditsLegacy: response = await self.client.get( "/wallets/default", cookies=self.session_cookies, ) response.raise_for_status() data = ( - Envelope[WalletGetWithAvailableCredits] + Envelope[WalletGetWithAvailableCreditsLegacy] .model_validate_json(response.text) .data ) @@ -546,14 +544,16 @@ async def get_default_wallet(self) -> WalletGetWithAvailableCredits: return data @_exception_mapper(_WALLET_STATUS_MAP) - async def get_wallet(self, *, wallet_id: int) -> WalletGetWithAvailableCredits: + async def get_wallet( + self, *, wallet_id: int + ) -> WalletGetWithAvailableCreditsLegacy: response = await self.client.get( f"/wallets/{wallet_id}", cookies=self.session_cookies, ) response.raise_for_status() data = ( - Envelope[WalletGetWithAvailableCredits] + Envelope[WalletGetWithAvailableCreditsLegacy] .model_validate_json(response.text) .data ) @@ -574,13 +574,13 @@ async def get_project_wallet(self, *, project_id: ProjectID) -> WalletGet: # PRODUCTS ------------------------------------------------- @_exception_mapper({status.HTTP_404_NOT_FOUND: ProductPriceNotFoundError}) - async def get_product_price(self) -> GetCreditPrice: + async def get_product_price(self) -> GetCreditPriceLegacy: response = await self.client.get( "/credits-price", cookies=self.session_cookies, ) response.raise_for_status() - data = Envelope[GetCreditPrice].model_validate_json(response.text).data + data = Envelope[GetCreditPriceLegacy].model_validate_json(response.text).data assert data is not None # nosec return data diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index bd14faf087e..274869d094a 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -14,12 +14,14 @@ from fastapi import status from fastapi.encoders import jsonable_encoder from models_library.api_schemas_api_server.api_keys import ApiKeyInDB -from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from models_library.generics import Envelope from models_library.users import UserID from models_library.wallets import WalletStatus from pydantic import PositiveInt from simcore_service_api_server._meta import API_VTAG +from simcore_service_api_server.models.schemas.model_adapter import ( + WalletGetWithAvailableCreditsLegacy, +) async def test_product_webserver( @@ -46,8 +48,8 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): return httpx.Response( status.HTTP_200_OK, json=jsonable_encoder( - Envelope[WalletGetWithAvailableCredits]( - data=WalletGetWithAvailableCredits( + Envelope[WalletGetWithAvailableCreditsLegacy]( + data=WalletGetWithAvailableCreditsLegacy( wallet_id=wallet_id, name="my_wallet", description="this is my wallet", diff --git a/services/api-server/tests/unit/test_api_solver_jobs.py b/services/api-server/tests/unit/test_api_solver_jobs.py index 3f1e642a6ad..4ed52877cfa 100644 --- a/services/api-server/tests/unit/test_api_solver_jobs.py +++ b/services/api-server/tests/unit/test_api_solver_jobs.py @@ -14,8 +14,6 @@ from fastapi import status from fastapi.encoders import jsonable_encoder from httpx import AsyncClient -from models_library.api_schemas_webserver.resource_usage import PricingUnitGet -from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from models_library.generics import Envelope from pydantic import TypeAdapter from pytest_simcore.helpers.httpx_calls_capture_models import ( @@ -25,6 +23,10 @@ ) from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.schemas.jobs import Job, JobStatus +from simcore_service_api_server.models.schemas.model_adapter import ( + PricingUnitGetLegacy, + WalletGetWithAvailableCreditsLegacy, +) from simcore_service_api_server.models.schemas.solvers import Solver from simcore_service_api_server.services.director_v2 import ComputationTaskGet @@ -182,7 +184,7 @@ def _get_pricing_unit_side_effect( ) if capture_file == "get_job_pricing_unit_success.json": assert response.status_code == status.HTTP_200_OK - _ = TypeAdapter(PricingUnitGet).validate_python(response.json()) + _ = TypeAdapter(PricingUnitGetLegacy).validate_python(response.json()) elif capture_file == "get_job_pricing_unit_invalid_job.json": assert response.status_code == status.HTTP_404_NOT_FOUND elif capture_file == "get_job_pricing_unit_invalid_solver.json": @@ -417,7 +419,7 @@ def _wallet_side_effect( capture: HttpApiCallCaptureModel, ): wallet = ( - TypeAdapter(Envelope[WalletGetWithAvailableCredits]) + TypeAdapter(Envelope[WalletGetWithAvailableCreditsLegacy]) .validate_python(capture.response_body) .data ) @@ -425,7 +427,7 @@ def _wallet_side_effect( wallet.available_credits = ( Decimal(10.0) if sufficient_credits else Decimal(-10.0) ) - envelope = Envelope[WalletGetWithAvailableCredits]() + envelope = Envelope[WalletGetWithAvailableCreditsLegacy]() envelope.data = wallet return jsonable_encoder(envelope) diff --git a/services/api-server/tests/unit/test_api_wallets.py b/services/api-server/tests/unit/test_api_wallets.py index cad3bf5e285..bf6f7167f23 100644 --- a/services/api-server/tests/unit/test_api_wallets.py +++ b/services/api-server/tests/unit/test_api_wallets.py @@ -10,12 +10,14 @@ import pytest from fastapi import status from httpx import AsyncClient -from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, ) from simcore_service_api_server._meta import API_VTAG +from simcore_service_api_server.models.schemas.model_adapter import ( + WalletGetWithAvailableCreditsLegacy, +) @pytest.mark.parametrize( @@ -52,8 +54,8 @@ def _get_wallet_side_effect( response = await client.get(f"{API_VTAG}/wallets/{wallet_id}", auth=auth) if "success" in capture: assert response.status_code == 200 - wallet: WalletGetWithAvailableCredits = ( - WalletGetWithAvailableCredits.model_validate(response.json()) + wallet: WalletGetWithAvailableCreditsLegacy = ( + WalletGetWithAvailableCreditsLegacy.model_validate(response.json()) ) assert wallet.wallet_id == wallet_id elif "failure" in capture: @@ -77,4 +79,4 @@ async def test_get_default_wallet( response = await client.get(f"{API_VTAG}/wallets/default", auth=auth) assert response.status_code == status.HTTP_200_OK - _ = WalletGetWithAvailableCredits.model_validate(response.json()) + _ = WalletGetWithAvailableCreditsLegacy.model_validate(response.json()) diff --git a/services/catalog/openapi.json b/services/catalog/openapi.json index c5663631059..5f5204053cd 100644 --- a/services/catalog/openapi.json +++ b/services/catalog/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-catalog", "description": "Manages and maintains a catalog of all published components (e.g. macro-algorithms, scripts, etc)", - "version": "0.5.0" + "version": "0.6.0" }, "paths": { "/": { @@ -628,17 +628,11 @@ }, "image": { "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", "title": "Image", "description": "Url to the badge" }, "url": { "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", "title": "Url", "description": "Link to the status" } @@ -3082,7 +3076,8 @@ "type": "null" } ], - "title": "Owner" + "title": "Owner", + "description": "None when the owner email cannot be found in the database" } }, "type": "object", @@ -3549,7 +3544,10 @@ "thumbnail": { "anyOf": [ { - "type": "string" + "type": "string", + "maxLength": 2083, + "minLength": 1, + "format": "uri" }, { "type": "null" diff --git a/services/director-v2/openapi.json b/services/director-v2/openapi.json index c1b38416efe..63418baabe5 100644 --- a/services/director-v2/openapi.json +++ b/services/director-v2/openapi.json @@ -1458,17 +1458,13 @@ }, "url": { "type": "string", - "minLength": 1, - "format": "uri", "title": "Url", "description": "the link where to get the status of the task" }, "stop_url": { "anyOf": [ { - "type": "string", - "minLength": 1, - "format": "uri" + "type": "string" }, { "type": "null" @@ -2447,6 +2443,9 @@ }, "type": "array" }, + "propertyNames": { + "format": "uuid" + }, "type": "object", "title": "Adjacency List", "description": "The adjacency list of the current pipeline in terms of {NodeID: [successor NodeID]}" @@ -2469,6 +2468,9 @@ "additionalProperties": { "$ref": "#/components/schemas/NodeState" }, + "propertyNames": { + "format": "uuid" + }, "type": "object", "title": "Node States", "description": "The states of each of the computational nodes in the pipeline" @@ -2978,9 +2980,16 @@ "description": "contains harware information so we know on which hardware to run the service" }, "product_name": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Product Name", - "description": "Current product upon which this service is scheduled. If set to None, the current product is undefined. Mostly for backwards compatibility" + "description": "Current product upon which this service is scheduledIf set to None, the current product is undefined. Mostly for backwards compatibility" } }, "additionalProperties": true, @@ -3000,7 +3009,8 @@ "service_resources", "request_dns", "request_scheme", - "request_simcore_user_agent" + "request_simcore_user_agent", + "proxy_service_name" ], "title": "SchedulerData" }, @@ -3106,9 +3116,7 @@ "download_link": { "anyOf": [ { - "type": "string", - "minLength": 1, - "format": "uri" + "type": "string" }, { "type": "null" @@ -3128,8 +3136,15 @@ "properties": { "nodes_outputs": { "additionalProperties": { + "propertyNames": { + "maxLength": 100, + "minLength": 1 + }, "type": "object" }, + "propertyNames": { + "format": "uuid" + }, "type": "object", "title": "Nodes Outputs" } diff --git a/services/director/src/simcore_service_director/api/Makefile b/services/director/src/simcore_service_director/api/Makefile index f3e3c172ddc..b9eb75b95b0 100644 --- a/services/director/src/simcore_service_director/api/Makefile +++ b/services/director/src/simcore_service_director/api/Makefile @@ -32,9 +32,3 @@ ${OAS_TARGET}: ${OAS_SOURCES} .update-schemas --outfile $@ \ --type yaml \ "${API_SPECS_DIR}/${APP_NAME}/openapi.yaml" - - -.PHONY: openapi-specs -openapi-specs: ${OAS_TARGET} ## creates and validates OpenAPI specs - # Validating bundled '${OAS_TARGET}' - @swagger-cli validate $< diff --git a/services/dynamic-scheduler/openapi.json b/services/dynamic-scheduler/openapi.json index b375bb8729e..0b593da90d1 100644 --- a/services/dynamic-scheduler/openapi.json +++ b/services/dynamic-scheduler/openapi.json @@ -2,14 +2,14 @@ "openapi": "3.1.0", "info": { "title": "simcore-service-dynamic-scheduler web API", - "description": " Service that manages lifecycle of dynamic services", + "description": "Service that manages lifecycle of dynamic services", "version": "1.0.0" }, "paths": { - "/": { + "/health": { "get": { "summary": "Healthcheck", - "operationId": "healthcheck__get", + "operationId": "healthcheck_health_get", "responses": { "200": { "description": "Successful Response", @@ -59,11 +59,24 @@ "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Version" }, + "released": { + "anyOf": [ + { + "additionalProperties": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Released", + "description": "Maps every route's path tag with a released version" + }, "docs_url": { "type": "string", - "maxLength": 2083, - "minLength": 1, - "format": "uri", "title": "Docs Url" } }, @@ -73,12 +86,7 @@ "version", "docs_url" ], - "title": "Meta", - "example": { - "name": "simcore_service_dynamic_scheduler", - "version": "2.4.45", - "docs_url": "https://foo.io/doc" - } + "title": "Meta" } } } diff --git a/services/payments/gateway/openapi.json b/services/payments/gateway/openapi.json deleted file mode 100644 index 20b8c11bbbc..00000000000 --- a/services/payments/gateway/openapi.json +++ /dev/null @@ -1,1039 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "title": "osparc-compliant payment-gateway", - "version": "0.3.0" - }, - "paths": { - "/init": { - "post": { - "tags": [ - "payment" - ], - "summary": "Init Payment", - "operationId": "init_payment", - "parameters": [ - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InitPayment" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PaymentInitiated" - } - } - } - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/pay": { - "get": { - "tags": [ - "payment" - ], - "summary": "Get Payment Form", - "operationId": "get_payment_form", - "parameters": [ - { - "name": "id", - "in": "query", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 100, - "title": "Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "text/html": { - "schema": { - "type": "string" - } - } - } - }, - "4XX": { - "content": { - "text/html": { - "schema": { - "type": "string" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/cancel": { - "post": { - "tags": [ - "payment" - ], - "summary": "Cancel Payment", - "operationId": "cancel_payment", - "parameters": [ - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PaymentInitiated" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PaymentCancelled" - } - } - } - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/payment-methods:init": { - "post": { - "tags": [ - "payment-method" - ], - "summary": "Init Payment Method", - "operationId": "init_payment_method", - "parameters": [ - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InitPaymentMethod" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PaymentMethodInitiated" - } - } - } - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/payment-methods/form": { - "get": { - "tags": [ - "payment-method" - ], - "summary": "Get Form Payment Method", - "operationId": "get_form_payment_method", - "parameters": [ - { - "name": "id", - "in": "query", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 100, - "title": "Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "text/html": { - "schema": { - "type": "string" - } - } - } - }, - "4XX": { - "content": { - "text/html": { - "schema": { - "type": "string" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/payment-methods:batchGet": { - "post": { - "tags": [ - "payment-method" - ], - "summary": "Batch Get Payment Methods", - "operationId": "batch_get_payment_methods", - "parameters": [ - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchGetPaymentMethods" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PaymentMethodsBatch" - } - } - } - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/payment-methods/{id}": { - "get": { - "tags": [ - "payment-method" - ], - "summary": "Get Payment Method", - "operationId": "get_payment_method", - "parameters": [ - { - "name": "id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 100, - "title": "Id" - } - }, - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GetPaymentMethod" - } - } - } - }, - "404": { - "description": "Payment method not found: It was not added or incomplete (i.e. create flow failed or canceled)", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - } - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - }, - "delete": { - "tags": [ - "payment-method" - ], - "summary": "Delete Payment Method", - "operationId": "delete_payment_method", - "parameters": [ - { - "name": "id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 100, - "title": "Id" - } - }, - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "responses": { - "204": { - "description": "Successful Response" - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - } - }, - "/payment-methods/{id}:pay": { - "post": { - "tags": [ - "payment-method" - ], - "summary": "Pay With Payment Method", - "operationId": "pay_with_payment_method", - "parameters": [ - { - "name": "id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 100, - "title": "Id" - } - }, - { - "name": "x-init-api-secret", - "in": "header", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "X-Init-Api-Secret" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InitPayment" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AckPaymentWithPaymentMethod" - } - } - } - }, - "4XX": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorModel" - } - } - }, - "description": "Client Error" - } - } - } - } - }, - "components": { - "schemas": { - "AckPaymentWithPaymentMethod": { - "properties": { - "success": { - "type": "boolean", - "title": "Success" - }, - "message": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Message" - }, - "provider_payment_id": { - "anyOf": [ - { - "type": "string", - "maxLength": 100, - "minLength": 1 - }, - { - "type": "null" - } - ], - "title": "Provider Payment Id", - "description": "Payment ID from the provider (e.g. stripe payment ID)" - }, - "invoice_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Invoice Url", - "description": "Link to invoice is required when success=true" - }, - "invoice_pdf": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Invoice Pdf", - "description": "Link to invoice PDF" - }, - "stripe_invoice_id": { - "anyOf": [ - { - "type": "string", - "maxLength": 100, - "minLength": 1 - }, - { - "type": "null" - } - ], - "title": "Stripe Invoice Id", - "description": "Stripe invoice ID" - }, - "stripe_customer_id": { - "anyOf": [ - { - "type": "string", - "maxLength": 100, - "minLength": 1 - }, - { - "type": "null" - } - ], - "title": "Stripe Customer Id", - "description": "Stripe customer ID" - }, - "payment_id": { - "anyOf": [ - { - "type": "string", - "maxLength": 100, - "minLength": 1 - }, - { - "type": "null" - } - ], - "title": "Payment Id", - "description": "Payment ID from the gateway" - } - }, - "type": "object", - "required": [ - "success" - ], - "title": "AckPaymentWithPaymentMethod", - "example": { - "invoice_url": "https://invoices.com/id=12345", - "payment_id": "D19EE68B-B007-4B61-A8BC-32B7115FB244", - "provider_payment_id": "pi_123ABC", - "success": true - } - }, - "BatchGetPaymentMethods": { - "properties": { - "payment_methods_ids": { - "items": { - "type": "string", - "maxLength": 100, - "minLength": 1 - }, - "type": "array", - "title": "Payment Methods Ids" - } - }, - "type": "object", - "required": [ - "payment_methods_ids" - ], - "title": "BatchGetPaymentMethods" - }, - "ErrorModel": { - "properties": { - "message": { - "type": "string", - "title": "Message" - }, - "exception": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Exception" - }, - "file": { - "anyOf": [ - { - "type": "string", - "format": "path" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "File" - }, - "line": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Line" - }, - "trace": { - "anyOf": [ - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Trace" - } - }, - "type": "object", - "required": [ - "message" - ], - "title": "ErrorModel" - }, - "GetPaymentMethod": { - "properties": { - "id": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "Id" - }, - "card_holder_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Card Holder Name" - }, - "card_number_masked": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Card Number Masked" - }, - "card_type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Card Type" - }, - "expiration_month": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Expiration Month" - }, - "expiration_year": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Expiration Year" - }, - "created": { - "type": "string", - "format": "date-time", - "title": "Created" - } - }, - "type": "object", - "required": [ - "id", - "created" - ], - "title": "GetPaymentMethod" - }, - "InitPayment": { - "properties": { - "amount_dollars": { - "anyOf": [ - { - "type": "number", - "exclusiveMaximum": true, - "exclusiveMinimum": true, - "maximum": 1000000.0, - "minimum": 0.0 - }, - { - "type": "string" - } - ], - "title": "Amount Dollars" - }, - "credits": { - "anyOf": [ - { - "type": "number", - "exclusiveMaximum": true, - "exclusiveMinimum": true, - "maximum": 1000000.0, - "minimum": 0.0 - }, - { - "type": "string" - } - ], - "title": "Credits", - "describe": "This is equal to `quantity` field in Stripe" - }, - "user_name": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "User Name" - }, - "user_email": { - "type": "string", - "format": "email", - "title": "User Email" - }, - "user_address": { - "$ref": "#/components/schemas/UserInvoiceAddress" - }, - "wallet_name": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "Wallet Name" - }, - "stripe_price_id": { - "type": "string", - "title": "Stripe Price Id" - }, - "stripe_tax_rate_id": { - "type": "string", - "title": "Stripe Tax Rate Id" - }, - "stripe_tax_exempt_value": { - "$ref": "#/components/schemas/StripeTaxExempt" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "amount_dollars", - "credits", - "user_name", - "user_email", - "user_address", - "wallet_name", - "stripe_price_id", - "stripe_tax_rate_id", - "stripe_tax_exempt_value" - ], - "title": "InitPayment" - }, - "InitPaymentMethod": { - "properties": { - "method": { - "type": "string", - "const": "CC", - "title": "Method", - "default": "CC" - }, - "user_name": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "User Name" - }, - "user_email": { - "type": "string", - "format": "email", - "title": "User Email" - }, - "wallet_name": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "Wallet Name" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "user_name", - "user_email", - "wallet_name" - ], - "title": "InitPaymentMethod" - }, - "PaymentCancelled": { - "properties": { - "message": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Message" - } - }, - "type": "object", - "title": "PaymentCancelled" - }, - "PaymentInitiated": { - "properties": { - "payment_id": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "Payment Id" - } - }, - "type": "object", - "required": [ - "payment_id" - ], - "title": "PaymentInitiated" - }, - "PaymentMethodInitiated": { - "properties": { - "payment_method_id": { - "type": "string", - "maxLength": 100, - "minLength": 1, - "title": "Payment Method Id" - } - }, - "type": "object", - "required": [ - "payment_method_id" - ], - "title": "PaymentMethodInitiated" - }, - "PaymentMethodsBatch": { - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/GetPaymentMethod" - }, - "type": "array", - "title": "Items" - } - }, - "type": "object", - "required": [ - "items" - ], - "title": "PaymentMethodsBatch" - }, - "StripeTaxExempt": { - "type": "string", - "enum": [ - "exempt", - "none", - "reverse" - ], - "title": "StripeTaxExempt" - }, - "UserInvoiceAddress": { - "properties": { - "line1": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Line1" - }, - "state": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "State" - }, - "postal_code": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Postal Code" - }, - "city": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "City" - }, - "country": { - "type": "string", - "title": "Country", - "description": "Currently validated in webserver via pycountry library. Two letter country code alpha_2 expected." - } - }, - "type": "object", - "required": [ - "country" - ], - "title": "UserInvoiceAddress" - } - } - } -} diff --git a/services/resource-usage-tracker/openapi.json b/services/resource-usage-tracker/openapi.json index 3ee8f2eeb73..91ee050d3d9 100644 --- a/services/resource-usage-tracker/openapi.json +++ b/services/resource-usage-tracker/openapi.json @@ -458,7 +458,7 @@ "$ref": "#/components/schemas/UnitExtraInfo" }, "current_cost_per_unit": { - "type": "number", + "type": "string", "title": "Current Cost Per Unit" }, "current_cost_per_unit_id": { diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 829aa2be9c2..485fb15c931 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -711,8 +711,6 @@ paths: in: query required: false schema: - enum: - - std const: std type: string default: std @@ -1010,9 +1008,7 @@ paths: - type: string minLength: 1 maxLength: 100 - - enum: - - current - const: current + - const: current type: string title: Product Name responses: @@ -1038,9 +1034,7 @@ paths: - type: string minLength: 1 maxLength: 100 - - enum: - - current - const: current + - const: current type: string title: Product Name - name: template_id @@ -4546,7 +4540,7 @@ paths: '403': description: ProjectInvalidRightsError '404': - description: UserDefaultWalletNotFoundError, ProjectNotFoundError + description: ProjectNotFoundError, UserDefaultWalletNotFoundError '409': description: ProjectTooManyProjectOpenedError '422': @@ -5452,7 +5446,7 @@ paths: schema: anyOf: - $ref: '#/components/schemas/Envelope_FileUploadSchema_' - - $ref: '#/components/schemas/Envelope_Url_' + - $ref: '#/components/schemas/Envelope_AnyUrl_' title: Response Upload File delete: tags: @@ -5840,9 +5834,7 @@ paths: anyOf: - type: integer - type: string - - enum: - - HEAD - const: HEAD + - const: HEAD type: string title: Ref Id - name: project_uuid @@ -6626,7 +6618,9 @@ components: limits: $ref: '#/components/schemas/Limits' queued: - type: boolean + anyOf: + - type: boolean + - type: 'null' title: Queued type: object required: @@ -6766,16 +6760,12 @@ components: url: anyOf: - type: string - minLength: 1 - format: uri - type: 'null' title: Url description: Link to current resource diagnostics_url: anyOf: - type: string - minLength: 1 - format: uri - type: 'null' title: Diagnostics Url description: Link to diagnostics report sub-resource. This MIGHT take some @@ -6876,9 +6866,6 @@ components: thumbnail: anyOf: - type: string - maxLength: 2083 - minLength: 1 - format: uri - type: 'null' title: Thumbnail description: @@ -7169,9 +7156,6 @@ components: title: Parents Ids url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Url type: object required: @@ -7297,15 +7281,11 @@ components: task url: type: string - minLength: 1 - format: uri title: Url description: the link where to get the status of the task stop_url: anyOf: - type: string - minLength: 1 - format: uri - type: 'null' title: Stop Url description: the link where to stop the task @@ -7565,6 +7545,20 @@ components: additionalProperties: false type: object title: EmptyModel + Envelope_AnyUrl_: + properties: + data: + anyOf: + - type: string + - type: 'null' + title: Data + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[AnyUrl] Envelope_AppStatusCheck_: properties: data: @@ -8248,22 +8242,6 @@ components: title: Error type: object title: Envelope[Union[WalletGet, NoneType]] - Envelope_Url_: - properties: - data: - anyOf: - - type: string - minLength: 1 - format: uri - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[Url] Envelope_UserProfile_: properties: data: @@ -8429,6 +8407,8 @@ components: type: integer exclusiveMinimum: true minimum: 0 + propertyNames: + const: comment_id type: object - type: 'null' title: Data @@ -8445,6 +8425,8 @@ components: anyOf: - additionalProperties: $ref: '#/components/schemas/Activity' + propertyNames: + format: uuid type: object - type: 'null' title: Data @@ -8461,6 +8443,8 @@ components: anyOf: - additionalProperties: $ref: '#/components/schemas/ProjectInputGet' + propertyNames: + format: uuid type: object - type: 'null' title: Data @@ -8477,6 +8461,8 @@ components: anyOf: - additionalProperties: $ref: '#/components/schemas/ProjectOutputGet' + propertyNames: + format: uuid type: object - type: 'null' title: Data @@ -9261,8 +9247,6 @@ components: file_size: anyOf: - type: integer - enum: - - -1 const: -1 - type: integer minimum: 0 @@ -9321,8 +9305,6 @@ components: properties: state: type: string - minLength: 1 - format: uri title: State type: object required: @@ -9357,13 +9339,9 @@ components: properties: abort_upload: type: string - minLength: 1 - format: uri title: Abort Upload complete_upload: type: string - minLength: 1 - format: uri title: Complete Upload type: object required: @@ -9379,8 +9357,6 @@ components: urls: items: type: string - minLength: 1 - format: uri type: array title: Urls links: @@ -9740,8 +9716,6 @@ components: thumbnail: anyOf: - type: string - minLength: 1 - format: uri - type: 'null' title: Thumbnail description: url to the group thumbnail @@ -9985,9 +9959,6 @@ components: title: Created invitationLink: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Invitationlink type: object required: @@ -10062,7 +10033,6 @@ components: type: string enum: - VIP_MODEL - const: VIP_MODEL title: LicensedResourceType Limits: properties: @@ -10292,6 +10262,10 @@ components: thumbnail: anyOf: - type: string + - type: string + maxLength: 2083 + minLength: 1 + format: uri - type: 'null' title: Thumbnail description: url of the latest screenshot of the node @@ -10302,7 +10276,6 @@ components: title: Runhash description: the hex digest of the resolved inputs +outputs hash at the time when the last outputs were generated - nullable: true inputs: anyOf: - type: object @@ -10366,7 +10339,6 @@ components: - type: 'null' title: Parent description: Parent's (group-nodes') node ID s. Used to group - nullable: true position: anyOf: - $ref: '#/components/schemas/Position' @@ -10433,7 +10405,6 @@ components: title: Runhash description: the hex digest of the resolved inputs +outputs hash at the time when the last outputs were generated - nullable: true inputs: anyOf: - type: object @@ -10497,7 +10468,6 @@ components: - type: 'null' title: Parent description: Parent's (group-nodes') node ID s. Used to group - nullable: true position: anyOf: - $ref: '#/components/schemas/Position' @@ -10598,6 +10568,7 @@ components: exclusiveMinimum: true title: Serviceport description: port to access the service within the network + default: 8080 maximum: 65535 minimum: 0 serviceBasepath: @@ -10634,7 +10605,6 @@ components: - serviceKey - serviceVersion - serviceHost - - servicePort - serviceState - userId title: NodeGet @@ -10642,8 +10612,6 @@ components: properties: serviceState: type: string - enum: - - idle const: idle title: Servicestate serviceUuid: @@ -10662,8 +10630,6 @@ components: properties: serviceState: type: string - enum: - - unknown const: unknown title: Servicestate serviceUuid: @@ -10709,6 +10675,7 @@ components: inputs: type: object title: Inputs + default: {} inputsRequired: anyOf: - items: @@ -10770,15 +10737,9 @@ components: properties: thumbnail_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Thumbnail Url file_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: File Url mimetype: anyOf: @@ -11153,9 +11114,6 @@ components: title: Paymentmethodid paymentMethodFormUrl: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Paymentmethodformurl description: Link to external site that holds the payment submission form type: object @@ -11213,9 +11171,6 @@ components: invoiceUrl: anyOf: - type: string - maxLength: 2083 - minLength: 1 - format: uri - type: 'null' title: Invoiceurl type: object @@ -11271,6 +11226,8 @@ components: type: string format: uuid type: array + propertyNames: + format: uuid type: object title: Adjacency List description: 'The adjacency list of the current pipeline in terms of {NodeID: @@ -11287,6 +11244,8 @@ components: node_states: additionalProperties: $ref: '#/components/schemas/NodeState' + propertyNames: + format: uuid type: object title: Node States description: The states of each of the computational nodes in the pipeline @@ -11405,8 +11364,6 @@ components: properties: link: type: string - minLength: 1 - format: uri title: Link type: object required: @@ -11459,7 +11416,6 @@ components: type: string enum: - TIER - const: TIER title: PricingPlanClassification PricingPlanToServiceAdminGet: properties: @@ -11695,6 +11651,9 @@ components: accessRights: additionalProperties: $ref: '#/components/schemas/AccessRights' + propertyNames: + maxLength: 100 + minLength: 1 type: object title: Accessrights tags: @@ -11746,12 +11705,7 @@ components: thumbnail: anyOf: - type: string - maxLength: 2083 - minLength: 1 - format: uri - type: string - enum: - - '' const: '' title: Thumbnail creationDate: @@ -11772,6 +11726,9 @@ components: accessRights: additionalProperties: $ref: '#/components/schemas/AccessRights' + propertyNames: + maxLength: 100 + minLength: 1 type: object title: Accessrights tags: @@ -11936,9 +11893,6 @@ components: project where this iteration is run workcopy_project_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Workcopy Project Url description: reference to a working copy project type: object @@ -11971,9 +11925,6 @@ components: project where this iteration is run workcopy_project_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Workcopy Project Url description: reference to a working copy project results: @@ -12002,12 +11953,7 @@ components: thumbnail: anyOf: - type: string - maxLength: 2083 - minLength: 1 - format: uri - type: string - enum: - - '' const: '' title: Thumbnail creationDate: @@ -12028,6 +11974,9 @@ components: accessRights: additionalProperties: $ref: '#/components/schemas/AccessRights' + propertyNames: + maxLength: 100 + minLength: 1 type: object title: Accessrights tags: @@ -12225,6 +12174,9 @@ components: anyOf: - additionalProperties: $ref: '#/components/schemas/AccessRights' + propertyNames: + maxLength: 100 + minLength: 1 type: object - type: 'null' title: Accessrights @@ -12255,9 +12207,6 @@ components: properties: url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Url is_public: type: boolean @@ -12492,9 +12441,6 @@ components: title: Project Uuid url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Url type: object required: @@ -12650,6 +12596,7 @@ components: exclusiveMinimum: true title: Service Port description: the service swarm internal port + default: 8080 maximum: 65535 minimum: 0 published_port: @@ -12687,7 +12634,6 @@ components: - project_id - service_uuid - service_host - - service_port - service_state title: RunningDynamicServiceDetails RunningState: @@ -12744,9 +12690,6 @@ components: description: Long description of the service thumbnail: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Thumbnail description: Url to service thumbnail file_extensions: @@ -12757,9 +12700,6 @@ components: description: File extensions that this service can process view_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: View Url description: Redirection to open a service in osparc (see /view) type: object @@ -13191,7 +13131,6 @@ components: type: string enum: - services - const: services title: ServicesAggregatedUsagesType SimCoreFileLink: properties: @@ -13670,7 +13609,7 @@ components: - type: string format: email - type: 'null' - title: 'From ' + title: From description: Email sender to: type: string @@ -13939,8 +13878,6 @@ components: product: anyOf: - type: string - enum: - - UNDEFINED const: UNDEFINED - type: string title: Product @@ -13948,8 +13885,6 @@ components: resource_id: anyOf: - type: string - enum: - - '' const: '' - type: string title: Resource Id @@ -13957,8 +13892,6 @@ components: user_from_id: anyOf: - type: 'null' - enum: - - null - type: integer exclusiveMinimum: true minimum: 0 @@ -14005,8 +13938,6 @@ components: product: anyOf: - type: string - enum: - - UNDEFINED const: UNDEFINED - type: string title: Product @@ -14014,8 +13945,6 @@ components: resource_id: anyOf: - type: string - enum: - - '' const: '' - type: string title: Resource Id @@ -14023,8 +13952,6 @@ components: user_from_id: anyOf: - type: 'null' - enum: - - null - type: integer exclusiveMinimum: true minimum: 0 @@ -14160,9 +14087,6 @@ components: description: Identifier for the file type view_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: View Url description: Base url to execute viewer. Needs appending file_size,[file_name] and download_link as query parameters @@ -14318,9 +14242,6 @@ components: paymentFormUrl: anyOf: - type: string - maxLength: 2083 - minLength: 1 - format: uri - type: 'null' title: Paymentformurl description: Link to external site that holds the payment submission form.None @@ -14384,15 +14305,9 @@ components: default: {} url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Url checkpoint_url: type: string - maxLength: 2083 - minLength: 1 - format: uri title: Checkpoint Url type: object required: From be48a0d3425fde6d7e0d310978d006406e388ccc Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:11:00 +0100 Subject: [PATCH 4/9] =?UTF-8?q?=E2=AC=86=EF=B8=8FUpgrade=20dask=20related?= =?UTF-8?q?=20libraries=20and=20services=20(=E2=9A=A0=EF=B8=8F=F0=9F=9A=A8?= =?UTF-8?q?)=20(#6873)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .codecov.yml | 1 + .github/workflows/ci-testing-deploy.yml | 283 +++++++----------- .../requirements/_base.txt | 39 +-- .../requirements/_test.txt | 23 +- .../requirements/_tools.txt | 33 +- .../src/pytest_simcore/dask_scheduler.py | 1 + services/autoscaling/requirements/_base.txt | 65 ++-- services/autoscaling/requirements/_test.txt | 12 +- services/autoscaling/requirements/_tools.txt | 5 +- .../clusters-keeper/requirements/_base.txt | 65 ++-- .../clusters-keeper/requirements/_test.txt | 12 +- .../clusters-keeper/requirements/_tools.txt | 5 +- services/dask-sidecar/requirements/_base.txt | 167 ++++++----- .../requirements/_dask-distributed.txt | 28 +- services/dask-sidecar/requirements/_test.txt | 98 +++--- services/dask-sidecar/requirements/_tools.txt | 37 ++- services/director-v2/requirements/_base.txt | 73 ++--- services/director-v2/requirements/_test.txt | 28 +- services/director-v2/requirements/_tools.txt | 8 +- .../modules/dask_client.py | 54 +++- .../tests/unit/test_modules_dask_client.py | 72 ++--- 21 files changed, 530 insertions(+), 579 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index ca81ee39226..02666df0a13 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -22,6 +22,7 @@ component_management: statuses: - type: project target: auto + threshold: 1% branches: - "!master" individual_components: diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 63623a2c521..277a131e5c2 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -347,16 +347,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/webserver.bash test_with_db 01 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-02: needs: changes @@ -392,16 +389,13 @@ jobs: run: ./ci/github/unit-testing/webserver.bash install - name: test run: ./ci/github/unit-testing/webserver.bash test_with_db 02 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-03: needs: changes @@ -437,16 +431,13 @@ jobs: run: ./ci/github/unit-testing/webserver.bash install - name: test run: ./ci/github/unit-testing/webserver.bash test_with_db 03 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-webserver-04: needs: changes @@ -482,7 +473,7 @@ jobs: run: ./ci/github/unit-testing/webserver.bash install - name: test run: ./ci/github/unit-testing/webserver.bash test_with_db 04 - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -530,16 +521,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/storage.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-agent: needs: changes @@ -580,16 +568,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/agent.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-api: needs: changes @@ -625,16 +610,13 @@ jobs: run: ./ci/github/unit-testing/api.bash install - name: test run: ./ci/github/unit-testing/api.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-api-server: needs: changes @@ -676,16 +658,13 @@ jobs: - name: OAS backwards compatibility check if: ${{ !cancelled() }} run: ./ci/github/unit-testing/api-server.bash openapi-diff - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-autoscaling: needs: changes @@ -724,16 +703,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/autoscaling.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-catalog: needs: changes @@ -778,16 +754,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/catalog/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-clusters-keeper: needs: changes @@ -837,16 +810,13 @@ jobs: source .venv/bin/activate && \ pushd services/clusters-keeper && \ make test-ci-unit - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-datcore-adapter: needs: changes @@ -891,16 +861,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/datcore-adapter/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-director: needs: changes @@ -945,16 +912,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/director/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-director-v2: needs: changes @@ -999,16 +963,13 @@ jobs: with: name: ${{ github.job }}_docker_logs path: ./services/director-v2/test_failures - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-aws-library: needs: changes @@ -1047,16 +1008,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/aws-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dask-task-models-library: needs: changes @@ -1095,16 +1053,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dask-task-models-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dask-sidecar: needs: changes @@ -1143,16 +1098,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dask-sidecar.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-payments: needs: changes @@ -1191,16 +1143,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/payments.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dynamic-scheduler: needs: changes @@ -1239,16 +1188,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dynamic-scheduler.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-resource-usage-tracker: needs: changes @@ -1297,16 +1243,13 @@ jobs: source .venv/bin/activate && \ pushd services/resource-usage-tracker && \ make test-ci-unit - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-dynamic-sidecar: needs: changes @@ -1345,16 +1288,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/dynamic-sidecar.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-efs-guardian: needs: changes @@ -1404,16 +1344,13 @@ jobs: source .venv/bin/activate && \ pushd services/efs-guardian && \ make test-ci-unit - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-python-linting: needs: changes @@ -1487,16 +1424,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/postgres-database.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-invitations: needs: changes @@ -1535,16 +1469,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/invitations.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-service-integration: needs: changes @@ -1583,16 +1514,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/service-integration.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-service-library: needs: changes @@ -1631,16 +1559,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/service-library.bash test_all - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-settings-library: needs: changes @@ -1679,16 +1604,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/settings-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-models-library: needs: changes @@ -1726,16 +1648,13 @@ jobs: run: ./ci/github/unit-testing/models-library.bash typecheck - name: test run: ./ci/github/unit-testing/models-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-common-library: needs: changes @@ -1774,7 +1693,7 @@ jobs: run: ./ci/github/unit-testing/common-library.bash typecheck - name: test run: ./ci/github/unit-testing/common-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 with: flags: unittests #optional @@ -1817,16 +1736,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/notifications-library.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-test-simcore-sdk: needs: changes @@ -1867,16 +1783,13 @@ jobs: - name: test if: ${{ !cancelled() }} run: ./ci/github/unit-testing/simcore-sdk.bash test - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: flags: unittests #optional - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} + unit-tests: # NOTE: this is a github required status check! @@ -1982,7 +1895,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/webserver.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2045,7 +1959,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/webserver.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2108,7 +2023,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/director-v2.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2180,7 +2096,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/director-v2.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2245,7 +2162,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/dynamic-sidecar.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: @@ -2308,7 +2226,8 @@ jobs: - name: cleanup if: ${{ !cancelled() }} run: ./ci/github/integration-testing/simcore-sdk.bash clean_up - - uses: codecov/codecov-action@v5.0.7 + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt index 7e813c8de2e..3f7021f3ad6 100644 --- a/packages/dask-task-models-library/requirements/_base.txt +++ b/packages/dask-task-models-library/requirements/_base.txt @@ -11,21 +11,21 @@ click==8.1.7 # dask # distributed # typer -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # dask # distributed -dask==2024.9.0 +dask==2024.12.0 # via # -r requirements/_base.in # distributed -distributed==2024.9.0 +distributed==2024.12.0 # via dask -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via pydantic -fsspec==2024.9.0 +fsspec==2024.10.0 # via dask idna==3.10 # via email-validator @@ -42,7 +42,7 @@ jinja2==3.1.4 # distributed jsonschema==4.23.0 # via -r requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema locket==1.0.0 # via @@ -50,13 +50,13 @@ locket==1.0.0 # partd markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 mdurl==0.1.2 # via markdown-it-py msgpack==1.1.0 # via distributed -orjson==3.10.7 +orjson==3.10.12 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -68,13 +68,13 @@ orjson==3.10.7 # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.1 +packaging==24.2 # via # dask # distributed partd==1.4.2 # via dask -psutil==6.0.0 +psutil==6.1.0 # via distributed pydantic==2.10.3 # via @@ -94,7 +94,7 @@ pydantic==2.10.3 # pydantic-settings pydantic-core==2.27.1 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.0 # via # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -124,37 +124,38 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -rich==13.8.1 +rich==13.9.4 # via # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.20.0 +rpds-py==0.22.3 # via # jsonschema # referencing shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sortedcontainers==2.4.0 # via distributed tblib==3.0.0 # via distributed -toolz==0.12.1 +toolz==1.0.0 # via # dask # distributed # partd -tornado==6.4.1 +tornado==6.4.2 # via distributed -typer==0.12.5 +typer==0.15.1 # via -r requirements/../../../packages/settings-library/requirements/_base.in -types-python-dateutil==2.9.0.20240906 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # pydantic # pydantic-core + # pydantic-extra-types # typer urllib3==2.2.3 # via @@ -167,5 +168,5 @@ urllib3==2.2.3 # distributed zict==3.0.0 # via distributed -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/packages/dask-task-models-library/requirements/_test.txt b/packages/dask-task-models-library/requirements/_test.txt index b0593212939..21f3ea3ccc4 100644 --- a/packages/dask-task-models-library/requirements/_test.txt +++ b/packages/dask-task-models-library/requirements/_test.txt @@ -1,31 +1,31 @@ -appdirs==1.4.4 - # via pint -coverage==7.6.1 +coverage==7.6.8 # via # -r requirements/_test.in # pytest-cov -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in flexcache==0.3 # via pint -flexparser==0.3.1 +flexparser==0.4 # via pint icdiff==2.0.7 # via pytest-icdiff iniconfig==2.0.0 # via pytest -packaging==24.1 +packaging==24.2 # via # -c requirements/_base.txt # pytest # pytest-sugar -pint==0.24.3 +pint==0.24.4 # via -r requirements/_test.in +platformdirs==4.3.6 + # via pint pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -38,7 +38,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-icdiff==0.9 # via -r requirements/_test.in @@ -59,15 +59,16 @@ pyyaml==6.0.2 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in -six==1.16.0 +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar typing-extensions==4.12.2 # via # -c requirements/_base.txt + # faker # flexcache # flexparser # pint diff --git a/packages/dask-task-models-library/requirements/_tools.txt b/packages/dask-task-models-library/requirements/_tools.txt index 2b96415451e..7bd7ea01f5a 100644 --- a/packages/dask-task-models-library/requirements/_tools.txt +++ b/packages/dask-task-models-library/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -13,13 +13,13 @@ click==8.1.7 # -c requirements/_base.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -27,7 +27,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -35,7 +35,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.1 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -43,20 +43,21 @@ packaging==24.1 # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt platformdirs==4.3.6 # via + # -c requirements/_test.txt # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -66,9 +67,9 @@ pyyaml==6.0.2 # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit -ruff==0.6.7 +ruff==0.8.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.1.0 +setuptools==75.6.0 # via pip-tools tomlkit==0.13.2 # via pylint @@ -77,7 +78,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py index 54019faca11..03c63c07d34 100644 --- a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py +++ b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py @@ -72,6 +72,7 @@ async def dask_spec_local_cluster( asynchronous=True, name="pytest_dask_spec_local_cluster", ) as cluster: + print("Cluster dashboard link: ", cluster.dashboard_link) scheduler_address = URL(cluster.scheduler_address) monkeypatch.setenv( "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL", diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index dec4d478b86..e0a298107ed 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -131,12 +131,12 @@ click==8.1.7 # distributed # typer # uvicorn -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in @@ -147,7 +147,7 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -170,7 +170,7 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -220,7 +220,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -272,7 +272,7 @@ locket==1.0.0 # partd markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # jinja2 @@ -286,7 +286,7 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -303,19 +303,19 @@ opentelemetry-api==1.26.0 # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore @@ -324,41 +324,42 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.49b2 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-botocore==0.47b0 +opentelemetry-instrumentation-botocore==0.49b2 # via -r requirements/../../../packages/aws-library/requirements/_base.in -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-httpx==0.47b0 +opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-logging==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-propagator-aws-xray==1.0.1 # via opentelemetry-instrumentation-botocore -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi @@ -366,7 +367,7 @@ opentelemetry-semantic-conventions==0.47b0 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -414,12 +415,13 @@ orjson==3.10.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 +packaging==24.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq partd==1.4.2 @@ -432,11 +434,11 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +protobuf==5.29.0 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -537,7 +539,7 @@ python-dateutil==2.9.0.post0 # botocore python-dotenv==1.0.1 # via pydantic-settings -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -619,8 +621,6 @@ rpds-py==0.18.1 # referencing s3transfer==0.10.1 # via boto3 -setuptools==74.0.0 - # via opentelemetry-instrumentation sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 @@ -670,7 +670,7 @@ tenacity==8.5.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -678,7 +678,7 @@ toolz==0.12.1 # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed @@ -755,6 +755,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis yarl==1.9.4 # via @@ -767,7 +768,7 @@ zict==3.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # importlib-metadata diff --git a/services/autoscaling/requirements/_test.txt b/services/autoscaling/requirements/_test.txt index b6342993557..d6bdd354de7 100644 --- a/services/autoscaling/requirements/_test.txt +++ b/services/autoscaling/requirements/_test.txt @@ -147,7 +147,7 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 # via fakeredis -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 @@ -164,7 +164,7 @@ openapi-spec-validator==0.7.1 # via moto orderly-set==5.2.2 # via deepdiff -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # pytest @@ -177,7 +177,7 @@ ply==3.11 # via jsonpath-ng pprintpp==0.4.0 # via pytest-icdiff -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/_base.txt # -r requirements/_test.in @@ -228,7 +228,7 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -273,9 +273,7 @@ s3transfer==0.10.1 # -c requirements/_base.txt # boto3 setuptools==74.0.0 - # via - # -c requirements/_base.txt - # moto + # via moto six==1.16.0 # via # -c requirements/_base.txt diff --git a/services/autoscaling/requirements/_tools.txt b/services/autoscaling/requirements/_tools.txt index 94d944fd4d1..b38a6583787 100644 --- a/services/autoscaling/requirements/_tools.txt +++ b/services/autoscaling/requirements/_tools.txt @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -61,7 +61,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -72,7 +72,6 @@ ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt setuptools==74.0.0 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index b5e0a2e1b12..cb76b03f0a6 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -129,12 +129,12 @@ click==8.1.7 # distributed # typer # uvicorn -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in @@ -145,7 +145,7 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -168,7 +168,7 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -218,7 +218,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -270,7 +270,7 @@ locket==1.0.0 # partd markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # jinja2 @@ -284,7 +284,7 @@ multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -301,19 +301,19 @@ opentelemetry-api==1.26.0 # opentelemetry-propagator-aws-xray # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore @@ -322,41 +322,42 @@ opentelemetry-instrumentation==0.47b0 # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.49b2 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-botocore==0.47b0 +opentelemetry-instrumentation-botocore==0.49b2 # via -r requirements/../../../packages/aws-library/requirements/_base.in -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-httpx==0.47b0 +opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-logging==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-propagator-aws-xray==1.0.1 # via opentelemetry-instrumentation-botocore -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-botocore # opentelemetry-instrumentation-fastapi @@ -364,7 +365,7 @@ opentelemetry-semantic-conventions==0.47b0 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -412,12 +413,13 @@ orjson==3.10.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 +packaging==24.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq partd==1.4.2 @@ -430,11 +432,11 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +protobuf==5.29.0 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -535,7 +537,7 @@ python-dateutil==2.9.0.post0 # botocore python-dotenv==1.0.1 # via pydantic-settings -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -617,8 +619,6 @@ rpds-py==0.18.1 # referencing s3transfer==0.10.1 # via boto3 -setuptools==74.0.0 - # via opentelemetry-instrumentation sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 @@ -668,7 +668,7 @@ tenacity==8.5.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -676,7 +676,7 @@ toolz==0.12.1 # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed @@ -753,6 +753,7 @@ wrapt==1.16.0 # aiobotocore # deprecated # opentelemetry-instrumentation + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis yarl==1.9.4 # via @@ -765,7 +766,7 @@ zict==3.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # importlib-metadata diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 0af083a1485..155d8b2b918 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -163,7 +163,7 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 # via fakeredis -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 @@ -185,7 +185,7 @@ openapi-spec-validator==0.7.1 # via moto orderly-set==5.2.2 # via deepdiff -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # pytest @@ -197,7 +197,7 @@ pluggy==1.5.0 # via pytest ply==3.11 # via jsonpath-ng -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/_base.txt # -r requirements/_test.in @@ -242,7 +242,7 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -287,9 +287,7 @@ s3transfer==0.10.1 # -c requirements/_base.txt # boto3 setuptools==74.0.0 - # via - # -c requirements/_base.txt - # moto + # via moto six==1.16.0 # via # -c requirements/_base.txt diff --git a/services/clusters-keeper/requirements/_tools.txt b/services/clusters-keeper/requirements/_tools.txt index 94d944fd4d1..b38a6583787 100644 --- a/services/clusters-keeper/requirements/_tools.txt +++ b/services/clusters-keeper/requirements/_tools.txt @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -61,7 +61,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -72,7 +72,6 @@ ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt setuptools==74.0.0 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index 0096bb1b261..20d9a1196c0 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -1,20 +1,22 @@ -aio-pika==9.4.1 +aio-pika==9.5.3 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiobotocore==2.13.0 +aiobotocore==2.15.2 # via s3fs -aiocache==0.12.2 +aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.24.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -aiohttp==3.9.5 +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.10 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -38,15 +40,15 @@ aiohttp==3.9.5 # aiodocker # fsspec # s3fs -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.7.0 # via # fast-depends # faststream @@ -56,18 +58,18 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing -blosc==1.11.1 +blosc==1.11.2 # via -r requirements/_base.in -bokeh==3.4.1 +bokeh==3.6.2 # via dask -botocore==1.34.106 +botocore==1.35.36 # via aiobotocore -certifi==2024.7.4 +certifi==2024.8.30 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -88,63 +90,65 @@ certifi==2024.7.4 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via # dask # distributed # typer -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # dask # distributed -contourpy==1.2.1 +contourpy==1.3.1 # via bokeh -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/constraints.txt # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in # -r requirements/_base.in # distributed -deprecated==1.2.14 +deprecated==1.2.15 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via dask -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic +exceptiongroup==1.2.2 + # via aio-pika fast-depends==2.4.12 # via faststream -faststream==0.5.31 +faststream==0.5.33 # via -r requirements/../../../packages/service-library/requirements/_base.in -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -r requirements/_base.in # dask # s3fs -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.0 +grpcio==1.68.1 # via opentelemetry-exporter-otlp-proto-grpc -idna==3.7 +idna==3.10 # via # anyio # email-validator # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # dask # opentelemetry-api @@ -173,12 +177,12 @@ jinja2==3.1.4 # distributed jmespath==1.0.1 # via botocore -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via jsonschema locket==1.0.0 # via @@ -188,22 +192,22 @@ lz4==4.3.3 # via -r requirements/_base.in markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 mdurl==0.1.2 # via markdown-it-py msgpack==1.1.0 # via distributed -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl -numpy==1.26.4 +numpy==2.1.3 # via # bokeh # contourpy # pandas -opentelemetry-api==1.26.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc @@ -214,45 +218,46 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.28.2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-logging==0.47b0 +opentelemetry-instrumentation-logging==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.47b0 +opentelemetry-instrumentation-redis==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.49b2 # via opentelemetry-instrumentation-requests -orjson==3.10.3 +orjson==3.10.12 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -284,30 +289,35 @@ orjson==3.10.3 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in -packaging==24.0 +packaging==24.2 # via # bokeh # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq -pandas==2.2.2 +pandas==2.2.3 # via bokeh partd==1.4.2 # via dask -pillow==10.3.0 +pillow==11.0.0 # via bokeh -prometheus-client==0.20.0 +prometheus-client==0.21.1 # via -r requirements/_base.in -protobuf==4.25.4 +propcache==0.2.1 + # via + # aiohttp + # yarl +protobuf==5.29.1 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # distributed -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -350,7 +360,7 @@ pydantic==2.10.2 # pydantic-settings pydantic-core==2.27.1 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.0 # via # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -374,7 +384,7 @@ pydantic-settings==2.6.1 # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via @@ -383,9 +393,9 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via pydantic-settings -pytz==2024.1 +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -409,7 +419,7 @@ pyyaml==6.0.1 # bokeh # dask # distributed -redis==5.0.4 +redis==5.2.0 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -430,7 +440,7 @@ redis==5.0.4 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in -referencing==0.29.3 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -440,23 +450,21 @@ repro-zipfile==0.3.1 # -r requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.4 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.1 +rpds-py==0.22.3 # via # jsonschema # referencing -s3fs==2024.5.0 +s3fs==2024.10.0 # via fsspec -setuptools==74.0.0 - # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -464,37 +472,38 @@ sortedcontainers==2.4.0 # via distributed tblib==3.0.0 # via distributed -tenacity==8.5.0 +tenacity==9.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # bokeh # distributed -tqdm==4.66.4 +tqdm==4.67.1 # via -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.15.1 # via # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # aiodebug - # aiodocker + # anyio # faststream # opentelemetry-sdk # pydantic # pydantic-core + # pydantic-extra-types # typer -tzdata==2024.1 +tzdata==2024.2 # via pandas urllib3==2.2.3 # via @@ -519,15 +528,15 @@ urllib3==2.2.3 # botocore # distributed # requests -wrapt==1.16.0 +wrapt==1.17.0 # via # aiobotocore # deprecated # opentelemetry-instrumentation # opentelemetry-instrumentation-redis -xyzservices==2024.4.0 +xyzservices==2024.9.0 # via bokeh -yarl==1.9.4 +yarl==1.18.3 # via # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika @@ -535,5 +544,5 @@ yarl==1.9.4 # aiormq zict==3.0.0 # via distributed -zipp==3.18.2 +zipp==3.21.0 # via importlib-metadata diff --git a/services/dask-sidecar/requirements/_dask-distributed.txt b/services/dask-sidecar/requirements/_dask-distributed.txt index 78a222ea415..2e37ea361da 100644 --- a/services/dask-sidecar/requirements/_dask-distributed.txt +++ b/services/dask-sidecar/requirements/_dask-distributed.txt @@ -1,4 +1,4 @@ -blosc==1.11.1 +blosc==1.11.2 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in @@ -7,25 +7,25 @@ click==8.1.7 # -c requirements/./_base.txt # dask # distributed -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/./_base.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in # distributed -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/./_base.txt # dask -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/./_base.txt # dask -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/./_base.txt # dask @@ -42,7 +42,7 @@ lz4==4.3.3 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/./_base.txt # jinja2 @@ -50,11 +50,11 @@ msgpack==1.1.0 # via # -c requirements/./_base.txt # distributed -numpy==1.26.4 +numpy==2.1.3 # via # -c requirements/./_base.txt # -r requirements/_dask-distributed.in -packaging==24.0 +packaging==24.2 # via # -c requirements/./_base.txt # dask @@ -63,11 +63,11 @@ partd==1.4.2 # via # -c requirements/./_base.txt # dask -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/./_base.txt # distributed -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/./_base.txt # dask @@ -80,13 +80,13 @@ tblib==3.0.0 # via # -c requirements/./_base.txt # distributed -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/./_base.txt # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/./_base.txt # distributed @@ -98,7 +98,7 @@ zict==3.0.0 # via # -c requirements/./_base.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/./_base.txt # importlib-metadata diff --git a/services/dask-sidecar/requirements/_test.txt b/services/dask-sidecar/requirements/_test.txt index 25ef41fa488..57070471558 100644 --- a/services/dask-sidecar/requirements/_test.txt +++ b/services/dask-sidecar/requirements/_test.txt @@ -4,38 +4,38 @@ annotated-types==0.7.0 # pydantic antlr4-python3-runtime==4.13.2 # via moto -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements/_base.txt # jsonschema # referencing -aws-sam-translator==1.89.0 +aws-sam-translator==1.94.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto -blinker==1.8.2 +blinker==1.9.0 # via flask -boto3==1.34.106 +boto3==1.35.36 # via # aws-sam-translator # moto -botocore==1.34.106 +botocore==1.35.36 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -certifi==2024.7.4 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # requests cffi==1.17.1 # via cryptography -cfn-lint==1.10.3 +cfn-lint==1.20.2 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests @@ -43,11 +43,11 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.1 +coverage==7.6.8 # via # -r requirements/_test.in # pytest-cov -cryptography==43.0.1 +cryptography==44.0.0 # via # -c requirements/../../../requirements/constraints.txt # joserfc @@ -57,19 +57,19 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in -flask==3.0.3 +flask==3.1.0 # via # flask-cors # moto flask-cors==5.0.0 # via moto -graphql-core==3.2.4 +graphql-core==3.2.5 # via moto icdiff==2.0.7 # via pytest-icdiff -idna==3.7 +idna==3.10 # via # -c requirements/_base.txt # requests @@ -88,53 +88,53 @@ jmespath==1.0.1 # -c requirements/_base.txt # boto3 # botocore -joserfc==1.0.0 +joserfc==1.0.1 # via moto jsondiff==2.2.1 # via moto jsonpatch==1.33 # via cfn-lint -jsonpath-ng==1.6.1 +jsonpath-ng==1.7.0 # via moto jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -c requirements/_base.txt # aws-sam-translator + # jsonschema-spec # openapi-schema-validator # openapi-spec-validator -jsonschema-path==0.3.3 +jsonschema-spec==0.1.3 # via openapi-spec-validator -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via # -c requirements/_base.txt # jsonschema - # openapi-schema-validator lazy-object-proxy==1.10.0 # via openapi-spec-validator -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 # werkzeug -moto==5.0.15 +moto==5.0.22 # via -r requirements/_test.in mpmath==1.3.0 # via sympy -networkx==3.3 +networkx==3.4.2 # via cfn-lint -openapi-schema-validator==0.6.2 +openapi-schema-validator==0.4.3 # via openapi-spec-validator -openapi-spec-validator==0.7.1 +openapi-spec-validator==0.5.5 # via moto -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # pytest # pytest-sugar pathable==0.4.3 - # via jsonschema-path + # via jsonschema-spec pluggy==1.5.0 # via pytest ply==3.11 @@ -145,7 +145,7 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -154,13 +154,13 @@ pydantic-core==2.27.1 # via # -c requirements/_base.txt # pydantic -pyftpdlib==2.0.0 +pyftpdlib==2.0.1 # via pytest-localftpserver -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via pytest-localftpserver -pyparsing==3.1.4 +pyparsing==3.2.0 # via moto -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -174,7 +174,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-icdiff==0.9 # via -r requirements/_test.in @@ -196,53 +196,49 @@ python-dotenv==1.0.1 # via # -c requirements/_base.txt # -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # cfn-lint # jsondiff - # jsonschema-path + # jsonschema-spec # moto # responses -referencing==0.29.3 +referencing==0.35.1 # via # -c requirements/_base.txt # jsonschema - # jsonschema-path # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via cfn-lint requests==2.32.3 # via # -c requirements/_base.txt # docker - # jsonschema-path # moto # responses responses==0.25.3 # via moto rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.18.1 +rpds-py==0.22.3 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.2 +s3transfer==0.10.4 # via boto3 -setuptools==74.0.0 - # via - # -c requirements/_base.txt - # moto -six==1.16.0 +setuptools==75.6.0 + # via moto +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil # rfc3339-validator sympy==1.13.3 # via cfn-lint -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar types-aiofiles==24.1.0.20240626 # via -r requirements/_test.in @@ -251,6 +247,8 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # aws-sam-translator # cfn-lint + # faker + # jsonschema-spec # pydantic # pydantic-core urllib3==2.2.3 @@ -261,13 +259,13 @@ urllib3==2.2.3 # docker # requests # responses -werkzeug==3.0.4 +werkzeug==3.1.3 # via # flask # moto -wrapt==1.16.0 +wrapt==1.17.0 # via # -c requirements/_base.txt # aws-xray-sdk -xmltodict==0.13.0 +xmltodict==0.14.2 # via moto diff --git a/services/dask-sidecar/requirements/_tools.txt b/services/dask-sidecar/requirements/_tools.txt index 94d944fd4d1..200f2c403cd 100644 --- a/services/dask-sidecar/requirements/_tools.txt +++ b/services/dask-sidecar/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,13 +14,13 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -28,7 +28,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -44,7 +44,7 @@ packaging==24.0 # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -53,26 +53,25 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.6.7 +ruff==0.8.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.6.0 # via - # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 @@ -82,9 +81,9 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -watchdog==5.0.2 +watchdog==6.0.0 # via -r requirements/_tools.in -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 0d514a1b886..b27992058e3 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -108,7 +108,7 @@ attrs==23.2.0 # referencing bidict==0.23.1 # via python-socketio -blosc==1.11.1 +blosc==1.11.2 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt certifi==2024.2.2 # via @@ -158,12 +158,12 @@ click==8.1.7 # distributed # typer # uvicorn -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed -dask==2024.5.1 +dask==2024.12.0 # via # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -174,7 +174,7 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -distributed==2024.5.1 +distributed==2024.12.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -205,7 +205,7 @@ frozenlist==1.4.1 # via # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -272,7 +272,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask @@ -373,7 +373,7 @@ mako==1.3.5 # alembic markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # jinja2 @@ -391,9 +391,9 @@ multidict==6.0.5 # yarl networkx==3.3 # via -r requirements/_base.in -numpy==1.26.4 +numpy==2.1.3 # via -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt -opentelemetry-api==1.27.0 +opentelemetry-api==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -411,19 +411,19 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.27.0 +opentelemetry-exporter-otlp==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.27.0 +opentelemetry-exporter-otlp-proto-common==1.28.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.27.0 +opentelemetry-exporter-otlp-proto-grpc==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.28.2 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.48b0 +opentelemetry-instrumentation==0.49b2 # via # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-asgi @@ -434,45 +434,46 @@ opentelemetry-instrumentation==0.48b0 # opentelemetry-instrumentation-logging # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-aiopg==0.48b0 +opentelemetry-instrumentation-aiopg==0.49b2 # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in -opentelemetry-instrumentation-asgi==0.48b0 +opentelemetry-instrumentation-asgi==0.49b2 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-asyncpg==0.48b0 +opentelemetry-instrumentation-asyncpg==0.49b2 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in -opentelemetry-instrumentation-dbapi==0.48b0 +opentelemetry-instrumentation-dbapi==0.49b2 # via opentelemetry-instrumentation-aiopg -opentelemetry-instrumentation-fastapi==0.48b0 +opentelemetry-instrumentation-fastapi==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-httpx==0.48b0 +opentelemetry-instrumentation-httpx==0.49b2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-logging==0.48b0 +opentelemetry-instrumentation-logging==0.49b2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-redis==0.48b0 +opentelemetry-instrumentation-redis==0.49b2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-instrumentation-requests==0.48b0 +opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.27.0 +opentelemetry-proto==1.28.2 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.27.0 +opentelemetry-sdk==1.28.2 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.48b0 +opentelemetry-semantic-conventions==0.49b2 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-dbapi @@ -481,7 +482,7 @@ opentelemetry-semantic-conventions==0.48b0 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.48b0 +opentelemetry-util-http==0.49b2 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi @@ -549,12 +550,13 @@ orjson==3.10.3 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in # fastapi -packaging==24.0 +packaging==24.2 # via # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask # distributed + # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq partd==1.4.2 @@ -569,11 +571,11 @@ prometheus-client==0.20.0 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +protobuf==5.29.0 # via # googleapis-common-protos # opentelemetry-proto -psutil==6.0.0 +psutil==6.1.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -715,7 +717,7 @@ python-multipart==0.0.9 # via fastapi python-socketio==5.11.2 # via -r requirements/_base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -821,8 +823,6 @@ rpds-py==0.18.1 # via # jsonschema # referencing -setuptools==74.0.0 - # via opentelemetry-instrumentation shellingham==1.5.4 # via typer simple-websocket==1.0.0 @@ -924,7 +924,7 @@ tenacity==8.5.0 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in @@ -932,7 +932,7 @@ toolz==0.12.1 # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed @@ -1059,6 +1059,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation # opentelemetry-instrumentation-aiopg # opentelemetry-instrumentation-dbapi + # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis wsproto==1.2.0 # via simple-websocket @@ -1075,7 +1076,7 @@ zict==3.0.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # importlib-metadata diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index 6194851e0c2..0f4d71204bc 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -67,7 +67,7 @@ click==8.1.7 # -c requirements/_base.txt # dask # distributed -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via # -c requirements/_base.txt # dask @@ -76,12 +76,12 @@ contourpy==1.3.0 # via bokeh coverage==7.6.1 # via pytest-cov -dask==2024.5.1 +dask==2024.12.0 # via # -c requirements/_base.txt # -r requirements/_test.in # distributed -distributed==2024.5.1 +distributed==2024.12.0 # via # -c requirements/_base.txt # dask @@ -98,7 +98,7 @@ frozenlist==1.4.1 # -c requirements/_base.txt # aiohttp # aiosignal -fsspec==2024.5.0 +fsspec==2024.10.0 # via # -c requirements/_base.txt # dask @@ -128,7 +128,7 @@ idna==3.7 # httpx # requests # yarl -importlib-metadata==7.1.0 +importlib-metadata==8.5.0 # via # -c requirements/_base.txt # dask @@ -155,7 +155,7 @@ mako==1.3.5 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # alembic -markupsafe==2.1.5 +markupsafe==3.0.2 # via # -c requirements/_base.txt # jinja2 @@ -174,14 +174,14 @@ mypy==1.12.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -numpy==1.26.4 +numpy==2.1.3 # via # -c requirements/_base.txt # bokeh # contourpy # pandas # types-networkx -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # bokeh @@ -204,7 +204,7 @@ pluggy==1.5.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -psutil==6.0.0 +psutil==6.1.0 # via # -c requirements/_base.txt # distributed @@ -241,7 +241,7 @@ python-dateutil==2.9.0.post0 # pandas pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -283,18 +283,18 @@ tblib==3.0.0 # via # -c requirements/_base.txt # distributed -toolz==0.12.1 +toolz==1.0.0 # via # -c requirements/_base.txt # dask # distributed # partd -tornado==6.4 +tornado==6.4.2 # via # -c requirements/_base.txt # bokeh # distributed -types-networkx==3.2.1.20240918 +types-networkx==3.4.2.20241115 # via -r requirements/_test.in types-psycopg2==2.9.21.20240819 # via -r requirements/_test.in @@ -332,7 +332,7 @@ zict==3.0.0 # via # -c requirements/_base.txt # distributed -zipp==3.18.2 +zipp==3.21.0 # via # -c requirements/_base.txt # importlib-metadata diff --git a/services/director-v2/requirements/_tools.txt b/services/director-v2/requirements/_tools.txt index 9588b0ee960..a4dfa668c25 100644 --- a/services/director-v2/requirements/_tools.txt +++ b/services/director-v2/requirements/_tools.txt @@ -39,7 +39,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.2 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -64,7 +64,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -74,9 +74,7 @@ pyyaml==6.0.1 ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt setuptools==74.0.0 - # via - # -c requirements/_base.txt - # pip-tools + # via pip-tools tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py index 96505371754..181c6c22a6d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dask_client.py @@ -16,7 +16,7 @@ from copy import deepcopy from dataclasses import dataclass, field from http.client import HTTPException -from typing import Any, cast +from typing import Any, Final, cast import dask.typing import distributed @@ -99,7 +99,7 @@ } -_DASK_DEFAULT_TIMEOUT_S = 1 +_DASK_DEFAULT_TIMEOUT_S: Final[int] = 5 _UserCallbackInSepThread = Callable[[], None] @@ -263,6 +263,9 @@ def _comp_sidecar_fct( ) # NOTE: the callback is running in a secondary thread, and takes a future as arg task_future.add_done_callback(lambda _: callback()) + await distributed.Variable(job_id, client=self.backend.client).set( + task_future + ) await dask_utils.wrap_client_async_routine( self.backend.client.publish_dataset(task_future, name=job_id) @@ -450,23 +453,34 @@ def _get_pipeline_statuses( DaskSchedulerTaskState | None, task_statuses.get(job_id, "lost") ) if dask_status == "erred": - # find out if this was a cancellation - exception = await distributed.Future(job_id).exception( - timeout=_DASK_DEFAULT_TIMEOUT_S - ) - assert isinstance(exception, Exception) # nosec - - if isinstance(exception, TaskCancelledError): - running_states.append(DaskClientTaskState.ABORTED) - else: - assert exception # nosec + try: + # find out if this was a cancellation + var = distributed.Variable(job_id, client=self.backend.client) + future: distributed.Future = await var.get( + timeout=_DASK_DEFAULT_TIMEOUT_S + ) + exception = await future.exception(timeout=_DASK_DEFAULT_TIMEOUT_S) + assert isinstance(exception, Exception) # nosec + + if isinstance(exception, TaskCancelledError): + running_states.append(DaskClientTaskState.ABORTED) + else: + assert exception # nosec + _logger.warning( + "Task %s completed in error:\n%s\nTrace:\n%s", + job_id, + exception, + "".join(traceback.format_exception(exception)), + ) + running_states.append(DaskClientTaskState.ERRED) + except TimeoutError: _logger.warning( - "Task %s completed in error:\n%s\nTrace:\n%s", + "Task %s could not be retrieved from dask-scheduler, it is lost\n" + "TIP:If the task was unpublished this can happen, or if the dask-scheduler was restarted.", job_id, - exception, - "".join(traceback.format_exception(exception)), ) - running_states.append(DaskClientTaskState.ERRED) + running_states.append(DaskClientTaskState.LOST) + elif dask_status is None: running_states.append(DaskClientTaskState.LOST) else: @@ -522,13 +536,21 @@ async def get_task_result(self, job_id: str) -> TaskOutputData: async def release_task_result(self, job_id: str) -> None: _logger.debug("releasing results for %s", f"{job_id=}") try: + # NOTE: The distributed Variable holds the future of the tasks in the dask-scheduler + # Alas, deleting the variable is done asynchronously and there is no way to ensure + # the variable was effectively deleted. + # This is annoying as one can re-create the variable without error. + var = distributed.Variable(job_id, client=self.backend.client) + var.delete() # first check if the key exists await dask_utils.wrap_client_async_routine( self.backend.client.get_dataset(name=job_id) ) + await dask_utils.wrap_client_async_routine( self.backend.client.unpublish_dataset(name=job_id) ) + except KeyError: _logger.warning("Unknown task cannot be unpublished: %s", f"{job_id=}") diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index 83939689808..f8e1ccd6c61 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -9,7 +9,7 @@ import traceback from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from dataclasses import dataclass -from typing import Any, NoReturn +from typing import Any, NoReturn, cast from unittest import mock from uuid import uuid4 @@ -91,7 +91,7 @@ async def _assert_wait_for_task_status( job_id: str, dask_client: DaskClient, expected_status: DaskClientTaskState, - timeout: int | None = None, + timeout: int | None = None, # noqa: ASYNC109 ): async for attempt in AsyncRetrying( reraise=True, @@ -104,24 +104,20 @@ async def _assert_wait_for_task_status( f"waiting for task to be {expected_status=}, " f"Attempt={attempt.retry_state.attempt_number}" ) - current_task_status = (await dask_client.get_tasks_status([job_id]))[0] - assert isinstance(current_task_status, DaskClientTaskState) - print(f"{current_task_status=} vs {expected_status=}") - if ( - current_task_status is DaskClientTaskState.ERRED - and expected_status - not in [ - DaskClientTaskState.ERRED, - DaskClientTaskState.LOST, - ] - ): + got = (await dask_client.get_tasks_status([job_id]))[0] + assert isinstance(got, DaskClientTaskState) + print(f"{got=} vs {expected_status=}") + if got is DaskClientTaskState.ERRED and expected_status not in [ + DaskClientTaskState.ERRED, + DaskClientTaskState.LOST, + ]: try: # we can fail fast here # this will raise and we catch the Assertion to not reraise too long await dask_client.get_task_result(job_id) except AssertionError as exc: raise RuntimeError from exc - assert current_task_status is expected_status + assert got is expected_status @pytest.fixture @@ -364,7 +360,9 @@ def fct_that_raise_cancellation_error() -> NoReturn: async def test_dask_does_not_report_base_exception_in_task(dask_client: DaskClient): def fct_that_raise_base_exception() -> NoReturn: err_msg = "task triggers a base exception, but dask does not care..." - raise BaseException(err_msg) # pylint: disable=broad-exception-raised + raise BaseException( # pylint: disable=broad-exception-raised # noqa: TRY002 + err_msg + ) future = dask_client.backend.client.submit(fct_that_raise_base_exception) # NOTE: Since asyncio.CancelledError is derived from BaseException and the worker code checks Exception only @@ -402,7 +400,7 @@ def comp_run_metadata(faker: Faker) -> RunMetadataDict: return RunMetadataDict( product_name=faker.pystr(), simcore_user_agent=faker.pystr(), - ) | faker.pydict(allowed_types=(str,)) + ) | cast(dict[str, str], faker.pydict(allowed_types=(str,))) @pytest.fixture @@ -418,6 +416,9 @@ def task_labels(comp_run_metadata: RunMetadataDict) -> ContainerLabelsDict: @pytest.fixture def hardware_info() -> HardwareInfo: + assert "json_schema_extra" in HardwareInfo.model_config + assert isinstance(HardwareInfo.model_config["json_schema_extra"], dict) + assert isinstance(HardwareInfo.model_config["json_schema_extra"]["examples"], list) return HardwareInfo.model_validate( HardwareInfo.model_config["json_schema_extra"]["examples"][0] ) @@ -476,7 +477,9 @@ def fake_sidecar_fct( assert node_params.node_requirements.ram assert "product_name" in comp_run_metadata assert "simcore_user_agent" in comp_run_metadata - assert image_params.fake_tasks[node_id].node_requirements is not None + node_requirements = image_params.fake_tasks[node_id].node_requirements + assert node_requirements + node_id_to_job_ids = await dask_client.send_computation_tasks( user_id=user_id, project_id=project_id, @@ -491,8 +494,8 @@ def fake_sidecar_fct( f"{to_simcore_runtime_docker_label_key('user-id')}": f"{user_id}", f"{to_simcore_runtime_docker_label_key('project-id')}": f"{project_id}", f"{to_simcore_runtime_docker_label_key('node-id')}": f"{node_id}", - f"{to_simcore_runtime_docker_label_key('cpu-limit')}": f"{image_params.fake_tasks[node_id].node_requirements.cpu}", - f"{to_simcore_runtime_docker_label_key('memory-limit')}": f"{image_params.fake_tasks[node_id].node_requirements.ram}", + f"{to_simcore_runtime_docker_label_key('cpu-limit')}": f"{node_requirements.cpu}", + f"{to_simcore_runtime_docker_label_key('memory-limit')}": f"{node_requirements.ram}", f"{to_simcore_runtime_docker_label_key('product-name')}": f"{comp_run_metadata['product_name']}", f"{to_simcore_runtime_docker_label_key('simcore-user-agent')}": f"{comp_run_metadata['simcore_user_agent']}", f"{to_simcore_runtime_docker_label_key('swarm-stack-name')}": "undefined-label", @@ -605,7 +608,9 @@ def fake_sidecar_fct( ) assert published_computation_task[0].node_id in image_params.fake_tasks # creating a new future shows that it is not done???? - assert not distributed.Future(published_computation_task[0].job_id).done() + assert not distributed.Future( + published_computation_task[0].job_id, client=dask_client.backend.client + ).done() # as the task is published on the dask-scheduler when sending, it shall still be published on the dask scheduler list_of_persisted_datasets = await dask_client.backend.client.list_datasets() # type: ignore @@ -628,7 +633,9 @@ def fake_sidecar_fct( assert isinstance(task_result, TaskOutputData) assert task_result.get("some_output_key") == 123 # try to create another future and this one is already done - assert distributed.Future(published_computation_task[0].job_id).done() + assert distributed.Future( + published_computation_task[0].job_id, client=dask_client.backend.client + ).done() async def test_abort_computation_tasks( @@ -1013,9 +1020,6 @@ def fake_remote_fct( assert len(published_computation_task) == 1 assert published_computation_task[0].node_id in cpu_image.fake_tasks - # let's get a dask future for the task here so dask will not remove the task from the scheduler at the end - computation_future = distributed.Future(key=published_computation_task[0].job_id) - assert computation_future await _assert_wait_for_task_status( published_computation_task[0].job_id, @@ -1034,15 +1038,11 @@ def fake_remote_fct( ) # release the task results await dask_client.release_task_result(published_computation_task[0].job_id) - # the task is still present since we hold a future here - await _assert_wait_for_task_status( - published_computation_task[0].job_id, - dask_client, - DaskClientTaskState.ERRED if fail_remote_fct else DaskClientTaskState.SUCCESS, - ) - # removing the future will let dask eventually delete the task from its memory, so its status becomes undefined - del computation_future + await asyncio.sleep( + 5 # NOTE: here we wait to be sure that the dask-scheduler properly updates its state + ) + # the task is gone, since the distributed Variable was removed above await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, @@ -1103,9 +1103,13 @@ def fake_remote_fct( assert len(published_computation_task) == 1 assert published_computation_task[0].node_id in cpu_image.fake_tasks - computation_future = distributed.Future(published_computation_task[0].job_id) + computation_future = distributed.Future( + published_computation_task[0].job_id, client=dask_client.backend.client + ) print("--> waiting for job to finish...") - await distributed.wait(computation_future, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # type: ignore + await distributed.wait( + computation_future, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS + ) assert computation_future.done() print("job finished, now checking that we received the publications...") From fcb92ea8e565123a3303f1c1ea8e5f8979873141 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:23:30 +0100 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=90=9B=20ensure=20api=20spec=20ci=20j?= =?UTF-8?q?obs=20only=20run=20during=20prs=20(#6912)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-testing-deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 277a131e5c2..e2eabc80994 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -2643,7 +2643,7 @@ jobs: system-api-specs: needs: [changes] - if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }} + if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }} timeout-minutes: 10 name: "[sys] check api-specs are up to date" runs-on: ubuntu-latest @@ -2669,7 +2669,7 @@ jobs: system-backwards-compatibility: needs: [changes, system-api-specs] - if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }} + if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }} timeout-minutes: 10 name: "[sys] api-server backwards compatibility" runs-on: ubuntu-latest @@ -2693,7 +2693,7 @@ jobs: api-spec-backwards-compatibility: needs: [changes, system-api-specs] - if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }} + if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }} continue-on-error: true timeout-minutes: 10 name: "api-specs-backwards-compatibility" From 9a15e0cd78efa861927a7d8d0770109a890ec247 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:59:49 +0100 Subject: [PATCH 6/9] =?UTF-8?q?=E2=9C=A8=20web-api:=20user's=20privacy=20s?= =?UTF-8?q?ettings=20(#6904)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/PULL_REQUEST_TEMPLATE.md | 6 +- api/specs/web-server/_users.py | 15 +- .../api_schemas_webserver/users.py | 130 +++++++++ .../38c9ac332c58_new_user_privacy_columns.py | 45 ++++ .../simcore_postgres_database/models/users.py | 62 ++++- .../services/webserver.py | 33 ++- .../tests/unit/_with_db/test_api_user.py | 10 +- .../osparc/desktop/account/ProfilePage.js | 2 +- services/web/server/VERSION | 2 +- services/web/server/setup.cfg | 2 +- .../api/v0/openapi.yaml | 67 ++++- .../application_settings.py | 3 +- .../users/_handlers.py | 15 +- .../users/_models.py | 47 ++++ .../simcore_service_webserver/users/api.py | 69 +++-- .../users/exceptions.py | 4 + .../users/schemas.py | 88 +----- .../tests/unit/isolated/test_users_models.py | 81 ++++-- .../03/login/test_login_registration.py | 2 +- .../tests/unit/with_dbs/03/test_users.py | 255 +++++++++++++----- .../test_studies_dispatcher_handlers.py | 1 - .../test_studies_dispatcher_studies_access.py | 2 - 22 files changed, 696 insertions(+), 245 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/users.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/38c9ac332c58_new_user_privacy_columns.py create mode 100644 services/web/server/src/simcore_service_webserver/users/_models.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e6f34058bd7..c142fd7d463 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -22,6 +22,9 @@ or from https://gitmoji.dev/ ## What do these changes do? + ## Related issue/s @@ -31,9 +34,6 @@ or from https://gitmoji.dev/ - resolves ITISFoundation/osparc-issues#428 - fixes #26 - - If openapi changes are provided, optionally point to the swagger editor with new changes - Example [openapi.json specs](https://editor.swagger.io/?url=https://raw.githubusercontent.com//osparc-simcore/is1133/create-api-for-creation-of-pricing-plan/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml) --> diff --git a/api/specs/web-server/_users.py b/api/specs/web-server/_users.py index f1204f15f35..c161a7aa69a 100644 --- a/api/specs/web-server/_users.py +++ b/api/specs/web-server/_users.py @@ -7,6 +7,7 @@ from typing import Annotated from fastapi import APIRouter, Depends, status +from models_library.api_schemas_webserver.users import ProfileGet, ProfileUpdate from models_library.api_schemas_webserver.users_preferences import PatchRequestBody from models_library.generics import Envelope from models_library.user_preferences import PreferenceIdentifier @@ -24,8 +25,6 @@ from simcore_service_webserver.users._tokens_handlers import _TokenPathParams from simcore_service_webserver.users.schemas import ( PermissionGet, - ProfileGet, - ProfileUpdate, ThirdPartyToken, TokenCreate, ) @@ -41,7 +40,7 @@ async def get_my_profile(): ... -@router.put( +@router.patch( "/me", status_code=status.HTTP_204_NO_CONTENT, ) @@ -49,6 +48,16 @@ async def update_my_profile(_profile: ProfileUpdate): ... +@router.put( + "/me", + status_code=status.HTTP_204_NO_CONTENT, + deprecated=True, + description="Use PATCH instead", +) +async def replace_my_profile(_profile: ProfileUpdate): + ... + + @router.patch( "/me/preferences/{preference_id}", status_code=status.HTTP_204_NO_CONTENT, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/users.py b/packages/models-library/src/models_library/api_schemas_webserver/users.py new file mode 100644 index 00000000000..ae7b9f89504 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_webserver/users.py @@ -0,0 +1,130 @@ +import re +from datetime import date +from enum import Enum +from typing import Annotated, Literal + +from models_library.api_schemas_webserver.groups import MyGroupsGet +from models_library.api_schemas_webserver.users_preferences import AggregatedPreferences +from models_library.basic_types import IDStr +from models_library.emails import LowerCaseEmailStr +from models_library.users import FirstNameStr, LastNameStr, UserID +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from ._base import InputSchema, OutputSchema + + +class ProfilePrivacyGet(OutputSchema): + hide_fullname: bool + hide_email: bool + + +class ProfilePrivacyUpdate(InputSchema): + hide_fullname: bool | None = None + hide_email: bool | None = None + + +class ProfileGet(BaseModel): + # WARNING: do not use InputSchema until front-end is updated! + id: UserID + user_name: Annotated[ + IDStr, Field(description="Unique username identifier", alias="userName") + ] + first_name: FirstNameStr | None = None + last_name: LastNameStr | None = None + login: LowerCaseEmailStr + + role: Literal["ANONYMOUS", "GUEST", "USER", "TESTER", "PRODUCT_OWNER", "ADMIN"] + groups: MyGroupsGet | None = None + gravatar_id: Annotated[str | None, Field(deprecated=True)] = None + + expiration_date: Annotated[ + date | None, + Field( + description="If user has a trial account, it sets the expiration date, otherwise None", + alias="expirationDate", + ), + ] = None + + privacy: ProfilePrivacyGet + preferences: AggregatedPreferences + + model_config = ConfigDict( + # NOTE: old models have an hybrid between snake and camel cases! + # Should be unified at some point + populate_by_name=True, + json_schema_extra={ + "examples": [ + { + "id": 42, + "login": "bla@foo.com", + "userName": "bla42", + "role": "admin", # pre + "expirationDate": "2022-09-14", # optional + "preferences": {}, + "privacy": {"hide_fullname": 0, "hide_email": 1}, + }, + ] + }, + ) + + @field_validator("role", mode="before") + @classmethod + def _to_upper_string(cls, v): + if isinstance(v, str): + return v.upper() + if isinstance(v, Enum): + return v.name.upper() + return v + + +class ProfileUpdate(BaseModel): + # WARNING: do not use InputSchema until front-end is updated! + first_name: FirstNameStr | None = None + last_name: LastNameStr | None = None + user_name: Annotated[IDStr | None, Field(alias="userName")] = None + + privacy: ProfilePrivacyUpdate | None = None + + model_config = ConfigDict( + json_schema_extra={ + "example": { + "first_name": "Pedro", + "last_name": "Crespo", + } + } + ) + + @field_validator("user_name") + @classmethod + def _validate_user_name(cls, value: str): + # Ensure valid characters (alphanumeric + . _ -) + if not re.match(r"^[a-zA-Z][a-zA-Z0-9._-]*$", value): + msg = f"Username '{value}' must start with a letter and can only contain letters, numbers and '_', '.' or '-'." + raise ValueError(msg) + + # Ensure no consecutive special characters + if re.search(r"[_.-]{2,}", value): + msg = f"Username '{value}' cannot contain consecutive special characters like '__'." + raise ValueError(msg) + + # Ensure it doesn't end with a special character + if {value[0], value[-1]}.intersection({"_", "-", "."}): + msg = f"Username '{value}' cannot end or start with a special character." + raise ValueError(msg) + + # Check reserved words (example list; extend as needed) + reserved_words = { + "admin", + "root", + "system", + "null", + "undefined", + "support", + "moderator", + # NOTE: add here extra via env vars + } + if any(w in value.lower() for w in reserved_words): + msg = f"Username '{value}' cannot be used." + raise ValueError(msg) + + return value diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/38c9ac332c58_new_user_privacy_columns.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/38c9ac332c58_new_user_privacy_columns.py new file mode 100644 index 00000000000..4d3e141e769 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/38c9ac332c58_new_user_privacy_columns.py @@ -0,0 +1,45 @@ +"""new user privacy columns + +Revision ID: 38c9ac332c58 +Revises: e5555076ef50 +Create Date: 2024-12-05 14:29:27.739650+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "38c9ac332c58" +down_revision = "e5555076ef50" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "users", + sa.Column( + "privacy_hide_fullname", + sa.Boolean(), + server_default=sa.text("true"), + nullable=False, + ), + ) + op.add_column( + "users", + sa.Column( + "privacy_hide_email", + sa.Boolean(), + server_default=sa.text("true"), + nullable=False, + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("users", "privacy_hide_email") + op.drop_column("users", "privacy_hide_fullname") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/users.py b/packages/postgres-database/src/simcore_postgres_database/models/users.py index 61b8c321130..d42568d772f 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/users.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/users.py @@ -2,6 +2,7 @@ from functools import total_ordering import sqlalchemy as sa +from sqlalchemy.sql import expression from ._common import RefActions from .base import metadata @@ -67,6 +68,9 @@ class UserStatus(str, Enum): users = sa.Table( "users", metadata, + # + # User Identifiers ------------------ + # sa.Column( "id", sa.BigInteger(), @@ -77,8 +81,23 @@ class UserStatus(str, Enum): "name", sa.String(), nullable=False, - doc="username is a unique short user friendly identifier e.g. pcrespov, sanderegg, GitHK, ...", + doc="username is a unique short user friendly identifier e.g. pcrespov, sanderegg, GitHK, ..." + "This identifier **is public**.", ), + sa.Column( + "primary_gid", + sa.BigInteger(), + sa.ForeignKey( + "groups.gid", + name="fk_users_gid_groups", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), + doc="User's group ID", + ), + # + # User Information ------------------ + # sa.Column( "first_name", sa.String(), @@ -102,37 +121,52 @@ class UserStatus(str, Enum): doc="Confirmed user phone used e.g. to send a code for a two-factor-authentication." "NOTE: new policy (NK) is that the same phone can be reused therefore it does not has to be unique", ), + # + # User Secrets ------------------ + # sa.Column( "password_hash", sa.String(), nullable=False, doc="Hashed password", ), - sa.Column( - "primary_gid", - sa.BigInteger(), - sa.ForeignKey( - "groups.gid", - name="fk_users_gid_groups", - onupdate=RefActions.CASCADE, - ondelete=RefActions.RESTRICT, - ), - doc="User's group ID", - ), + # + # User Account ------------------ + # sa.Column( "status", sa.Enum(UserStatus), nullable=False, default=UserStatus.CONFIRMATION_PENDING, - doc="Status of the user account. SEE UserStatus", + doc="Current status of the user's account", ), sa.Column( "role", sa.Enum(UserRole), nullable=False, default=UserRole.USER, - doc="Use for role-base authorization", + doc="Used for role-base authorization", + ), + # + # User Privacy Rules ------------------ + # + sa.Column( + "privacy_hide_fullname", + sa.Boolean, + nullable=False, + server_default=expression.true(), + doc="If true, it hides users.first_name, users.last_name to others", + ), + sa.Column( + "privacy_hide_email", + sa.Boolean, + nullable=False, + server_default=expression.true(), + doc="If true, it hides users.email to others", ), + # + # Timestamps --------------- + # sa.Column( "created_at", sa.DateTime(), diff --git a/services/api-server/src/simcore_service_api_server/services/webserver.py b/services/api-server/src/simcore_service_api_server/services/webserver.py index cba31689654..9301b5ce42c 100644 --- a/services/api-server/src/simcore_service_api_server/services/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services/webserver.py @@ -29,6 +29,8 @@ ProjectInputUpdate, ) from models_library.api_schemas_webserver.resource_usage import PricingPlanGet +from models_library.api_schemas_webserver.users import ProfileGet as WebProfileGet +from models_library.api_schemas_webserver.users import ProfileUpdate as WebProfileUpdate from models_library.api_schemas_webserver.wallets import WalletGet from models_library.generics import Envelope from models_library.projects import ProjectID @@ -77,7 +79,7 @@ PricingUnitGetLegacy, WalletGetWithAvailableCreditsLegacy, ) -from ..models.schemas.profiles import Profile, ProfileUpdate +from ..models.schemas.profiles import Profile, ProfileUpdate, UserRoleEnum from ..models.schemas.solvers import SolverKeyId from ..models.schemas.studies import StudyPort from ..utils.client_base import BaseServiceClientApi, setup_client_instance @@ -243,17 +245,34 @@ async def _wait_for_long_running_task_results(self, lrt_response: httpx.Response async def get_me(self) -> Profile: response = await self.client.get("/me", cookies=self.session_cookies) response.raise_for_status() - profile: Profile | None = ( - Envelope[Profile].model_validate_json(response.text).data + + got: WebProfileGet | None = ( + Envelope[WebProfileGet].model_validate_json(response.text).data + ) + assert got is not None # nosec + + return Profile( + first_name=got.first_name, + last_name=got.last_name, + id=got.id, + login=got.login, + role=UserRoleEnum(got.role), + groups=got.groups.model_dump() if got.groups else None, # type: ignore + gravatar_id=got.gravatar_id, ) - assert profile is not None # nosec - return profile @_exception_mapper(_PROFILE_STATUS_MAP) async def update_me(self, *, profile_update: ProfileUpdate) -> Profile: - response = await self.client.put( + + update = WebProfileUpdate.model_construct( + _fields_set=profile_update.model_fields_set, + first_name=profile_update.first_name, + last_name=profile_update.last_name, + ) + + response = await self.client.patch( "/me", - json=profile_update.model_dump(exclude_none=True), + json=update.model_dump(exclude_unset=True), cookies=self.session_cookies, ) response.raise_for_status() diff --git a/services/api-server/tests/unit/_with_db/test_api_user.py b/services/api-server/tests/unit/_with_db/test_api_user.py index 0a42177867b..24836d1b3cd 100644 --- a/services/api-server/tests/unit/_with_db/test_api_user.py +++ b/services/api-server/tests/unit/_with_db/test_api_user.py @@ -4,12 +4,12 @@ import json -from copy import deepcopy import httpx import pytest import respx from fastapi import FastAPI +from models_library.api_schemas_webserver.users import ProfileGet as WebProfileGet from respx import MockRouter from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.core.settings import ApplicationSettings @@ -32,7 +32,9 @@ def mocked_webserver_service_api(app: FastAPI): ) as respx_mock: # NOTE: webserver-api uses the same schema as api-server! # in-memory fake data - me = deepcopy(Profile.model_config["json_schema_extra"]["example"]) + me: dict = WebProfileGet.model_json_schema()["examples"][0] + me["first_name"] = "James" + me["last_name"] = "Maxwell" def _get_me(request): return httpx.Response(status.HTTP_200_OK, json={"data": me}) @@ -43,12 +45,10 @@ def _update_me(request: httpx.Request): return httpx.Response(status.HTTP_200_OK, json={"data": me}) respx_mock.get("/me", name="get_me").mock(side_effect=_get_me) - respx_mock.put("/me", name="update_me").mock(side_effect=_update_me) + respx_mock.patch("/me", name="update_me").mock(side_effect=_update_me) yield respx_mock - del me - async def test_get_profile( client: httpx.AsyncClient, diff --git a/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js b/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js index 465e4f0f391..a4f9b773ea3 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js +++ b/services/static-webserver/client/source/class/osparc/desktop/account/ProfilePage.js @@ -156,7 +156,7 @@ qx.Class.define("osparc.desktop.account.ProfilePage", { if (this.__userProfileData["first_name"] !== model.getFirstName() || this.__userProfileData["last_name"] !== model.getLastName()) { if (namesValidator.validate()) { - const profileReq = new osparc.io.request.ApiRequest("/me", "PUT"); + const profileReq = new osparc.io.request.ApiRequest("/me", "PATCH"); profileReq.setRequestData({ "first_name": model.getFirstName(), "last_name": model.getLastName() diff --git a/services/web/server/VERSION b/services/web/server/VERSION index 421ab545d9a..a758a09aae5 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.47.0 +0.48.0 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 628685e0ad5..0b6157ef959 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.47.0 +current_version = 0.48.0 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 485fb15c931..91516c62ee8 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: simcore-service-webserver description: Main service with an interface (http-API & websockets) to the web front-end - version: 0.47.0 + version: 0.48.0 servers: - url: '' description: webserver @@ -1088,6 +1088,22 @@ paths: schema: $ref: '#/components/schemas/Envelope_ProfileGet_' put: + tags: + - user + summary: Replace My Profile + description: Use PATCH instead + operationId: replace_my_profile + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProfileUpdate' + required: true + responses: + '204': + description: Successful Response + deprecated: true + patch: tags: - user summary: Update My Profile @@ -11520,6 +11536,12 @@ components: exclusiveMinimum: true title: Id minimum: 0 + userName: + type: string + maxLength: 100 + minLength: 1 + title: Username + description: Unique username identifier first_name: anyOf: - type: string @@ -11555,6 +11577,7 @@ components: - type: string - type: 'null' title: Gravatar Id + deprecated: true expirationDate: anyOf: - type: string @@ -11563,6 +11586,8 @@ components: title: Expirationdate description: If user has a trial account, it sets the expiration date, otherwise None + privacy: + $ref: '#/components/schemas/ProfilePrivacyGet' preferences: additionalProperties: $ref: '#/components/schemas/Preference' @@ -11571,10 +11596,39 @@ components: type: object required: - id + - userName - login - role + - privacy - preferences title: ProfileGet + ProfilePrivacyGet: + properties: + hideFullname: + type: boolean + title: Hidefullname + hideEmail: + type: boolean + title: Hideemail + type: object + required: + - hideFullname + - hideEmail + title: ProfilePrivacyGet + ProfilePrivacyUpdate: + properties: + hideFullname: + anyOf: + - type: boolean + - type: 'null' + title: Hidefullname + hideEmail: + anyOf: + - type: boolean + - type: 'null' + title: Hideemail + type: object + title: ProfilePrivacyUpdate ProfileUpdate: properties: first_name: @@ -11589,6 +11643,17 @@ components: maxLength: 255 - type: 'null' title: Last Name + userName: + anyOf: + - type: string + maxLength: 100 + minLength: 1 + - type: 'null' + title: Username + privacy: + anyOf: + - $ref: '#/components/schemas/ProfilePrivacyUpdate' + - type: 'null' type: object title: ProfileUpdate example: diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index b15cd73e1c8..07941e1e92c 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -117,13 +117,12 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_LOGLEVEL: Annotated[ LogLevel, Field( - default=LogLevel.WARNING.value, validation_alias=AliasChoices( "WEBSERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" ), # NOTE: suffix '_LOGLEVEL' is used overall ), - ] + ] = LogLevel.WARNING WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, validation_alias=AliasChoices( diff --git a/services/web/server/src/simcore_service_webserver/users/_handlers.py b/services/web/server/src/simcore_service_webserver/users/_handlers.py index 4d69e9ffaab..d67d772e0ee 100644 --- a/services/web/server/src/simcore_service_webserver/users/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/users/_handlers.py @@ -2,6 +2,7 @@ import logging from aiohttp import web +from models_library.api_schemas_webserver.users import ProfileGet, ProfileUpdate from models_library.users import UserID from pydantic import BaseModel, Field from servicelib.aiohttp import status @@ -25,9 +26,9 @@ from .exceptions import ( AlreadyPreRegisteredError, MissingGroupExtraPropertiesForProductError, + UserNameDuplicateError, UserNotFoundError, ) -from .schemas import ProfileGet, ProfileUpdate _logger = logging.getLogger(__name__) @@ -48,6 +49,10 @@ async def wrapper(request: web.Request) -> web.StreamResponse: except UserNotFoundError as exc: raise web.HTTPNotFound(reason=f"{exc}") from exc + + except UserNameDuplicateError as exc: + raise web.HTTPConflict(reason=f"{exc}") from exc + except MissingGroupExtraPropertiesForProductError as exc: error_code = exc.error_code() user_error_msg = FMSG_MISSING_CONFIG_WITH_OEC.format(error_code=error_code) @@ -75,15 +80,19 @@ async def get_my_profile(request: web.Request) -> web.Response: return envelope_json_response(profile) -@routes.put(f"/{API_VTAG}/me", name="update_my_profile") +@routes.patch(f"/{API_VTAG}/me", name="update_my_profile") +@routes.put( + f"/{API_VTAG}/me", name="replace_my_profile" # deprecated. Use patch instead +) @login_required @permission_required("user.profile.update") @_handle_users_exceptions async def update_my_profile(request: web.Request) -> web.Response: req_ctx = UsersRequestContext.model_validate(request) profile_update = await parse_request_body_as(ProfileUpdate, request) + await api.update_user_profile( - request.app, req_ctx.user_id, profile_update, as_patch=False + request.app, user_id=req_ctx.user_id, update=profile_update ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/users/_models.py b/services/web/server/src/simcore_service_webserver/users/_models.py new file mode 100644 index 00000000000..cd9de6a873c --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/users/_models.py @@ -0,0 +1,47 @@ +from typing import Annotated, Any, Self + +from pydantic import BaseModel, ConfigDict, Field + +# +# DB models +# + + +def flatten_dict(d: dict, parent_key="", sep="_"): + items = [] + for key, value in d.items(): + new_key = f"{parent_key}{sep}{key}" if parent_key else key + if isinstance(value, dict): + # Recursively process nested dictionaries + items.extend(flatten_dict(value, new_key, sep=sep).items()) + else: + items.append((new_key, value)) + return dict(items) + + +class ToUserUpdateDB(BaseModel): + """ + Maps ProfileUpdate api-model into UserUpdate db-model + """ + + # NOTE: field names are UserDB columns + # NOTE: aliases are ProfileUpdate field names + + name: Annotated[str | None, Field(alias="user_name")] = None + first_name: str | None = None + last_name: str | None = None + + privacy_hide_fullname: bool | None = None + privacy_hide_email: bool | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_api(cls, profile_update) -> Self: + # The mapping of embed fields to flatten keys is done here + return cls.model_validate( + flatten_dict(profile_update.model_dump(exclude_unset=True, by_alias=False)) + ) + + def to_db(self) -> dict[str, Any]: + return self.model_dump(exclude_unset=True, by_alias=False) diff --git a/services/web/server/src/simcore_service_webserver/users/api.py b/services/web/server/src/simcore_service_webserver/users/api.py index 50dfdc4e12d..7fc2c138204 100644 --- a/services/web/server/src/simcore_service_webserver/users/api.py +++ b/services/web/server/src/simcore_service_webserver/users/api.py @@ -9,29 +9,39 @@ from collections import deque from typing import Any, NamedTuple, TypedDict +import simcore_postgres_database.errors as db_errors import sqlalchemy as sa from aiohttp import web from aiopg.sa.engine import Engine from aiopg.sa.result import RowProxy +from models_library.api_schemas_webserver.users import ( + ProfileGet, + ProfilePrivacyGet, + ProfileUpdate, +) from models_library.basic_types import IDStr from models_library.products import ProductName from models_library.users import GroupID, UserID from pydantic import EmailStr, TypeAdapter, ValidationError -from simcore_postgres_database.models.users import UserRole +from simcore_postgres_database.models.groups import GroupType, groups, user_to_groups +from simcore_postgres_database.models.users import UserRole, users from simcore_postgres_database.utils_groups_extra_properties import ( GroupExtraPropertiesNotFoundError, ) -from ..db.models import GroupType, groups, user_to_groups, users from ..db.plugin import get_database_engine from ..groups.models import convert_groups_db_to_schema from ..login.storage import AsyncpgStorage, get_plugin_storage from ..security.api import clean_auth_policy_cache from . import _db from ._api import get_user_credentials, get_user_invoice_address, set_user_as_deleted +from ._models import ToUserUpdateDB from ._preferences_api import get_frontend_user_preferences_aggregation -from .exceptions import MissingGroupExtraPropertiesForProductError, UserNotFoundError -from .schemas import ProfileGet, ProfileUpdate +from .exceptions import ( + MissingGroupExtraPropertiesForProductError, + UserNameDuplicateError, + UserNotFoundError, +) _logger = logging.getLogger(__name__) @@ -40,7 +50,7 @@ def _parse_as_user(user_id: Any) -> UserID: try: return TypeAdapter(UserID).validate_python(user_id) except ValidationError as err: - raise UserNotFoundError(uid=user_id) from err + raise UserNotFoundError(uid=user_id, user_id=user_id) from err async def get_user_profile( @@ -59,15 +69,12 @@ async def get_user_profile( async with engine.acquire() as conn: row: RowProxy + async for row in conn.execute( sa.select(users, groups, user_to_groups.c.access_rights) .select_from( - sa.join( - users, - sa.join( - user_to_groups, groups, user_to_groups.c.gid == groups.c.gid - ), - users.c.id == user_to_groups.c.uid, + users.join(user_to_groups, users.c.id == user_to_groups.c.uid).join( + groups, user_to_groups.c.gid == groups.c.gid ) ) .where(users.c.id == user_id) @@ -77,10 +84,13 @@ async def get_user_profile( if not user_profile: user_profile = { "id": row.users_id, + "user_name": row.users_name, "first_name": row.users_first_name, "last_name": row.users_last_name, "login": row.users_email, "role": row.users_role, + "privacy_hide_fullname": row.users_privacy_hide_fullname, + "privacy_hide_email": row.users_privacy_hide_email, "expiration_date": ( row.users_expires_at.date() if row.users_expires_at else None ), @@ -128,6 +138,7 @@ async def get_user_profile( return ProfileGet( id=user_profile["id"], + user_name=user_profile["user_name"], first_name=user_profile["first_name"], last_name=user_profile["last_name"], login=user_profile["login"], @@ -137,6 +148,10 @@ async def get_user_profile( "organizations": user_standard_groups, "all": all_group, }, + privacy=ProfilePrivacyGet( + hide_fullname=user_profile["privacy_hide_fullname"], + hide_email=user_profile["privacy_hide_email"], + ), preferences=preferences, **optional, ) @@ -144,32 +159,30 @@ async def get_user_profile( async def update_user_profile( app: web.Application, + *, user_id: UserID, update: ProfileUpdate, - *, - as_patch: bool = True, ) -> None: """ - Keyword Arguments: - as_patch -- set False if PUT and True if PATCH (default: {True}) - Raises: UserNotFoundError + UserNameAlreadyExistsError """ user_id = _parse_as_user(user_id) - async with get_database_engine(app).acquire() as conn: - to_update = update.model_dump( - include={ - "first_name", - "last_name", - }, - exclude_unset=as_patch, - ) - resp = await conn.execute( - users.update().where(users.c.id == user_id).values(**to_update) - ) - assert resp.rowcount == 1 # nosec + if updated_values := ToUserUpdateDB.from_api(update).to_db(): + async with get_database_engine(app).acquire() as conn: + query = users.update().where(users.c.id == user_id).values(**updated_values) + + try: + + resp = await conn.execute(query) + assert resp.rowcount == 1 # nosec + + except db_errors.UniqueViolation as err: + raise UserNameDuplicateError( + user_name=updated_values.get("name") + ) from err async def get_user_role(app: web.Application, user_id: UserID) -> UserRole: diff --git a/services/web/server/src/simcore_service_webserver/users/exceptions.py b/services/web/server/src/simcore_service_webserver/users/exceptions.py index 39791ea39fe..653cfeca719 100644 --- a/services/web/server/src/simcore_service_webserver/users/exceptions.py +++ b/services/web/server/src/simcore_service_webserver/users/exceptions.py @@ -21,6 +21,10 @@ def __init__(self, *, uid: int | None = None, email: str | None = None, **ctx: A self.email = email +class UserNameDuplicateError(UsersBaseError): + msg_template = "Username {user_name} is already in use. Violates unique constraint" + + class TokenNotFoundError(UsersBaseError): msg_template = "Token for service {service_id} not found" diff --git a/services/web/server/src/simcore_service_webserver/users/schemas.py b/services/web/server/src/simcore_service_webserver/users/schemas.py index e2da8da8ed7..8ad46a5c317 100644 --- a/services/web/server/src/simcore_service_webserver/users/schemas.py +++ b/services/web/server/src/simcore_service_webserver/users/schemas.py @@ -1,16 +1,7 @@ -from datetime import date -from typing import Literal from uuid import UUID from models_library.api_schemas_webserver._base import OutputSchema -from models_library.api_schemas_webserver.groups import MyGroupsGet -from models_library.api_schemas_webserver.users_preferences import AggregatedPreferences -from models_library.emails import LowerCaseEmailStr -from models_library.users import FirstNameStr, LastNameStr, UserID -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from simcore_postgres_database.models.users import UserRole - -from ..utils import gravatar_hash +from pydantic import BaseModel, ConfigDict, Field # @@ -41,83 +32,6 @@ class TokenCreate(ThirdPartyToken): ... -# -# PROFILE resource -# - - -class ProfileUpdate(BaseModel): - first_name: FirstNameStr | None = None - last_name: LastNameStr | None = None - - model_config = ConfigDict( - json_schema_extra={ - "example": { - "first_name": "Pedro", - "last_name": "Crespo", - } - } - ) - - -class ProfileGet(BaseModel): - id: UserID - first_name: FirstNameStr | None = None - last_name: LastNameStr | None = None - login: LowerCaseEmailStr - role: Literal["ANONYMOUS", "GUEST", "USER", "TESTER", "PRODUCT_OWNER", "ADMIN"] - groups: MyGroupsGet | None = None - gravatar_id: str | None = None - expiration_date: date | None = Field( - default=None, - description="If user has a trial account, it sets the expiration date, otherwise None", - alias="expirationDate", - ) - preferences: AggregatedPreferences - - model_config = ConfigDict( - # NOTE: old models have an hybrid between snake and camel cases! - # Should be unified at some point - populate_by_name=True, - json_schema_extra={ - "examples": [ - { - "id": 1, - "login": "bla@foo.com", - "role": "Admin", - "gravatar_id": "205e460b479e2e5b48aec07710c08d50", - "preferences": {}, - }, - { - "id": 42, - "login": "bla@foo.com", - "role": UserRole.ADMIN.value, - "expirationDate": "2022-09-14", - "preferences": {}, - }, - ] - }, - ) - - @model_validator(mode="before") - @classmethod - def _auto_generate_gravatar(cls, values): - gravatar_id = values.get("gravatar_id") - email = values.get("login") - if not gravatar_id and email: - values["gravatar_id"] = gravatar_hash(email) - return values - - @field_validator("role", mode="before") - @classmethod - def _to_upper_string(cls, v): - if isinstance(v, str): - return v.upper() - if isinstance(v, UserRole): - return v.name.upper() - return v - - # # Permissions # diff --git a/services/web/server/tests/unit/isolated/test_users_models.py b/services/web/server/tests/unit/isolated/test_users_models.py index a5670b5054e..8ff676476ee 100644 --- a/services/web/server/tests/unit/isolated/test_users_models.py +++ b/services/web/server/tests/unit/isolated/test_users_models.py @@ -1,3 +1,8 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + from copy import deepcopy from datetime import UTC, datetime from pprint import pformat @@ -5,12 +10,18 @@ import pytest from faker import Faker +from models_library.api_schemas_webserver.users import ( + ProfileGet, + ProfilePrivacyGet, + ProfileUpdate, +) from models_library.generics import Envelope from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import BaseModel from servicelib.rest_constants import RESPONSE_MODEL_POLICY from simcore_postgres_database.models.users import UserRole -from simcore_service_webserver.users.schemas import ProfileGet, ThirdPartyToken +from simcore_service_webserver.users._models import ToUserUpdateDB +from simcore_service_webserver.users.schemas import ThirdPartyToken @pytest.mark.parametrize( @@ -39,38 +50,46 @@ def test_user_models_examples( assert model_array_enveloped.error is None -def test_profile_get_expiration_date(faker: Faker): - fake_expiration = datetime.now(UTC) +@pytest.fixture +def fake_profile_get(faker: Faker) -> ProfileGet: + fake_profile: dict[str, Any] = faker.simple_profile() + first, last = fake_profile["name"].rsplit(maxsplit=1) - profile = ProfileGet( - id=1, - login=faker.email(), - role=UserRole.ADMIN, - expiration_date=fake_expiration.date(), + return ProfileGet( + id=faker.pyint(), + first_name=first, + last_name=last, + user_name=fake_profile["username"], + login=fake_profile["mail"], + role="USER", + privacy=ProfilePrivacyGet(hide_fullname=True, hide_email=True), preferences={}, ) + +def test_profile_get_expiration_date(fake_profile_get: ProfileGet): + fake_expiration = datetime.now(UTC) + + profile = fake_profile_get.model_copy( + update={"expiration_date": fake_expiration.date()} + ) + assert fake_expiration.date() == profile.expiration_date body = jsonable_encoder(profile.model_dump(exclude_unset=True, by_alias=True)) assert body["expirationDate"] == fake_expiration.date().isoformat() -def test_auto_compute_gravatar(faker: Faker): +def test_auto_compute_gravatar__deprecated(fake_profile_get: ProfileGet): - profile = ProfileGet( - id=faker.pyint(), - first_name=faker.first_name(), - last_name=faker.last_name(), - login=faker.email(), - role="USER", - preferences={}, - ) + profile = fake_profile_get.model_copy() envelope = Envelope[Any](data=profile) data = envelope.model_dump(**RESPONSE_MODEL_POLICY)["data"] - assert data["gravatar_id"] + assert ( + "gravatar_id" not in data + ), f"{ProfileGet.model_fields['gravatar_id'].deprecated=}" assert data["id"] == profile.id assert data["first_name"] == profile.first_name assert data["last_name"] == profile.last_name @@ -81,7 +100,7 @@ def test_auto_compute_gravatar(faker: Faker): @pytest.mark.parametrize("user_role", [u.name for u in UserRole]) def test_profile_get_role(user_role: str): - for example in ProfileGet.model_config["json_schema_extra"]["examples"]: + for example in ProfileGet.model_json_schema()["examples"]: data = deepcopy(example) data["role"] = user_role m1 = ProfileGet(**data) @@ -95,10 +114,12 @@ def test_parsing_output_of_get_user_profile(): result_from_db_query_and_composition = { "id": 1, "login": "PtN5Ab0uv@guest-at-osparc.io", + "userName": "PtN5Ab0uv", "first_name": "PtN5Ab0uv", "last_name": "", "role": "Guest", "gravatar_id": "9d5e02c75fcd4bce1c8861f219f7f8a5", + "privacy": {"hide_email": True, "hide_fullname": False}, "groups": { "me": { "gid": 2, @@ -136,3 +157,25 @@ def test_parsing_output_of_get_user_profile(): profile = ProfileGet.model_validate(result_from_db_query_and_composition) assert "password" not in profile.model_dump(exclude_unset=True) + + +def test_mapping_update_models_from_rest_to_db(): + + profile_update = ProfileUpdate.model_validate( + # request payload + { + "first_name": "foo", + "userName": "foo1234", + "privacy": {"hideFullname": False}, + } + ) + + # to db + profile_update_db = ToUserUpdateDB.from_api(profile_update) + + # expected + assert profile_update_db.to_db() == { + "first_name": "foo", + "name": "foo1234", + "privacy_hide_fullname": False, + } diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py index d99f8f1f297..762642dfb5c 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_registration.py @@ -9,6 +9,7 @@ import pytest from aiohttp.test_utils import TestClient from faker import Faker +from models_library.api_schemas_webserver.users import ProfileGet from models_library.products import ProductName from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_error, assert_status @@ -31,7 +32,6 @@ get_plugin_settings, ) from simcore_service_webserver.login.storage import AsyncpgStorage -from simcore_service_webserver.users.schemas import ProfileGet @pytest.fixture diff --git a/services/web/server/tests/unit/with_dbs/03/test_users.py b/services/web/server/tests/unit/with_dbs/03/test_users.py index 391053b40ac..4e2829c6fce 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users.py @@ -18,6 +18,7 @@ from aiopg.sa.connection import SAConnection from faker import Faker from models_library.api_schemas_webserver.auth import AccountRequestInfo +from models_library.api_schemas_webserver.users import ProfileGet from models_library.generics import Envelope from psycopg2 import OperationalError from pytest_simcore.helpers.assert_checks import assert_status @@ -38,7 +39,6 @@ PreUserProfile, UserProfile, ) -from simcore_service_webserver.users.schemas import ProfileGet, ProfileUpdate @pytest.fixture @@ -59,16 +59,52 @@ def app_environment( "user_role,expected", [ (UserRole.ANONYMOUS, status.HTTP_401_UNAUTHORIZED), - (UserRole.GUEST, status.HTTP_200_OK), - (UserRole.USER, status.HTTP_200_OK), - (UserRole.TESTER, status.HTTP_200_OK), + *((r, status.HTTP_200_OK) for r in UserRole if r >= UserRole.GUEST), ], ) -async def test_get_profile( +async def test_access_rights_on_get_profile( + user_role: UserRole, + logged_user: UserInfoDict, + client: TestClient, + expected: HTTPStatus, +): + assert client.app + + url = client.app.router["get_my_profile"].url_for() + assert url.path == "/v0/me" + + resp = await client.get(f"{url}") + await assert_status(resp, expected) + + +@pytest.mark.parametrize( + "user_role,expected", + [ + (UserRole.ANONYMOUS, status.HTTP_401_UNAUTHORIZED), + (UserRole.GUEST, status.HTTP_403_FORBIDDEN), + *((r, status.HTTP_204_NO_CONTENT) for r in UserRole if r >= UserRole.USER), + ], +) +async def test_access_update_profile( logged_user: UserInfoDict, client: TestClient, user_role: UserRole, expected: HTTPStatus, +): + assert client.app + + url = client.app.router["update_my_profile"].url_for() + assert url.path == "/v0/me" + + resp = await client.patch(f"{url}", json={"last_name": "Foo"}) + await assert_status(resp, expected) + + +@pytest.mark.parametrize("user_role", [UserRole.USER]) +async def test_get_profile( + logged_user: UserInfoDict, + client: TestClient, + user_role: UserRole, primary_group: dict[str, Any], standard_groups: list[dict[str, Any]], all_group: dict[str, str], @@ -78,84 +114,171 @@ async def test_get_profile( url = client.app.router["get_my_profile"].url_for() assert url.path == "/v0/me" - resp = await client.get(url.path) - data, error = await assert_status(resp, expected) + resp = await client.get(f"{url}") + data, error = await assert_status(resp, status.HTTP_200_OK) - # check enveloped - e = Envelope[ProfileGet].model_validate(await resp.json()) - assert e.error == error - assert ( - e.data.model_dump(**RESPONSE_MODEL_POLICY, mode="json") == data - if e.data - else e.data == data - ) + resp_model = Envelope[ProfileGet].model_validate(await resp.json()) - if not error: - profile = ProfileGet.model_validate(data) - - product_group = { - "accessRights": {"delete": False, "read": False, "write": False}, - "description": "osparc product group", - "gid": 2, - "inclusionRules": {}, - "label": "osparc", - "thumbnail": None, - } - - assert profile.login == logged_user["email"] - assert profile.gravatar_id - assert profile.first_name == logged_user.get("first_name", None) - assert profile.last_name == logged_user.get("last_name", None) - assert profile.role == user_role.name - assert profile.groups - - got_profile_groups = profile.groups.model_dump( - **RESPONSE_MODEL_POLICY, mode="json" - ) - assert got_profile_groups["me"] == primary_group - assert got_profile_groups["all"] == all_group + assert resp_model.data.model_dump(**RESPONSE_MODEL_POLICY, mode="json") == data + assert resp_model.error is None - sorted_by_group_id = functools.partial(sorted, key=lambda d: d["gid"]) - assert sorted_by_group_id( - got_profile_groups["organizations"] - ) == sorted_by_group_id([*standard_groups, product_group]) + profile = resp_model.data - assert profile.preferences == await get_frontend_user_preferences_aggregation( - client.app, user_id=logged_user["id"], product_name="osparc" - ) + product_group = { + "accessRights": {"delete": False, "read": False, "write": False}, + "description": "osparc product group", + "gid": 2, + "inclusionRules": {}, + "label": "osparc", + "thumbnail": None, + } + assert profile.login == logged_user["email"] + assert profile.first_name == logged_user.get("first_name", None) + assert profile.last_name == logged_user.get("last_name", None) + assert profile.role == user_role.name + assert profile.groups -@pytest.mark.parametrize( - "user_role,expected", - [ - (UserRole.ANONYMOUS, status.HTTP_401_UNAUTHORIZED), - (UserRole.GUEST, status.HTTP_403_FORBIDDEN), - (UserRole.USER, status.HTTP_204_NO_CONTENT), - (UserRole.TESTER, status.HTTP_204_NO_CONTENT), - ], -) + got_profile_groups = profile.groups.model_dump(**RESPONSE_MODEL_POLICY, mode="json") + assert got_profile_groups["me"] == primary_group + assert got_profile_groups["all"] == all_group + + sorted_by_group_id = functools.partial(sorted, key=lambda d: d["gid"]) + assert sorted_by_group_id( + got_profile_groups["organizations"] + ) == sorted_by_group_id([*standard_groups, product_group]) + + assert profile.preferences == await get_frontend_user_preferences_aggregation( + client.app, user_id=logged_user["id"], product_name="osparc" + ) + + +@pytest.mark.parametrize("user_role", [UserRole.USER]) async def test_update_profile( logged_user: UserInfoDict, client: TestClient, user_role: UserRole, - expected: HTTPStatus, ): assert client.app + resp = await client.get("/v0/me") + data, _ = await assert_status(resp, status.HTTP_200_OK) + + assert data["role"] == user_role.name + before = deepcopy(data) + url = client.app.router["update_my_profile"].url_for() assert url.path == "/v0/me" + resp = await client.patch( + f"{url}", + json={ + "last_name": "Foo", + }, + ) + _, error = await assert_status(resp, status.HTTP_204_NO_CONTENT) + + assert not error + + resp = await client.get("/v0/me") + data, _ = await assert_status(resp, status.HTTP_200_OK) + + assert data["last_name"] == "Foo" + + def _copy(data: dict, exclude: set) -> dict: + return {k: v for k, v in data.items() if k not in exclude} + + exclude = {"last_name"} + assert _copy(data, exclude) == _copy(before, exclude) + + +@pytest.mark.parametrize("user_role", [UserRole.USER]) +async def test_profile_workflow( + logged_user: UserInfoDict, + client: TestClient, + user_role: UserRole, +): + assert client.app + + url = client.app.router["get_my_profile"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + my_profile = ProfileGet.model_validate(data) + + url = client.app.router["update_my_profile"].url_for() + resp = await client.patch( + f"{url}", + json={ + "first_name": "Odei", # NOTE: still not camecase! + "userName": "odei123", + "privacy": {"hideFullname": False}, + }, + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + url = client.app.router["get_my_profile"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + updated_profile = ProfileGet.model_validate(data) + + assert updated_profile.first_name != my_profile.first_name + assert updated_profile.last_name == my_profile.last_name + assert updated_profile.login == my_profile.login - resp = await client.put(url.path, json={"last_name": "Foo"}) - _, error = await assert_status(resp, expected) + assert updated_profile.user_name != my_profile.user_name + assert updated_profile.user_name == "odei123" - if not error: - resp = await client.get(f"{url}") - data, _ = await assert_status(resp, status.HTTP_200_OK) + assert updated_profile.privacy != my_profile.privacy + assert updated_profile.privacy.hide_email == my_profile.privacy.hide_email + assert updated_profile.privacy.hide_fullname != my_profile.privacy.hide_fullname - # This is a PUT! i.e. full replace of profile variable fields! - assert data["first_name"] == ProfileUpdate.model_fields["first_name"].default - assert data["last_name"] == "Foo" - assert data["role"] == user_role.name + +@pytest.mark.parametrize("user_role", [UserRole.USER]) +@pytest.mark.parametrize("invalid_username", ["", "_foo", "superadmin", "foo..-123"]) +async def test_update_wrong_user_name( + logged_user: UserInfoDict, + client: TestClient, + user_role: UserRole, + invalid_username: str, +): + assert client.app + + url = client.app.router["update_my_profile"].url_for() + resp = await client.patch( + f"{url}", + json={ + "userName": invalid_username, + }, + ) + await assert_status(resp, status.HTTP_422_UNPROCESSABLE_ENTITY) + + +@pytest.mark.parametrize("user_role", [UserRole.USER]) +async def test_update_existing_user_name( + user: UserInfoDict, + logged_user: UserInfoDict, + client: TestClient, + user_role: UserRole, +): + assert client.app + + other_username = user["name"] + assert other_username != logged_user["name"] + + # update with SAME username (i.e. existing) + url = client.app.router["get_my_profile"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + + assert data["userName"] == logged_user["name"] + + url = client.app.router["update_my_profile"].url_for() + resp = await client.patch( + f"{url}", + json={ + "userName": other_username, + }, + ) + await assert_status(resp, status.HTTP_409_CONFLICT) @pytest.fixture @@ -219,7 +342,7 @@ async def test_get_profile_with_failing_db_connection( (UserRole.PRODUCT_OWNER, status.HTTP_200_OK), ], ) -async def test_only_product_owners_can_access_users_api( +async def test_access_rights_on_search_users_only_product_owners_can_access( client: TestClient, logged_user: UserInfoDict, expected: HTTPStatus, diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index af534e9a481..6f8853337ee 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -424,7 +424,6 @@ async def test_dispatch_study_anonymously( data, _ = await assert_status(response, status.HTTP_200_OK) assert data["login"].endswith("guest-at-osparc.io") - assert data["gravatar_id"] assert data["role"].upper() == UserRole.GUEST.name # guest user only a copy of the template project diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index 3cb82c2bf20..cff892d7f00 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -282,7 +282,6 @@ async def test_access_study_anonymously( data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["login"].endswith("guest-at-osparc.io") - assert data["gravatar_id"] assert data["role"].upper() == UserRole.GUEST.name # guest user only a copy of the template project @@ -448,7 +447,6 @@ async def _test_guest_user_workflow(request_index): data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["login"].endswith("guest-at-osparc.io") - assert data["gravatar_id"] assert data["role"].upper() == UserRole.GUEST.name # guest user only a copy of the template project From 186c07be8a125516ac0c056d8c7dfb65df2571a6 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Mon, 9 Dec 2024 08:57:33 +0100 Subject: [PATCH 7/9] =?UTF-8?q?=E2=AC=86=EF=B8=8FFull=20upgrade=20autoscal?= =?UTF-8?q?ing/clusters-keeper=20(#6915)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/autoscaling/requirements/_base.txt | 133 +++++++++--------- services/autoscaling/requirements/_test.in | 1 + services/autoscaling/requirements/_test.txt | 110 ++++++++------- services/autoscaling/requirements/_tools.txt | 32 ++--- services/autoscaling/tests/unit/conftest.py | 2 +- .../clusters-keeper/requirements/_base.txt | 133 +++++++++--------- .../clusters-keeper/requirements/_test.txt | 116 ++++++++------- .../clusters-keeper/requirements/_tools.txt | 32 ++--- .../clusters-keeper/tests/unit/conftest.py | 2 +- 9 files changed, 293 insertions(+), 268 deletions(-) diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index e0a298107ed..9600358f524 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -1,12 +1,12 @@ -aio-pika==9.4.1 +aio-pika==9.5.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aioboto3==13.1.0 +aioboto3==13.2.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in -aiobotocore==2.13.1 +aiobotocore==2.15.2 # via aioboto3 -aiocache==0.12.2 +aiocache==0.12.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in @@ -16,17 +16,19 @@ aiodebug==2.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.24.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aioboto3 -aiohttp==3.9.5 +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.10 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -54,15 +56,15 @@ aiohttp==3.9.5 # -c requirements/../../../requirements/constraints.txt # aiobotocore # aiodocker -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.7.0 # via # fast-depends # faststream @@ -79,21 +81,21 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing -boto3==1.34.131 +boto3==1.35.36 # via aiobotocore -botocore==1.34.131 +botocore==1.35.36 # via # aiobotocore # boto3 # s3transfer -botocore-stubs==1.34.94 +botocore-stubs==1.35.76 # via types-aiobotocore -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -122,7 +124,7 @@ certifi==2024.2.2 # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -141,7 +143,7 @@ dask==2024.12.0 # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in # distributed -deprecated==1.2.14 +deprecated==1.2.15 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -151,22 +153,23 @@ distributed==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic +exceptiongroup==1.2.2 + # via aio-pika fast-depends==2.4.12 # via faststream -fastapi==0.115.5 +fastapi==0.115.6 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator -faststream==0.5.31 +faststream==0.5.33 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal @@ -174,19 +177,19 @@ fsspec==2024.10.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.0 +grpcio==1.68.1 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.5 +httpcore==1.0.7 # via httpx -httpx==0.27.0 +httpx==0.28.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -213,7 +216,7 @@ httpx==0.27.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in -idna==3.7 +idna==3.10 # via # anyio # email-validator @@ -257,13 +260,13 @@ jmespath==1.0.1 # via # boto3 # botocore -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via jsonschema locket==1.0.0 # via @@ -282,7 +285,7 @@ msgpack==1.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl @@ -344,7 +347,7 @@ opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-propagator-aws-xray==1.0.1 +opentelemetry-propagator-aws-xray==1.0.2 # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.28.2 # via @@ -373,7 +376,7 @@ opentelemetry-util-http==0.49b2 # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-requests -orjson==3.10.3 +orjson==3.10.12 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -428,13 +431,17 @@ partd==1.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask -prometheus-client==0.20.0 +prometheus-client==0.21.1 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 +prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==5.29.0 +propcache==0.2.1 + # via + # aiohttp + # yarl +protobuf==5.29.1 # via # googleapis-common-protos # opentelemetry-proto @@ -444,7 +451,7 @@ psutil==6.1.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # distributed -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -499,7 +506,7 @@ pydantic==2.10.2 # pydantic-settings pydantic-core==2.27.1 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -529,7 +536,7 @@ pydantic-settings==2.6.1 # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -570,7 +577,7 @@ pyyaml==6.0.2 # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed -redis==5.0.4 +redis==5.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -598,7 +605,7 @@ redis==5.0.4 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -referencing==0.29.3 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -608,34 +615,32 @@ repro-zipfile==0.3.1 # -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.1 +rpds-py==0.22.3 # via # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.4 # via boto3 -sh==2.0.6 +sh==2.1.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio sortedcontainers==2.4.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -starlette==0.41.2 +starlette==0.41.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -662,11 +667,12 @@ starlette==0.41.2 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi + # prometheus-fastapi-instrumentator tblib==3.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -tenacity==8.5.0 +tenacity==9.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -682,37 +688,38 @@ tornado==6.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -tqdm==4.66.4 +tqdm==4.67.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.15.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -types-aiobotocore==2.13.0 +types-aiobotocore==2.15.2.post3 # via -r requirements/../../../packages/aws-library/requirements/_base.in -types-aiobotocore-ec2==2.13.0 +types-aiobotocore-ec2==2.15.2 # via types-aiobotocore -types-aiobotocore-s3==2.13.0 +types-aiobotocore-s3==2.15.2.post1 # via types-aiobotocore -types-aiobotocore-ssm==2.13.0 +types-aiobotocore-ssm==2.15.2 # via types-aiobotocore -types-awscrt==0.20.9 +types-awscrt==0.23.3 # via botocore-stubs -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # aiodebug - # aiodocker + # anyio # fastapi # faststream # opentelemetry-sdk # pydantic # pydantic-core + # pydantic-extra-types # typer # types-aiobotocore # types-aiobotocore-ec2 @@ -748,16 +755,16 @@ urllib3==2.2.3 # botocore # distributed # requests -uvicorn==0.29.0 +uvicorn==0.32.1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -wrapt==1.16.0 +wrapt==1.17.0 # via # aiobotocore # deprecated # opentelemetry-instrumentation # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis -yarl==1.9.4 +yarl==1.18.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/autoscaling/requirements/_test.in b/services/autoscaling/requirements/_test.in index 2ef80abf6d1..7aecc03cd3e 100644 --- a/services/autoscaling/requirements/_test.in +++ b/services/autoscaling/requirements/_test.in @@ -18,6 +18,7 @@ deepdiff docker faker fakeredis[lua] +flaky httpx moto[server] psutil diff --git a/services/autoscaling/requirements/_test.txt b/services/autoscaling/requirements/_test.txt index d6bdd354de7..28a6d7a45e9 100644 --- a/services/autoscaling/requirements/_test.txt +++ b/services/autoscaling/requirements/_test.txt @@ -4,40 +4,40 @@ annotated-types==0.7.0 # pydantic antlr4-python3-runtime==4.13.2 # via moto -anyio==4.3.0 +anyio==4.7.0 # via # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements/_base.txt # jsonschema # referencing -aws-sam-translator==1.89.0 +aws-sam-translator==1.94.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto -blinker==1.8.2 +blinker==1.9.0 # via flask -boto3==1.34.131 +boto3==1.35.36 # via # -c requirements/_base.txt # aws-sam-translator # moto -botocore==1.34.131 +botocore==1.35.36 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -botocore-stubs==1.34.94 +botocore-stubs==1.35.76 # via # -c requirements/_base.txt # types-aiobotocore -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -46,9 +46,9 @@ certifi==2024.2.2 # requests cffi==1.17.1 # via cryptography -cfn-lint==1.10.3 +cfn-lint==1.20.2 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests @@ -56,11 +56,11 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.1 +coverage==7.6.9 # via # -r requirements/_test.in # pytest-cov -cryptography==43.0.1 +cryptography==44.0.0 # via # -c requirements/../../../requirements/constraints.txt # joserfc @@ -71,27 +71,29 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in -fakeredis==2.24.1 +fakeredis==2.26.1 # via -r requirements/_test.in -flask==3.0.3 +flaky==3.8.1 + # via -r requirements/_test.in +flask==3.1.0 # via # flask-cors # moto flask-cors==5.0.0 # via moto -graphql-core==3.2.4 +graphql-core==3.2.5 # via moto h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.5 +httpcore==1.0.7 # via # -c requirements/_base.txt # httpx -httpx==0.27.0 +httpx==0.28.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -99,7 +101,7 @@ httpx==0.27.0 # respx icdiff==2.0.7 # via pytest-icdiff -idna==3.7 +idna==3.10 # via # -c requirements/_base.txt # anyio @@ -120,29 +122,29 @@ jmespath==1.0.1 # -c requirements/_base.txt # boto3 # botocore -joserfc==1.0.0 +joserfc==1.0.1 # via moto jsondiff==2.2.1 # via moto jsonpatch==1.33 # via cfn-lint -jsonpath-ng==1.6.1 +jsonpath-ng==1.7.0 # via moto jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -c requirements/_base.txt # aws-sam-translator + # jsonschema-spec # openapi-schema-validator # openapi-spec-validator -jsonschema-path==0.3.3 +jsonschema-spec==0.1.3 # via openapi-spec-validator -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via # -c requirements/_base.txt # jsonschema - # openapi-schema-validator lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 @@ -156,11 +158,11 @@ moto==5.0.15 # via -r requirements/_test.in mpmath==1.3.0 # via sympy -networkx==3.3 +networkx==3.4.2 # via cfn-lint -openapi-schema-validator==0.6.2 +openapi-schema-validator==0.4.3 # via openapi-spec-validator -openapi-spec-validator==0.7.1 +openapi-spec-validator==0.5.5 # via moto orderly-set==5.2.2 # via deepdiff @@ -170,7 +172,7 @@ packaging==24.2 # pytest # pytest-sugar pathable==0.4.3 - # via jsonschema-path + # via jsonschema-spec pluggy==1.5.0 # via pytest ply==3.11 @@ -185,7 +187,7 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -194,9 +196,9 @@ pydantic-core==2.27.1 # via # -c requirements/_base.txt # pydantic -pyparsing==3.1.4 +pyparsing==3.2.0 # via moto -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -208,7 +210,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-icdiff==0.9 # via -r requirements/_test.in @@ -234,27 +236,25 @@ pyyaml==6.0.2 # -c requirements/_base.txt # cfn-lint # jsondiff - # jsonschema-path + # jsonschema-spec # moto # responses -redis==5.0.4 +redis==5.2.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # fakeredis -referencing==0.29.3 +referencing==0.35.1 # via # -c requirements/_base.txt # jsonschema - # jsonschema-path # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via cfn-lint requests==2.32.3 # via # -c requirements/_base.txt # docker - # jsonschema-path # moto # responses responses==0.25.3 @@ -263,18 +263,18 @@ respx==0.21.1 # via -r requirements/_test.in rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.18.1 +rpds-py==0.22.3 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.4 # via # -c requirements/_base.txt # boto3 -setuptools==74.0.0 +setuptools==75.6.0 # via moto -six==1.16.0 +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil @@ -284,34 +284,33 @@ sniffio==1.3.1 # -c requirements/_base.txt # anyio # asgi-lifespan - # httpx sortedcontainers==2.4.0 # via # -c requirements/_base.txt # fakeredis sympy==1.13.3 # via cfn-lint -termcolor==2.4.0 +termcolor==2.5.0 # via pytest-sugar -types-aiobotocore==2.13.0 +types-aiobotocore==2.15.2.post3 # via # -c requirements/_base.txt # -r requirements/_test.in -types-aiobotocore-ec2==2.13.0 +types-aiobotocore-ec2==2.15.2 # via # -c requirements/_base.txt # types-aiobotocore -types-aiobotocore-iam==2.13.3 +types-aiobotocore-iam==2.15.2 # via types-aiobotocore -types-aiobotocore-s3==2.13.0 +types-aiobotocore-s3==2.15.2.post1 # via # -c requirements/_base.txt # types-aiobotocore -types-aiobotocore-ssm==2.13.0 +types-aiobotocore-ssm==2.15.2 # via # -c requirements/_base.txt # types-aiobotocore -types-awscrt==0.20.9 +types-awscrt==0.23.3 # via # -c requirements/_base.txt # botocore-stubs @@ -320,8 +319,11 @@ types-pyyaml==6.0.12.20240917 typing-extensions==4.12.2 # via # -c requirements/_base.txt + # anyio # aws-sam-translator # cfn-lint + # faker + # jsonschema-spec # pydantic # pydantic-core # types-aiobotocore @@ -337,13 +339,13 @@ urllib3==2.2.3 # docker # requests # responses -werkzeug==3.0.4 +werkzeug==3.1.3 # via # flask # moto -wrapt==1.16.0 +wrapt==1.17.0 # via # -c requirements/_base.txt # aws-xray-sdk -xmltodict==0.13.0 +xmltodict==0.14.2 # via moto diff --git a/services/autoscaling/requirements/_tools.txt b/services/autoscaling/requirements/_tools.txt index b38a6583787..200f2c403cd 100644 --- a/services/autoscaling/requirements/_tools.txt +++ b/services/autoscaling/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,13 +14,13 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -28,7 +28,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -44,7 +44,7 @@ packaging==24.2 # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -53,11 +53,11 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -68,9 +68,9 @@ pyyaml==6.0.2 # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.6.7 +ruff==0.8.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.6.0 # via # -c requirements/_test.txt # pip-tools @@ -81,9 +81,9 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -watchdog==5.0.2 +watchdog==6.0.0 # via -r requirements/_tools.in -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index c0d89ff995a..eccee9967b5 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -419,7 +419,7 @@ def service_monitored_labels( @pytest.fixture async def async_client(initialized_app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: async with httpx.AsyncClient( - app=initialized_app, + transport=httpx.ASGITransport(app=initialized_app), base_url=f"http://{initialized_app.title}.testserver.io", headers={"Content-Type": "application/json"}, ) as client: diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index cb76b03f0a6..ea2355739bd 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -1,12 +1,12 @@ -aio-pika==9.4.1 +aio-pika==9.5.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aioboto3==13.1.0 +aioboto3==13.2.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in -aiobotocore==2.13.1 +aiobotocore==2.15.2 # via aioboto3 -aiocache==0.12.2 +aiocache==0.12.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in @@ -15,16 +15,18 @@ aiodebug==2.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.24.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aioboto3 -aiohttp==3.9.5 +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.10 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -52,15 +54,15 @@ aiohttp==3.9.5 # -c requirements/../../../requirements/constraints.txt # aiobotocore # aiodocker -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.7.0 # via # fast-depends # faststream @@ -77,21 +79,21 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing -boto3==1.34.131 +boto3==1.35.36 # via aiobotocore -botocore==1.34.131 +botocore==1.35.36 # via # aiobotocore # boto3 # s3transfer -botocore-stubs==1.34.94 +botocore-stubs==1.35.76 # via types-aiobotocore -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -120,7 +122,7 @@ certifi==2024.2.2 # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -139,7 +141,7 @@ dask==2024.12.0 # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # -r requirements/_base.in # distributed -deprecated==1.2.14 +deprecated==1.2.15 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -149,22 +151,23 @@ distributed==2024.12.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic +exceptiongroup==1.2.2 + # via aio-pika fast-depends==2.4.12 # via faststream -fastapi==0.115.5 +fastapi==0.115.6 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in - # prometheus-fastapi-instrumentator -faststream==0.5.31 +faststream==0.5.33 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal @@ -172,19 +175,19 @@ fsspec==2024.10.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.0 +grpcio==1.68.1 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.5 +httpcore==1.0.7 # via httpx -httpx==0.27.0 +httpx==0.28.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -211,7 +214,7 @@ httpx==0.27.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in -idna==3.7 +idna==3.10 # via # anyio # email-validator @@ -255,13 +258,13 @@ jmespath==1.0.1 # via # boto3 # botocore -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via jsonschema locket==1.0.0 # via @@ -280,7 +283,7 @@ msgpack==1.1.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl @@ -342,7 +345,7 @@ opentelemetry-instrumentation-requests==0.49b2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-propagator-aws-xray==1.0.1 +opentelemetry-propagator-aws-xray==1.0.2 # via opentelemetry-instrumentation-botocore opentelemetry-proto==1.28.2 # via @@ -371,7 +374,7 @@ opentelemetry-util-http==0.49b2 # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-requests -orjson==3.10.3 +orjson==3.10.12 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -426,13 +429,17 @@ partd==1.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # dask -prometheus-client==0.20.0 +prometheus-client==0.21.1 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator -prometheus-fastapi-instrumentator==6.1.0 +prometheus-fastapi-instrumentator==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==5.29.0 +propcache==0.2.1 + # via + # aiohttp + # yarl +protobuf==5.29.1 # via # googleapis-common-protos # opentelemetry-proto @@ -442,7 +449,7 @@ psutil==6.1.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # distributed -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -497,7 +504,7 @@ pydantic==2.10.2 # pydantic-settings pydantic-core==2.27.1 # via pydantic -pydantic-extra-types==2.9.0 +pydantic-extra-types==2.10.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in @@ -527,7 +534,7 @@ pydantic-settings==2.6.1 # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==5.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -568,7 +575,7 @@ pyyaml==6.0.2 # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed -redis==5.0.4 +redis==5.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -596,7 +603,7 @@ redis==5.0.4 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -referencing==0.29.3 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -606,34 +613,32 @@ repro-zipfile==0.3.1 # -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.1 +rpds-py==0.22.3 # via # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.4 # via boto3 -sh==2.0.6 +sh==2.1.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio sortedcontainers==2.4.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -starlette==0.41.2 +starlette==0.41.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -660,11 +665,12 @@ starlette==0.41.2 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi + # prometheus-fastapi-instrumentator tblib==3.0.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -tenacity==8.5.0 +tenacity==9.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -680,37 +686,38 @@ tornado==6.4.2 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -tqdm==4.66.4 +tqdm==4.67.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.15.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -types-aiobotocore==2.13.0 +types-aiobotocore==2.15.2.post3 # via -r requirements/../../../packages/aws-library/requirements/_base.in -types-aiobotocore-ec2==2.13.0 +types-aiobotocore-ec2==2.15.2 # via types-aiobotocore -types-aiobotocore-s3==2.13.0 +types-aiobotocore-s3==2.15.2.post1 # via types-aiobotocore -types-aiobotocore-ssm==2.13.1 +types-aiobotocore-ssm==2.15.2 # via types-aiobotocore -types-awscrt==0.20.9 +types-awscrt==0.23.3 # via botocore-stubs -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.12.2 # via # aiodebug - # aiodocker + # anyio # fastapi # faststream # opentelemetry-sdk # pydantic # pydantic-core + # pydantic-extra-types # typer # types-aiobotocore # types-aiobotocore-ec2 @@ -746,16 +753,16 @@ urllib3==2.2.3 # botocore # distributed # requests -uvicorn==0.29.0 +uvicorn==0.32.1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -wrapt==1.16.0 +wrapt==1.17.0 # via # aiobotocore # deprecated # opentelemetry-instrumentation # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis -yarl==1.9.4 +yarl==1.18.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 155d8b2b918..1f80746a22a 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -1,8 +1,12 @@ -aiodocker==0.21.0 +aiodocker==0.24.0 # via # -c requirements/_base.txt # -r requirements/_test.in -aiohttp==3.9.5 +aiohappyeyeballs==2.4.4 + # via + # -c requirements/_base.txt + # aiohttp +aiohttp==3.11.10 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -17,37 +21,37 @@ annotated-types==0.7.0 # pydantic antlr4-python3-runtime==4.13.2 # via moto -anyio==4.3.0 +anyio==4.7.0 # via # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements/_base.txt # aiohttp # jsonschema # referencing -aws-sam-translator==1.89.0 +aws-sam-translator==1.94.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto -blinker==1.8.2 +blinker==1.9.0 # via flask -boto3==1.34.131 +boto3==1.35.36 # via # -c requirements/_base.txt # aws-sam-translator # moto -botocore==1.34.131 +botocore==1.35.36 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -56,9 +60,9 @@ certifi==2024.2.2 # requests cffi==1.17.1 # via cryptography -cfn-lint==1.10.3 +cfn-lint==1.20.2 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests @@ -66,16 +70,16 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.1 +coverage==7.6.9 # via # -r requirements/_test.in # pytest-cov -cryptography==43.0.1 +cryptography==44.0.0 # via # -c requirements/../../../requirements/constraints.txt # joserfc # moto -debugpy==1.8.5 +debugpy==1.8.9 # via -r requirements/_test.in deepdiff==8.0.1 # via -r requirements/_test.in @@ -83,38 +87,38 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in -fakeredis==2.24.1 +fakeredis==2.26.1 # via -r requirements/_test.in -flask==3.0.3 +flask==3.1.0 # via # flask-cors # moto flask-cors==5.0.0 # via moto -frozenlist==1.4.1 +frozenlist==1.5.0 # via # -c requirements/_base.txt # aiohttp # aiosignal -graphql-core==3.2.4 +graphql-core==3.2.5 # via moto h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.5 +httpcore==1.0.7 # via # -c requirements/_base.txt # httpx -httpx==0.27.0 +httpx==0.28.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in # respx -idna==3.7 +idna==3.10 # via # -c requirements/_base.txt # anyio @@ -136,29 +140,29 @@ jmespath==1.0.1 # -c requirements/_base.txt # boto3 # botocore -joserfc==1.0.0 +joserfc==1.0.1 # via moto jsondiff==2.2.1 # via moto jsonpatch==1.33 # via cfn-lint -jsonpath-ng==1.6.1 +jsonpath-ng==1.7.0 # via moto jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -c requirements/_base.txt # aws-sam-translator + # jsonschema-spec # openapi-schema-validator # openapi-spec-validator -jsonschema-path==0.3.3 +jsonschema-spec==0.1.3 # via openapi-spec-validator -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2024.10.1 # via # -c requirements/_base.txt # jsonschema - # openapi-schema-validator lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 @@ -168,20 +172,20 @@ markupsafe==3.0.2 # -c requirements/_base.txt # jinja2 # werkzeug -moto==5.0.15 +moto==5.0.22 # via -r requirements/_test.in mpmath==1.3.0 # via sympy -multidict==6.0.5 +multidict==6.1.0 # via # -c requirements/_base.txt # aiohttp # yarl -networkx==3.3 +networkx==3.4.2 # via cfn-lint -openapi-schema-validator==0.6.2 +openapi-schema-validator==0.4.3 # via openapi-spec-validator -openapi-spec-validator==0.7.1 +openapi-spec-validator==0.5.5 # via moto orderly-set==5.2.2 # via deepdiff @@ -192,11 +196,16 @@ packaging==24.2 parse==1.20.2 # via -r requirements/_test.in pathable==0.4.3 - # via jsonschema-path + # via jsonschema-spec pluggy==1.5.0 # via pytest ply==3.11 # via jsonpath-ng +propcache==0.2.1 + # via + # -c requirements/_base.txt + # aiohttp + # yarl psutil==6.1.0 # via # -c requirements/_base.txt @@ -205,7 +214,7 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==2.10.2 +pydantic==2.10.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -214,9 +223,9 @@ pydantic-core==2.27.1 # via # -c requirements/_base.txt # pydantic -pyparsing==3.1.4 +pyparsing==3.2.0 # via moto -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -226,7 +235,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-mock==3.14.0 # via -r requirements/_test.in @@ -248,27 +257,25 @@ pyyaml==6.0.2 # -c requirements/_base.txt # cfn-lint # jsondiff - # jsonschema-path + # jsonschema-spec # moto # responses -redis==5.0.4 +redis==5.2.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # fakeredis -referencing==0.29.3 +referencing==0.35.1 # via # -c requirements/_base.txt # jsonschema - # jsonschema-path # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via cfn-lint requests==2.32.3 # via # -c requirements/_base.txt # docker - # jsonschema-path # moto # responses responses==0.25.3 @@ -277,18 +284,18 @@ respx==0.21.1 # via -r requirements/_test.in rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.18.1 +rpds-py==0.22.3 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.4 # via # -c requirements/_base.txt # boto3 -setuptools==74.0.0 +setuptools==75.6.0 # via moto -six==1.16.0 +six==1.17.0 # via # -c requirements/_base.txt # python-dateutil @@ -298,7 +305,6 @@ sniffio==1.3.1 # -c requirements/_base.txt # anyio # asgi-lifespan - # httpx sortedcontainers==2.4.0 # via # -c requirements/_base.txt @@ -310,9 +316,11 @@ types-pyyaml==6.0.12.20240917 typing-extensions==4.12.2 # via # -c requirements/_base.txt - # aiodocker + # anyio # aws-sam-translator # cfn-lint + # faker + # jsonschema-spec # pydantic # pydantic-core urllib3==2.2.3 @@ -323,17 +331,17 @@ urllib3==2.2.3 # docker # requests # responses -werkzeug==3.0.4 +werkzeug==3.1.3 # via # flask # moto -wrapt==1.16.0 +wrapt==1.17.0 # via # -c requirements/_base.txt # aws-xray-sdk -xmltodict==0.13.0 +xmltodict==0.14.2 # via moto -yarl==1.9.4 +yarl==1.18.3 # via # -c requirements/_base.txt # aiohttp diff --git a/services/clusters-keeper/requirements/_tools.txt b/services/clusters-keeper/requirements/_tools.txt index b38a6583787..200f2c403cd 100644 --- a/services/clusters-keeper/requirements/_tools.txt +++ b/services/clusters-keeper/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,13 +14,13 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -28,7 +28,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -44,7 +44,7 @@ packaging==24.2 # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -53,11 +53,11 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -68,9 +68,9 @@ pyyaml==6.0.2 # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.6.7 +ruff==0.8.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.6.0 # via # -c requirements/_test.txt # pip-tools @@ -81,9 +81,9 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -watchdog==5.0.2 +watchdog==6.0.0 # via -r requirements/_tools.in -wheel==0.44.0 +wheel==0.45.1 # via pip-tools diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 432d743fb0c..890968f4d83 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -262,7 +262,7 @@ def app_settings(initialized_app: FastAPI) -> ApplicationSettings: @pytest.fixture async def async_client(initialized_app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: async with httpx.AsyncClient( - app=initialized_app, + transport=httpx.ASGITransport(app=initialized_app), base_url=f"http://{initialized_app.title}.testserver.io", headers={"Content-Type": "application/json"}, ) as client: From 66a42172b7b59b523f06a99037e88137ef46c9a9 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:42:01 +0100 Subject: [PATCH 8/9] =?UTF-8?q?=E2=9A=97=EF=B8=8FCI:=20Separate=20image=20?= =?UTF-8?q?building=20of=20frontend=20(#6918)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-testing-deploy.yml | 81 ++++++++++++++-------- Makefile | 15 ++-- packages/service-integration/Dockerfile | 6 +- requirements/tools/Dockerfile | 3 +- services/agent/Dockerfile | 6 +- services/api-server/Dockerfile | 7 +- services/autoscaling/Dockerfile | 6 +- services/catalog/Dockerfile | 7 +- services/clusters-keeper/Dockerfile | 6 +- services/dask-sidecar/Dockerfile | 7 +- services/datcore-adapter/Dockerfile | 7 +- services/director-v2/Dockerfile | 7 +- services/director/Dockerfile | 7 +- services/dynamic-scheduler/Dockerfile | 7 +- services/dynamic-sidecar/Dockerfile | 3 +- services/efs-guardian/Dockerfile | 6 +- services/invitations/Dockerfile | 7 +- services/migration/Dockerfile | 3 +- services/payments/Dockerfile | 7 +- services/resource-usage-tracker/Dockerfile | 7 +- services/storage/Dockerfile | 7 +- services/web/Dockerfile | 7 +- 22 files changed, 108 insertions(+), 111 deletions(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index e2eabc80994..2fd149a15e8 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -272,20 +272,43 @@ jobs: python: ["3.11"] os: [ubuntu-22.04] fail-fast: false - name: "[build] docker images" + name: "[build] docker images (excluding frontend)" steps: - - name: Remove unused software + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: expose github runtime for buildx + uses: crazy-max/ghaction-github-runtime@v3 + - name: show system environs + run: ./ci/helpers/show_system_versions.bash + - name: build images run: | - echo "Available storage before:" - sudo df -h - echo - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - echo "Available storage after:" - sudo df -h - echo + export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) + mkdir --parents /${{ runner.temp }}/build + make build local-dest=/${{ runner.temp }}/build exclude=static-webserver + - name: upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend + path: /${{ runner.temp }}/build + + build-test-images-frontend: + # this step comes first, so that it is executed as first job in push calls + # in PR calls this step is anyway skipped + needs: changes + if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }} + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-22.04] + fail-fast: false + name: "[build] docker images (frontend-only)" + steps: - uses: actions/checkout@v4 - name: setup docker buildx id: buildx @@ -300,11 +323,11 @@ jobs: run: | export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) mkdir --parents /${{ runner.temp }}/build - make build local-dest=/${{ runner.temp }}/build + make build local-dest=/${{ runner.temp }}/build target=static-webserver - name: upload build artifacts uses: actions/upload-artifact@v4 with: - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-frontend path: /${{ runner.temp }}/build unit-test-webserver-01: @@ -1868,7 +1891,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -1932,7 +1955,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -1996,7 +2019,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2067,7 +2090,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2133,7 +2156,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2199,7 +2222,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2257,7 +2280,7 @@ jobs: run: echo "::notice All good!" system-test-public-api: - needs: [changes, build-test-images] + needs: [changes, build-test-images, build-test-images-frontend] if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }} timeout-minutes: 25 # if this timeout gets too small, then split the tests name: "[sys] public api" @@ -2286,7 +2309,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-* path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2317,7 +2340,7 @@ jobs: run: ./ci/github/system-testing/public-api.bash clean_up system-test-swarm-deploy: - needs: [changes, build-test-images] + needs: [changes, build-test-images, build-test-images-frontend] if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }} timeout-minutes: 30 # if this timeout gets too small, then split the tests name: "[sys] deploy simcore" @@ -2346,7 +2369,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-* path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2382,7 +2405,7 @@ jobs: run: ./ci/github/system-testing/swarm-deploy.bash clean_up system-test-e2e: - needs: [changes, build-test-images] + needs: [changes, build-test-images, build-test-images-frontend] if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }} timeout-minutes: 30 # if this timeout gets too small, then split the tests name: "[sys] e2e" @@ -2417,7 +2440,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-* path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 @@ -2462,7 +2485,7 @@ jobs: run: ./ci/github/system-testing/e2e.bash clean_up system-test-e2e-playwright: - needs: [changes, build-test-images] + needs: [changes, build-test-images, build-test-images-frontend] if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }} timeout-minutes: 30 # if this timeout gets too small, then split the tests name: "[sys] e2e-playwright" @@ -2497,7 +2520,7 @@ jobs: - name: download docker images uses: actions/download-artifact@v4 with: - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-* path: /${{ runner.temp }}/build - name: load docker images run: make load-images local-src=/${{ runner.temp }}/build @@ -2616,7 +2639,7 @@ jobs: with: action: actions/download-artifact@v4 with: | - name: docker-buildx-images-${{ runner.os }}-${{ github.sha }} + pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-* path: /${{ runner.temp }}/build attempt_limit: 5 attempt_delay: 1000 diff --git a/Makefile b/Makefile index 1ecccd09739..b640fe626b8 100644 --- a/Makefile +++ b/Makefile @@ -153,6 +153,7 @@ DOCKER_TARGET_PLATFORMS ?= linux/amd64 comma := , define _docker_compose_build +$(eval INCLUDED_SERVICES := $(filter-out $(exclude), $(SERVICES_NAMES_TO_BUILD))) \ export BUILD_TARGET=$(if $(findstring -devel,$@),development,production) &&\ pushd services &&\ $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ @@ -160,7 +161,7 @@ $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ export $(subst -,_,$(shell echo $(service) | tr a-z A-Z))_VERSION=$(shell cat services/$(service)/VERSION);\ ,) \ )\ -docker buildx bake \ +docker buildx bake --allow=fs.read=.. \ $(if $(findstring -devel,$@),,\ --set *.platform=$(DOCKER_TARGET_PLATFORMS) \ )\ @@ -172,7 +173,7 @@ docker buildx bake \ )\ )\ $(if $(push),--push,) \ - $(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),) \ + $(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),$(INCLUDED_SERVICES)) \ $(if $(findstring -nc,$@),--no-cache,\ $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ --set $(service).cache-to=type=gha$(comma)mode=max$(comma)scope=$(service) \ @@ -183,22 +184,20 @@ endef rebuild: build-nc # alias build build-nc: .env ## Builds production images and tags them as 'local/{service-name}:production'. For single target e.g. 'make target=webserver build'. To export to a folder: `make local-dest=/tmp/build` - # Building service$(if $(target),,s) $(target) + # Building service$(if $(target),,s) $(target) $(if $(exclude),excluding,) $(exclude) @$(_docker_compose_build) # List production images @docker images --filter="reference=local/*:production" load-images: guard-local-src ## loads images from local-src - # loading from images from $(local-src)... - @$(foreach service, $(SERVICES_NAMES_TO_BUILD),\ - docker load --input $(local-src)/$(service).tar; \ - ) + # loading from any tar images from $(local-src)... + @find $(local-src) -name '*.tar' -print0 | xargs -0 -n1 -P $(shell nproc) --no-run-if-empty --verbose docker load --input # all images loaded @docker images build-devel build-devel-nc: .env ## Builds development images and tags them as 'local/{service-name}:development'. For single target e.g. 'make target=webserver build-devel' ifeq ($(target),) - # Building services + # Building services $(if $(exclude),excluding,) $(exclude) @$(_docker_compose_build) else ifeq ($(findstring static-webserver,$(target)),static-webserver) diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile index b41f4d07b0b..e62c0a5271e 100644 --- a/packages/service-integration/Dockerfile +++ b/packages/service-integration/Dockerfile @@ -12,8 +12,7 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ @@ -54,8 +53,7 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" FROM base AS build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/requirements/tools/Dockerfile b/requirements/tools/Dockerfile index 0901b5e8302..1d5638b363b 100644 --- a/requirements/tools/Dockerfile +++ b/requirements/tools/Dockerfile @@ -16,8 +16,7 @@ FROM python:${PYTHON_VERSION}-slim-bookworm AS base ENV VIRTUAL_ENV=/home/scu/.venv -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update \ && apt-get -y install --no-install-recommends\ diff --git a/services/agent/Dockerfile b/services/agent/Dockerfile index 9b5d031affd..3c7469ac92b 100644 --- a/services/agent/Dockerfile +++ b/services/agent/Dockerfile @@ -20,8 +20,7 @@ LABEL maintainer=GitHK # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -71,8 +70,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/api-server/Dockerfile b/services/api-server/Dockerfile index 933df727fc6..52bb197a36d 100644 --- a/services/api-server/Dockerfile +++ b/services/api-server/Dockerfile @@ -19,13 +19,13 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -67,8 +67,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/autoscaling/Dockerfile b/services/autoscaling/Dockerfile index 4ace322f5f9..f094e802b04 100644 --- a/services/autoscaling/Dockerfile +++ b/services/autoscaling/Dockerfile @@ -23,8 +23,7 @@ ENV DOCKER_APT_VERSION="5:26.1.4-1~debian.12~bookworm" # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ @@ -87,8 +86,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/catalog/Dockerfile b/services/catalog/Dockerfile index 8fb6418b3ce..cea9aac8e52 100644 --- a/services/catalog/Dockerfile +++ b/services/catalog/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -68,8 +68,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/clusters-keeper/Dockerfile b/services/clusters-keeper/Dockerfile index e75ba032ac0..7011f9b527c 100644 --- a/services/clusters-keeper/Dockerfile +++ b/services/clusters-keeper/Dockerfile @@ -23,8 +23,7 @@ ENV DOCKER_APT_VERSION="5:26.1.4-1~debian.12~bookworm" # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ @@ -87,8 +86,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/dask-sidecar/Dockerfile b/services/dask-sidecar/Dockerfile index 5ea8e90d545..38ff5e5bcb7 100644 --- a/services/dask-sidecar/Dockerfile +++ b/services/dask-sidecar/Dockerfile @@ -22,8 +22,7 @@ LABEL maintainer=sanderegg # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ @@ -31,6 +30,7 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ curl \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -78,8 +78,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/datcore-adapter/Dockerfile b/services/datcore-adapter/Dockerfile index e83bf01c37e..f2ddebe5382 100644 --- a/services/datcore-adapter/Dockerfile +++ b/services/datcore-adapter/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=sanderegg # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -68,8 +68,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/director-v2/Dockerfile b/services/director-v2/Dockerfile index 5388dbb6f73..3093c28134f 100644 --- a/services/director-v2/Dockerfile +++ b/services/director-v2/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -68,8 +68,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/director/Dockerfile b/services/director/Dockerfile index 247188f6360..5d9bdcad811 100644 --- a/services/director/Dockerfile +++ b/services/director/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=sanderegg # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -68,8 +68,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/dynamic-scheduler/Dockerfile b/services/dynamic-scheduler/Dockerfile index bffb3808bdd..f99fca55598 100644 --- a/services/dynamic-scheduler/Dockerfile +++ b/services/dynamic-scheduler/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -67,8 +67,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index a6e9fc0a4a1..a5173e7f19a 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -108,8 +108,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/efs-guardian/Dockerfile b/services/efs-guardian/Dockerfile index 82174799ca5..e4341d48e02 100644 --- a/services/efs-guardian/Dockerfile +++ b/services/efs-guardian/Dockerfile @@ -23,8 +23,7 @@ ENV DOCKER_APT_VERSION="5:26.1.4-1~debian.12~bookworm" # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ @@ -105,8 +104,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/invitations/Dockerfile b/services/invitations/Dockerfile index 0422ca73cbd..c17ce3f3bbb 100644 --- a/services/invitations/Dockerfile +++ b/services/invitations/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -67,8 +67,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/migration/Dockerfile b/services/migration/Dockerfile index 30a20c6df93..fe262597d07 100644 --- a/services/migration/Dockerfile +++ b/services/migration/Dockerfile @@ -40,8 +40,7 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # -------------------------------------------- FROM base AS build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/payments/Dockerfile b/services/payments/Dockerfile index 32fada11361..c7d3ef40026 100644 --- a/services/payments/Dockerfile +++ b/services/payments/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -67,8 +67,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/resource-usage-tracker/Dockerfile b/services/resource-usage-tracker/Dockerfile index 528ecabfd3e..14a0583afee 100644 --- a/services/resource-usage-tracker/Dockerfile +++ b/services/resource-usage-tracker/Dockerfile @@ -20,13 +20,13 @@ LABEL maintainer=sanderegg # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -68,8 +68,7 @@ FROM base AS build ENV SC_BUILD_TARGET=build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/storage/Dockerfile b/services/storage/Dockerfile index 8c9f0e381c0..5eb25f4e9a3 100644 --- a/services/storage/Dockerfile +++ b/services/storage/Dockerfile @@ -19,13 +19,13 @@ LABEL maintainer=mguidon # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -75,8 +75,7 @@ FROM base AS build ENV SC_BUILD_TARGET build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/services/web/Dockerfile b/services/web/Dockerfile index e7ff6b3fe41..542d0b34ed2 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -21,14 +21,14 @@ LABEL maintainer=pcrespov # for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux && \ apt-get update && \ apt-get install -y --no-install-recommends \ libmagic1 \ gosu \ && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ # verify that the binary works && gosu nobody true @@ -78,8 +78,7 @@ FROM base AS build ENV SC_BUILD_TARGET build -RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ - --mount=type=cache,target=/var/lib/apt,mode=0755,sharing=private \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ From 1ab9b998154137362c62702f1cd2009771a35ae4 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:54:12 +0100 Subject: [PATCH 9/9] =?UTF-8?q?=E2=9A=97=EF=B8=8F=20remove=20api=20checks?= =?UTF-8?q?=20from=20system=20tests=20(#6920)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-testing-deploy.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 2fd149a15e8..831b6d3415a 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -2596,9 +2596,7 @@ jobs: system-test-e2e-playwright, system-test-environment-setup, system-test-public-api, - system-test-swarm-deploy, - system-api-specs, - system-backwards-compatibility + system-test-swarm-deploy ] runs-on: ubuntu-latest steps: