From e50f9726a578152158a23bc9cc0034c8695cf5b0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 May 2024 16:32:41 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=E2=99=BB=EF=B8=8F=20New=20fields=20fo?= =?UTF-8?q?r=20service=20metadata=20(#5902)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_schemas_catalog/services.py | 6 +- .../src/models_library/services.py | 203 +++++++++--------- .../src/models_library/services_db.py | 4 +- packages/service-integration/VERSION | 2 +- packages/service-integration/setup.cfg | 2 +- .../_compose_spec_model_autogenerated.py | 2 +- .../src/service_integration/cli.py | 4 +- .../service_integration/commands/compose.py | 32 +-- .../service_integration/commands/config.py | 18 +- .../service_integration/commands/metadata.py | 3 +- .../commands/run_creator.py | 16 +- .../src/service_integration/commands/test.py | 3 +- .../src/service_integration/errors.py | 2 +- .../src/service_integration/oci_image_spec.py | 9 +- .../src/service_integration/osparc_config.py | 94 ++------ .../service_integration/osparc_image_specs.py | 11 +- .../osparc_runtime_specs.py | 5 +- .../pytest_plugin/docker_integration.py | 15 +- .../pytest_plugin/folder_structure.py | 4 +- .../pytest_plugin/validation_data.py | 6 +- .../src/service_integration/settings.py | 6 +- .../src/service_integration/versioning.py | 12 +- .../service-integration/tests/conftest.py | 19 +- .../tests/data/metadata.yml | 6 +- .../tests/test__usecase_jupytermath.py | 6 +- .../service-integration/tests/test_cli.py | 2 +- .../tests/test_command_compose.py | 48 ++--- .../tests/test_command_config.py | 5 +- .../tests/test_command_metadata.py | 2 +- .../tests/test_labels_annotations.py | 4 +- .../tests/test_oci_image_spec.py | 6 +- .../tests/test_osparc_config.py | 10 +- .../tests/test_osparc_image_specs.py | 8 +- .../tests/test_osparc_runtime_specs.py | 2 +- .../tests/test_versioning.py | 12 +- .../test_api_routers_solvers_jobs.py | 4 +- services/web/server/VERSION | 2 +- services/web/server/setup.cfg | 2 +- .../api/v0/openapi.yaml | 44 +++- 39 files changed, 317 insertions(+), 324 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index bf3ca20a25c..f491eb6bcaf 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -3,12 +3,12 @@ from pydantic import Extra from ..emails import LowerCaseEmailStr -from ..services import ServiceDockerData, ServiceMetaData +from ..services import BaseServiceMetaData, ServiceDockerData from ..services_access import ServiceAccessRights from ..services_resources import ServiceResourcesDict -class ServiceUpdate(ServiceMetaData, ServiceAccessRights): +class ServiceUpdate(BaseServiceMetaData, ServiceAccessRights): class Config: schema_extra: ClassVar[dict[str, Any]] = { "example": { @@ -61,7 +61,7 @@ class Config: class ServiceGet( - ServiceDockerData, ServiceAccessRights, ServiceMetaData + ServiceDockerData, ServiceAccessRights, BaseServiceMetaData ): # pylint: disable=too-many-ancestors owner: LowerCaseEmailStr | None diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 821ecef9a74..144d4b22d04 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -18,7 +18,7 @@ validator, ) -from .basic_regex import VERSION_RE +from .basic_regex import SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS, VERSION_RE from .boot_options import BootOption, BootOptions from .emails import LowerCaseEmailStr from .services_constants import FILENAME_RE, PROPERTY_TYPE_RE @@ -480,25 +480,104 @@ def validate_thumbnail(cls, value): # pylint: disable=no-self-argument,no-self- ServiceOutputsDict: TypeAlias = dict[ServicePortKey, ServiceOutput] +_EXAMPLE = { + "name": "oSparc Python Runner", + "key": "simcore/services/comp/osparc-python-runner", + "type": "computational", + "integration-version": "1.0.0", + "progress_regexp": "^(?:\\[?PROGRESS\\]?:?)?\\s*(?P[0-1]?\\.\\d+|\\d+\\s*(?P%))", + "version": "1.7.0", + "description": "oSparc Python Runner", + "contact": "smith@company.com", + "authors": [ + { + "name": "John Smith", + "email": "smith@company.com", + "affiliation": "Company", + }, + { + "name": "Richard Brown", + "email": "brown@uni.edu", + "affiliation": "University", + }, + ], + "inputs": { + "input_1": { + "displayOrder": 1, + "label": "Input data", + "description": "Any code, requirements or data file", + "type": "data:*/*", + } + }, + "outputs": { + "output_1": { + "displayOrder": 1, + "label": "Output data", + "description": "All data produced by the script is zipped as output_data.zip", + "type": "data:*/*", + "fileToKeyMap": {"output_data.zip": "output_1"}, + } + }, +} + +_EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER = { + **_EXAMPLE, + "description": "oSparc Python Runner with boot options", + "inputs": { + "input_1": { + "label": "Input data", + "description": "Any code, requirements or data file", + "type": "data:*/*", + } + }, + "outputs": { + "output_1": { + "label": "Output data", + "description": "All data produced by the script is zipped as output_data.zip", + "type": "data:*/*", + "fileToKeyMap": {"output_data.zip": "output_1"}, + } + }, + "boot-options": { + "example_service_defined_boot_mode": BootOption.Config.schema_extra["examples"][ + 0 + ], + "example_service_defined_theme_selection": BootOption.Config.schema_extra[ + "examples" + ][1], + }, + "min-visible-inputs": 2, +} + + class ServiceDockerData(ServiceKeyVersion, _BaseServiceCommonDataModel): """ Static metadata for a service injected in the image labels - This is one to one with node-meta-v0.0.1.json + NOTE: This model is serialized in .osparc/metadata.yml and in the labels of the docker image """ - integration_version: str | None = Field( + version_display: str | None = Field( None, - alias="integration-version", - description="integration version number", - regex=VERSION_RE, - examples=["1.0.0"], + description="A user-friendly or marketing name for the release." + " This can be used to reference the release in a more readable and recognizable format, such as 'Matterhorn Release,' 'Spring Update,' or 'Holiday Edition.'" + " This name is not used for version comparison but is useful for communication and documentation purposes.", ) - progress_regexp: str | None = Field( + + release_date: datetime | None = Field( None, - alias="progress_regexp", - description="regexp pattern for detecting computational service's progress", + description="A timestamp when the specific version of the service was released." + " This field helps in tracking the timeline of releases and understanding the sequence of updates." + " A timestamp string should be formatted as YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]", ) + + integration_version: str | None = Field( + None, + alias="integration-version", + description="This version is used to maintain backward compatibility when there are changes in the way a service is integrated into the framework", + regex=SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS, + ) + service_type: ServiceType = Field( ..., alias="type", @@ -526,6 +605,7 @@ class ServiceDockerData(ServiceKeyVersion, _BaseServiceCommonDataModel): alias="boot-options", description="Service defined boot options. These get injected in the service as env variables.", ) + min_visible_inputs: NonNegativeInt | None = Field( None, alias="min-visible-inputs", @@ -535,108 +615,33 @@ class ServiceDockerData(ServiceKeyVersion, _BaseServiceCommonDataModel): ), ) + progress_regexp: str | None = Field( + None, + alias="progress_regexp", + description="regexp pattern for detecting computational service's progress", + ) + class Config: description = "Description of a simcore node 'class' with input and output" extra = Extra.forbid - frozen = False # it inherits from ServiceKeyVersion. + frozen = False # overrides config from ServiceKeyVersion. + allow_population_by_field_name = True schema_extra: ClassVar[dict[str, Any]] = { "examples": [ - { - "name": "oSparc Python Runner", - "key": "simcore/services/comp/osparc-python-runner", - "type": "computational", - "integration-version": "1.0.0", - "progress_regexp": "^(?:\\[?PROGRESS\\]?:?)?\\s*(?P[0-1]?\\.\\d+|\\d+\\s*(?P%))", - "version": "1.7.0", - "description": "oSparc Python Runner", - "contact": "smith@company.com", - "authors": [ - { - "name": "John Smith", - "email": "smith@company.com", - "affiliation": "Company", - }, - { - "name": "Richard Brown", - "email": "brown@uni.edu", - "affiliation": "University", - }, - ], - "inputs": { - "input_1": { - "displayOrder": 1, - "label": "Input data", - "description": "Any code, requirements or data file", - "type": "data:*/*", - } - }, - "outputs": { - "output_1": { - "displayOrder": 1, - "label": "Output data", - "description": "All data produced by the script is zipped as output_data.zip", - "type": "data:*/*", - "fileToKeyMap": {"output_data.zip": "output_1"}, - } - }, - }, + _EXAMPLE, + _EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER, # latest { - "name": "oSparc Python Runner", - "key": "simcore/services/comp/osparc-python-runner", - "type": "computational", - "integration-version": "1.0.0", - "progress_regexp": "^(?:\\[?PROGRESS\\]?:?)?\\s*(?P[0-1]?\\.\\d+|\\d+\\s*(?P%))", - "version": "1.7.0", - "description": "oSparc Python Runner with boot options", - "contact": "smith@company.com", - "authors": [ - { - "name": "John Smith", - "email": "smith@company.com", - "affiliation": "Company", - }, - { - "name": "Richard Brown", - "email": "brown@uni.edu", - "affiliation": "University", - }, - ], - "inputs": { - "input_1": { - "label": "Input data", - "description": "Any code, requirements or data file", - "type": "data:*/*", - } - }, - "outputs": { - "output_1": { - "label": "Output data", - "description": "All data produced by the script is zipped as output_data.zip", - "type": "data:*/*", - "fileToKeyMap": {"output_data.zip": "output_1"}, - } - }, - "boot-options": { - "example_service_defined_boot_mode": BootOption.Config.schema_extra[ - "examples" - ][ - 0 - ], - "example_service_defined_theme_selection": BootOption.Config.schema_extra[ - "examples" - ][ - 1 - ], - }, - "min-visible-inputs": 2, + **_EXAMPLE_W_BOOT_OPTIONS_AND_NO_DISPLAY_ORDER, + "version_display": "Matterhorn Release", + "release_date": "2024-05-31T13:45:30", }, ] } -class ServiceMetaData(_BaseServiceCommonDataModel): +class BaseServiceMetaData(_BaseServiceCommonDataModel): # Overrides all fields of _BaseServiceCommonDataModel: # - for a partial update all members must be Optional # FIXME: if API entry needs a schema to allow partial updates (e.g. patch/put), diff --git a/packages/models-library/src/models_library/services_db.py b/packages/models-library/src/models_library/services_db.py index 0e5353353ae..e4ceae79c1f 100644 --- a/packages/models-library/src/models_library/services_db.py +++ b/packages/models-library/src/models_library/services_db.py @@ -9,7 +9,7 @@ from pydantic import Field from pydantic.types import PositiveInt -from .services import ServiceKeyVersion, ServiceMetaData +from .services import BaseServiceMetaData, ServiceKeyVersion from .services_access import ServiceGroupAccessRights # ------------------------------------------------------------------- @@ -18,7 +18,7 @@ # - table services_access_rights -class ServiceMetaDataAtDB(ServiceKeyVersion, ServiceMetaData): +class ServiceMetaDataAtDB(ServiceKeyVersion, BaseServiceMetaData): # for a partial update all members must be Optional classifiers: list[str] | None = Field([]) owner: PositiveInt | None diff --git a/packages/service-integration/VERSION b/packages/service-integration/VERSION index 21e8796a09d..ee90284c27f 100644 --- a/packages/service-integration/VERSION +++ b/packages/service-integration/VERSION @@ -1 +1 @@ -1.0.3 +1.0.4 diff --git a/packages/service-integration/setup.cfg b/packages/service-integration/setup.cfg index af7998eb1a3..a6c19f847ef 100644 --- a/packages/service-integration/setup.cfg +++ b/packages/service-integration/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.3 +current_version = 1.0.4 commit = True message = service-integration version: {current_version} → {new_version} tag = False diff --git a/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py b/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py index 56ad2faf2df..a390a469a41 100644 --- a/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py +++ b/packages/service-integration/src/service_integration/_compose_spec_model_autogenerated.py @@ -519,7 +519,7 @@ class Config: ) = None oom_kill_disable: bool | None = None oom_score_adj: conint(ge=-1000, le=1000) | None = None - pid: str | None | None = None + pid: str | None = None pids_limit: float | str | None = None platform: str | None = None ports: list[PortInt | str | Port] | None = None diff --git a/packages/service-integration/src/service_integration/cli.py b/packages/service-integration/src/service_integration/cli.py index bf5d0101119..a257be65c14 100644 --- a/packages/service-integration/src/service_integration/cli.py +++ b/packages/service-integration/src/service_integration/cli.py @@ -14,7 +14,7 @@ def _version_callback(value: bool): if value: rich.print(__version__) - raise typer.Exit() + raise typer.Exit @app.callback() @@ -38,7 +38,7 @@ def main( ), ): """o2s2parc service integration library""" - assert version or not version # nosec + assert isinstance(version, bool | None) # nosec overrides = {} if registry_name: diff --git a/packages/service-integration/src/service_integration/commands/compose.py b/packages/service-integration/src/service_integration/commands/compose.py index 15ddae6094e..3904828cad5 100644 --- a/packages/service-integration/src/service_integration/commands/compose.py +++ b/packages/service-integration/src/service_integration/commands/compose.py @@ -10,7 +10,12 @@ from ..compose_spec_model import ComposeSpecification from ..oci_image_spec import LS_LABEL_PREFIX, OCI_LABEL_PREFIX -from ..osparc_config import DockerComposeOverwriteCfg, MetaConfig, RuntimeConfig +from ..osparc_config import ( + OSPARC_CONFIG_DIRNAME, + DockerComposeOverwriteConfig, + MetadataConfig, + RuntimeConfig, +) from ..osparc_image_specs import create_image_spec from ..settings import AppSettings @@ -20,10 +25,7 @@ def _run_git(*args) -> str: """:raises CalledProcessError""" return subprocess.run( # nosec - [ - "git", - ] - + list(args), + ["git", *list(args)], capture_output=True, encoding="utf8", check=True, @@ -60,15 +62,15 @@ def create_docker_compose_image_spec( config_basedir = meta_config_path.parent # required - meta_cfg = MetaConfig.from_yaml(meta_config_path) + meta_cfg = MetadataConfig.from_yaml(meta_config_path) # required if docker_compose_overwrite_path: - docker_compose_overwrite_cfg = DockerComposeOverwriteCfg.from_yaml( + docker_compose_overwrite_cfg = DockerComposeOverwriteConfig.from_yaml( docker_compose_overwrite_path ) else: - docker_compose_overwrite_cfg = DockerComposeOverwriteCfg.create_default( + docker_compose_overwrite_cfg = DockerComposeOverwriteConfig.create_default( service_name=meta_cfg.service_name() ) @@ -88,7 +90,8 @@ def create_docker_compose_image_spec( (config_basedir / f"{OCI_LABEL_PREFIX}.yml").read_text() ) if not oci_spec: - raise ValueError("Undefined OCI image spec") + msg = "Undefined OCI image spec" + raise ValueError(msg) oci_labels = to_labels(oci_spec, prefix_key=OCI_LABEL_PREFIX) extra_labels.update(oci_labels) @@ -118,7 +121,7 @@ def create_docker_compose_image_spec( "config", "--get", "remote.origin.url" ) - compose_spec = create_image_spec( + return create_image_spec( settings, meta_cfg, docker_compose_overwrite_cfg, @@ -126,13 +129,11 @@ def create_docker_compose_image_spec( extra_labels=extra_labels, ) - return compose_spec - def main( ctx: typer.Context, config_path: Path = typer.Option( - ".osparc", + OSPARC_CONFIG_DIRNAME, "-m", "--metadata", help="osparc config file or folder. " @@ -149,7 +150,8 @@ def main( # TODO: all these MUST be replaced by osparc_config.ConfigFilesStructure if not config_path.exists(): - raise typer.BadParameter("Invalid path to metadata file or folder") + msg = "Invalid path to metadata file or folder" + raise typer.BadParameter(msg) if config_path.is_dir(): # equivalent to 'basedir/**/metadata.yml' @@ -162,7 +164,7 @@ def main( configs_kwargs_map: dict[str, dict[str, Path]] = {} - for meta_config in sorted(list(basedir.rglob(config_pattern))): + for meta_config in sorted(basedir.rglob(config_pattern)): config_name = meta_config.parent.name configs_kwargs_map[config_name] = {} diff --git a/packages/service-integration/src/service_integration/commands/config.py b/packages/service-integration/src/service_integration/commands/config.py index dc321430a12..e1e5b8ef5b1 100644 --- a/packages/service-integration/src/service_integration/commands/config.py +++ b/packages/service-integration/src/service_integration/commands/config.py @@ -9,7 +9,12 @@ from pydantic.main import BaseModel from ..compose_spec_model import ComposeSpecification -from ..osparc_config import DockerComposeOverwriteCfg, MetaConfig, RuntimeConfig +from ..osparc_config import ( + OSPARC_CONFIG_DIRNAME, + DockerComposeOverwriteConfig, + MetadataConfig, + RuntimeConfig, +) def create_osparc_specs( @@ -53,17 +58,18 @@ def _save(service_name: str, filename: Path, model: BaseModel): labels = dict(item.strip().split("=") for item in build_labels) elif isinstance(build_labels, dict): labels = build_labels - elif labels__root__ := getattr(build_labels, "__root__"): + elif labels__root__ := build_labels.__root__: assert isinstance(labels__root__, dict) # nosec labels = labels__root__ else: - raise ValueError(f"Invalid build labels {build_labels}") + msg = f"Invalid build labels {build_labels}" + raise ValueError(msg) - meta_cfg = MetaConfig.from_labels_annotations(labels) + meta_cfg = MetadataConfig.from_labels_annotations(labels) _save(service_name, metadata_path, meta_cfg) docker_compose_overwrite_cfg = ( - DockerComposeOverwriteCfg.create_default( + DockerComposeOverwriteConfig.create_default( service_name=meta_cfg.service_name() ) ) @@ -94,7 +100,7 @@ def main( ): """Creates osparc config from complete docker compose-spec""" # TODO: sync defaults among CLI commands - config_dir = from_spec_file.parent / ".osparc" + config_dir = from_spec_file.parent / OSPARC_CONFIG_DIRNAME project_cfg_path = config_dir / "docker-compose.overwrite.yml" meta_cfg_path = config_dir / "metadata.yml" runtime_cfg_path = config_dir / "runtime.yml" diff --git a/packages/service-integration/src/service_integration/commands/metadata.py b/packages/service-integration/src/service_integration/commands/metadata.py index 3e26c455c57..eb6e153b7f5 100644 --- a/packages/service-integration/src/service_integration/commands/metadata.py +++ b/packages/service-integration/src/service_integration/commands/metadata.py @@ -7,6 +7,7 @@ import yaml from models_library.services import ServiceDockerData +from ..osparc_config import OSPARC_CONFIG_DIRNAME from ..versioning import bump_version_string from ..yaml_utils import ordered_safe_dump, ordered_safe_load @@ -57,7 +58,7 @@ def get_version( TargetVersionChoices.SEMANTIC_VERSION ), metadata_file: Path = typer.Option( - ".osparc/metadata.yml", + f"{OSPARC_CONFIG_DIRNAME}/metadata.yml", help="The metadata yaml file", ), ): diff --git a/packages/service-integration/src/service_integration/commands/run_creator.py b/packages/service-integration/src/service_integration/commands/run_creator.py index cfcb6a6b5fb..3b08948eeec 100644 --- a/packages/service-integration/src/service_integration/commands/run_creator.py +++ b/packages/service-integration/src/service_integration/commands/run_creator.py @@ -4,6 +4,8 @@ import typer import yaml +from ..osparc_config import OSPARC_CONFIG_DIRNAME + def get_input_config(metadata_file: Path) -> dict: inputs = {} @@ -16,7 +18,7 @@ def get_input_config(metadata_file: Path) -> dict: def main( metadata_file: Path = typer.Option( - ".osparc/metadata.yml", + f"{OSPARC_CONFIG_DIRNAME}/metadata.yml", "--metadata", help="The metadata yaml of the node", ), @@ -50,19 +52,19 @@ def main( ] input_config = get_input_config(metadata_file) for input_key, input_value in input_config.items(): + input_key_upper = f"{input_key}".upper() + if "data:" in input_value["type"]: filename = input_key if "fileToKeyMap" in input_value and len(input_value["fileToKeyMap"]) > 0: filename, _ = next(iter(input_value["fileToKeyMap"].items())) - input_script.append( - f"{str(input_key).upper()}=$INPUT_FOLDER/{str(filename)}" - ) - input_script.append(f"export {str(input_key).upper()}") + input_script.append(f"{input_key_upper}=$INPUT_FOLDER/{filename}") + input_script.append(f"export {input_key_upper}") else: input_script.append( - f"{str(input_key).upper()}=$(< \"$json_input\" jq '.{input_key}')" + f"{input_key_upper}=$(< \"$json_input\" jq '.{input_key}')" ) - input_script.append(f"export {str(input_key).upper()}") + input_script.append(f"export {input_key_upper}") input_script.extend( [ diff --git a/packages/service-integration/src/service_integration/commands/test.py b/packages/service-integration/src/service_integration/commands/test.py index b08c5a85c03..3bf25551dc2 100644 --- a/packages/service-integration/src/service_integration/commands/test.py +++ b/packages/service-integration/src/service_integration/commands/test.py @@ -14,7 +14,8 @@ def main( """Runs tests against service directory""" if not service_dir.exists(): - raise typer.BadParameter("Invalid path to service directory") + msg = "Invalid path to service directory" + raise typer.BadParameter(msg) rich.print(f"Testing '{service_dir.resolve()}' ...") error_code = pytest_runner.main(service_dir=service_dir, extra_args=[]) diff --git a/packages/service-integration/src/service_integration/errors.py b/packages/service-integration/src/service_integration/errors.py index 33ebe3eebfc..e9a857edc1c 100644 --- a/packages/service-integration/src/service_integration/errors.py +++ b/packages/service-integration/src/service_integration/errors.py @@ -5,5 +5,5 @@ class ServiceIntegrationError(PydanticErrorMixin, RuntimeError): pass -class ConfigNotFound(ServiceIntegrationError): +class ConfigNotFoundError(ServiceIntegrationError): msg_template = "could not find any osparc config under {basedir}" diff --git a/packages/service-integration/src/service_integration/oci_image_spec.py b/packages/service-integration/src/service_integration/oci_image_spec.py index ba1cf3c8b77..e07a5e4cafc 100644 --- a/packages/service-integration/src/service_integration/oci_image_spec.py +++ b/packages/service-integration/src/service_integration/oci_image_spec.py @@ -37,6 +37,10 @@ } +def _underscore_as_dot(field_name: str): + return field_name.replace("_", ".") + + class OciImageSpecAnnotations(BaseModel): # TODO: review and polish constraints @@ -98,7 +102,7 @@ class OciImageSpecAnnotations(BaseModel): ) class Config: - alias_generator = lambda field_name: field_name.replace("_", ".") + alias_generator = _underscore_as_dot allow_population_by_field_name = True extra = Extra.forbid @@ -153,5 +157,4 @@ def to_oci_data(self) -> dict[str, Any]: set(self.__fields__.keys()) ) # nosec - oci_data = {_TO_OCI[key]: value for key, value in convertable_data.items()} - return oci_data + return {_TO_OCI[key]: value for key, value in convertable_data.items()} diff --git a/packages/service-integration/src/service_integration/osparc_config.py b/packages/service-integration/src/service_integration/osparc_config.py index 514b9d26194..17c5f1d181f 100644 --- a/packages/service-integration/src/service_integration/osparc_config.py +++ b/packages/service-integration/src/service_integration/osparc_config.py @@ -8,13 +8,13 @@ - config should provide enough information about that context to allow - build an image - run an container - on a single command call. + on a single command call. - """ import logging from pathlib import Path -from typing import Any, ClassVar, Literal, NamedTuple +from typing import Any, Literal from models_library.callbacks_mapping import CallbacksMapping from models_library.service_settings_labels import ( @@ -43,13 +43,12 @@ from pydantic.main import BaseModel from .compose_spec_model import ComposeSpecification -from .errors import ConfigNotFound from .settings import AppSettings from .yaml_utils import yaml_safe_load _logger = logging.getLogger(__name__) -CONFIG_FOLDER_NAME = ".osparc" +OSPARC_CONFIG_DIRNAME = ".osparc" SERVICE_KEY_FORMATS = { @@ -58,20 +57,14 @@ } -## MODELS --------------------------------------------------------------------------------- -# -# Project config -> stored in repo's basedir/.osparc -# - - -class DockerComposeOverwriteCfg(ComposeSpecification): - """picks up configurations used to overwrite the docker-compuse output""" +class DockerComposeOverwriteConfig(ComposeSpecification): + """Content of docker-compose.overwrite.yml configuration file""" @classmethod def create_default( cls, service_name: str | None = None - ) -> "DockerComposeOverwriteCfg": - model: "DockerComposeOverwriteCfg" = cls.parse_obj( + ) -> "DockerComposeOverwriteConfig": + model: "DockerComposeOverwriteConfig" = cls.parse_obj( { "services": { service_name: { @@ -85,16 +78,17 @@ def create_default( return model @classmethod - def from_yaml(cls, path: Path) -> "DockerComposeOverwriteCfg": + def from_yaml(cls, path: Path) -> "DockerComposeOverwriteConfig": with path.open() as fh: data = yaml_safe_load(fh) - model: "DockerComposeOverwriteCfg" = cls.parse_obj(data) + model: "DockerComposeOverwriteConfig" = cls.parse_obj(data) return model -class MetaConfig(ServiceDockerData): - """Details about general info and I/O configuration of the service +class MetadataConfig(ServiceDockerData): + """Content of metadata.yml configuration file + Details about general info and I/O configuration of the service Necessary for both image- and runtime-spec """ @@ -109,18 +103,18 @@ def check_contact_in_authors(cls, v, values): return v @classmethod - def from_yaml(cls, path: Path) -> "MetaConfig": + def from_yaml(cls, path: Path) -> "MetadataConfig": with path.open() as fh: data = yaml_safe_load(fh) - model: "MetaConfig" = cls.parse_obj(data) + model: "MetadataConfig" = cls.parse_obj(data) return model @classmethod - def from_labels_annotations(cls, labels: dict[str, str]) -> "MetaConfig": + def from_labels_annotations(cls, labels: dict[str, str]) -> "MetadataConfig": data = from_labels( labels, prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False ) - model: "MetaConfig" = cls.parse_obj(data) + model: "MetadataConfig" = cls.parse_obj(data) return model def to_labels_annotations(self) -> dict[str, str]: @@ -194,7 +188,7 @@ class Config: allow_population_by_field_name = True -def _get_alias_generator(field_name: str) -> str: +def _underscore_as_minus(field_name: str) -> str: return field_name.replace("_", "-") @@ -240,7 +234,7 @@ def ensure_compatibility(cls, v): return v class Config: - alias_generator = _get_alias_generator + alias_generator = _underscore_as_minus allow_population_by_field_name = True extra = Extra.forbid @@ -261,55 +255,3 @@ def to_labels_annotations(self) -> dict[str, str]: prefix_key=OSPARC_LABEL_PREFIXES[1], ) return labels - - -## FILES ----------------------------------------------------------- - - -class ConfigFileDescriptor(NamedTuple): - glob_pattern: str - required: bool = True - - -class ConfigFilesStructure: - """ - Defines config file structure and how they - map to the models - """ - - FILES_GLOBS: ClassVar[dict] = { - DockerComposeOverwriteCfg.__name__: ConfigFileDescriptor( - glob_pattern="docker-compose.overwrite.y*ml", required=False - ), - MetaConfig.__name__: ConfigFileDescriptor(glob_pattern="metadata.y*ml"), - RuntimeConfig.__name__: ConfigFileDescriptor(glob_pattern="runtime.y*ml"), - } - - @staticmethod - def config_file_path(scope: Literal["user", "project"]) -> Path: - basedir = Path.cwd() # assumes project is in CWD - if scope == "user": - basedir = Path.home() - return basedir / ".osparc" / "service-integration.json" - - def search(self, start_dir: Path) -> dict[str, Path]: - """Tries to match of any of file layouts - and returns associated config files - """ - found = { - configtype: list(start_dir.rglob(pattern)) - for configtype, (pattern, required) in self.FILES_GLOBS.items() - if required - } - - if not found: - raise ConfigNotFound(basedir=start_dir) - - raise NotImplementedError("TODO") - - # TODO: - # scenarios: - # .osparc/meta, [runtime] - # .osparc/{service-name}/meta, [runtime] - - # metadata is required, runtime is optional? diff --git a/packages/service-integration/src/service_integration/osparc_image_specs.py b/packages/service-integration/src/service_integration/osparc_image_specs.py index ec7747ec9f5..df97e7c18b1 100644 --- a/packages/service-integration/src/service_integration/osparc_image_specs.py +++ b/packages/service-integration/src/service_integration/osparc_image_specs.py @@ -9,17 +9,17 @@ Service, ) -from .osparc_config import DockerComposeOverwriteCfg, MetaConfig, RuntimeConfig +from .osparc_config import DockerComposeOverwriteConfig, MetadataConfig, RuntimeConfig from .settings import AppSettings def create_image_spec( settings: AppSettings, - meta_cfg: MetaConfig, - docker_compose_overwrite_cfg: DockerComposeOverwriteCfg, + meta_cfg: MetadataConfig, + docker_compose_overwrite_cfg: DockerComposeOverwriteConfig, runtime_cfg: RuntimeConfig | None = None, *, - extra_labels: dict[str, str] = None, + extra_labels: dict[str, str] | None = None, **_context ) -> ComposeSpecification: """Creates the image-spec provided the osparc-config and a given context (e.g. development) @@ -46,10 +46,9 @@ def create_image_spec( ) build_spec = BuildItem(**overwrite_options) - compose_spec = ComposeSpecification( + return ComposeSpecification( version=settings.COMPOSE_VERSION, services={ service_name: Service(image=meta_cfg.image_name(settings), build=build_spec) }, ) - return compose_spec diff --git a/packages/service-integration/src/service_integration/osparc_runtime_specs.py b/packages/service-integration/src/service_integration/osparc_runtime_specs.py index a9ffc331f5f..56e33db0d79 100644 --- a/packages/service-integration/src/service_integration/osparc_runtime_specs.py +++ b/packages/service-integration/src/service_integration/osparc_runtime_specs.py @@ -5,6 +5,5 @@ # # -raise NotImplementedError( - "SEE prototype in packages/service-integration/tests/test_osparc_runtime_specs.py" -) +msg = "SEE prototype in packages/service-integration/tests/test_osparc_runtime_specs.py" +raise NotImplementedError(msg) diff --git a/packages/service-integration/src/service_integration/pytest_plugin/docker_integration.py b/packages/service-integration/src/service_integration/pytest_plugin/docker_integration.py index 854ba67782b..6b6f1ec19d7 100644 --- a/packages/service-integration/src/service_integration/pytest_plugin/docker_integration.py +++ b/packages/service-integration/src/service_integration/pytest_plugin/docker_integration.py @@ -8,10 +8,10 @@ import shutil import urllib.error import urllib.request +from collections.abc import Iterator from contextlib import suppress from pathlib import Path from pprint import pformat -from typing import Iterator import docker import jsonschema @@ -119,11 +119,10 @@ def host_folders(temporary_path: Path) -> dict: @pytest.fixture def container_variables() -> dict: # of type INPUT_FOLDER=/home/scu/data/input - env = { + return { f"{str(folder).upper()}_FOLDER": (_CONTAINER_FOLDER / folder).as_posix() for folder in _FOLDER_NAMES } - return env @pytest.fixture @@ -224,7 +223,7 @@ def assert_container_runs( list_of_files = [ x.name for x in validation_folders[folder].iterdir() - if not ".gitkeep" in x.name + if ".gitkeep" not in x.name ] for file_name in list_of_files: assert Path( @@ -244,14 +243,12 @@ def assert_container_runs( continue # test if the generated files are the ones expected list_of_files = [ - x.name for x in host_folders[folder].iterdir() if not ".gitkeep" in x.name + x.name for x in host_folders[folder].iterdir() if ".gitkeep" not in x.name ] for file_name in list_of_files: assert Path( validation_folders[folder] / file_name - ).exists(), "{} is not present in {}".format( - file_name, validation_folders[folder] - ) + ).exists(), f"{file_name} is not present in {validation_folders[folder]}" _, _, errors = filecmp.cmpfiles( host_folders[folder], validation_folders[folder], @@ -274,7 +271,7 @@ def assert_container_runs( for key, value in io_simcore_labels["outputs"].items(): assert "type" in value # rationale: files are on their own and other types are in inputs.json - if not "data:" in value["type"]: + if "data:" not in value["type"]: # check that keys are available assert key in output_cfg else: diff --git a/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py b/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py index dc1e57fbee4..47969490661 100644 --- a/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py +++ b/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py @@ -5,6 +5,8 @@ import pytest +from ..osparc_config import OSPARC_CONFIG_DIRNAME + @pytest.fixture(scope="session") def project_slug_dir(request: pytest.FixtureRequest) -> Path: @@ -15,7 +17,7 @@ def project_slug_dir(request: pytest.FixtureRequest) -> Path: assert isinstance(root_dir, Path) assert root_dir.exists() - assert any(root_dir.glob(".osparc")) + assert any(root_dir.glob(OSPARC_CONFIG_DIRNAME)) return root_dir diff --git a/packages/service-integration/src/service_integration/pytest_plugin/validation_data.py b/packages/service-integration/src/service_integration/pytest_plugin/validation_data.py index 4128b8004bf..e5cc87da4a2 100644 --- a/packages/service-integration/src/service_integration/pytest_plugin/validation_data.py +++ b/packages/service-integration/src/service_integration/pytest_plugin/validation_data.py @@ -3,8 +3,8 @@ # pylint: disable=unused-variable import json +from collections.abc import Iterator from pathlib import Path -from typing import Iterator import pytest import yaml @@ -63,7 +63,7 @@ def assert_validation_data_follows_definition( assert "type" in value # rationale: files are on their own and other types are in inputs.json - if not "data:" in value["type"]: + if "data:" not in value["type"]: # check that keys are available assert key in validation_cfg, f"missing {key} in validation config file" else: @@ -99,7 +99,7 @@ def assert_validation_data_follows_definition( "boolean": bool, "string": str, } - if not "data:" in label_cfg[key]["type"]: + if "data:" not in label_cfg[key]["type"]: # check the type is correct expected_type = label2types[label_cfg[key]["type"]] assert isinstance( diff --git a/packages/service-integration/src/service_integration/settings.py b/packages/service-integration/src/service_integration/settings.py index e93c33dbc0e..70c971c8db9 100644 --- a/packages/service-integration/src/service_integration/settings.py +++ b/packages/service-integration/src/service_integration/settings.py @@ -1,12 +1,10 @@ -from typing import Optional - from pydantic import BaseModel, BaseSettings, Field, SecretStr class Registry(BaseModel): url_or_prefix: str - user: Optional[str] = None - password: Optional[SecretStr] = None + user: str | None = None + password: SecretStr | None = None # NOTE: image names w/o a prefix default in dockerhub registry diff --git a/packages/service-integration/src/service_integration/versioning.py b/packages/service-integration/src/service_integration/versioning.py index f990e4633d8..3ed56868e50 100644 --- a/packages/service-integration/src/service_integration/versioning.py +++ b/packages/service-integration/src/service_integration/versioning.py @@ -1,6 +1,7 @@ import re from datetime import datetime -from typing import Pattern +from re import Pattern +from typing import Any, ClassVar from models_library.basic_regex import SEMANTIC_VERSION_RE_W_CAPTURE_GROUPS from packaging.version import Version @@ -20,7 +21,8 @@ def bump_version_string(current_version: str, bump: str) -> str: # CAN ONLY bump releases not pre/post/dev releases if version.is_devrelease or version.is_postrelease or version.is_prerelease: - raise NotImplementedError("Can only bump released versions") + msg = "Can only bump released versions" + raise NotImplementedError(msg) major, minor, patch = version.major, version.minor, version.micro if bump == "major": @@ -32,7 +34,6 @@ def bump_version_string(current_version: str, bump: str) -> str: return new_version -# TODO: from https://github.com/ITISFoundation/osparc-simcore/issues/2409 # ### versioning # a single version number does not suffice. Instead we should have a set of versions that describes "what is inside the container" # - service version (following semantic versioning): for the published service @@ -40,6 +41,7 @@ def bump_version_string(current_version: str, bump: str) -> str: # - executable name: the public name of the wrapped program (e.g. matlab) # - executable version: the version of the program (e.g. matlab 2020b) # - further libraries version dump (e.g. requirements.txt, etc) +# SEE from https://github.com/ITISFoundation/osparc-simcore/issues/2409 class ExecutableVersionInfo(BaseModel): @@ -51,7 +53,7 @@ class ExecutableVersionInfo(BaseModel): released: datetime class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "display_name": "SEMCAD X", "display_version": "Matterhorn Student Edition 1", @@ -71,7 +73,7 @@ class ServiceVersionInfo(BaseModel): released: datetime = Field(..., description="Publication/release date") class Config: - schema_extra = { + schema_extra: ClassVar[dict[str, Any]] = { "example": { "version": "1.0.0", # e.g. first time released as an osparc "integration_version": "2.1.0", diff --git a/packages/service-integration/tests/conftest.py b/packages/service-integration/tests/conftest.py index 2c210da61e6..07e4652b2ea 100644 --- a/packages/service-integration/tests/conftest.py +++ b/packages/service-integration/tests/conftest.py @@ -4,8 +4,8 @@ import shutil import sys +from collections.abc import Callable from pathlib import Path -from typing import Callable import pytest import service_integration @@ -18,7 +18,9 @@ "pytest_simcore.pytest_global_environs", ] -CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +_CURRENT_DIR = ( + Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +) @pytest.fixture(scope="session") @@ -30,24 +32,27 @@ def package_dir() -> Path: @pytest.fixture(scope="session") def tests_data_dir() -> Path: - pdir = CURRENT_DIR / "data" + pdir = _CURRENT_DIR / "data" assert pdir.exists() return pdir @pytest.fixture -def project_file_path(tests_data_dir, tmp_path) -> Path: +def docker_compose_overwrite_path(tests_data_dir, tmp_path) -> Path: + name = "docker-compose.overwrite.yml" dst = shutil.copy( - src=tests_data_dir / "docker-compose.overwrite.yml", - dst=tmp_path / "docker-compose.overwrite.yml", + src=tests_data_dir / name, + dst=tmp_path / name, ) return Path(dst) @pytest.fixture def metadata_file_path(tests_data_dir, tmp_path) -> Path: + name = "metadata.yml" dst = shutil.copy( - src=tests_data_dir / "metadata.yml", dst=tmp_path / "metadata.yml" + src=tests_data_dir / name, + dst=tmp_path / name, ) return Path(dst) diff --git a/packages/service-integration/tests/data/metadata.yml b/packages/service-integration/tests/data/metadata.yml index 46c3ee99a5e..0b9ffc61cd8 100644 --- a/packages/service-integration/tests/data/metadata.yml +++ b/packages/service-integration/tests/data/metadata.yml @@ -1,9 +1,11 @@ -name: oSparc Python Runner +name: Sim4Life Python Runner key: simcore/services/dynamic/osparc-python-runner type: computational integration-version: 1.0.0 version: 1.1.0 -description: oSparc Python Runner +version_display: "Sim4Life Release V7.2" +release_date: "2024-05-31T13:45:30" +description: Python Runner with Sim4Life contact: sylvain@foo.com authors: - name: Mr X diff --git a/packages/service-integration/tests/test__usecase_jupytermath.py b/packages/service-integration/tests/test__usecase_jupytermath.py index b1fb1cceb4f..e49f9b0512a 100644 --- a/packages/service-integration/tests/test__usecase_jupytermath.py +++ b/packages/service-integration/tests/test__usecase_jupytermath.py @@ -7,8 +7,9 @@ import os import shutil import subprocess +from collections.abc import Callable, Iterable from pathlib import Path -from typing import Any, Callable, Iterable +from typing import Any import pytest import yaml @@ -118,10 +119,9 @@ def compose_spec_reference(tests_data_dir: Path) -> dict[str, Any]: Digest: sha256:279a297b49f1fddb26289d205d4ba5acca1bb8e7bedadcfce00f821873935c03 Status: Downloaded newer image for itisfoundation/ci-service-integration-library:v1.0.1-dev-25 """ - compose_spec = yaml.safe_load( + return yaml.safe_load( (tests_data_dir / "docker-compose_jupyter-math_ad51f53.yml").read_text() ) - return compose_spec def test_ooil_compose_wo_arguments( diff --git a/packages/service-integration/tests/test_cli.py b/packages/service-integration/tests/test_cli.py index aa2844823c3..4eee418ec14 100644 --- a/packages/service-integration/tests/test_cli.py +++ b/packages/service-integration/tests/test_cli.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable from service_integration import __version__ diff --git a/packages/service-integration/tests/test_command_compose.py b/packages/service-integration/tests/test_command_compose.py index 562ae7c3708..371d8a9dbdc 100644 --- a/packages/service-integration/tests/test_command_compose.py +++ b/packages/service-integration/tests/test_command_compose.py @@ -3,31 +3,19 @@ # pylint: disable=unused-variable import os +from collections.abc import Callable from pathlib import Path -from typing import Callable -import pytest import yaml - - -@pytest.fixture -def compose_file_path(metadata_file_path: Path) -> Path: - # TODO: should pass with non-existing docker-compose-meta.yml file - compose_file_path: Path = metadata_file_path.parent / "docker-compose-meta.yml" - assert not compose_file_path.exists() - - # minimal - compose_file_path.write_text( - yaml.dump({"services": {"osparc-python-runner": {"build": {"labels": {}}}}}) - ) - return compose_file_path +from service_integration.compose_spec_model import ComposeSpecification +from service_integration.osparc_config import MetadataConfig def test_make_docker_compose_meta( run_program_with_args: Callable, - project_file_path: Path, + docker_compose_overwrite_path: Path, metadata_file_path: Path, - compose_file_path: Path, + tmp_path: Path, ): """ docker-compose-build.yml: $(metatada) @@ -35,25 +23,33 @@ def test_make_docker_compose_meta( simcore-service-integrator compose --metadata $< --to-spec-file $@ """ + target_compose_specs = tmp_path / "docker-compose.yml" + metadata_cfg = MetadataConfig.from_yaml(metadata_file_path) + result = run_program_with_args( "compose", "--metadata", str(metadata_file_path), "--to-spec-file", - compose_file_path, + target_compose_specs, ) assert result.exit_code == os.EX_OK, result.output - assert compose_file_path.exists() + # produces a compose spec + assert target_compose_specs.exists() - compose_cfg = yaml.safe_load(compose_file_path.read_text()) - metadata_cfg = yaml.safe_load(metadata_file_path.read_text()) + # valid compose specs + compose_cfg = ComposeSpecification.parse_obj( + yaml.safe_load(target_compose_specs.read_text()) + ) + assert compose_cfg.services - # TODO: compare labels vs metadata - service_name = metadata_cfg["key"].split("/")[-1] - compose_labels = compose_cfg["services"][service_name]["build"]["labels"] + # compose labels vs metadata fild + compose_labels = compose_cfg.services[metadata_cfg.service_name()].build.labels assert compose_labels - # schema of expected + assert isinstance(compose_labels.__root__, dict) - # deserialize content and should fit metadata_cfg + assert ( + MetadataConfig.from_labels_annotations(compose_labels.__root__) == metadata_cfg + ) diff --git a/packages/service-integration/tests/test_command_config.py b/packages/service-integration/tests/test_command_config.py index ea2b984aafe..08967ba63e6 100644 --- a/packages/service-integration/tests/test_command_config.py +++ b/packages/service-integration/tests/test_command_config.py @@ -3,11 +3,12 @@ # pylint: disable=unused-variable import os import shutil +from collections.abc import Callable from pathlib import Path -from typing import Callable import pytest import yaml +from service_integration.osparc_config import OSPARC_CONFIG_DIRNAME @pytest.fixture @@ -21,7 +22,7 @@ def tmp_compose_spec(tests_data_dir: Path, tmp_path: Path): def test_create_new_osparc_config( run_program_with_args: Callable, tmp_compose_spec: Path ): - osparc_dir = tmp_compose_spec.parent / ".osparc" + osparc_dir = tmp_compose_spec.parent / OSPARC_CONFIG_DIRNAME assert not osparc_dir.exists() result = run_program_with_args( diff --git a/packages/service-integration/tests/test_command_metadata.py b/packages/service-integration/tests/test_command_metadata.py index 97376a5e633..24073dcbc42 100644 --- a/packages/service-integration/tests/test_command_metadata.py +++ b/packages/service-integration/tests/test_command_metadata.py @@ -3,8 +3,8 @@ # pylint: disable=unused-variable import os +from collections.abc import Callable from pathlib import Path -from typing import Callable import pytest import yaml diff --git a/packages/service-integration/tests/test_labels_annotations.py b/packages/service-integration/tests/test_labels_annotations.py index 708e120e973..f92b9e75c0e 100644 --- a/packages/service-integration/tests/test_labels_annotations.py +++ b/packages/service-integration/tests/test_labels_annotations.py @@ -3,7 +3,6 @@ # pylint: disable=unused-variable from pathlib import Path -from pprint import pprint from typing import Any import pytest @@ -20,14 +19,13 @@ def metadata_config(tests_data_dir: Path): return config -@pytest.mark.parametrize("trim_key_head", (True, False)) +@pytest.mark.parametrize("trim_key_head", [True, False]) def test_to_and_from_labels(metadata_config: dict[str, Any], trim_key_head: bool): metadata_labels = to_labels( metadata_config, prefix_key="swiss.itisfoundation", trim_key_head=trim_key_head ) print(f"\n{trim_key_head=:*^100}") - pprint(metadata_labels) assert all(key.startswith("swiss.itisfoundation.") for key in metadata_labels) diff --git a/packages/service-integration/tests/test_oci_image_spec.py b/packages/service-integration/tests/test_oci_image_spec.py index 4207c005199..ef2bd8b47d9 100644 --- a/packages/service-integration/tests/test_oci_image_spec.py +++ b/packages/service-integration/tests/test_oci_image_spec.py @@ -8,7 +8,7 @@ LabelSchemaAnnotations, OciImageSpecAnnotations, ) -from service_integration.osparc_config import MetaConfig +from service_integration.osparc_config import MetadataConfig def test_label_schema_to_oci_conversion(monkeypatch): @@ -27,10 +27,10 @@ def test_create_annotations_from_metadata(tests_data_dir: Path): # recover from docker labels # - meta_cfg = MetaConfig.from_yaml(tests_data_dir / "metadata.yml") + meta_cfg = MetadataConfig.from_yaml(tests_data_dir / "metadata.yml") # map io_spec to OCI image-spec - oic_image_spec = OciImageSpecAnnotations( + OciImageSpecAnnotations( authors=", ".join([f"{a.name} ({a.email})" for a in meta_cfg.authors]) ) diff --git a/packages/service-integration/tests/test_osparc_config.py b/packages/service-integration/tests/test_osparc_config.py index cddc93d9e9b..e993bc25392 100644 --- a/packages/service-integration/tests/test_osparc_config.py +++ b/packages/service-integration/tests/test_osparc_config.py @@ -10,7 +10,11 @@ import pytest import yaml from models_library.service_settings_labels import SimcoreServiceSettingLabelEntry -from service_integration.osparc_config import MetaConfig, RuntimeConfig, SettingsItem +from service_integration.osparc_config import ( + MetadataConfig, + RuntimeConfig, + SettingsItem, +) @pytest.fixture @@ -44,7 +48,7 @@ def labels(tests_data_dir: Path, labels_fixture_name: str) -> dict[str, str]: def test_load_from_labels( labels: dict[str, str], labels_fixture_name: str, tmp_path: Path ): - meta_cfg = MetaConfig.from_labels_annotations(labels) + meta_cfg = MetadataConfig.from_labels_annotations(labels) runtime_cfg = RuntimeConfig.from_labels_annotations(labels) assert runtime_cfg.callbacks_mapping is not None @@ -56,7 +60,7 @@ def test_load_from_labels( config_path = ( tmp_path / f"{model.__class__.__name__.lower()}-{labels_fixture_name}.yml" ) - with open(config_path, "wt") as fh: + with open(config_path, "w") as fh: data = json.loads( model.json(exclude_unset=True, by_alias=True, exclude_none=True) ) diff --git a/packages/service-integration/tests/test_osparc_image_specs.py b/packages/service-integration/tests/test_osparc_image_specs.py index b3ee3ada466..f5777ada3f2 100644 --- a/packages/service-integration/tests/test_osparc_image_specs.py +++ b/packages/service-integration/tests/test_osparc_image_specs.py @@ -9,8 +9,8 @@ from pydantic import BaseModel from service_integration.compose_spec_model import BuildItem, Service from service_integration.osparc_config import ( - DockerComposeOverwriteCfg, - MetaConfig, + DockerComposeOverwriteConfig, + MetadataConfig, RuntimeConfig, ) from service_integration.osparc_image_specs import create_image_spec @@ -27,10 +27,10 @@ def test_create_image_spec_impl(tests_data_dir: Path, settings: AppSettings): # image-spec for devel, prod, ... # load & parse osparc configs - docker_compose_overwrite_cfg = DockerComposeOverwriteCfg.from_yaml( + docker_compose_overwrite_cfg = DockerComposeOverwriteConfig.from_yaml( tests_data_dir / "docker-compose.overwrite.yml" ) - meta_cfg = MetaConfig.from_yaml(tests_data_dir / "metadata-dynamic.yml") + meta_cfg = MetadataConfig.from_yaml(tests_data_dir / "metadata-dynamic.yml") runtime_cfg = RuntimeConfig.from_yaml(tests_data_dir / "runtime.yml") assert runtime_cfg.callbacks_mapping is not None diff --git a/packages/service-integration/tests/test_osparc_runtime_specs.py b/packages/service-integration/tests/test_osparc_runtime_specs.py index f7cd59f8ec7..74d63e15e5b 100644 --- a/packages/service-integration/tests/test_osparc_runtime_specs.py +++ b/packages/service-integration/tests/test_osparc_runtime_specs.py @@ -85,7 +85,7 @@ def test_create_runtime_spec_impl(tests_data_dir: Path): data["deploy"] = {"placement": {"constraints": item.value}} else: - assert False, item + raise AssertionError(item) print(Service(**data).json(exclude_unset=True, indent=2)) diff --git a/packages/service-integration/tests/test_versioning.py b/packages/service-integration/tests/test_versioning.py index a4172e62461..01c36e49082 100644 --- a/packages/service-integration/tests/test_versioning.py +++ b/packages/service-integration/tests/test_versioning.py @@ -20,13 +20,11 @@ def test_pep404_compare_versions(): assert Version("0.6a9dev") < Version("0.6a9") # same release but one is pre-release - assert ( - Version("2.1-rc2").release == Version("2.1").release - and Version("2.1-rc2").is_prerelease - ) + assert Version("2.1-rc2").release == Version("2.1").release + assert Version("2.1-rc2").is_prerelease -BUMP_PARAMS = [ +_BUMP_PARAMS = [ # "upgrade,current_version,new_version", ("patch", "1.1.1", "1.1.2"), ("minor", "1.1.1", "1.2.0"), @@ -36,7 +34,7 @@ def test_pep404_compare_versions(): @pytest.mark.parametrize( "bump,current_version,new_version", - BUMP_PARAMS, + _BUMP_PARAMS, ) def test_bump_version_string( bump: str, @@ -48,7 +46,7 @@ def test_bump_version_string( @pytest.mark.parametrize( "model_cls", - (ExecutableVersionInfo, ServiceVersionInfo), + [ExecutableVersionInfo, ServiceVersionInfo], ) def test_version_info_model_examples(model_cls, model_cls_examples): for name, example in model_cls_examples.items(): diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index 7f56eac79ad..db57f42dc97 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -313,8 +313,8 @@ async def test_run_solver_job( example = next( e - for e in ServiceDockerData.Config.schema_extra["examples"][::-1] - if "boot" in e["description"] + for e in ServiceDockerData.Config.schema_extra["examples"] + if "boot-options" in e ) mocked_catalog_service_api.get( diff --git a/services/web/server/VERSION b/services/web/server/VERSION index e373c4adece..f57373a053b 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.40.3 +0.40.4 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 55164b0810f..d660bde7d6f 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.40.3 +current_version = 0.40.4 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 734c8b55102..1d65163f669 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.0.2 info: title: simcore-service-webserver description: Main service with an interface (http-API & websockets) to the web front-end - version: 0.40.3 + version: 0.40.4 servers: - url: '' description: webserver @@ -7667,6 +7667,13 @@ components: title: Inputs type: object description: values of input properties + inputsRequired: + title: Inputsrequired + type: array + items: + pattern: ^[-_a-zA-Z0-9]+$ + type: string + description: Defines inputs that are required in order to run the service inputsUnits: title: Inputsunits type: object @@ -7905,6 +7912,12 @@ components: inputs: title: Inputs type: object + inputsRequired: + title: Inputsrequired + type: array + items: + pattern: ^[-_a-zA-Z0-9]+$ + type: string inputNodes: title: Inputnodes type: array @@ -11441,15 +11454,27 @@ components: pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ type: string description: service version number + versionDisplay: + title: Versiondisplay + type: string + description: A user-friendly or marketing name for the release. This can + be used to reference the release in a more readable and recognizable format, + such as 'Matterhorn Release,' 'Spring Update,' or 'Holiday Edition.' This + name is not used for version comparison but is useful for communication + and documentation purposes. + releaseDate: + title: Releasedate + type: string + description: The date when the specific version of the service was released. + This field helps in tracking the timeline of releases and understanding + the sequence of updates. The date should be formatted in YYYY-MM-DD format + for consistency and easy sorting. + format: date integration-version: title: Integration-Version pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ type: string - description: integration version number - progress_regexp: - title: Progress Regexp - type: string - description: regexp pattern for detecting computational service's progress + description: Defines which version of the integration workflow should use type: allOf: - $ref: '#/components/schemas/ServiceType' @@ -11489,6 +11514,10 @@ components: type: integer description: The number of 'data type inputs' displayed by default in the UI. When None all 'data type inputs' are displayed. + progress_regexp: + title: Progress Regexp + type: string + description: regexp pattern for detecting computational service's progress owner: title: Owner type: string @@ -11496,7 +11525,8 @@ components: description: 'Static metadata for a service injected in the image labels - This is one to one with node-meta-v0.0.1.json' + NOTE: This model serialized in .osparc/metadata.yml and in the labels of the + docker image' example: name: File Picker description: description