diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0761eaac13a..464d513d684 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -49,8 +49,9 @@ Add here YOUR checklist/notes to guide and monitor the progress of the case! e.g. -- [ ] Openapi changes? ``make openapi-specs``, ``git commit ...`` and then ``make version-*``) -- [ ] Database migration script? ``cd packages/postgres-database``, ``make setup-commit``, ``sc-pg review -m "my changes"`` +- [ ] ``make version-*`` +- [ ] ``make openapi.json`` +- [ ] ``cd packages/postgres-database``, ``make setup-commit``, ``sc-pg review -m "my changes"`` - [ ] Unit tests for the changes exist - [ ] Runs in the swarm - [ ] Documentation reflects the changes diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index f1b9d7e3a85..c6072089a2e 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -114,6 +114,9 @@ jobs: - 'packages/**' api: - 'api/**' + api-server: + - 'packages/**' + - 'services/api-server/**' autoscaling: - 'packages/**' - 'services/autoscaling/**' @@ -1748,12 +1751,7 @@ jobs: deploy: name: deploy to dockerhub if: github.event_name == 'push' - needs: - [ - unit-tests, - integration-tests, - system-tests - ] + needs: [unit-tests, integration-tests, system-tests] runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/services/api-server/Makefile b/services/api-server/Makefile index 28ce479dcbc..20ccba061a5 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -87,10 +87,11 @@ run-fake-devel: # starts a fake server in a dev-container .PHONY: openapi-specs openapi.json openapi-specs: openapi.json openapi.json: .env - # generating openapi specs file under $< + # generating openapi specs file under $< (NOTE: Skips DEV FEATURES since this OAS is the 'offically released'!) @set -o allexport; \ source .env; \ set +o allexport; \ + export API_SERVER_DEV_FEATURES_ENABLED=0; \ python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ # validates OAS file: $@ diff --git a/services/api-server/VERSION b/services/api-server/VERSION index 1d0ba9ea182..267577d47e4 100644 --- a/services/api-server/VERSION +++ b/services/api-server/VERSION @@ -1 +1 @@ -0.4.0 +0.4.1 diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 862b62a460d..eb1d6709c75 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "osparc.io web API", "description": "osparc-simcore public web API specifications", - "version": "0.4.0", + "version": "0.4.1", "x-logo": { "url": "https://raw.githubusercontent.com/ITISFoundation/osparc-manual/b809d93619512eb60c827b7e769c6145758378d0/_media/osparc-logo.svg", "altText": "osparc-simcore logo" @@ -1000,7 +1000,7 @@ "solvers" ], "summary": "Get Job Output Logfile", - "description": "Special extra output with persistent logs file for the solver run.\n\nNOTE: this is not a log stream but a predefined output that is only\navailable after the job is done.", + "description": "Special extra output with persistent logs file for the solver run.\n\nNOTE: this is not a log stream but a predefined output that is only\navailable after the job is done.\n\nNew in *version 0.4.0*", "operationId": "get_job_output_logfile", "parameters": [ { @@ -1637,7 +1637,14 @@ "title": "Location", "type": "array", "items": { - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] } }, "msg": { diff --git a/services/api-server/setup.cfg b/services/api-server/setup.cfg index d56175b6486..f04cee32aef 100644 --- a/services/api-server/setup.cfg +++ b/services/api-server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.0 +current_version = 0.4.1 commit = True message = services/api-server version: {current_version} → {new_version} tag = False diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 186009bd457..3998102a97f 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -6,9 +6,10 @@ from httpx import HTTPStatusError from pydantic import ValidationError from pydantic.errors import PydanticValueError +from servicelib.error_codes import create_error_code -from ...core.settings import ApplicationSettings -from ...models.schemas.solvers import Solver, SolverKeyId, VersionStr +from ...core.settings import ApplicationSettings, BasicSettings +from ...models.schemas.solvers import Solver, SolverKeyId, SolverPort, VersionStr from ...modules.catalog import CatalogApi from ..dependencies.application import get_reverse_url_mapper, get_settings from ..dependencies.authentication import get_current_user_id @@ -17,7 +18,7 @@ logger = logging.getLogger(__name__) router = APIRouter() - +settings = BasicSettings.create_from_envs() ## SOLVERS ----------------------------------------------------------------------------------------- # @@ -163,3 +164,49 @@ async def get_solver_release( status_code=status.HTTP_404_NOT_FOUND, detail=f"Solver {solver_key}:{version} not found", ) from err + + +@router.get( + "/{solver_key:path}/releases/{version}/ports", + response_model=list[SolverPort], + include_in_schema=settings.API_SERVER_DEV_FEATURES_ENABLED, +) +async def list_solver_ports( + solver_key: SolverKeyId, + version: VersionStr, + user_id: int = Depends(get_current_user_id), + catalog_client: CatalogApi = Depends(get_api_client(CatalogApi)), + app_settings: ApplicationSettings = Depends(get_settings), +): + """Lists inputs and outputs of a given solver + + New in *version 0.5.0* (only with API_SERVER_DEV_FEATURES_ENABLED=1) + """ + try: + + ports = await catalog_client.get_solver_ports( + user_id, + solver_key, + version, + product_name=app_settings.API_SERVER_DEFAULT_PRODUCT_NAME, + ) + return ports + + except ValidationError as err: + error_code = create_error_code(err) + logger.exception( + "Corrupted port data for service %s [%s]", + f"{solver_key}:{version}", + f"{error_code}", + extra={"error_code": error_code}, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Port definition of {solver_key}:{version} seems corrupted [{error_code}]", + ) from err + + except HTTPStatusError as err: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Ports for solver {solver_key}:{version} not found", + ) from err diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 63adc16766a..06085d90358 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -322,6 +322,8 @@ async def get_job_output_logfile( NOTE: this is not a log stream but a predefined output that is only available after the job is done. + + New in *version 0.4.0* """ logs_urls: dict[NodeName, DownloadLink] = await director2_api.get_computation_logs( diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index cee4f46732b..0de7a91e03f 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -108,6 +108,7 @@ def init_app(settings: Optional[ApplicationSettings] = None) -> FastAPI: api_router = create_router(settings) app.include_router(api_router, prefix=f"/{API_VTAG}") + # NOTE: cleanup all OpenAPIs https://github.com/ITISFoundation/osparc-simcore/issues/3487 use_route_names_as_operation_ids(app) config_all_loggers() return app diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index 932c2660553..4ad3d6d3e5f 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -72,10 +72,12 @@ def base_url(self) -> str: # MAIN SETTINGS -------------------------------------------- -class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): +class BasicSettings(BaseCustomSettings, MixinLoggingSettings): - # DOCKER - SC_BOOT_MODE: Optional[BootModeEnum] + # DEVELOPMENT + API_SERVER_DEV_FEATURES_ENABLED: bool = Field( + False, env=["API_SERVER_DEV_FEATURES_ENABLED", "FAKE_API_SERVER_ENABLED"] + ) # LOGGING LOG_LEVEL: LogLevel = Field( @@ -83,6 +85,20 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): env=["API_SERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], ) + # DEBUGGING + API_SERVER_REMOTE_DEBUG_PORT: int = 3000 + + @validator("LOG_LEVEL", pre=True) + @classmethod + def _validate_loglevel(cls, value) -> str: + return cls.validate_log_level(value) + + +class ApplicationSettings(BasicSettings): + + # DOCKER BOOT + SC_BOOT_MODE: Optional[BootModeEnum] + # POSTGRES API_SERVER_POSTGRES: Optional[PostgresSettings] = Field(auto_default_from_env=True) @@ -101,10 +117,6 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # DIAGNOSTICS API_SERVER_TRACING: Optional[TracingSettings] = Field(auto_default_from_env=True) - API_SERVER_DEV_FEATURES_ENABLED: bool = Field( - False, env=["API_SERVER_DEV_FEATURES_ENABLED", "FAKE_API_SERVER_ENABLED"] - ) - API_SERVER_REMOTE_DEBUG_PORT: int = 3000 @cached_property def debug(self) -> bool: @@ -114,8 +126,3 @@ def debug(self) -> bool: BootModeEnum.DEVELOPMENT, BootModeEnum.LOCAL, ] - - @validator("LOG_LEVEL", pre=True) - @classmethod - def _validate_loglevel(cls, value) -> str: - return cls.validate_log_level(value) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index c557dcef7bf..fd0bb9a628a 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -1,7 +1,8 @@ import urllib.parse -from typing import Optional, Union +from typing import Any, Literal, Optional, Union import packaging.version +from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE from models_library.services import COMPUTATIONAL_SERVICE_KEY_RE, ServiceDockerData from packaging.version import LegacyVersion, Version from pydantic import BaseModel, Extra, Field, HttpUrl, constr @@ -104,3 +105,36 @@ def resource_name(self) -> str: @classmethod def compose_resource_name(cls, solver_key, solver_version) -> str: return compose_resource_name("solvers", solver_key, "releases", solver_version) + + +PortKindStr = Literal["input", "output"] + + +class SolverPort(BaseModel): + key: str = Field( + ..., + description="port identifier name", + regex=PUBLIC_VARIABLE_NAME_RE, + title="Key name", + ) + kind: PortKindStr + content_schema: Optional[dict[str, Any]] = Field( + None, + description="jsonschema for the port's value. SEE https://json-schema.org", + ) + + class Config: + extra = Extra.ignore + schema_extra = { + "example": { + "key": "input_2", + "kind": "input", + "content_schema": { + "title": "Sleep interval", + "type": "integer", + "x_unit": "second", + "minimum": 0, + "maximum": 5, + }, + } + } diff --git a/services/api-server/src/simcore_service_api_server/modules/catalog.py b/services/api-server/src/simcore_service_api_server/modules/catalog.py index 03efe210b3e..8e13edf0656 100644 --- a/services/api-server/src/simcore_service_api_server/modules/catalog.py +++ b/services/api-server/src/simcore_service_api_server/modules/catalog.py @@ -6,10 +6,16 @@ from fastapi import FastAPI from models_library.services import ServiceDockerData, ServiceType -from pydantic import EmailStr, Extra, ValidationError +from pydantic import EmailStr, Extra, ValidationError, parse_obj_as from settings_library.catalog import CatalogSettings -from ..models.schemas.solvers import LATEST_VERSION, Solver, SolverKeyId, VersionStr +from ..models.schemas.solvers import ( + LATEST_VERSION, + Solver, + SolverKeyId, + SolverPort, + VersionStr, +) from ..utils.client_base import BaseServiceClientApi, setup_client_instance ## from ..utils.client_decorators import JSON, handle_errors, handle_retry @@ -133,6 +139,25 @@ async def get_solver( return service.to_solver() + async def get_solver_ports( + self, user_id: int, name: SolverKeyId, version: VersionStr, *, product_name: str + ): + + assert version != LATEST_VERSION # nosec + + service_key = urllib.parse.quote_plus(name) + service_version = version + + resp = await self.client.get( + f"/services/{service_key}/{service_version}/ports", + params={"user_id": user_id}, + headers={"x-simcore-products-name": product_name}, + ) + resp.raise_for_status() + + solver_ports = parse_obj_as(list[SolverPort], resp.json()) + return solver_ports + async def list_latest_releases( self, user_id: int, *, product_name: str ) -> list[Solver]: diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 62974eb1ed9..2c67a91a788 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -3,7 +3,10 @@ # pylint: disable=unused-variable -from typing import Iterator +import json +from copy import deepcopy +from pathlib import Path +from typing import Any, Iterator import pytest import respx @@ -13,11 +16,24 @@ from simcore_service_api_server.core.settings import ApplicationSettings +@pytest.fixture(scope="session") +def catalog_service_openapi_specs(osparc_simcore_services_dir: Path) -> dict[str, Any]: + + openapi_path = osparc_simcore_services_dir / "catalog" / "openapi.json" + openapi_specs = json.loads(openapi_path.read_text()) + return openapi_specs + + @pytest.fixture -def mocked_catalog_service_api(app: FastAPI) -> Iterator[MockRouter]: +def mocked_catalog_service_api( + app: FastAPI, catalog_service_openapi_specs: dict[str, Any] +) -> Iterator[MockRouter]: settings: ApplicationSettings = app.state.settings assert settings.API_SERVER_CATALOG + openapi = deepcopy(catalog_service_openapi_specs) + schemas = openapi["components"]["schemas"] + # pylint: disable=not-context-manager with respx.mock( base_url=settings.API_SERVER_CATALOG.base_url, @@ -25,8 +41,11 @@ def mocked_catalog_service_api(app: FastAPI) -> Iterator[MockRouter]: assert_all_mocked=True, ) as respx_mock: + respx_mock.get("/v0/meta").respond(200, json=schemas["Meta"]["example"]) + + # ---- respx_mock.get( - "/services?user_id=1&details=false", name="list_services" + "/v0/services?user_id=1&details=false", name="list_services" ).respond( 200, json=[ @@ -42,4 +61,20 @@ def mocked_catalog_service_api(app: FastAPI) -> Iterator[MockRouter]: ], ) + # ----- + # NOTE: we could use https://python-jsonschema.readthedocs.io/en/stable/ + # + + respx_mock.get( + # NOTE: regex does not work even if tested https://regex101.com/r/drVAGr/1 + # path__regex=r"/v0/services/(?P[\w/%]+)/(?P[\d\.]+)/ports\?user_id=(?P\d+)", + path__startswith="/v0/services/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/2.1.4/ports", + name="list_service_ports", + ).respond( + 200, + json=[ + schemas["ServicePortGet"]["example"], + ], + ) + yield respx_mock diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index 59c789c385b..d35c2c30d34 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -6,6 +6,7 @@ import httpx import pytest import simcore_service_api_server.api.routes.solvers +from pytest_mock import MockFixture from respx import MockRouter from simcore_service_api_server.models.schemas.solvers import Solver from starlette import status @@ -15,7 +16,7 @@ async def test_list_solvers( client: httpx.AsyncClient, mocked_catalog_service_api: MockRouter, - mocker, + mocker: MockFixture, ): warn = mocker.patch.object( simcore_service_api_server.api.routes.solvers.logger, "warning" @@ -60,3 +61,29 @@ async def test_list_solvers( assert f"GET latest {solver.id}" in resp2.json()["errors"][0] # assert Solver(**resp2.json()) == Solver(**resp3.json()) + + +async def test_list_solver_ports( + mocked_catalog_service_api: MockRouter, + client: httpx.AsyncClient, + auth: httpx.BasicAuth, +): + resp = await client.get( + "/v0/solvers/simcore/services/comp/itis/sleeper/releases/2.1.4/ports", + auth=auth, + ) + assert resp.status_code == status.HTTP_200_OK + + assert resp.json() == [ + { + "key": "input_1", + "kind": "input", + "content_schema": { + "title": "Sleep interval", + "type": "integer", + "x_unit": "second", + "minimum": 0, + "maximum": 5, + }, + } + ] diff --git a/services/api-server/tests/unit/test_models_schemas_solvers.py b/services/api-server/tests/unit/test_models_schemas_solvers.py index 6418492ef1e..551d2a1d6d1 100644 --- a/services/api-server/tests/unit/test_models_schemas_solvers.py +++ b/services/api-server/tests/unit/test_models_schemas_solvers.py @@ -6,10 +6,14 @@ from pprint import pformat import pytest -from simcore_service_api_server.models.schemas.solvers import Solver, Version +from simcore_service_api_server.models.schemas.solvers import ( + Solver, + SolverPort, + Version, +) -@pytest.mark.parametrize("model_cls", (Solver,)) +@pytest.mark.parametrize("model_cls", (Solver, SolverPort)) def test_solvers_model_examples(model_cls, model_cls_examples): for name, example in model_cls_examples.items(): print(name, ":", pformat(example)) diff --git a/services/web/server/src/simcore_service_webserver/login/handlers.py b/services/web/server/src/simcore_service_webserver/login/handlers.py index 953b97597b0..6467cfcf894 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers.py @@ -240,7 +240,7 @@ async def register_phone(request: web.Request): ) raise web.HTTPServiceUnavailable( - reason=f"Currently our system cannot register phones ({error_code})", + reason=f"Currently our system cannot register phone numbers ({error_code})", content_type=MIMETYPE_APPLICATION_JSON, ) from e diff --git a/tests/public-api/examples/file_uploads.py b/tests/public-api/examples/file_uploads.py new file mode 100644 index 00000000000..7a80d4057c6 --- /dev/null +++ b/tests/public-api/examples/file_uploads.py @@ -0,0 +1,57 @@ +""" + + $ cd examples + $ make install-ci + $ make .env + +""" +import os +import tempfile +from pathlib import Path + +import osparc +from dotenv import load_dotenv +from osparc.models import File + +load_dotenv() +cfg = osparc.Configuration( + host=os.environ.get("OSPARC_API_URL", "http://127.0.0.1:8006"), + username=os.environ["OSPARC_API_KEY"], + password=os.environ["OSPARC_API_SECRET"], +) +print("Entrypoint", cfg.host) + + +GB = 1024 * 1024 * 1024 # 1GB in bytes + + +def generate_big_sparse_file(filename, size): + with open(filename, "wb") as f: + f.seek(size - 1) + f.write(b"\1") + + +# NOTE: +# +# This script reproduces OverflowError in the client due to ssl comms +# SEE https://github.com/ITISFoundation/osparc-issues/issues/617#issuecomment-1204916094 + +with osparc.ApiClient(cfg) as api_client: + + with tempfile.TemporaryDirectory( + suffix="_public_api__examples__file_uploads" + ) as tmpdir: + local_path = Path(tmpdir) / "large_file.dat" + generate_big_sparse_file(local_path, size=5 * GB) + + assert local_path.exists() + assert local_path.stat().st_size == 5 * GB + + files_api = osparc.FilesApi(api_client) + + uploaded_file: File = files_api.upload_file(f"{local_path}") + print(f"{uploaded_file=}") + + file_in_server = files_api.get_file(uploaded_file.id) + print(f"{file_in_server=}") + assert file_in_server == uploaded_file