diff --git a/.env.template b/.env.template index 11d230c..3533663 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,5 @@ +OMOTES_ID=omotes-rest + RABBITMQ_HOSTNAME=localhost RABBITMQ_PORT=5672 RABBITMQ_USERNAME=omotes diff --git a/dev-requirements.txt b/dev-requirements.txt index 699ff26..fffe3ac 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --constraint=requirements.txt --extra=dev --output-file=dev-requirements.txt pyproject.toml # -aio-pika==9.3.1 +aio-pika==9.4.2 # via # -c requirements.txt # omotes-sdk-python -aiormq==6.7.7 +aiormq==6.8.0 # via # -c requirements.txt # aio-pika @@ -63,6 +63,8 @@ colorama==0.4.6 # omotes-rest (pyproject.toml) coverage[toml]==7.4.4 # via pytest-cov +exceptiongroup==1.2.1 + # via pytest flake8==6.0.0 # via # flake8-docstrings @@ -121,19 +123,10 @@ kombu==5.3.5 # via # -c requirements.txt # celery -mako==1.3.2 - # via - # -c requirements.txt - # pdoc3 -markdown==3.6 - # via - # -c requirements.txt - # pdoc3 markupsafe==2.1.3 # via # -c requirements.txt # jinja2 - # mako # werkzeug marshmallow==3.20.1 # via @@ -168,11 +161,11 @@ mypy-extensions==1.0.0 # black # mypy # typing-inspect -omotes-sdk-protocol==0.0.8 +omotes-sdk-protocol==0.1.1 # via # -c requirements.txt # omotes-sdk-python -omotes-sdk-python==0.0.14 +omotes-sdk-python==2.0.2 # via # -c requirements.txt # omotes-rest (pyproject.toml) @@ -186,16 +179,12 @@ packaging==23.1 # pytest # setuptools-git-versioning # webargs -pamqp==3.2.1 +pamqp==3.3.0 # via # -c requirements.txt # aiormq pathspec==0.12.1 # via black -pdoc3==0.10.0 - # via - # -c requirements.txt - # streamcapture platformdirs==4.2.0 # via black pluggy==1.4.0 @@ -246,7 +235,7 @@ sqlalchemy[mypy]==2.0.28 # via # -c requirements.txt # omotes-rest (pyproject.toml) -streamcapture==1.2.2 +streamcapture==1.2.5 # via # -c requirements.txt # omotes-sdk-python @@ -254,16 +243,27 @@ structlog==23.1.0 # via # -c requirements.txt # omotes-rest (pyproject.toml) +toml==0.10.2 + # via setuptools-git-versioning tomli==2.0.1 - # via black + # via + # black + # build + # coverage + # flake8-pyproject + # mypy + # pyproject-hooks + # pytest types-flask-cors==4.0.0.20240405 # via omotes-rest (pyproject.toml) types-protobuf==4.24.0.20240311 # via omotes-rest (pyproject.toml) -typing-extensions==4.7.1 +typing-extensions==4.11.0 # via # -c requirements.txt + # marshmallow-dataclass # mypy + # omotes-sdk-python # sqlalchemy # typing-inspect typing-inspect==0.9.0 diff --git a/docker-compose.yml b/docker-compose.yml index 38905be..7d8be73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: omotes-rest: # build: . - image: ghcr.io/project-omotes/omotes_rest:0.0.8 + image: ghcr.io/project-omotes/omotes_rest:0.0.9 networks: - omotes - mapeditor-net diff --git a/pyproject.toml b/pyproject.toml index 150407d..2f8c5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "python-dotenv ~= 1.0.0", "structlog ~= 23.1.0", "SQLAlchemy == 2.0.28", - "omotes-sdk-python ~= 0.0.14", + "omotes-sdk-python ~= 2.0.2", ] [project.optional-dependencies] @@ -74,7 +74,7 @@ enabled = true [tool.pytest.ini_options] addopts = """--cov=omotes_rest --cov-report html --cov-report term-missing \ ---cov-fail-under 2""" +--cov-fail-under 40""" [tool.coverage.run] source = ["src"] diff --git a/requirements.txt b/requirements.txt index 93f224a..2d97cda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements.txt pyproject.toml # -aio-pika==9.3.1 +aio-pika==9.4.2 # via omotes-sdk-python -aiormq==6.7.7 +aiormq==6.8.0 # via aio-pika amqp==5.2.0 # via kombu @@ -57,14 +57,9 @@ jinja2==3.1.2 # via flask kombu==5.3.5 # via celery -mako==1.3.2 - # via pdoc3 -markdown==3.6 - # via pdoc3 markupsafe==2.1.3 # via # jinja2 - # mako # werkzeug marshmallow==3.20.1 # via @@ -82,9 +77,9 @@ multidict==6.0.5 # via yarl mypy-extensions==1.0.0 # via typing-inspect -omotes-sdk-protocol==0.0.8 +omotes-sdk-protocol==0.1.1 # via omotes-sdk-python -omotes-sdk-python==0.0.14 +omotes-sdk-python==2.0.2 # via omotes-rest (pyproject.toml) packaging==23.1 # via @@ -92,10 +87,8 @@ packaging==23.1 # gunicorn # marshmallow # webargs -pamqp==3.2.1 +pamqp==3.3.0 # via aiormq -pdoc3==0.10.0 - # via streamcapture prompt-toolkit==3.0.43 # via click-repl protobuf==4.25.3 @@ -110,12 +103,14 @@ six==1.16.0 # via python-dateutil sqlalchemy==2.0.28 # via omotes-rest (pyproject.toml) -streamcapture==1.2.2 +streamcapture==1.2.5 # via omotes-sdk-python structlog==23.1.0 # via omotes-rest (pyproject.toml) -typing-extensions==4.7.1 +typing-extensions==4.11.0 # via + # marshmallow-dataclass + # omotes-sdk-python # sqlalchemy # typing-inspect typing-inspect==0.9.0 diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh index 9d4fb69..1dbaade 100755 --- a/scripts/install_dependencies.sh +++ b/scripts/install_dependencies.sh @@ -1,3 +1,8 @@ -. .venv/bin/activate +#!/bin/bash -pip-sync requirements.txt +if [[ "$OSTYPE" != "win32" && "$OSTYPE" != "msys" ]]; then + echo "Activating .venv first." + . .venv/bin/activate +fi + +pip-sync ./dev-requirements.txt ./requirements.txt diff --git a/scripts/update_dependencies.sh b/scripts/update_dependencies.sh index 0a54063..e285b32 100755 --- a/scripts/update_dependencies.sh +++ b/scripts/update_dependencies.sh @@ -1,3 +1,8 @@ #!/bin/bash -. .venv/bin/activate -pip-compile -o requirements.txt ./pyproject.toml + +if [[ "$OSTYPE" != "win32" && "$OSTYPE" != "msys" ]]; then + . .venv/bin/activate +fi + +pip-compile --output-file=requirements.txt pyproject.toml +pip-compile --extra=dev -c requirements.txt --output-file=dev-requirements.txt pyproject.toml diff --git a/src/omotes_rest/__init__.py b/src/omotes_rest/__init__.py index d3602dd..57a0aa5 100644 --- a/src/omotes_rest/__init__.py +++ b/src/omotes_rest/__init__.py @@ -44,8 +44,10 @@ def create_app(object_name: str) -> Flask: # Register blueprints. from omotes_rest.apis.job import api as job_api + from omotes_rest.apis.workflow import api as workflow_api api.register_blueprint(job_api) + api.register_blueprint(workflow_api) CORS(app, resources={r"/*": {"origins": "*"}}) diff --git a/src/omotes_rest/apis/api_dataclasses.py b/src/omotes_rest/apis/api_dataclasses.py index f84b6a7..e7dfccd 100644 --- a/src/omotes_rest/apis/api_dataclasses.py +++ b/src/omotes_rest/apis/api_dataclasses.py @@ -8,6 +8,18 @@ from marshmallow_dataclass import add_schema, dataclass +@add_schema +@dataclass +class WorkflowResponse: + """Response with available workflows.""" + + Schema: ClassVar[Type[Schema]] = Schema + + workflow_type_name: str + workflow_type_description_name: str + workflow_parameters: dict[str, Any] | None + + class JobRestStatus(Enum): """Possible job status.""" diff --git a/src/omotes_rest/apis/job.py b/src/omotes_rest/apis/job.py index 2a4be73..f00d0eb 100644 --- a/src/omotes_rest/apis/job.py +++ b/src/omotes_rest/apis/job.py @@ -1,4 +1,3 @@ -import base64 import logging import uuid @@ -103,8 +102,6 @@ def get(self, job_id: str) -> JobResultResponse: """Return job result with output ESDL (can be None).""" job_uuid = uuid.UUID(job_id) output_esdl = current_app.rest_if.get_job_output_esdl(job_uuid) - if output_esdl: - output_esdl = base64.b64encode(bytes(output_esdl, "utf-8")).decode("utf-8") return JobResultResponse(job_id=job_uuid, output_esdl=output_esdl) diff --git a/src/omotes_rest/apis/workflow.py b/src/omotes_rest/apis/workflow.py new file mode 100644 index 0000000..0813ba8 --- /dev/null +++ b/src/omotes_rest/apis/workflow.py @@ -0,0 +1,26 @@ +import logging + +from flask_smorest import Blueprint +from flask.views import MethodView +from flask import jsonify, Response + +from omotes_rest.typed_app import current_app + + +logger = logging.getLogger("omotes_rest") + +api = Blueprint( + "Workflow", + "Workflow", + url_prefix="/workflow", + description="Omotes workflows: retrieve the available workflow and their properties", +) + + +@api.route("/") +class WorkflowAPI(MethodView): + """Requests.""" + + def get(self) -> Response: + """Return a summary of all workflows with parameter jsonforms format.""" + return jsonify(current_app.rest_if.get_workflows_jsonforms_format()) diff --git a/src/omotes_rest/config.py b/src/omotes_rest/config.py index 73f0fae..22399dc 100644 --- a/src/omotes_rest/config.py +++ b/src/omotes_rest/config.py @@ -1,7 +1,7 @@ import os -class POSTGRESConfig: +class PostgresConfig: """Retrieve POSTGRES configuration from environment variables.""" host: str diff --git a/src/omotes_rest/main.py b/src/omotes_rest/main.py index 9ff52fd..147a7b7 100644 --- a/src/omotes_rest/main.py +++ b/src/omotes_rest/main.py @@ -12,7 +12,6 @@ from omotes_rest import create_app from omotes_rest.rest_interface import RestInterface from omotes_rest.settings import EnvSettings -from omotes_rest.workflows import WORKFLOW_TYPE_MANAGER from omotes_rest.typed_app import current_app @@ -87,7 +86,7 @@ def post_fork(_: Arbiter, __: SyncWorker) -> None: with app.app_context(): """current_app is only within the app context""" - current_app.rest_if = RestInterface(WORKFLOW_TYPE_MANAGER) + current_app.rest_if = RestInterface() """Interface for this Omotes Rest service.""" current_app.rest_if.start() diff --git a/src/omotes_rest/postgres_interface.py b/src/omotes_rest/postgres_interface.py index 319c462..3984dfe 100644 --- a/src/omotes_rest/postgres_interface.py +++ b/src/omotes_rest/postgres_interface.py @@ -11,7 +11,7 @@ import logging from omotes_rest.apis.api_dataclasses import JobRestStatus, JobInput from omotes_rest.db_models.job_rest import JobRest -from omotes_rest.config import POSTGRESConfig +from omotes_rest.config import PostgresConfig logger = logging.getLogger("omotes_rest") @@ -60,7 +60,7 @@ def session_scope(do_expunge: bool = False) -> Generator[SQLSession, None, None] Session.remove() -def initialize_db(application_name: str, config: POSTGRESConfig) -> Engine: +def initialize_db(application_name: str, config: PostgresConfig) -> Engine: """Initialize the database connection by creating the engine. Also configure the default session maker. @@ -112,12 +112,12 @@ class PostgresInterface: in this interface must set up a Session (scope) separately. """ - db_config: POSTGRESConfig + db_config: PostgresConfig """Configuration on how to connect to the database.""" engine: Engine """Engine for starting connections to the database.""" - def __init__(self, postgres_config: POSTGRESConfig) -> None: + def __init__(self, postgres_config: PostgresConfig) -> None: """Create the PostgreSQL interface.""" self.db_config = postgres_config diff --git a/src/omotes_rest/rest_interface.py b/src/omotes_rest/rest_interface.py index 0c3bc59..22fd2d3 100644 --- a/src/omotes_rest/rest_interface.py +++ b/src/omotes_rest/rest_interface.py @@ -1,6 +1,6 @@ -import base64 import uuid -from datetime import timedelta +from datetime import timedelta, datetime +from typing import Union from omotes_sdk.omotes_interface import OmotesInterface from omotes_sdk.internal.common.config import EnvRabbitMQConfig @@ -10,14 +10,20 @@ JobProgressUpdate, JobStatusUpdate, ) -from omotes_sdk.workflow_type import WorkflowTypeManager +from omotes_sdk.workflow_type import ( + StringParameter, + BooleanParameter, + IntegerParameter, + FloatParameter, + DateTimeParameter, +) import logging from omotes_rest.postgres_interface import PostgresInterface -from omotes_rest.config import POSTGRESConfig +from omotes_rest.config import PostgresConfig from omotes_rest.apis.api_dataclasses import JobInput, JobStatusResponse from omotes_rest.db_models.job_rest import JobRestStatus, JobRest -from omotes_rest.workflows import FRONTEND_NAME_TO_OMOTES_WORKFLOW_NAME +from omotes_rest.settings import EnvSettings logger = logging.getLogger("omotes_rest") @@ -29,17 +35,13 @@ class RestInterface: """Interface to Omotes.""" postgres_if: PostgresInterface """Interface to Omotes rest postgres.""" - workflow_type_manager: WorkflowTypeManager - """Interface to Omotes.""" - def __init__(self, workflow_type_manager: WorkflowTypeManager): - """Create the omotes rest interface. - - :param workflow_type_manager: All available OMOTES workflow types. - """ - self.omotes_if = OmotesInterface(EnvRabbitMQConfig(), workflow_type_manager) - self.postgres_if = PostgresInterface(POSTGRESConfig()) - self.workflow_type_manager = workflow_type_manager + def __init__( + self, + ) -> None: + """Create the omotes rest interface.""" + self.omotes_if = OmotesInterface(EnvRabbitMQConfig(), EnvSettings.omotes_id()) + self.postgres_if = PostgresInterface(PostgresConfig()) def start(self) -> None: """Start the omotes rest interface.""" @@ -108,23 +110,99 @@ def handle_on_job_progress_update(self, job: Job, progress_update: JobProgressUp progress_message=progress_update.message, ) + def get_workflows_jsonforms_format(self) -> list: + """Get the available workflows with jsonforms schema for the non-ESDL parameters. + + :return: dictionary response. + """ + workflows = [] + for _workflow in self.omotes_if.get_workflow_type_manager().get_all_workflows(): + properties = dict() + required_props = [] + if _workflow.workflow_parameters: + for _parameter in _workflow.workflow_parameters: + jsonforms_schema: dict[ + str, Union[str, float, datetime, list[dict[str, str]]] + ] = dict() + if _parameter.title: + jsonforms_schema["title"] = _parameter.title + if _parameter.description: + jsonforms_schema["description"] = _parameter.description + + if isinstance(_parameter, StringParameter): + jsonforms_schema["type"] = "string" + if _parameter.default: + jsonforms_schema["default"] = _parameter.default + if _parameter.enum_options: + one_of_list = [] + for enum_option in _parameter.enum_options: + one_of_list.append( + dict(const=enum_option.key_name, title=enum_option.display_name) + ) + jsonforms_schema["oneOf"] = one_of_list + elif isinstance(_parameter, BooleanParameter): + jsonforms_schema["type"] = "boolean" + if _parameter.default: + jsonforms_schema["default"] = _parameter.default + elif isinstance(_parameter, IntegerParameter): + jsonforms_schema["type"] = "integer" + if _parameter.default: + jsonforms_schema["default"] = _parameter.default + if _parameter.minimum: + jsonforms_schema["minimum"] = _parameter.minimum + if _parameter.maximum: + jsonforms_schema["maximum"] = _parameter.maximum + elif isinstance(_parameter, FloatParameter): + jsonforms_schema["type"] = "number" + if _parameter.default: + jsonforms_schema["default"] = _parameter.default + if _parameter.minimum: + jsonforms_schema["minimum"] = _parameter.minimum + if _parameter.maximum: + jsonforms_schema["maximum"] = _parameter.maximum + elif isinstance(_parameter, DateTimeParameter): + jsonforms_schema["type"] = "string" + jsonforms_schema["format"] = "date-time" + if _parameter.default: + jsonforms_schema["default"] = _parameter.default.isoformat() + else: + raise NotImplementedError( + f"Parameter type {type(_parameter)} not supported" + ) + properties[_parameter.key_name] = jsonforms_schema + required_props.append(_parameter.key_name) + + if properties: + workflows.append( + dict( + id=_workflow.workflow_type_name, + description=_workflow.workflow_type_description_name, + schema=dict(type="object", properties=properties, required=required_props), + ) + ) + else: + workflows.append( + dict( + id=_workflow.workflow_type_name, + description=_workflow.workflow_type_description_name, + ) + ) + return workflows + def submit_job(self, job_input: JobInput) -> JobStatusResponse: """When a job has a progress update. :param job_input: JobInput dataclass with job input. :return: JobStatusResponse. """ - workflow_type = self.workflow_type_manager.get_workflow_by_name( - FRONTEND_NAME_TO_OMOTES_WORKFLOW_NAME[job_input.workflow_type] + workflow_type = self.omotes_if.get_workflow_type_manager().get_workflow_by_name( + job_input.workflow_type ) if not workflow_type: raise RuntimeError(f"Unknown workflow type {job_input.workflow_type}") - esdlstr_bytes = job_input.input_esdl.encode("utf-8") - esdlstr_base64_bytes = base64.b64decode(esdlstr_bytes) - esdl_str = esdlstr_base64_bytes.decode("utf-8") job = self.omotes_if.submit_job( - esdl=esdl_str, + esdl=job_input.input_esdl, params_dict=job_input.input_params_dict, workflow_type=workflow_type, job_timeout=timedelta(seconds=job_input.timeout_after_s), @@ -136,7 +214,7 @@ def submit_job(self, job_input: JobInput) -> JobStatusResponse: self.postgres_if.put_new_job( job_id=job.id, job_input=job_input, - esdl_input=esdl_str, + esdl_input=job_input.input_esdl, ) return JobStatusResponse(job_id=job.id, status=JobRestStatus.REGISTERED) @@ -164,8 +242,8 @@ def cancel_job(self, job_id: uuid.UUID) -> bool: job_in_db = self.get_job(job_id) if job_in_db: - workflow_type = self.workflow_type_manager.get_workflow_by_name( - FRONTEND_NAME_TO_OMOTES_WORKFLOW_NAME[job_in_db.workflow_type] + workflow_type = self.omotes_if.get_workflow_type_manager().get_workflow_by_name( + job_in_db.workflow_type ) if not workflow_type: raise RuntimeError(f"Unknown workflow type {job_in_db.workflow_type}") diff --git a/src/omotes_rest/settings.py b/src/omotes_rest/settings.py index 2ab4f87..a697134 100644 --- a/src/omotes_rest/settings.py +++ b/src/omotes_rest/settings.py @@ -29,6 +29,11 @@ def is_production() -> bool: """Check if production.""" return EnvSettings.env() == "prod" + @staticmethod + def omotes_id() -> str: + """Env var.""" + return os.getenv("OMOTES_ID", "omotes-rest") + class Config(object): """Generic config for all environments.""" diff --git a/src/omotes_rest/workflows.py b/src/omotes_rest/workflows.py deleted file mode 100644 index a47176f..0000000 --- a/src/omotes_rest/workflows.py +++ /dev/null @@ -1,34 +0,0 @@ -from omotes_sdk.workflow_type import WorkflowTypeManager, WorkflowType - -# TODO to be retrieved via de omotes_sdk in the future -WORKFLOW_TYPE_MANAGER = WorkflowTypeManager( - possible_workflows=[ - WorkflowType( - workflow_type_name="grow_optimizer_default", - workflow_type_description_name="Grow Optimizer", - ), - WorkflowType( - workflow_type_name="grow_simulator", workflow_type_description_name="Grow Simulator" - ), - WorkflowType( - workflow_type_name="grow_optimizer_no_heat_losses", - workflow_type_description_name="Grow Optimizer without heat losses", - ), - WorkflowType( - workflow_type_name="grow_optimizer_with_pressure", - workflow_type_description_name="Grow Optimizer with pressure drops", - ), - WorkflowType( - workflow_type_name="simulator", - workflow_type_description_name="High fidelity simulator", - ), - ] -) - -FRONTEND_NAME_TO_OMOTES_WORKFLOW_NAME = { - "Draft Design - Quickscan Validation": "grow_optimizer_no_heat_losses", - "Draft Design - Optimization": "grow_optimizer_default", - "Draft Design - Optimization with Pressure Drops": "grow_optimizer_with_pressure", - "Draft Design - Simulation with Source Merit Order": "grow_simulator", - "Conceptual Design - Simulation": "simulator", -} diff --git a/unit_test/test_main.py b/unit_test/test_main.py index f1d03c9..0e7393e 100644 --- a/unit_test/test_main.py +++ b/unit_test/test_main.py @@ -1,5 +1,5 @@ import unittest -from omotes_rest.config import POSTGRESConfig +from omotes_rest.config import PostgresConfig class MyTest(unittest.TestCase): @@ -7,7 +7,7 @@ def test__construct_postgres_config__no_exception(self) -> None: # Arrange # Act - result = POSTGRESConfig() + result = PostgresConfig() # Assert self.assertIsNotNone(result)