From 4441e7590b56ebb48b1514ed488dfbbc584b52d5 Mon Sep 17 00:00:00 2001 From: Maximilian Jugl Date: Thu, 7 Mar 2024 11:27:56 +0100 Subject: [PATCH 1/3] chore(deps): add `tomli` as explicit dependency, populate pyproject.toml with name and description --- poetry.lock | 2 +- pyproject.toml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5688582..5b848ec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1222,4 +1222,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "86001e87854a6a7dc6824299d474fe3aff66dba603a71b36ee25945505fb6abf" +content-hash = "0f86079ae3068be6edf5844ffaab3c98cb1c9c8bf18764863ec9b7e9d106fbc1" diff --git a/pyproject.toml b/pyproject.toml index ec64404..3718001 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] -name = "python-template-repo" +name = "flame-result-service" version = "0.1.0" -description = "" +description = "Service for handling intermediate files and submitting result files to the FLAME Hub." authors = ["Maximilian Jugl "] readme = "README.md" packages = [{ include = "project" }] @@ -20,6 +20,7 @@ python-multipart = "^0.0.9" click = "^8.1.7" jwcrypto = "^1.5.6" httpx = "^0.26.0" +tomli = "^2.0.1" [tool.poetry.group.dev.dependencies] pytest = "^7.4.3" From dfc1919a0c923678b0cd85a0c15ad30093f9a69f Mon Sep 17 00:00:00 2001 From: Maximilian Jugl Date: Thu, 7 Mar 2024 11:28:26 +0100 Subject: [PATCH 2/3] feat: add README.md and pyproject.toml to Dockerfile so that they can be sourced in container --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2dbafdc..23ea19c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ WORKDIR /app COPY ./config/ ./config/ COPY --from=builder /tmp/requirements.txt ./ +COPY pyproject.toml README.md ./ COPY ./project/ ./project/ RUN pip install -r requirements.txt From 432e346f271366d6b1815f99858b6e6245e8b26a Mon Sep 17 00:00:00 2001 From: Maximilian Jugl Date: Thu, 7 Mar 2024 11:29:20 +0100 Subject: [PATCH 3/3] docs: add docs to endpoints and update README --- README.md | 84 +++++++++++++++++++++++++++++++------- project/routers/scratch.py | 20 ++++++++- project/routers/upload.py | 9 ++++ project/server.py | 37 ++++++++++++++++- 4 files changed, 132 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 37e644b..82ba419 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,69 @@ -# Node Result Service +The FLAME Node Result Service is responsible for handling result files for federated analyses within FLAME. +It uses a local object storage to store intermediate files, as well as to enqueue files for upload to the FLAME Hub. + +# Setup + +You will need access to a MinIO instance and an identification provider that offers a JWKS endpoint for the access +tokens it issues. + +For manual installation, you will need Python 3.10 or higher and [Poetry](https://python-poetry.org/) installed. +Clone the repository and run `poetry install` in the root directory. +Create a copy of `.env.example`, name it `.env` and configure to your needs. +Finally, use the command line tool `flame-result` to start the service. ``` -$ flame-result server --reload +$ git clone https://github.com/PrivateAIM/node-result-service.git +$ cd node-result-service +$ poetry install +$ cp .env.example .env +$ poetry shell +$ flame-result server ``` -## Running tests +Alternatively, if you're using +Docker, [pull a recent image from the GitHub container registry](https://github.com/PrivateAIM/node-result-service/pkgs/container/node-result-service). +Pass in the configuration options using `-e` flags and forward port 8080 from your host to the container. + +``` +$ docker run --rm -p 8080:8080 -e HUB__AUTH_USERNAME=admin \ + -e HUB__AUTH_PASSWORD=super_secret \ + -e MINIO__ENDPOINT=localhost:9000 \ + -e MINIO__ACCESS_KEY=admin \ + -e MINIO__SECRET_KEY=super_secret \ + -e MINIO__BUCKET=flame \ + -e MINIO__USE_SSL=false \ + -e OIDC__CERTS_URL="http://my.idp.org/realms/flame/protocol/openid-connect/certs" \ + ghcr.io/privateaim/node-result-service:sha-c1970cf +``` + +# Configuration + +The following table shows all available configuration options. + +| **Environment variable** | **Description** | **Default** | **Required** | +|----------------------------|----------------------------------------------------------|-----------------------------|:------------:| +| HUB__API_BASE_URL | Base URL for the FLAME Hub API | https://api.privateaim.net | | +| HUB__AUTH_BASE_URL | Base URL for the FLAME Auth API | https://auth.privateaim.net | | +| HUB__AUTH_USERNAME | Username to use for obtaining access tokens | | x | +| HUB__AUTH_PASSWORD | Password to use for obtaining access tokens | | x | +| MINIO__ENDPOINT | MinIO S3 API endpoint (without scheme) | | x | +| MINIO__ACCESS_KEY | Access key for interacting with MinIO S3 API | | x | +| MINIO__SECRET_KEY | Secret key for interacting with MinIO S3 API | | x | +| MINIO__BUCKET | Name of S3 bucket to store result files in | | x | +| MINIO__REGION | Region of S3 bucket to store result files in | us-east-1 | | +| MINIO__USE_SSL | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0 | | +| OIDC__CERTS_URL | URL to OIDC-complaint JWKS endpoint for validating JWTs | | x | +| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with | client_id | | -Integration tests require at least one MinIO instance. -For testing purposes, it is fine to run tests against a single MinIO instance targeting two different buckets. -These buckets must exist before testing. -Configuration is passed in using environment variables. -The names are the same, except they are prepended with `PYTEST__`. -At least, the MinIO endpoint, access key, secret key and bucket name need to be provided. -If unspecified, the region is set to `us-east-1` and HTTPS is disabled. +## Note on running tests + +When running tests, environment variables must be overwritten by prefixing them with `PYTEST__`. +OIDC does not need to be configured, since an OIDC-compatible endpoint will be spawned alongside the tests that are +being run. +A [pre-generated keypair](tests/assets/keypair.pem) is used for this purpose. +This allows all tests to generate valid JWTs as well as the service to validate them. +The keypair is for development purposes only and should not be used in a productive setting. +Therefore `pytest` should be invoked as follows. ``` $ PYTEST__MINIO__ENDPOINT="localhost:9000" \ @@ -23,8 +74,11 @@ $ PYTEST__MINIO__ENDPOINT="localhost:9000" \ PYTEST__HUB__AUTH_PASSWORD="XXXXXXXX" pytest ``` -OIDC does not need to be configured. -Running integration tests will spawn a minimal webserver that provides an OIDC-compliant JWKS endpoint. -A [pre-generated keypair](tests/assets/keypair.pem) is used for this purpose. -This allows all tests to generate valid JWTs as well as the service to validate them. -The keypair is for development purposes only and should not be used in a productive setting. +Some tests need to be run against live infrastructure. +Since a proper test instance is not available yet, these tests are hidden behind a flag and are not explicitly run in +CI. +To run these tests, append `-m live` to the command above. + +# License + +The FLAME Node Result Service is released under the Apache 2.0 license. diff --git a/project/routers/scratch.py b/project/routers/scratch.py index 1eefbb8..4fb7eef 100644 --- a/project/routers/scratch.py +++ b/project/routers/scratch.py @@ -23,6 +23,8 @@ class ScratchUploadResponse(BaseModel): @router.put( "/", response_model=ScratchUploadResponse, + summary="Upload file to local object storage", + operation_id="putIntermediateFile", ) async def upload_to_scratch( client_id: Annotated[str, Depends(get_client_id)], @@ -31,6 +33,12 @@ async def upload_to_scratch( minio: Annotated[Minio, Depends(get_local_minio)], request: Request, ): + """Upload a file to the local S3 instance. + The file is not forwarded to the FLAME hub. + Responds with a 200 on success and a link to the endpoint for fetching the uploaded file. + + This endpoint is to be used for submitting intermediate results of a federated analysis. + """ object_id = str(uuid.uuid4()) minio.put_object( @@ -51,13 +59,23 @@ async def upload_to_scratch( ) -@router.get("/{object_id}") +@router.get( + "/{object_id}", + summary="Get file from local object storage", + operation_id="getIntermediateFile", +) async def read_from_scratch( client_id: Annotated[str, Depends(get_client_id)], object_id: uuid.UUID, settings: Annotated[Settings, Depends(get_settings)], minio: Annotated[Minio, Depends(get_local_minio)], ): + """Get a file from the local S3 instance. + The file must have previously been uploaded using the PUT method of this endpoint. + Responds with a 200 on success and the requested file in the response body. + + This endpoint is to be used for retrieving intermediate results of a federated analysis. + """ try: response = minio.get_object( settings.minio.bucket, f"scratch/{client_id}/{object_id}" diff --git a/project/routers/upload.py b/project/routers/upload.py index b8ec969..f232e77 100644 --- a/project/routers/upload.py +++ b/project/routers/upload.py @@ -64,6 +64,8 @@ async def __bg_upload_to_remote( @router.put( "/", status_code=status.HTTP_204_NO_CONTENT, + summary="Upload file to submit to Hub", + operation_id="putResultFile", ) async def upload_to_remote( client_id: Annotated[str, Depends(get_client_id)], @@ -73,6 +75,13 @@ async def upload_to_remote( local_minio: Annotated[Minio, Depends(get_local_minio)], api_access_token: Annotated[AccessToken, Depends(get_access_token)], ): + """Upload a file to the local S3 instance and send it to FLAME Hub in the background. + The request is successful if the file was uploaded to the local S3 instance. + Responds with a 204 on success. + + This endpoint is to be used for submitting final results of a federated analysis. + + Currently, there is no way of determining the status or progress of the upload to the FLAME Hub.""" object_id = str(uuid.uuid4()) object_name = f"upload/{client_id}/{object_id}" diff --git a/project/server.py b/project/server.py index ad36c28..cb3cdfb 100644 --- a/project/server.py +++ b/project/server.py @@ -2,7 +2,9 @@ import logging.config import os.path from contextlib import asynccontextmanager +from pathlib import Path +import tomli from fastapi import FastAPI from project.routers import upload, scratch @@ -23,11 +25,42 @@ async def lifespan(app: FastAPI): yield -app = FastAPI(lifespan=lifespan) +with open(Path(__file__).parent.parent / "pyproject.toml", mode="rb") as f: + pyproject_data = tomli.load(f) +with open(Path(__file__).parent.parent / "README.md", mode="r") as f: + app_description = f.read() -@app.get("/healthz") +app_version = pyproject_data["tool"]["poetry"]["version"] +app_summary = pyproject_data["tool"]["poetry"]["description"] + +app = FastAPI( + title="FLAME Node Result Service", + summary=app_summary, + version=app_version, + lifespan=lifespan, + description=app_description, + license_info={ + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0.html", + "identifier": "Apache-2.0", + }, + openapi_tags=[ + { + "name": "upload", + "description": "Upload files for submission to FLAME hub", + }, + { + "name": "scratch", + "description": "Upload files to local object storage", + }, + ], +) + + +@app.get("/healthz", summary="Check service readiness", operation_id="getHealth") async def do_healthcheck(): + """Check whether the service is ready to process requests. Responds with a 200 on success.""" return {"status": "ok"}