Merge pull request #26 from PrivateAIM/24-improve-endpoint-documentation

Improve endpoint documentation

Dockerfile

@@ -12,6 +12,7 @@ WORKDIR /app
 
 COPY ./config/ ./config/
 COPY --from=builder /tmp/requirements.txt ./
+COPY pyproject.toml README.md ./
 COPY ./project/ ./project/
 
 RUN pip install -r requirements.txt

README.md

@@ -1,18 +1,69 @@
-# Node Result Service
+The FLAME Node Result Service is responsible for handling result files for federated analyses within FLAME.
+It uses a local object storage to store intermediate files, as well as to enqueue files for upload to the FLAME Hub.
+
+# Setup
+
+You will need access to a MinIO instance and an identification provider that offers a JWKS endpoint for the access
+tokens it issues.
+
+For manual installation, you will need Python 3.10 or higher and [Poetry](https://python-poetry.org/) installed.
+Clone the repository and run `poetry install` in the root directory.
+Create a copy of `.env.example`, name it `.env` and configure to your needs.
+Finally, use the command line tool `flame-result` to start the service.
 
 ```
-$ flame-result server --reload
+$ git clone https://github.com/PrivateAIM/node-result-service.git
+$ cd node-result-service
+$ poetry install
+$ cp .env.example .env
+$ poetry shell
+$ flame-result server
 ```
 
-## Running tests
+Alternatively, if you're using
+Docker, [pull a recent image from the GitHub container registry](https://github.com/PrivateAIM/node-result-service/pkgs/container/node-result-service).
+Pass in the configuration options using `-e` flags and forward port 8080 from your host to the container.
+
+```
+$ docker run --rm -p 8080:8080 -e HUB__AUTH_USERNAME=admin \
+    -e HUB__AUTH_PASSWORD=super_secret \
+    -e MINIO__ENDPOINT=localhost:9000 \
+    -e MINIO__ACCESS_KEY=admin \
+    -e MINIO__SECRET_KEY=super_secret \
+    -e MINIO__BUCKET=flame \
+    -e MINIO__USE_SSL=false \
+    -e OIDC__CERTS_URL="http://my.idp.org/realms/flame/protocol/openid-connect/certs" \
+    ghcr.io/privateaim/node-result-service:sha-c1970cf
+```
+
+# Configuration
+
+The following table shows all available configuration options.
+
+| **Environment variable**   | **Description**                                          | **Default**                 | **Required** |
+|----------------------------|----------------------------------------------------------|-----------------------------|:------------:|
+| HUB__API_BASE_URL          | Base URL for the FLAME Hub API                           | https://api.privateaim.net  |              |
+| HUB__AUTH_BASE_URL         | Base URL for the FLAME Auth API                          | https://auth.privateaim.net |              |
+| HUB__AUTH_USERNAME         | Username to use for obtaining access tokens              |                             |      x       |
+| HUB__AUTH_PASSWORD         | Password to use for obtaining access tokens              |                             |      x       |
+| MINIO__ENDPOINT            | MinIO S3 API endpoint (without scheme)                   |                             |      x       |
+| MINIO__ACCESS_KEY          | Access key for interacting with MinIO S3 API             |                             |      x       |
+| MINIO__SECRET_KEY          | Secret key for interacting with MinIO S3 API             |                             |      x       |
+| MINIO__BUCKET              | Name of S3 bucket to store result files in               |                             |      x       |
+| MINIO__REGION              | Region of S3 bucket to store result files in             | us-east-1                   |              |
+| MINIO__USE_SSL             | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0                           |              |
+| OIDC__CERTS_URL            | URL to OIDC-complaint JWKS endpoint for validating JWTs  |                             |      x       |
+| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with        | client_id                   |              |
 
-Integration tests require at least one MinIO instance.
-For testing purposes, it is fine to run tests against a single MinIO instance targeting two different buckets.
-These buckets must exist before testing.
-Configuration is passed in using environment variables.
-The names are the same, except they are prepended with `PYTEST__`.
-At least, the MinIO endpoint, access key, secret key and bucket name need to be provided.
-If unspecified, the region is set to `us-east-1` and HTTPS is disabled.
+## Note on running tests
+
+When running tests, environment variables must be overwritten by prefixing them with `PYTEST__`.
+OIDC does not need to be configured, since an OIDC-compatible endpoint will be spawned alongside the tests that are
+being run.
+A [pre-generated keypair](tests/assets/keypair.pem) is used for this purpose.
+This allows all tests to generate valid JWTs as well as the service to validate them.
+The keypair is for development purposes only and should not be used in a productive setting.
+Therefore `pytest` should be invoked as follows.
 
 ```
 $ PYTEST__MINIO__ENDPOINT="localhost:9000" \
@@ -23,8 +74,11 @@ $ PYTEST__MINIO__ENDPOINT="localhost:9000" \
     PYTEST__HUB__AUTH_PASSWORD="XXXXXXXX" pytest
 ```
 
-OIDC does not need to be configured.
-Running integration tests will spawn a minimal webserver that provides an OIDC-compliant JWKS endpoint.
-A [pre-generated keypair](tests/assets/keypair.pem) is used for this purpose.
-This allows all tests to generate valid JWTs as well as the service to validate them.
-The keypair is for development purposes only and should not be used in a productive setting.
+Some tests need to be run against live infrastructure.
+Since a proper test instance is not available yet, these tests are hidden behind a flag and are not explicitly run in
+CI.
+To run these tests, append `-m live` to the command above.
+
+# License
+
+The FLAME Node Result Service is released under the Apache 2.0 license.

project/routers/scratch.py

@@ -23,6 +23,8 @@ class ScratchUploadResponse(BaseModel):
 @router.put(
     "/",
     response_model=ScratchUploadResponse,
+    summary="Upload file to local object storage",
+    operation_id="putIntermediateFile",
 )
 async def upload_to_scratch(
     client_id: Annotated[str, Depends(get_client_id)],
@@ -31,6 +33,12 @@ async def upload_to_scratch(
     minio: Annotated[Minio, Depends(get_local_minio)],
     request: Request,
 ):
+    """Upload a file to the local S3 instance.
+    The file is not forwarded to the FLAME hub.
+    Responds with a 200 on success and a link to the endpoint for fetching the uploaded file.
+
+    This endpoint is to be used for submitting intermediate results of a federated analysis.
+    """
     object_id = str(uuid.uuid4())
 
     minio.put_object(
@@ -51,13 +59,23 @@ async def upload_to_scratch(
     )
 
 
-@router.get("/{object_id}")
+@router.get(
+    "/{object_id}",
+    summary="Get file from local object storage",
+    operation_id="getIntermediateFile",
+)
 async def read_from_scratch(
     client_id: Annotated[str, Depends(get_client_id)],
     object_id: uuid.UUID,
     settings: Annotated[Settings, Depends(get_settings)],
     minio: Annotated[Minio, Depends(get_local_minio)],
 ):
+    """Get a file from the local S3 instance.
+    The file must have previously been uploaded using the PUT method of this endpoint.
+    Responds with a 200 on success and the requested file in the response body.
+
+    This endpoint is to be used for retrieving intermediate results of a federated analysis.
+    """
     try:
         response = minio.get_object(
             settings.minio.bucket, f"scratch/{client_id}/{object_id}"

project/routers/upload.py

@@ -64,6 +64,8 @@ async def __bg_upload_to_remote(
 @router.put(
     "/",
     status_code=status.HTTP_204_NO_CONTENT,
+    summary="Upload file to submit to Hub",
+    operation_id="putResultFile",
 )
 async def upload_to_remote(
     client_id: Annotated[str, Depends(get_client_id)],
@@ -73,6 +75,13 @@ async def upload_to_remote(
     local_minio: Annotated[Minio, Depends(get_local_minio)],
     api_access_token: Annotated[AccessToken, Depends(get_access_token)],
 ):
+    """Upload a file to the local S3 instance and send it to FLAME Hub in the background.
+    The request is successful if the file was uploaded to the local S3 instance.
+    Responds with a 204 on success.
+
+    This endpoint is to be used for submitting final results of a federated analysis.
+
+    Currently, there is no way of determining the status or progress of the upload to the FLAME Hub."""
     object_id = str(uuid.uuid4())
     object_name = f"upload/{client_id}/{object_id}"
 

project/server.py

@@ -2,7 +2,9 @@
 import logging.config
 import os.path
 from contextlib import asynccontextmanager
+from pathlib import Path
 
+import tomli
 from fastapi import FastAPI
 
 from project.routers import upload, scratch
@@ -23,11 +25,42 @@ async def lifespan(app: FastAPI):
     yield
 
 
-app = FastAPI(lifespan=lifespan)
+with open(Path(__file__).parent.parent / "pyproject.toml", mode="rb") as f:
+    pyproject_data = tomli.load(f)
 
+with open(Path(__file__).parent.parent / "README.md", mode="r") as f:
+    app_description = f.read()
 
-@app.get("/healthz")
+app_version = pyproject_data["tool"]["poetry"]["version"]
+app_summary = pyproject_data["tool"]["poetry"]["description"]
+
+app = FastAPI(
+    title="FLAME Node Result Service",
+    summary=app_summary,
+    version=app_version,
+    lifespan=lifespan,
+    description=app_description,
+    license_info={
+        "name": "Apache 2.0",
+        "url": "https://www.apache.org/licenses/LICENSE-2.0.html",
+        "identifier": "Apache-2.0",
+    },
+    openapi_tags=[
+        {
+            "name": "upload",
+            "description": "Upload files for submission to FLAME hub",
+        },
+        {
+            "name": "scratch",
+            "description": "Upload files to local object storage",
+        },
+    ],
+)
+
+
+@app.get("/healthz", summary="Check service readiness", operation_id="getHealth")
 async def do_healthcheck():
+    """Check whether the service is ready to process requests. Responds with a 200 on success."""
     return {"status": "ok"}
 
 

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
-name = "python-template-repo"
+name = "flame-result-service"
 version = "0.1.0"
-description = ""
+description = "Service for handling intermediate files and submitting result files to the FLAME Hub."
 authors = ["Maximilian Jugl <[email protected]>"]
 readme = "README.md"
 packages = [{ include = "project" }]
@@ -20,6 +20,7 @@ python-multipart = "^0.0.9"
 click = "^8.1.7"
 jwcrypto = "^1.5.6"
 httpx = "^0.26.0"
+tomli = "^2.0.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.3"