feat: authorization

.github/workflows/lint.yml

@@ -22,6 +22,9 @@ jobs:
         run: pip install poetry
       - name: Install dependencies
         run: poetry install
-      - name: Lint
+      - name: Format check
         run: |
-          poetry run black --check transcriptomics_data_service tests
+          poetry run ruff format --check
+      - name: Lint check
+        run: |
+          poetry run ruff check

.gitignore

@@ -167,3 +167,6 @@ cython_debug/
 
 # tds-db
 pg_data
+
+# TDS custom authz module
+lib/*

.vscode/launch.json

@@ -8,6 +8,7 @@
             "name": "Python Debugger (TDS): Dev Container Attach",
             "type": "debugpy",
             "request": "attach",
+            "justMyCode": false,
             "connect": {
                 "host": "0.0.0.0",
                 "port": 9511

README.md

@@ -36,6 +36,14 @@ docker compose -f ./docker-compose.dev.yaml down
 
 You can then attach VS Code to the `tds` container, and use the preconfigured `Python Debugger (TDS)` for interactive debugging.
 
+## Authorization plugin
+
+The Transcriptomics Data Service is meant to be a reusable microservice that can be integrated in existing 
+stacks. Since authorization schemes vary across projects, TDS allows adopters to code their own authorization plugin, 
+enabling adopters to leverage their existing access control code, tools and policies.
+
+See the [authorization docs](./docs/authz.md) for more information on how to create and use the authz plugin with TDS.
+
 ## Endpoints
 
 * /service-info

authz_plugins/api_key/README.md

@@ -0,0 +1,22 @@
+# API key authorization example
+
+This sample authorization plugin authorizes requests with an API key header.
+
+## Contents
+- [Authz plugin](authz.module.py)
+- [Extra configuration](example.env)
+
+## Instructions
+
+```bash
+# Copy the module to the mount directory
+cp authz_plugins/api_key/authz.module.py lib/
+
+# Copy the extra environment variables file to the mount directory
+cp authz_plugins/api_key/example.env lib/.env
+
+# optional: modify the API key value in lib/.env
+
+# Start the service
+docker compose up --build
+```

authz_plugins/api_key/authz.module.py

@@ -0,0 +1,93 @@
+from logging import Logger
+from typing import Annotated, Any, Awaitable, Callable, Coroutine, Sequence
+from fastapi import Depends, FastAPI, HTTPException, Header, Request, Response
+from fastapi.responses import JSONResponse
+
+from transcriptomics_data_service.authz.middleware_base import BaseAuthzMiddleware
+from transcriptomics_data_service.config import Config, get_config
+from transcriptomics_data_service.logger import get_logger
+
+config = get_config()
+logger = get_logger(config)
+
+
+"""
+CUSTOM PLUGIN CONFIGURATION
+Extra configurations can be added to the config object by adding
+a '.env' file in the plugin mount directory.
+Variables placed there will be loaded as lowercase properties
+
+This variable's value can be accessed with: config.api_key
+API_KEY="fake-super-secret-api-key"
+"""
+
+
+class ApiKeyAuthzMiddleware(BaseAuthzMiddleware):
+    """
+    Concrete implementation of BaseAuthzMiddleware to authorize requests based on the provided API key.
+    """
+
+    def __init__(self, config: Config, logger: Logger) -> None:
+        super().__init__()
+        self.enabled = config.bento_authz_enabled
+        self.logger = logger
+
+        # Load the api_key from the config's extras
+        self.api_key = config.model_extra.get("api_key")
+        if self.api_key is None:
+            # prevents the server from starting if misconfigured
+            raise ValueError("Expected variable 'API_KEY' is not set in the plugin's .env")
+
+    # Middleware lifecycle
+
+    def attach(self, app: FastAPI):
+        app.middleware("http")(self.dispatch)
+
+    async def dispatch(
+        self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
+    ) -> Coroutine[Any, Any, Response]:
+        if not self.enabled:
+            return await call_next(request)
+
+        # Request was not checked for authz yet
+        try:
+            res = await call_next(request)
+        except HTTPException as e:
+            # Catch exceptions raised by authz functions
+            self.logger.error(e)
+            return JSONResponse(status_code=e.status_code, content=e.detail)
+
+        return res
+
+    # API KEY authorization
+
+    def _dep_check_api_key(self):
+        """
+        Dependency injection for the API key authorization.
+        The inner function checks the x_api_key header to validate the API key.
+        Raises an exception that should be caught and handled in the dispatch func.
+        """
+
+        async def _inner(x_api_key: Annotated[str, Header()]):
+            if x_api_key != self.api_key:
+                raise HTTPException(status_code=403, detail="Unauthorized: invalid API key")
+
+        return Depends(_inner)
+
+    def dep_ingest_router(self) -> Sequence[Depends]:
+        # Require API key check on the ingest router
+        return [self._dep_check_api_key()]
+
+    def dep_expression_router(self) -> Sequence[Depends]:
+        # Require API key check on the expressions router
+        return [self._dep_check_api_key()]
+
+    def dep_experiment_result_router(self) -> Sequence[Depends]:
+        # Require API key check on the experiment_result router
+        return [self._dep_check_api_key()]
+
+    # NOTE: With an all-or-nothing authz mechanism like an API key,
+    # we can place the authz checks at the router level to have a more concise module.
+
+
+authz_middleware = ApiKeyAuthzMiddleware(config, logger)

authz_plugins/api_key/example.env

@@ -0,0 +1 @@
+API_KEY="fake-super-secret-api-key"

authz_plugins/bento/README.md

@@ -0,0 +1,17 @@
+# Bento authorization
+
+This is the authorization middleware used to authorize TDS requests with the
+[Bento Authorization Service](https://github.com/bento-platform/bento_authorization_service).
+
+## Contents
+- [Authz plugin](./authz.module.py)
+
+## Instructions
+
+```bash
+# Copy the module to the mount directory
+cp authz_plugins/bento/authz.module.py lib/
+
+# Start the service
+docker compose up --build
+```

authz_plugins/bento/authz.module.py

@@ -0,0 +1,79 @@
+from typing import Annotated
+from bento_lib.auth.middleware.fastapi import FastApiAuthMiddleware
+from bento_lib.auth.permissions import P_INGEST_DATA, P_DELETE_DATA, P_QUERY_DATA, Permission
+from bento_lib.auth.resources import RESOURCE_EVERYTHING, build_resource
+from fastapi import Depends, Request
+
+from transcriptomics_data_service.config import get_config
+from transcriptomics_data_service.logger import get_logger
+from transcriptomics_data_service.authz.middleware_base import BaseAuthzMiddleware
+
+
+config = get_config()
+logger = get_logger(config)
+
+
+class BentoAuthzMiddleware(FastApiAuthMiddleware, BaseAuthzMiddleware):
+    """
+    Concrete implementation of BaseAuthzMiddleware to authorize with Bento's authorization service/model.
+    Extends the bento-lib FastApiAuthMiddleware, which includes all the middleware lifecycle and authorization logic.
+
+    Notes:
+        - This middleware plugin will only work with a Bento authorization-service.
+        - TDS should be able to perform HTTP requests on the authz service url: `config.bento_authz_service_url`
+    """
+
+    def _build_resource_from_id(self, experiment_result_id: str):
+        # Injectable for endpoints that use the 'experiment_result_id' param to create the authz Resource
+        # Ownsership of an experiment is baked-in the ExperimentResult's ID in Bento
+        # e.g. "<project-id>--<dataset-id>--<experiment_id>"
+        # TODO: come up with better delimiters
+        [project, dataset, experiment] = experiment_result_id.split("--")
+        self._logger.debug(
+            f"Injecting resource: project={project} dataset={dataset} experiment_result_id={experiment_result_id}"
+        )
+        return build_resource(project, dataset)
+
+    def _dep_require_permission_injected_resource(
+        self,
+        permission: Permission,
+    ):
+        # Given a permission and the injected resource, will evaluate if operation is allowed
+        async def inner(
+            request: Request,
+            resource: Annotated[dict, Depends(self._build_resource_from_id)],  # Inject resource
+        ):
+            await self.async_check_authz_evaluate(request, frozenset({permission}), resource, set_authz_flag=True)
+
+        return Depends(inner)
+
+    def _dep_perm_data_everything(self, permission: Permission):
+        return self.dep_require_permissions_on_resource(
+            permissions=frozenset({permission}),
+            resource=RESOURCE_EVERYTHING,
+        )
+
+    # INGESTION router paths
+
+    def dep_authz_ingest(self):
+        # User needs P_INGEST_DATA permission on the target resource (injected)
+        return [self._dep_require_permission_injected_resource(P_INGEST_DATA)]
+
+    def dep_authz_normalize(self):
+        return [self._dep_require_permission_injected_resource(P_INGEST_DATA)]
+
+    # EXPERIMENT RESULT router paths
+
+    def dep_authz_get_experiment_result(self):
+        return [self._dep_require_permission_injected_resource(P_QUERY_DATA)]
+
+    def dep_authz_delete_experiment_result(self):
+        return [self._dep_require_permission_injected_resource(P_DELETE_DATA)]
+
+    # EXPRESSIONS router paths
+
+    def dep_authz_expressions_list(self):
+        return [self._dep_perm_data_everything(P_QUERY_DATA)]
+
+
+authz_middleware = BentoAuthzMiddleware.build_from_fastapi_pydantic_config(config, logger)

authz_plugins/opa/README.md

@@ -0,0 +1,31 @@
+# OPA authorization example
+
+This sample authorization plugin showcases how to use OPA as the authorization service for TDS.
+While this could be done in pure Python with HTTP, 
+using the [OPA client for Python](https://github.com/Turall/OPA-python-client) is more convenient.
+
+As described in the authz module docs, additional dependencies can be provided for the authorization plugin.
+In this example, we include the OPA client as an additional dependency.
+
+Furthermore, the OPA server details are provided with extra environment configurations.
+
+## Contents
+- [Authz plugin](authz.module.py)
+- [Extra configuration](example.env)
+- [Additional python dependencies](requirements.txt)
+
+## Instructions
+
+```bash
+# Copy the module to the mount directory
+cp authz_plugins/opa/authz.module.py lib/
+
+# Copy the extra environment variable file to the mount directory
+cp authz_plugins/opa/example.env lib/.env
+
+# Copy the additional Python dependencies to the mount directory
+cp authz_plugins/opa/requirements.txt lib/
+
+# Start the service
+docker compose up --build
+```

authz_plugins/opa/authz.module.py

@@ -0,0 +1,93 @@
+from logging import Logger
+from typing import Any, Awaitable, Callable, Coroutine
+from fastapi import Depends, FastAPI, HTTPException, Request, Response
+from fastapi.responses import JSONResponse
+from opa_client.opa import OpaClient  # CUSTOM PLUGIN DEPENDENCY
+
+from transcriptomics_data_service.authz.middleware_base import BaseAuthzMiddleware
+from transcriptomics_data_service.config import Config, get_config
+from transcriptomics_data_service.logger import get_logger
+
+config = get_config()
+logger = get_logger(config)
+
+"""
+CUSTOM PLUGIN DEPENDENCY
+Extra dependencies can be added if the authz plugin requires them.
+In this example, the authz module imports the OPA client.
+Since OPA does not ship with TDS, a requirements.txt file must be placed under 'lib'.
+"""
+
+
+class OPAAuthzMiddleware(BaseAuthzMiddleware):
+    """
+    Concrete implementation of BaseAuthzMiddleware to authorize requests with OPA.
+    """
+
+    def __init__(self, config: Config, logger: Logger) -> None:
+        super().__init__()
+        self.enabled = config.bento_authz_enabled
+        self.logger = logger
+
+        # Get custom OPA configs from lib/.env
+        opa_host = config.model_extra.get("opa_host")
+        opa_port = int(config.model_extra.get("opa_host_port"))
+
+        # Init the OPA client with the server
+        self.opa_client = OpaClient(host=opa_host, port=opa_port)
+
+        # This is not pointing to a real OPA server.
+        # Will raise an exception if connection is invalid.
+        self.logger.info(self.opa_client.check_connection())
+
+    # Middleware lifecycle
+
+    def attach(self, app: FastAPI):
+        app.middleware("http")(self.dispatch)
+
+    async def dispatch(
+        self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
+    ) -> Coroutine[Any, Any, Response]:
+        if not self.enabled:
+            return await call_next(request)
+
+        try:
+            res = await call_next(request)
+        except HTTPException as e:
+            # Catch exceptions raised by authz functions
+            self.logger.error(e)
+            return JSONResponse(status_code=e.status_code, content=e.detail)
+
+        return res
+
+    # OPA authorization function
+    def _dep_check_opa(self):
+        async def inner(request: Request):
+            # Check the permission using the OPA client.
+            # We assume true for the sake of the demonstration
+            # authz_result = await self.opa_client.check_permission()
+            authz_result = True
+            if not authz_result:
+                raise HTTPException(status_code=403, detail="Unauthorized: policy evaluation failed")
+
+        return Depends(inner)
+
+    # Authz logic: OPA check injected at endpoint levels
+
+    def dep_authz_ingest(self):
+        return [self._dep_check_opa()]
+
+    def dep_authz_normalize(self):
+        return [self._dep_check_opa()]
+
+    def dep_authz_delete_experiment_result(self):
+        return [self._dep_check_opa()]
+
+    def dep_authz_expressions_list(self):
+        return [self._dep_check_opa()]
+
+    def dep_authz_get_experiment_result(self):
+        return [self._dep_check_opa()]
+
+
+authz_middleware = OPAAuthzMiddleware(config, logger)

authz_plugins/opa/example.env

@@ -0,0 +1,2 @@
+OPA_HOST=opa-server-ip
+OPA_HOST_PORT=8181

authz_plugins/opa/requirements.txt

@@ -0,0 +1 @@
+opa-python-client==2.0.0

docker-compose.dev.yaml

@@ -9,7 +9,7 @@ services:
     depends_on:
       - tds-db
     environment:
-      - BENTO_UID=1001
+      - BENTO_UID=${UID}
       - DATABASE_URI=postgres://tds_user:tds_password@tds-db:5432/tds_db
       - CORS_ORIGINS="*"
       - BENTO_AUTHZ_SERVICE_URL=""