feat: update to new FLAME Hub endpoints

.env.example

@@ -1,5 +1,6 @@
 # Hub endpoints
-HUB__API_BASE_URL=https://api.privateaim.net
+HUB__CORE_BASE_URL=https://core.privateaim.net
+HUB__STORAGE_BASE_URL=https://storage.privateaim.net
 HUB__AUTH_BASE_URL=https://auth.privateaim.net
 HUB__AUTH_USERNAME=foobar
 HUB__AUTH_PASSWORD=sup3r_s3cr3t

.github/workflows/integration-test.yml

@@ -5,6 +5,10 @@ env:
   MINIO_ROOT_PASSWORD: s3cr3t_p4ssw0rd
   MINIO_LOCAL_BUCKET_NAME: flame
   MINIO_REMOTE_BUCKET_NAME: upload
+  # Point these to the new dev instance of the FLAME Hub
+  HUB_CORE_BASE_URL: https://core.privateaim.dev
+  HUB_STORAGE_BASE_URL: https://storage.privateaim.dev
+  HUB_AUTH_BASE_URL: https://auth.privateaim.dev
 
 on:
   push:
@@ -41,6 +45,12 @@ jobs:
           MINIO__ENDPOINT: localhost:9000
           HUB__AUTH_USERNAME: ${{secrets.HUB_AUTH_USERNAME}}
           HUB__AUTH_PASSWORD: ${{secrets.HUB_AUTH_PASSWORD}}
+          # No tests against live infra are run here but these envs are set anyway to future-proof in case
+          # we ever end up testing against live infra. python-dotenv will not override existing envs so the prod
+          # URLs in the .env.example file shouldn't carry over here.
+          HUB__AUTH_BASE_URL: ${{env.HUB_AUTH_BASE_URL}}
+          HUB__CORE_BASE_URL: ${{env.HUB_CORE_BASE_URL}}
+          HUB__STORAGE_BASE_URL: ${{env.HUB_STORAGE_BASE_URL}}
 
     services:
       minio:

README.md

@@ -40,20 +40,21 @@ $ docker run --rm -p 8080:8080 -e HUB__AUTH_USERNAME=admin \
 
 The following table shows all available configuration options.
 
-| **Environment variable**   | **Description**                                          | **Default**                 | **Required** |
-|----------------------------|----------------------------------------------------------|-----------------------------|:------------:|
-| HUB__API_BASE_URL          | Base URL for the FLAME Hub API                           | https://api.privateaim.net  |              |
-| HUB__AUTH_BASE_URL         | Base URL for the FLAME Auth API                          | https://auth.privateaim.net |              |
-| HUB__AUTH_USERNAME         | Username to use for obtaining access tokens              |                             |      x       |
-| HUB__AUTH_PASSWORD         | Password to use for obtaining access tokens              |                             |      x       |
-| MINIO__ENDPOINT            | MinIO S3 API endpoint (without scheme)                   |                             |      x       |
-| MINIO__ACCESS_KEY          | Access key for interacting with MinIO S3 API             |                             |      x       |
-| MINIO__SECRET_KEY          | Secret key for interacting with MinIO S3 API             |                             |      x       |
-| MINIO__BUCKET              | Name of S3 bucket to store result files in               |                             |      x       |
-| MINIO__REGION              | Region of S3 bucket to store result files in             | us-east-1                   |              |
-| MINIO__USE_SSL             | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0                           |              |
-| OIDC__CERTS_URL            | URL to OIDC-complaint JWKS endpoint for validating JWTs  |                             |      x       |
-| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with        | client_id                   |              |
+| **Environment variable**   | **Description**                                          | **Default**                    | **Required** |
+|----------------------------|----------------------------------------------------------|--------------------------------|:------------:|
+| HUB__CORE_BASE_URL         | Base URL for the FLAME Core API                          | https://core.privateaim.net    |              |
+| HUB__STORAGE_BASE_URL      | Base URL for the FLAME Storage API                       | https://storage.privateaim.net |              |
+| HUB__AUTH_BASE_URL         | Base URL for the FLAME Auth API                          | https://auth.privateaim.net    |              |
+| HUB__AUTH_USERNAME         | Username to use for obtaining access tokens              |                                |      x       |
+| HUB__AUTH_PASSWORD         | Password to use for obtaining access tokens              |                                |      x       |
+| MINIO__ENDPOINT            | MinIO S3 API endpoint (without scheme)                   |                                |      x       |
+| MINIO__ACCESS_KEY          | Access key for interacting with MinIO S3 API             |                                |      x       |
+| MINIO__SECRET_KEY          | Secret key for interacting with MinIO S3 API             |                                |      x       |
+| MINIO__BUCKET              | Name of S3 bucket to store result files in               |                                |      x       |
+| MINIO__REGION              | Region of S3 bucket to store result files in             | us-east-1                      |              |
+| MINIO__USE_SSL             | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0                              |              |
+| OIDC__CERTS_URL            | URL to OIDC-complaint JWKS endpoint for validating JWTs  |                                |      x       |
+| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with        | client_id                      |              |
 
 ## Note on running tests
 

project/config.py

@@ -25,8 +25,9 @@ class OIDCConfig(BaseModel):
 
 
 class HubConfig(BaseModel):
-    api_base_url: HttpUrl = "https://api.privateaim.net"
+    core_base_url: HttpUrl = "https://core.privateaim.net"
     auth_base_url: HttpUrl = "https://auth.privateaim.net"
+    storage_base_url: HttpUrl = "https://storage.privateaim.net"
     auth_username: str
     auth_password: str
 

project/dependencies.py

@@ -12,7 +12,7 @@
 from starlette import status
 
 from project.config import Settings, MinioBucketConfig
-from project.hub import FlamePasswordAuthClient, FlameHubClient
+from project.hub import FlamePasswordAuthClient, FlameCoreClient, FlameStorageClient
 
 security = HTTPBearer()
 logger = logging.getLogger(__name__)
@@ -118,11 +118,18 @@ def get_auth_client(settings: Annotated[Settings, Depends(get_settings)]):
     )
 
 
-def get_api_client(
+def get_core_client(
     settings: Annotated[Settings, Depends(get_settings)],
     auth_client: Annotated[FlamePasswordAuthClient, Depends(get_auth_client)],
 ):
-    return FlameHubClient(
+    return FlameCoreClient(
         auth_client,
-        base_url=str(settings.hub.api_base_url),
+        base_url=str(settings.hub.core_base_url),
     )
+
+
+def get_storage_client(
+    settings: Annotated[Settings, Depends(get_settings)],
+    auth_client: Annotated[FlamePasswordAuthClient, Depends(get_auth_client)],
+):
+    return FlameStorageClient(auth_client, base_url=str(settings.hub.storage_base_url))

project/hub.py

@@ -194,11 +194,11 @@ def get_auth_bearer_header(self):
         }
 
 
-class FlameHubClient:
+class FlameCoreClient:
     def __init__(
         self,
         auth_client: FlamePasswordAuthClient,
-        base_url="https://api.privateaim.net",
+        base_url="https://core.privateaim.net",
     ):
         """
         Create a new client to interact with the FLAME Hub API.
@@ -369,89 +369,6 @@ def get_analysis_by_id(self, analysis_id: str | UUID) -> Analysis | None:
         r.raise_for_status()
         return Analysis(**r.json())
 
-    def get_bucket_list(self) -> ResourceList[Bucket]:
-        """
-        Get list of buckets.
-
-        Returns:
-            list of bucket resources
-        """
-        r = httpx.get(
-            self._format_url("/storage/buckets"),
-            headers=self.auth_client.get_auth_bearer_header(),
-        )
-
-        r.raise_for_status()
-        return ResourceList[Bucket](**r.json())
-
-    def get_bucket_by_id(self, bucket_id: str | UUID) -> Bucket | None:
-        """
-        Get a bucket by its ID.
-
-        Args:
-            bucket_id: ID of the bucket to get
-
-        Returns:
-            bucket resource, or *None* if no bucket was found
-        """
-        r = httpx.get(
-            self._format_url(f"/storage/buckets/{bucket_id}"),
-            headers=self.auth_client.get_auth_bearer_header(),
-        )
-
-        if r.status_code == status.HTTP_404_NOT_FOUND:
-            return None
-
-        r.raise_for_status()
-        return Bucket(**r.json())
-
-    def get_bucket_file_list(self) -> ResourceList[BucketFile]:
-        """
-        Get list of bucket files.
-
-        Returns:
-            list of bucket file resources
-        """
-        r = httpx.get(
-            self._format_url("/storage/bucket-files"),
-            headers=self.auth_client.get_auth_bearer_header(),
-        )
-
-        r.raise_for_status()
-        return ResourceList[BucketFile](**r.json())
-
-    def upload_to_bucket(
-        self,
-        bucket_id: str | UUID,
-        file_name: str,
-        file: bytes | BytesIO,
-        content_type: str = "application/octet-stream",
-    ) -> ResourceList[BucketFile]:
-        """
-        Upload a single file to a bucket.
-
-        Args:
-            bucket_id: ID of the bucket to upload the file to
-            file_name: file name
-            file: file contents
-            content_type: content type of the file (*application/octet-stream* by default)
-
-        Returns:
-            list of bucket file resources for the uploaded file
-        """
-        # wrap into BytesIO if raw bytes are passed in
-        if isinstance(file, bytes):
-            file = BytesIO(file)
-
-        r = httpx.post(
-            self._format_url(f"/storage/buckets/{bucket_id}/upload"),
-            headers=self.auth_client.get_auth_bearer_header(),
-            files={"file": (file_name, file, content_type)},
-        )
-
-        r.raise_for_status()
-        return ResourceList[BucketFile](**r.json())
-
     def get_analysis_bucket_file_list(self) -> ResourceList[AnalysisBucketFile]:
         """
         Get list of files that have been linked to an analysis.
@@ -531,6 +448,121 @@ def link_bucket_file_to_analysis(
         r.raise_for_status()
         return AnalysisBucketFile(**r.json())
 
+
+class FlameStorageClient:
+    def __init__(
+        self,
+        auth_client: FlamePasswordAuthClient,
+        base_url="https://storage.privateaim.net",
+    ):
+        """
+        Create a new client to interact with the FLAME Storage API.
+
+        Args:
+            auth_client: FLAME Auth API client to use
+            base_url: base API url
+        """
+        self.base_url = base_url
+        self.auth_client = auth_client
+
+        base_url_parts = urllib.parse.urlsplit(base_url)
+
+        self._base_scheme = base_url_parts[0]
+        self._base_netloc = base_url_parts[1]
+        self._base_path = base_url_parts[2]
+
+    def _format_url(self, path: str, query: dict[str, str] = None):
+        return build_url(
+            self._base_scheme,
+            self._base_netloc,
+            urllib.parse.urljoin(self._base_path, path),
+            query,
+            "",
+        )
+
+    def get_bucket_list(self) -> ResourceList[Bucket]:
+        """
+        Get list of buckets.
+
+        Returns:
+            list of bucket resources
+        """
+        r = httpx.get(
+            self._format_url("/buckets"),
+            headers=self.auth_client.get_auth_bearer_header(),
+        )
+
+        r.raise_for_status()
+        return ResourceList[Bucket](**r.json())
+
+    def get_bucket_by_id(self, bucket_id: str | UUID) -> Bucket | None:
+        """
+        Get a bucket by its ID.
+
+        Args:
+            bucket_id: ID of the bucket to get
+
+        Returns:
+            bucket resource, or *None* if no bucket was found
+        """
+        r = httpx.get(
+            self._format_url(f"/buckets/{bucket_id}"),
+            headers=self.auth_client.get_auth_bearer_header(),
+        )
+
+        if r.status_code == status.HTTP_404_NOT_FOUND:
+            return None
+
+        r.raise_for_status()
+        return Bucket(**r.json())
+
+    def get_bucket_file_list(self) -> ResourceList[BucketFile]:
+        """
+        Get list of bucket files.
+
+        Returns:
+            list of bucket file resources
+        """
+        r = httpx.get(
+            self._format_url("/bucket-files"),
+            headers=self.auth_client.get_auth_bearer_header(),
+        )
+
+        r.raise_for_status()
+        return ResourceList[BucketFile](**r.json())
+
+    def upload_to_bucket(
+        self,
+        bucket_id: str | UUID,
+        file_name: str,
+        file: bytes | BytesIO,
+        content_type: str = "application/octet-stream",
+    ) -> ResourceList[BucketFile]:
+        """
+        Upload a single file to a bucket.
+
+        Args:
+            bucket_id: ID of the bucket to upload the file to
+            file_name: file name
+            file: file contents
+            content_type: content type of the file (*application/octet-stream* by default)
+
+        Returns:
+            list of bucket file resources for the uploaded file
+        """
+        # wrap into BytesIO if raw bytes are passed in
+        if isinstance(file, bytes):
+            file = BytesIO(file)
+
+        r = httpx.post(
+            self._format_url(f"/buckets/{bucket_id}/upload"),
+            headers=self.auth_client.get_auth_bearer_header(),
+            files={"file": (file_name, file, content_type)},
+        )
+
+        r.raise_for_status()
+        return ResourceList[BucketFile](**r.json())
+
     def stream_bucket_file(self, bucket_file_id: str | UUID, chunk_size=1024):
         """
         Fetch the contents of a bucket file.
@@ -545,7 +577,7 @@ def stream_bucket_file(self, bucket_file_id: str | UUID, chunk_size=1024):
         """
         with httpx.stream(
             "GET",
-            self._format_url(f"/storage/bucket-files/{bucket_file_id}/stream"),
+            self._format_url(f"/bucket-files/{bucket_file_id}/stream"),
             headers=self.auth_client.get_auth_bearer_header(),
         ) as r:
             for b in r.iter_bytes(chunk_size=chunk_size):