Skip to content

Commit

Permalink
Merge pull request #52 from PrivateAIM/51-new-hub-endpoints
Browse files Browse the repository at this point in the history 
feat: update to new FLAME Hub endpoints
  • Loading branch information
@mjugl
mjugl authored Jul 18, 2024
2 parents 18ec695 + e061dd0 commit eccc246
Show file tree
Hide file tree
Showing 12 changed files with 228 additions and 159 deletions.
3 changes: 2 additions & 1 deletion .env.example
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Hub endpoints
HUB__API_BASE_URL=https://api.privateaim.net
HUB__CORE_BASE_URL=https://core.privateaim.net
HUB__STORAGE_BASE_URL=https://storage.privateaim.net
HUB__AUTH_BASE_URL=https://auth.privateaim.net
HUB__AUTH_USERNAME=foobar
HUB__AUTH_PASSWORD=sup3r_s3cr3t
Expand Down
10 changes: 10 additions & 0 deletions .github/workflows/integration-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ env:
MINIO_ROOT_PASSWORD: s3cr3t_p4ssw0rd
MINIO_LOCAL_BUCKET_NAME: flame
MINIO_REMOTE_BUCKET_NAME: upload
# Point these to the new dev instance of the FLAME Hub
HUB_CORE_BASE_URL: https://core.privateaim.dev
HUB_STORAGE_BASE_URL: https://storage.privateaim.dev
HUB_AUTH_BASE_URL: https://auth.privateaim.dev

on:
push:
Expand Down Expand Up @@ -41,6 +45,12 @@ jobs:
MINIO__ENDPOINT: localhost:9000
HUB__AUTH_USERNAME: ${{secrets.HUB_AUTH_USERNAME}}
HUB__AUTH_PASSWORD: ${{secrets.HUB_AUTH_PASSWORD}}
# No tests against live infra are run here but these envs are set anyway to future-proof in case
# we ever end up testing against live infra. python-dotenv will not override existing envs so the prod
# URLs in the .env.example file shouldn't carry over here.
HUB__AUTH_BASE_URL: ${{env.HUB_AUTH_BASE_URL}}
HUB__CORE_BASE_URL: ${{env.HUB_CORE_BASE_URL}}
HUB__STORAGE_BASE_URL: ${{env.HUB_STORAGE_BASE_URL}}

services:
minio:
Expand Down
29 changes: 15 additions & 14 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,21 @@ $ docker run --rm -p 8080:8080 -e HUB__AUTH_USERNAME=admin \

The following table shows all available configuration options.

| **Environment variable** | **Description** | **Default** | **Required** |
|----------------------------|----------------------------------------------------------|-----------------------------|:------------:|
| HUB__API_BASE_URL | Base URL for the FLAME Hub API | https://api.privateaim.net | |
| HUB__AUTH_BASE_URL | Base URL for the FLAME Auth API | https://auth.privateaim.net | |
| HUB__AUTH_USERNAME | Username to use for obtaining access tokens | | x |
| HUB__AUTH_PASSWORD | Password to use for obtaining access tokens | | x |
| MINIO__ENDPOINT | MinIO S3 API endpoint (without scheme) | | x |
| MINIO__ACCESS_KEY | Access key for interacting with MinIO S3 API | | x |
| MINIO__SECRET_KEY | Secret key for interacting with MinIO S3 API | | x |
| MINIO__BUCKET | Name of S3 bucket to store result files in | | x |
| MINIO__REGION | Region of S3 bucket to store result files in | us-east-1 | |
| MINIO__USE_SSL | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0 | |
| OIDC__CERTS_URL | URL to OIDC-complaint JWKS endpoint for validating JWTs | | x |
| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with | client_id | |
| **Environment variable** | **Description** | **Default** | **Required** |
|----------------------------|----------------------------------------------------------|--------------------------------|:------------:|
| HUB__CORE_BASE_URL | Base URL for the FLAME Core API | https://core.privateaim.net | |
| HUB__STORAGE_BASE_URL | Base URL for the FLAME Storage API | https://storage.privateaim.net | |
| HUB__AUTH_BASE_URL | Base URL for the FLAME Auth API | https://auth.privateaim.net | |
| HUB__AUTH_USERNAME | Username to use for obtaining access tokens | | x |
| HUB__AUTH_PASSWORD | Password to use for obtaining access tokens | | x |
| MINIO__ENDPOINT | MinIO S3 API endpoint (without scheme) | | x |
| MINIO__ACCESS_KEY | Access key for interacting with MinIO S3 API | | x |
| MINIO__SECRET_KEY | Secret key for interacting with MinIO S3 API | | x |
| MINIO__BUCKET | Name of S3 bucket to store result files in | | x |
| MINIO__REGION | Region of S3 bucket to store result files in | us-east-1 | |
| MINIO__USE_SSL | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0 | |
| OIDC__CERTS_URL | URL to OIDC-complaint JWKS endpoint for validating JWTs | | x |
| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with | client_id | |

## Note on running tests

Expand Down
3 changes: 2 additions & 1 deletion project/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ class OIDCConfig(BaseModel):


class HubConfig(BaseModel):
api_base_url: HttpUrl = "https://api.privateaim.net"
core_base_url: HttpUrl = "https://core.privateaim.net"
auth_base_url: HttpUrl = "https://auth.privateaim.net"
storage_base_url: HttpUrl = "https://storage.privateaim.net"
auth_username: str
auth_password: str

Expand Down
15 changes: 11 additions & 4 deletions project/dependencies.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from starlette import status

from project.config import Settings, MinioBucketConfig
from project.hub import FlamePasswordAuthClient, FlameHubClient
from project.hub import FlamePasswordAuthClient, FlameCoreClient, FlameStorageClient

security = HTTPBearer()
logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -118,11 +118,18 @@ def get_auth_client(settings: Annotated[Settings, Depends(get_settings)]):
)


def get_api_client(
def get_core_client(
settings: Annotated[Settings, Depends(get_settings)],
auth_client: Annotated[FlamePasswordAuthClient, Depends(get_auth_client)],
):
return FlameHubClient(
return FlameCoreClient(
auth_client,
base_url=str(settings.hub.api_base_url),
base_url=str(settings.hub.core_base_url),
)


def get_storage_client(
settings: Annotated[Settings, Depends(get_settings)],
auth_client: Annotated[FlamePasswordAuthClient, Depends(get_auth_client)],
):
return FlameStorageClient(auth_client, base_url=str(settings.hub.storage_base_url))
204 changes: 118 additions & 86 deletions project/hub.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,11 +194,11 @@ def get_auth_bearer_header(self):
}


class FlameHubClient:
class FlameCoreClient:
def __init__(
self,
auth_client: FlamePasswordAuthClient,
base_url="https://api.privateaim.net",
base_url="https://core.privateaim.net",
):
"""
Create a new client to interact with the FLAME Hub API.
Expand Down Expand Up @@ -369,89 +369,6 @@ def get_analysis_by_id(self, analysis_id: str | UUID) -> Analysis | None:
r.raise_for_status()
return Analysis(**r.json())

def get_bucket_list(self) -> ResourceList[Bucket]:
"""
Get list of buckets.
Returns:
list of bucket resources
"""
r = httpx.get(
self._format_url("/storage/buckets"),
headers=self.auth_client.get_auth_bearer_header(),
)

r.raise_for_status()
return ResourceList[Bucket](**r.json())

def get_bucket_by_id(self, bucket_id: str | UUID) -> Bucket | None:
"""
Get a bucket by its ID.
Args:
bucket_id: ID of the bucket to get
Returns:
bucket resource, or *None* if no bucket was found
"""
r = httpx.get(
self._format_url(f"/storage/buckets/{bucket_id}"),
headers=self.auth_client.get_auth_bearer_header(),
)

if r.status_code == status.HTTP_404_NOT_FOUND:
return None

r.raise_for_status()
return Bucket(**r.json())

def get_bucket_file_list(self) -> ResourceList[BucketFile]:
"""
Get list of bucket files.
Returns:
list of bucket file resources
"""
r = httpx.get(
self._format_url("/storage/bucket-files"),
headers=self.auth_client.get_auth_bearer_header(),
)

r.raise_for_status()
return ResourceList[BucketFile](**r.json())

def upload_to_bucket(
self,
bucket_id: str | UUID,
file_name: str,
file: bytes | BytesIO,
content_type: str = "application/octet-stream",
) -> ResourceList[BucketFile]:
"""
Upload a single file to a bucket.
Args:
bucket_id: ID of the bucket to upload the file to
file_name: file name
file: file contents
content_type: content type of the file (*application/octet-stream* by default)
Returns:
list of bucket file resources for the uploaded file
"""
# wrap into BytesIO if raw bytes are passed in
if isinstance(file, bytes):
file = BytesIO(file)

r = httpx.post(
self._format_url(f"/storage/buckets/{bucket_id}/upload"),
headers=self.auth_client.get_auth_bearer_header(),
files={"file": (file_name, file, content_type)},
)

r.raise_for_status()
return ResourceList[BucketFile](**r.json())

def get_analysis_bucket_file_list(self) -> ResourceList[AnalysisBucketFile]:
"""
Get list of files that have been linked to an analysis.
Expand Down Expand Up @@ -531,6 +448,121 @@ def link_bucket_file_to_analysis(
r.raise_for_status()
return AnalysisBucketFile(**r.json())


class FlameStorageClient:
def __init__(
self,
auth_client: FlamePasswordAuthClient,
base_url="https://storage.privateaim.net",
):
"""
Create a new client to interact with the FLAME Storage API.
Args:
auth_client: FLAME Auth API client to use
base_url: base API url
"""
self.base_url = base_url
self.auth_client = auth_client

base_url_parts = urllib.parse.urlsplit(base_url)

self._base_scheme = base_url_parts[0]
self._base_netloc = base_url_parts[1]
self._base_path = base_url_parts[2]

def _format_url(self, path: str, query: dict[str, str] = None):
return build_url(
self._base_scheme,
self._base_netloc,
urllib.parse.urljoin(self._base_path, path),
query,
"",
)

def get_bucket_list(self) -> ResourceList[Bucket]:
"""
Get list of buckets.
Returns:
list of bucket resources
"""
r = httpx.get(
self._format_url("/buckets"),
headers=self.auth_client.get_auth_bearer_header(),
)

r.raise_for_status()
return ResourceList[Bucket](**r.json())

def get_bucket_by_id(self, bucket_id: str | UUID) -> Bucket | None:
"""
Get a bucket by its ID.
Args:
bucket_id: ID of the bucket to get
Returns:
bucket resource, or *None* if no bucket was found
"""
r = httpx.get(
self._format_url(f"/buckets/{bucket_id}"),
headers=self.auth_client.get_auth_bearer_header(),
)

if r.status_code == status.HTTP_404_NOT_FOUND:
return None

r.raise_for_status()
return Bucket(**r.json())

def get_bucket_file_list(self) -> ResourceList[BucketFile]:
"""
Get list of bucket files.
Returns:
list of bucket file resources
"""
r = httpx.get(
self._format_url("/bucket-files"),
headers=self.auth_client.get_auth_bearer_header(),
)

r.raise_for_status()
return ResourceList[BucketFile](**r.json())

def upload_to_bucket(
self,
bucket_id: str | UUID,
file_name: str,
file: bytes | BytesIO,
content_type: str = "application/octet-stream",
) -> ResourceList[BucketFile]:
"""
Upload a single file to a bucket.
Args:
bucket_id: ID of the bucket to upload the file to
file_name: file name
file: file contents
content_type: content type of the file (*application/octet-stream* by default)
Returns:
list of bucket file resources for the uploaded file
"""
# wrap into BytesIO if raw bytes are passed in
if isinstance(file, bytes):
file = BytesIO(file)

r = httpx.post(
self._format_url(f"/buckets/{bucket_id}/upload"),
headers=self.auth_client.get_auth_bearer_header(),
files={"file": (file_name, file, content_type)},
)

r.raise_for_status()
return ResourceList[BucketFile](**r.json())

def stream_bucket_file(self, bucket_file_id: str | UUID, chunk_size=1024):
"""
Fetch the contents of a bucket file.
Expand All @@ -545,7 +577,7 @@ def stream_bucket_file(self, bucket_file_id: str | UUID, chunk_size=1024):
"""
with httpx.stream(
"GET",
self._format_url(f"/storage/bucket-files/{bucket_file_id}/stream"),
self._format_url(f"/bucket-files/{bucket_file_id}/stream"),
headers=self.auth_client.get_auth_bearer_header(),
) as r:
for b in r.iter_bytes(chunk_size=chunk_size):
Expand Down
Loading

0 comments on commit eccc246

Please sign in to comment.