From 561a02a0c277a2b11cb96d074f204d5a74e4aa13 Mon Sep 17 00:00:00 2001 From: Maximilian Jugl Date: Thu, 18 Jul 2024 15:34:36 +0200 Subject: [PATCH 1/2] feat: update to new FLAME Hub endpoints --- .env.example | 3 +- .github/workflows/integration-test.yml | 10 ++ README.md | 29 ++-- project/config.py | 3 +- project/dependencies.py | 15 +- project/hub.py | 206 ++++++++++++++----------- project/routers/final.py | 22 +-- project/routers/intermediate.py | 24 +-- tests/common/env.py | 10 +- tests/conftest.py | 35 +++-- tests/test_final.py | 6 +- tests/test_hub.py | 26 ++-- 12 files changed, 229 insertions(+), 160 deletions(-) diff --git a/.env.example b/.env.example index a48b73f..09e6306 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # Hub endpoints -HUB__API_BASE_URL=https://api.privateaim.net +HUB__CORE_BASE_URL=https://core.privateaim.net +HUB__STORAGE_BASE_URL=https://storage.privateaim.net HUB__AUTH_BASE_URL=https://auth.privateaim.net HUB__AUTH_USERNAME=foobar HUB__AUTH_PASSWORD=sup3r_s3cr3t diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9114185..2d217fb 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -5,6 +5,10 @@ env: MINIO_ROOT_PASSWORD: s3cr3t_p4ssw0rd MINIO_LOCAL_BUCKET_NAME: flame MINIO_REMOTE_BUCKET_NAME: upload + # Point these to the new dev instance of the FLAME Hub + HUB_CORE_BASE_URL: https://core.privateaim.dev + HUB_STORAGE_BASE_URL: https://storage.privateaim.dev + HUB_AUTH_BASE_URL: https://auth.privateaim.dev on: push: @@ -41,6 +45,12 @@ jobs: MINIO__ENDPOINT: localhost:9000 HUB__AUTH_USERNAME: ${{secrets.HUB_AUTH_USERNAME}} HUB__AUTH_PASSWORD: ${{secrets.HUB_AUTH_PASSWORD}} + # No tests against live infra are run here but these envs are set anyway to future-proof in case + # we ever end up testing against live infra. python-dotenv will not override existing envs so the prod + # URLs in the .env.example file shouldn't carry over here. + HUB__AUTH_BASE_URL: ${{env.HUB_AUTH_BASE_URL}} + HUB__CORE_BASE_URL: ${{env.HUB_CORE_BASE_URL}} + HUB__STORAGE_BASE_URL: ${{env.HUB_STORAGE_BASE_URL}} services: minio: diff --git a/README.md b/README.md index 7638f9a..d529fdf 100644 --- a/README.md +++ b/README.md @@ -40,20 +40,21 @@ $ docker run --rm -p 8080:8080 -e HUB__AUTH_USERNAME=admin \ The following table shows all available configuration options. -| **Environment variable** | **Description** | **Default** | **Required** | -|----------------------------|----------------------------------------------------------|-----------------------------|:------------:| -| HUB__API_BASE_URL | Base URL for the FLAME Hub API | https://api.privateaim.net | | -| HUB__AUTH_BASE_URL | Base URL for the FLAME Auth API | https://auth.privateaim.net | | -| HUB__AUTH_USERNAME | Username to use for obtaining access tokens | | x | -| HUB__AUTH_PASSWORD | Password to use for obtaining access tokens | | x | -| MINIO__ENDPOINT | MinIO S3 API endpoint (without scheme) | | x | -| MINIO__ACCESS_KEY | Access key for interacting with MinIO S3 API | | x | -| MINIO__SECRET_KEY | Secret key for interacting with MinIO S3 API | | x | -| MINIO__BUCKET | Name of S3 bucket to store result files in | | x | -| MINIO__REGION | Region of S3 bucket to store result files in | us-east-1 | | -| MINIO__USE_SSL | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0 | | -| OIDC__CERTS_URL | URL to OIDC-complaint JWKS endpoint for validating JWTs | | x | -| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with | client_id | | +| **Environment variable** | **Description** | **Default** | **Required** | +|----------------------------|----------------------------------------------------------|--------------------------------|:------------:| +| HUB__CORE_BASE_URL | Base URL for the FLAME Core API | https://core.privateaim.net | | +| HUB__STORAGE_BASE_URL | Base URL for the FLAME Storage API | https://storage.privateaim.net | | +| HUB__AUTH_BASE_URL | Base URL for the FLAME Auth API | https://auth.privateaim.net | | +| HUB__AUTH_USERNAME | Username to use for obtaining access tokens | | x | +| HUB__AUTH_PASSWORD | Password to use for obtaining access tokens | | x | +| MINIO__ENDPOINT | MinIO S3 API endpoint (without scheme) | | x | +| MINIO__ACCESS_KEY | Access key for interacting with MinIO S3 API | | x | +| MINIO__SECRET_KEY | Secret key for interacting with MinIO S3 API | | x | +| MINIO__BUCKET | Name of S3 bucket to store result files in | | x | +| MINIO__REGION | Region of S3 bucket to store result files in | us-east-1 | | +| MINIO__USE_SSL | Flag for en-/disabling encrypted traffic to MinIO S3 API | 0 | | +| OIDC__CERTS_URL | URL to OIDC-complaint JWKS endpoint for validating JWTs | | x | +| OIDC__CLIENT_ID_CLAIM_NAME | JWT claim to identify authenticated requests with | client_id | | ## Note on running tests diff --git a/project/config.py b/project/config.py index 4446a45..78f3d35 100644 --- a/project/config.py +++ b/project/config.py @@ -25,8 +25,9 @@ class OIDCConfig(BaseModel): class HubConfig(BaseModel): - api_base_url: HttpUrl = "https://api.privateaim.net" + core_base_url: HttpUrl = "https://core.privateaim.net" auth_base_url: HttpUrl = "https://auth.privateaim.net" + storage_base_url: HttpUrl = "https://storage.privateaim.net" auth_username: str auth_password: str diff --git a/project/dependencies.py b/project/dependencies.py index 1c75982..69ae92d 100644 --- a/project/dependencies.py +++ b/project/dependencies.py @@ -12,7 +12,7 @@ from starlette import status from project.config import Settings, MinioBucketConfig -from project.hub import FlamePasswordAuthClient, FlameHubClient +from project.hub import FlamePasswordAuthClient, FlameCoreClient, FlameStorageClient security = HTTPBearer() logger = logging.getLogger(__name__) @@ -118,11 +118,18 @@ def get_auth_client(settings: Annotated[Settings, Depends(get_settings)]): ) -def get_api_client( +def get_core_client( settings: Annotated[Settings, Depends(get_settings)], auth_client: Annotated[FlamePasswordAuthClient, Depends(get_auth_client)], ): - return FlameHubClient( + return FlameCoreClient( auth_client, - base_url=str(settings.hub.api_base_url), + base_url=str(settings.hub.core_base_url), ) + + +def get_storage_client( + settings: Annotated[Settings, Depends(get_settings)], + auth_client: Annotated[FlamePasswordAuthClient, Depends(get_auth_client)], +): + return FlameStorageClient(auth_client, base_url=str(settings.hub.storage_base_url)) diff --git a/project/hub.py b/project/hub.py index bfb5116..6c41e26 100644 --- a/project/hub.py +++ b/project/hub.py @@ -95,7 +95,7 @@ def __init__( self, username: str, password: str, - base_url="https://auth.privateaim.net", + base_url="https://core.privateaim.net", token_expiration_leeway_seconds=60, force_acquire_on_init=False, ): @@ -194,11 +194,11 @@ def get_auth_bearer_header(self): } -class FlameHubClient: +class FlameCoreClient: def __init__( self, auth_client: FlamePasswordAuthClient, - base_url="https://api.privateaim.net", + base_url="https://core.privateaim.net", ): """ Create a new client to interact with the FLAME Hub API. @@ -369,89 +369,6 @@ def get_analysis_by_id(self, analysis_id: str | UUID) -> Analysis | None: r.raise_for_status() return Analysis(**r.json()) - def get_bucket_list(self) -> ResourceList[Bucket]: - """ - Get list of buckets. - - Returns: - list of bucket resources - """ - r = httpx.get( - self._format_url("/storage/buckets"), - headers=self.auth_client.get_auth_bearer_header(), - ) - - r.raise_for_status() - return ResourceList[Bucket](**r.json()) - - def get_bucket_by_id(self, bucket_id: str | UUID) -> Bucket | None: - """ - Get a bucket by its ID. - - Args: - bucket_id: ID of the bucket to get - - Returns: - bucket resource, or *None* if no bucket was found - """ - r = httpx.get( - self._format_url(f"/storage/buckets/{bucket_id}"), - headers=self.auth_client.get_auth_bearer_header(), - ) - - if r.status_code == status.HTTP_404_NOT_FOUND: - return None - - r.raise_for_status() - return Bucket(**r.json()) - - def get_bucket_file_list(self) -> ResourceList[BucketFile]: - """ - Get list of bucket files. - - Returns: - list of bucket file resources - """ - r = httpx.get( - self._format_url("/storage/bucket-files"), - headers=self.auth_client.get_auth_bearer_header(), - ) - - r.raise_for_status() - return ResourceList[BucketFile](**r.json()) - - def upload_to_bucket( - self, - bucket_id: str | UUID, - file_name: str, - file: bytes | BytesIO, - content_type: str = "application/octet-stream", - ) -> ResourceList[BucketFile]: - """ - Upload a single file to a bucket. - - Args: - bucket_id: ID of the bucket to upload the file to - file_name: file name - file: file contents - content_type: content type of the file (*application/octet-stream* by default) - - Returns: - list of bucket file resources for the uploaded file - """ - # wrap into BytesIO if raw bytes are passed in - if isinstance(file, bytes): - file = BytesIO(file) - - r = httpx.post( - self._format_url(f"/storage/buckets/{bucket_id}/upload"), - headers=self.auth_client.get_auth_bearer_header(), - files={"file": (file_name, file, content_type)}, - ) - - r.raise_for_status() - return ResourceList[BucketFile](**r.json()) - def get_analysis_bucket_file_list(self) -> ResourceList[AnalysisBucketFile]: """ Get list of files that have been linked to an analysis. @@ -531,6 +448,121 @@ def link_bucket_file_to_analysis( r.raise_for_status() return AnalysisBucketFile(**r.json()) + +class FlameStorageClient: + def __init__( + self, + auth_client: FlamePasswordAuthClient, + base_url="https://storage.privateaim.net", + ): + """ + Create a new client to interact with the FLAME Storage API. + + Args: + auth_client: FLAME Auth API client to use + base_url: base API url + """ + self.base_url = base_url + self.auth_client = auth_client + + base_url_parts = urllib.parse.urlsplit(base_url) + + self._base_scheme = base_url_parts[0] + self._base_netloc = base_url_parts[1] + self._base_path = base_url_parts[2] + + def _format_url(self, path: str, query: dict[str, str] = None): + return build_url( + self._base_scheme, + self._base_netloc, + urllib.parse.urljoin(self._base_path, path), + query, + "", + ) + + def get_bucket_list(self) -> ResourceList[Bucket]: + """ + Get list of buckets. + + Returns: + list of bucket resources + """ + r = httpx.get( + self._format_url("/buckets"), + headers=self.auth_client.get_auth_bearer_header(), + ) + + r.raise_for_status() + return ResourceList[Bucket](**r.json()) + + def get_bucket_by_id(self, bucket_id: str | UUID) -> Bucket | None: + """ + Get a bucket by its ID. + + Args: + bucket_id: ID of the bucket to get + + Returns: + bucket resource, or *None* if no bucket was found + """ + r = httpx.get( + self._format_url(f"/buckets/{bucket_id}"), + headers=self.auth_client.get_auth_bearer_header(), + ) + + if r.status_code == status.HTTP_404_NOT_FOUND: + return None + + r.raise_for_status() + return Bucket(**r.json()) + + def get_bucket_file_list(self) -> ResourceList[BucketFile]: + """ + Get list of bucket files. + + Returns: + list of bucket file resources + """ + r = httpx.get( + self._format_url("/bucket-files"), + headers=self.auth_client.get_auth_bearer_header(), + ) + + r.raise_for_status() + return ResourceList[BucketFile](**r.json()) + + def upload_to_bucket( + self, + bucket_id: str | UUID, + file_name: str, + file: bytes | BytesIO, + content_type: str = "application/octet-stream", + ) -> ResourceList[BucketFile]: + """ + Upload a single file to a bucket. + + Args: + bucket_id: ID of the bucket to upload the file to + file_name: file name + file: file contents + content_type: content type of the file (*application/octet-stream* by default) + + Returns: + list of bucket file resources for the uploaded file + """ + # wrap into BytesIO if raw bytes are passed in + if isinstance(file, bytes): + file = BytesIO(file) + + r = httpx.post( + self._format_url(f"/buckets/{bucket_id}/upload"), + headers=self.auth_client.get_auth_bearer_header(), + files={"file": (file_name, file, content_type)}, + ) + + r.raise_for_status() + return ResourceList[BucketFile](**r.json()) + def stream_bucket_file(self, bucket_file_id: str | UUID, chunk_size=1024): """ Fetch the contents of a bucket file. @@ -545,7 +577,7 @@ def stream_bucket_file(self, bucket_file_id: str | UUID, chunk_size=1024): """ with httpx.stream( "GET", - self._format_url(f"/storage/bucket-files/{bucket_file_id}/stream"), + self._format_url(f"/bucket-files/{bucket_file_id}/stream"), headers=self.auth_client.get_auth_bearer_header(), ) as r: for b in r.iter_bytes(chunk_size=chunk_size): diff --git a/project/routers/final.py b/project/routers/final.py index ab957ab..7578015 100644 --- a/project/routers/final.py +++ b/project/routers/final.py @@ -12,9 +12,10 @@ get_local_minio, get_settings, get_client_id, - get_api_client, + get_core_client, + get_storage_client, ) -from project.hub import FlameHubClient +from project.hub import FlameCoreClient, FlameStorageClient router = APIRouter() logger = logging.getLogger(__name__) @@ -24,7 +25,8 @@ def __bg_upload_to_remote( minio: Minio, bucket_name: str, object_name: str, - api: FlameHubClient, + core_client: FlameCoreClient, + storage_client: FlameStorageClient, client_id: str, ): logger.info( @@ -40,10 +42,10 @@ def __bg_upload_to_remote( minio_resp = minio.get_object(bucket_name, object_name) # fetch analysis bucket - analysis_bucket = api.get_analysis_bucket(client_id, "RESULT") + analysis_bucket = core_client.get_analysis_bucket(client_id, "RESULT") # upload to remote - bucket_file_lst = api.upload_to_bucket( + bucket_file_lst = storage_client.upload_to_bucket( analysis_bucket.external_id, object_name, io.BytesIO(minio_resp.data), @@ -54,9 +56,9 @@ def __bg_upload_to_remote( assert len(bucket_file_lst.data) == 1 # fetch file s.t. it can be linked to result bucket bucket_file = bucket_file_lst.data[0] - analysis_bucket = api.get_analysis_bucket(client_id, "RESULT") + analysis_bucket = core_client.get_analysis_bucket(client_id, "RESULT") # link file to analysis - api.link_bucket_file_to_analysis( + core_client.link_bucket_file_to_analysis( analysis_bucket.id, bucket_file.id, bucket_file.name ) # remove from local minio @@ -80,7 +82,8 @@ async def submit_final_result_to_hub( background_tasks: BackgroundTasks, settings: Annotated[Settings, Depends(get_settings)], local_minio: Annotated[Minio, Depends(get_local_minio)], - api_client: Annotated[FlameHubClient, Depends(get_api_client)], + core_client: Annotated[FlameCoreClient, Depends(get_core_client)], + storage_client: Annotated[FlameStorageClient, Depends(get_storage_client)], ): """Upload a file as a final result to the FLAME Hub. Returns a 204 on success. @@ -101,6 +104,7 @@ async def submit_final_result_to_hub( local_minio, settings.minio.bucket, object_name, - api_client, + core_client, + storage_client, client_id, ) diff --git a/project/routers/intermediate.py b/project/routers/intermediate.py index bca2e04..6f62d53 100644 --- a/project/routers/intermediate.py +++ b/project/routers/intermediate.py @@ -15,9 +15,10 @@ get_settings, get_local_minio, get_client_id, - get_api_client, + get_core_client, + get_storage_client, ) -from project.hub import FlameHubClient +from project.hub import FlameCoreClient, FlameStorageClient router = APIRouter() logger = logging.getLogger(__name__) @@ -34,7 +35,8 @@ def __bg_upload_to_remote( minio: Minio, bucket_name: str, object_name: str, - api: FlameHubClient, + core_client: FlameCoreClient, + storage_client: FlameStorageClient, client_id: str, object_id: str, ): @@ -48,9 +50,9 @@ def __bg_upload_to_remote( minio_resp = minio.get_object(bucket_name, object_name) # fetch analysis bucket - analysis_bucket = api.get_analysis_bucket(client_id, "TEMP") + analysis_bucket = core_client.get_analysis_bucket(client_id, "TEMP") - bucket_file_lst = api.upload_to_bucket( + bucket_file_lst = storage_client.upload_to_bucket( analysis_bucket.external_id, object_name, io.BytesIO(minio_resp.data), @@ -59,7 +61,7 @@ def __bg_upload_to_remote( assert len(bucket_file_lst.data) == 1 bucket_file = bucket_file_lst.data[0] - api.link_bucket_file_to_analysis( + core_client.link_bucket_file_to_analysis( analysis_bucket.id, bucket_file.id, bucket_file.name ) object_id_to_hub_bucket_dict[object_id] = str(bucket_file.id) @@ -83,7 +85,8 @@ async def submit_intermediate_result_to_hub( settings: Annotated[Settings, Depends(get_settings)], minio: Annotated[Minio, Depends(get_local_minio)], request: Request, - api_client: Annotated[FlameHubClient, Depends(get_api_client)], + core_client: Annotated[FlameCoreClient, Depends(get_core_client)], + storage_client: Annotated[FlameStorageClient, Depends(get_storage_client)], background_tasks: BackgroundTasks, ): """Upload a file as an intermediate result to the FLAME Hub. @@ -107,7 +110,8 @@ async def submit_intermediate_result_to_hub( minio, settings.minio.bucket, object_name, - api_client, + core_client, + storage_client, client_id, object_id, ) @@ -132,7 +136,7 @@ async def submit_intermediate_result_to_hub( ) async def retrieve_intermediate_result_from_hub( object_id: uuid.UUID, - api_client: Annotated[FlameHubClient, Depends(get_api_client)], + storage_client: Annotated[FlameStorageClient, Depends(get_storage_client)], ): """Get an intermediate result as file from the FLAME Hub.""" object_id_str = str(object_id) @@ -149,7 +153,7 @@ async def retrieve_intermediate_result_from_hub( bucket_file_id = object_id_to_hub_bucket_dict[object_id_str] async def _stream_bucket_file(): - for b in api_client.stream_bucket_file(bucket_file_id): + for b in storage_client.stream_bucket_file(bucket_file_id): yield b return StreamingResponse(_stream_bucket_file()) diff --git a/tests/common/env.py b/tests/common/env.py index 22ef003..b10b7d3 100644 --- a/tests/common/env.py +++ b/tests/common/env.py @@ -10,12 +10,16 @@ def __get_env(env_name: str, val_def: str | None = None) -> str: return val -def hub_api_base_url(): - return __get_env("HUB__API_BASE_URL", "https://api.privateaim.net") +def hub_core_base_url(): + return __get_env("HUB__CORE_BASE_URL", "https://core.privateaim.dev") + + +def hub_storage_base_url(): + return __get_env("HUB__STORAGE_BASE_URL", "https://storage.privateaim.dev") def hub_auth_base_url(): - return __get_env("HUB__AUTH_BASE_URL", "https://auth.privateaim.net") + return __get_env("HUB__AUTH_BASE_URL", "https://auth.privateaim.dev") def hub_auth_username(): diff --git a/tests/conftest.py b/tests/conftest.py index 20f5265..be4ba2f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ from jwcrypto import jwk from starlette.testclient import TestClient -from project.hub import FlamePasswordAuthClient, FlameHubClient +from project.hub import FlamePasswordAuthClient, FlameCoreClient, FlameStorageClient from project.server import app from tests.common import env from tests.common.auth import get_oid_test_jwk @@ -67,56 +67,61 @@ def auth_client(): @pytest.fixture(scope="package") -def api_client(auth_client): - return FlameHubClient(auth_client, base_url=env.hub_api_base_url()) +def core_client(auth_client): + return FlameCoreClient(auth_client, base_url=env.hub_core_base_url()) + + +@pytest.fixture(scope="package") +def storage_client(auth_client): + return FlameStorageClient(auth_client, base_url=env.hub_storage_base_url()) @pytest.fixture -def project_id(api_client): +def project_id(core_client): project_name = next_prefixed_name() - project = api_client.create_project(project_name) + project = core_client.create_project(project_name) # check that project was successfully created assert project.name == project_name # check that project can be retrieved - project_get = api_client.get_project_by_id(project.id) + project_get = core_client.get_project_by_id(project.id) assert project_get.id == project.id # check that project appears in list - project_get_list = api_client.get_project_list() + project_get_list = core_client.get_project_list() assert any([p.id == project.id for p in project_get_list.data]) yield project.id # check that project can be deleted - api_client.delete_project(project.id) + core_client.delete_project(project.id) # check that project is no longer found - assert api_client.get_project_by_id(project.id) is None + assert core_client.get_project_by_id(project.id) is None @pytest.fixture -def analysis_id(api_client, project_id): +def analysis_id(core_client, project_id): analysis_name = next_prefixed_name() - analysis = api_client.create_analysis(analysis_name, project_id) + analysis = core_client.create_analysis(analysis_name, project_id) # check that analysis was created assert analysis.name == analysis_name assert analysis.project_id == project_id # check that GET on analysis works - analysis_get = api_client.get_analysis_by_id(analysis.id) + analysis_get = core_client.get_analysis_by_id(analysis.id) assert analysis_get.id == analysis.id # check that analysis appears in list - analysis_get_list = api_client.get_analysis_list() + analysis_get_list = core_client.get_analysis_list() assert any([a.id == analysis.id for a in analysis_get_list.data]) yield analysis.id # check that DELETE analysis works - api_client.delete_analysis(analysis.id) + core_client.delete_analysis(analysis.id) # check that analysis is no longer found - assert api_client.get_analysis_by_id(analysis.id) is None + assert core_client.get_analysis_by_id(analysis.id) is None diff --git a/tests/test_final.py b/tests/test_final.py index cf4bc27..f60963b 100644 --- a/tests/test_final.py +++ b/tests/test_final.py @@ -8,8 +8,8 @@ pytestmark = pytest.mark.live -def test_200_submit_to_upload(test_client, rng, api_client, analysis_id): - analysis_file_count_old = len(api_client.get_analysis_bucket_file_list().data) +def test_200_submit_to_upload(test_client, rng, core_client, analysis_id): + analysis_file_count_old = len(core_client.get_analysis_bucket_file_list().data) blob = next_random_bytes(rng) r = test_client.put( @@ -21,7 +21,7 @@ def test_200_submit_to_upload(test_client, rng, api_client, analysis_id): assert r.status_code == status.HTTP_204_NO_CONTENT def __check_analysis_file_count_increases(): - analysis_file_count_new = len(api_client.get_analysis_bucket_file_list().data) + analysis_file_count_new = len(core_client.get_analysis_bucket_file_list().data) return analysis_file_count_new > analysis_file_count_old assert eventually(__check_analysis_file_count_increases) diff --git a/tests/test_hub.py b/tests/test_hub.py index 3df399b..a7760a8 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -20,14 +20,14 @@ def test_auth_no_reissue(auth_client): @pytest.fixture -def result_bucket_id(analysis_id, api_client): +def result_bucket_id(analysis_id, core_client, storage_client): bucket_types: tuple[BucketType, ...] = ("CODE", "TEMP", "RESULT") # check that buckets are eventually created (happens asynchronously) def _check_buckets_exist(): for bucket_type in bucket_types: - analysis_bucket = api_client.get_analysis_bucket(analysis_id, bucket_type) - bucket = api_client.get_bucket_by_id(analysis_bucket.external_id) + analysis_bucket = core_client.get_analysis_bucket(analysis_id, bucket_type) + bucket = storage_client.get_bucket_by_id(analysis_bucket.external_id) if bucket is None: return False @@ -37,16 +37,16 @@ def _check_buckets_exist(): assert eventually(_check_buckets_exist) # bucket id is referenced from analysis bucket by its external_id prop - yield api_client.get_analysis_bucket(analysis_id, "RESULT").external_id + yield core_client.get_analysis_bucket(analysis_id, "RESULT").external_id @pytest.fixture -def uploaded_bucket_file(result_bucket_id, api_client, rng): +def uploaded_bucket_file(result_bucket_id, storage_client, rng): file_name = next_prefixed_name() file_blob = next_random_bytes(rng) # check that bucket file is created - bucket_file_created_list = api_client.upload_to_bucket( + bucket_file_created_list = storage_client.upload_to_bucket( result_bucket_id, file_name, file_blob ) assert len(bucket_file_created_list.data) == 1 @@ -57,19 +57,19 @@ def uploaded_bucket_file(result_bucket_id, api_client, rng): assert bucket_file.size == len(file_blob) # check that bucket file appears in list - bucket_file_list = api_client.get_bucket_file_list() + bucket_file_list = storage_client.get_bucket_file_list() assert any([bf.id == bucket_file.id for bf in bucket_file_list.data]) yield file_blob, bucket_file -def test_link_bucket_file_to_analysis(uploaded_bucket_file, analysis_id, api_client): +def test_link_bucket_file_to_analysis(uploaded_bucket_file, analysis_id, core_client): _, bucket_file = uploaded_bucket_file - analysis_bucket = api_client.get_analysis_bucket(analysis_id, "RESULT") + analysis_bucket = core_client.get_analysis_bucket(analysis_id, "RESULT") # check that the analysis file was created - analysis_file = api_client.link_bucket_file_to_analysis( + analysis_file = core_client.link_bucket_file_to_analysis( analysis_bucket.id, bucket_file.id, bucket_file.name ) @@ -77,14 +77,14 @@ def test_link_bucket_file_to_analysis(uploaded_bucket_file, analysis_id, api_cli assert analysis_file.external_id == bucket_file.id # check that it appears in the list - analysis_file_list = api_client.get_analysis_bucket_file_list() + analysis_file_list = core_client.get_analysis_bucket_file_list() assert any([af.id == analysis_file.id for af in analysis_file_list.data]) -def test_stream_bucket_file(uploaded_bucket_file, api_client): +def test_stream_bucket_file(uploaded_bucket_file, storage_client): file_blob, bucket_file = uploaded_bucket_file # default chunk size is 1024 and the blobs in these tests are 16 bytes large, so one call to next() # should fetch the blob in its entirety from hub - remote_file_blob = next(api_client.stream_bucket_file(bucket_file.id)) + remote_file_blob = next(storage_client.stream_bucket_file(bucket_file.id)) assert file_blob == remote_file_blob From e061dd06f0594061d9931be684379fb6b5e23b96 Mon Sep 17 00:00:00 2001 From: Maximilian Jugl Date: Thu, 18 Jul 2024 15:40:06 +0200 Subject: [PATCH 2/2] fix: change default base URL of FLAME password auth client --- project/hub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/hub.py b/project/hub.py index 6c41e26..d32506e 100644 --- a/project/hub.py +++ b/project/hub.py @@ -95,7 +95,7 @@ def __init__( self, username: str, password: str, - base_url="https://core.privateaim.net", + base_url="https://auth.privateaim.net", token_expiration_leeway_seconds=60, force_acquire_on_init=False, ):