Skip to content

Commit

Permalink
⬆️ Maintenance: Datcore-adapter grand update (#5600)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanderegg
sanderegg authored Apr 2, 2024
1 parent 795db0f commit d73165d
Show file tree
Hide file tree
Showing 7 changed files with 173 additions and 338 deletions.
313 changes: 82 additions & 231 deletions services/datcore-adapter/requirements/_base.txt
View file

Large diffs are not rendered by default.

57 changes: 30 additions & 27 deletions services/datcore-adapter/requirements/_test.txt
View file
Original file line number Diff line number Diff line change
@@ -1,44 +1,46 @@
anyio==3.6.2
# via httpcore
anyio==4.3.0
# via httpx
asgi-lifespan==2.1.0
certifi==2023.7.22
certifi==2024.2.2
# via
# httpcore
# httpx
# requests
charset-normalizer==3.1.0
charset-normalizer==3.3.2
# via requests
coverage==7.3.2
coverage==7.4.4
# via pytest-cov
exceptiongroup==1.1.3
# via pytest
exceptiongroup==1.2.0
# via
# anyio
# pytest
execnet==2.0.2
# via pytest-xdist
faker==19.13.0
h11==0.12.0
faker==24.4.0
h11==0.14.0
# via httpcore
httpcore==0.15.0
httpcore==1.0.5
# via httpx
httpx==0.24.0
httpx==0.27.0
# via respx
icdiff==2.0.7
# via pytest-icdiff
idna==3.4
idna==3.6
# via
# anyio
# httpx
# requests
iniconfig==2.0.0
# via pytest
packaging==23.2
packaging==24.0
# via
# pytest
# pytest-sugar
pluggy==1.3.0
pluggy==1.4.0
# via pytest
pprintpp==0.4.0
# via pytest-icdiff
pytest==7.4.3
pytest==8.1.1
# via
# pytest-asyncio
# pytest-cov
Expand All @@ -48,30 +50,31 @@ pytest==7.4.3
# pytest-sugar
# pytest-xdist
pytest-asyncio==0.21.1
pytest-cov==4.1.0
pytest-icdiff==0.8
pytest-cov==5.0.0
pytest-icdiff==0.9
pytest-instafail==0.5.0
pytest-mock==3.12.0
pytest-runner==6.0.0
pytest-sugar==0.9.7
pytest-xdist==3.3.1
python-dateutil==2.8.2
pytest-mock==3.14.0
pytest-runner==6.0.1
pytest-sugar==1.0.0
pytest-xdist==3.5.0
python-dateutil==2.9.0.post0
# via faker
requests==2.31.0
respx==0.20.2
respx==0.21.1
six==1.16.0
# via python-dateutil
sniffio==1.3.0
sniffio==1.3.1
# via
# anyio
# asgi-lifespan
# httpcore
# httpx
termcolor==2.3.0
termcolor==2.4.0
# via pytest-sugar
tomli==2.0.1
# via
# coverage
# pytest
urllib3==1.26.12
typing-extensions==4.10.0
# via anyio
urllib3==2.2.1
# via requests
62 changes: 21 additions & 41 deletions services/datcore-adapter/requirements/_tools.txt
View file
Original file line number Diff line number Diff line change
@@ -1,74 +1,56 @@
aiohttp==3.8.5
# via black
aiosignal==1.3.1
# via aiohttp
astroid==3.0.2
astroid==3.1.0
# via pylint
async-timeout==4.0.2
# via aiohttp
attrs==21.4.0
# via aiohttp
black==23.12.0
build==1.0.3
black==24.3.0
build==1.2.1
# via pip-tools
bump2version==1.0.1
cfgv==3.4.0
# via pre-commit
charset-normalizer==3.1.0
# via aiohttp
click==8.1.3
click==8.1.7
# via
# black
# pip-tools
dill==0.3.7
dill==0.3.8
# via pylint
distlib==0.3.8
# via virtualenv
filelock==3.13.1
filelock==3.13.3
# via virtualenv
frozenlist==1.3.3
# via
# aiohttp
# aiosignal
identify==2.5.33
identify==2.5.35
# via pre-commit
idna==3.4
# via yarl
isort==5.13.2
# via pylint
mccabe==0.7.0
# via pylint
multidict==6.0.3
# via
# aiohttp
# yarl
mypy-extensions==1.0.0
# via black
nodeenv==1.8.0
# via pre-commit
packaging==23.2
packaging==24.0
# via
# black
# build
pathspec==0.12.1
# via black
pip==24.0
# via pip-tools
pip-tools==7.3.0
platformdirs==4.1.0
pip-tools==7.4.1
platformdirs==4.2.0
# via
# black
# pylint
# virtualenv
pre-commit==3.6.0
pylint==3.0.3
pre-commit==3.7.0
pylint==3.1.0
pyproject-hooks==1.0.0
# via build
# via
# build
# pip-tools
pyyaml==6.0.1
# via
# pre-commit
# watchdog
ruff==0.1.8
ruff==0.3.5
setuptools==69.2.0
# via
# nodeenv
Expand All @@ -80,16 +62,14 @@ tomli==2.0.1
# pip-tools
# pylint
# pyproject-hooks
tomlkit==0.12.3
tomlkit==0.12.4
# via pylint
typing-extensions==4.4.0
typing-extensions==4.10.0
# via
# astroid
# black
virtualenv==20.25.0
virtualenv==20.25.1
# via pre-commit
watchdog==3.0.0
wheel==0.42.0
watchdog==4.0.0
wheel==0.43.0
# via pip-tools
yarl==1.9.2
# via aiohttp
41 changes: 20 additions & 21 deletions services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/datasets.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import logging
from typing import Final
from typing import Annotated, Final

from aiocache import cached
from fastapi import APIRouter, Depends, Header, Request
Expand Down Expand Up @@ -35,10 +35,10 @@
)
async def list_datasets(
request: Request,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
params: Params = Depends(),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
params: Annotated[Params, Depends()],
) -> Page[DatasetsOut]:
assert request # nosec
raw_params: RawParams = resolve_params(params).to_raw_params()
Expand All @@ -48,7 +48,7 @@ async def list_datasets(
limit=raw_params.limit,
offset=raw_params.offset,
)
return create_page(items=datasets, total=total, params=params)
return create_page(datasets, total=total, params=params)


@router.get(
Expand All @@ -65,10 +65,10 @@ async def list_datasets(
async def list_dataset_top_level_files(
request: Request,
dataset_id: str,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
params: Params = Depends(),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
params: Annotated[Params, Depends()],
) -> Page[FileMetaDataOut]:
assert request # nosec
raw_params: RawParams = resolve_params(params).to_raw_params()
Expand All @@ -80,7 +80,7 @@ async def list_dataset_top_level_files(
limit=raw_params.limit,
offset=raw_params.offset,
)
return create_page(items=file_metas, total=total, params=params)
return create_page(file_metas, total=total, params=params)


@router.get(
Expand All @@ -98,10 +98,10 @@ async def list_dataset_collection_files(
request: Request,
dataset_id: str,
collection_id: str,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
params: Params = Depends(),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
params: Annotated[Params, Depends()],
) -> Page[FileMetaDataOut]:
assert request # nosec
raw_params: RawParams = resolve_params(params).to_raw_params()
Expand All @@ -114,7 +114,7 @@ async def list_dataset_collection_files(
dataset_id=dataset_id,
collection_id=collection_id,
)
return create_page(items=file_metas, total=total, params=params)
return create_page(file_metas, total=total, params=params)


@router.get(
Expand All @@ -131,14 +131,13 @@ async def list_dataset_collection_files(
async def list_dataset_files_legacy(
request: Request,
dataset_id: str,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
) -> list[FileMetaDataOut]:
assert request # nosec
file_metas = await pennsieve_client.list_all_dataset_files(
return await pennsieve_client.list_all_dataset_files(
api_key=x_datcore_api_key,
api_secret=x_datcore_api_secret,
dataset_id=dataset_id,
)
return file_metas
20 changes: 10 additions & 10 deletions services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/files.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import logging
from typing import Any
from typing import Annotated, Any

from fastapi import APIRouter, Depends, Header, Request
from pydantic import AnyUrl, parse_obj_as
Expand All @@ -24,9 +24,9 @@
async def download_file(
request: Request,
file_id: str,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
) -> FileDownloadOut:
assert request # nosec
presigned_download_link = await pennsieve_client.get_presigned_download_link(
Expand All @@ -44,9 +44,9 @@ async def download_file(
async def delete_file(
request: Request,
file_id: str,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
):
assert request # nosec
await pennsieve_client.delete_object(
Expand All @@ -66,9 +66,9 @@ async def delete_file(
async def get_package(
request: Request,
package_id: str,
x_datcore_api_key: str = Header(..., description="Datcore API Key"),
x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
) -> list[dict[str, Any]]:
assert request # nosec
return await pennsieve_client.get_package_files(
Expand Down
11 changes: 6 additions & 5 deletions services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/health.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from datetime import datetime
from typing import Callable
from collections.abc import Callable
from datetime import datetime, timezone
from typing import Annotated

from fastapi import APIRouter, Depends
from models_library.app_diagnostics import AppStatusCheck
Expand All @@ -23,13 +24,13 @@
status_code=status.HTTP_200_OK,
)
async def get_service_alive():
return f"{__name__}@{datetime.utcnow().isoformat()}"
return f"{__name__}@{datetime.now(timezone.utc).isoformat()}"


@router.get("/ready", status_code=status.HTTP_200_OK, response_model=AppStatusCheck)
async def get_service_ready(
pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
url_for: Callable = Depends(get_reverse_url_mapper),
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
url_for: Annotated[Callable, Depends(get_reverse_url_mapper)],
):
pennsieve_health_ok = await pennsieve_client.is_responsive()
return AppStatusCheck(
Expand Down
Loading

0 comments on commit d73165d

Please sign in to comment.