Skip to content

Commit

Permalink
⬆️Migration: Storage (#6599)
Browse files Browse the repository at this point in the history
  • Loading branch information
sanderegg authored Oct 25, 2024
1 parent 82c69d0 commit 7410516
Show file tree
Hide file tree
Showing 46 changed files with 682 additions and 411 deletions.
11 changes: 5 additions & 6 deletions packages/aws-library/src/aws_library/s3/_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from boto3.s3.transfer import TransferConfig
from botocore import exceptions as botocore_exc
from botocore.client import Config
from common_library.pydantic_networks_extension import AnyUrlLegacy
from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart
from models_library.basic_types import SHA256Str
from pydantic import AnyUrl, ByteSize, TypeAdapter
Expand Down Expand Up @@ -255,7 +254,7 @@ async def create_single_presigned_download_link(
bucket: S3BucketName,
object_key: S3ObjectKey,
expiration_secs: int,
) -> str:
) -> AnyUrl:
# NOTE: ensure the bucket/object exists, this will raise if not
await self._client.head_bucket(Bucket=bucket)
await self._client.head_object(Bucket=bucket, Key=object_key)
Expand All @@ -264,20 +263,20 @@ async def create_single_presigned_download_link(
Params={"Bucket": bucket, "Key": object_key},
ExpiresIn=expiration_secs,
)
return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}"
return TypeAdapter(AnyUrl).validate_python(generated_link)

@s3_exception_handler(_logger)
async def create_single_presigned_upload_link(
self, *, bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int
) -> str:
) -> AnyUrl:
# NOTE: ensure the bucket/object exists, this will raise if not
await self._client.head_bucket(Bucket=bucket)
generated_link = await self._client.generate_presigned_url(
"put_object",
Params={"Bucket": bucket, "Key": object_key},
ExpiresIn=expiration_secs,
)
return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}"
return TypeAdapter(AnyUrl).validate_python(generated_link)

@s3_exception_handler(_logger)
async def create_multipart_upload_links(
Expand Down Expand Up @@ -474,6 +473,6 @@ def is_multipart(file_size: ByteSize) -> bool:

@staticmethod
def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl:
return TypeAdapter(AnyUrlLegacy).validate_python(
return TypeAdapter(AnyUrl).validate_python(
f"s3://{bucket}/{urllib.parse.quote(object_key)}"
)
12 changes: 8 additions & 4 deletions packages/aws-library/tests/test_s3_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -742,7 +742,7 @@ async def test_create_single_presigned_upload_link(
create_file_of_size: Callable[[ByteSize], Path],
default_expiration_time_seconds: int,
upload_to_presigned_link: Callable[
[Path, str, S3BucketName, S3ObjectKey], Awaitable[None]
[Path, AnyUrl, S3BucketName, S3ObjectKey], Awaitable[None]
],
):
file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1Mib"))
Expand Down Expand Up @@ -1289,20 +1289,24 @@ def test_is_multipart(file_size: ByteSize, expected_multipart: bool):
(
"some-bucket",
"an/object/separate/by/slashes",
"s3://some-bucket/an/object/separate/by/slashes",
TypeAdapter(AnyUrl).validate_python(
"s3://some-bucket/an/object/separate/by/slashes"
),
),
(
"some-bucket",
"an/object/separate/by/slashes-?/3#$",
r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24",
TypeAdapter(AnyUrl).validate_python(
r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24"
),
),
],
)
def test_compute_s3_url(
bucket: S3BucketName, object_key: S3ObjectKey, expected_s3_url: AnyUrl
):
assert (
str(SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key))
SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key)
== expected_s3_url
)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
from typing import Annotated, TypeAlias

from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, HttpUrl
from pydantic import AfterValidator, AnyHttpUrl, HttpUrl
from pydantic_core import Url


def _strip_last_slash(url: Url) -> str:
return f"{url}".rstrip("/")


AnyUrlLegacy: TypeAlias = Annotated[
AnyUrl,
AfterValidator(_strip_last_slash),
]

AnyHttpUrlLegacy: TypeAlias = Annotated[
AnyHttpUrl,
AfterValidator(_strip_last_slash),
Expand Down
25 changes: 14 additions & 11 deletions packages/models-library/src/models_library/api_schemas_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from datetime import datetime
from enum import Enum
from typing import Annotated, Any, TypeAlias
from typing import Annotated, Any, Literal, Self, TypeAlias
from uuid import UUID

from pydantic import (
Expand Down Expand Up @@ -107,6 +107,10 @@ class DatasetMetaDataGet(BaseModel):
)


UNDEFINED_SIZE_TYPE: TypeAlias = Literal[-1]
UNDEFINED_SIZE: UNDEFINED_SIZE_TYPE = -1


# /locations/{location_id}/files/metadata:
# /locations/{location_id}/files/{file_id}/metadata:
class FileMetaDataGet(BaseModel):
Expand All @@ -130,8 +134,8 @@ class FileMetaDataGet(BaseModel):
)
created_at: datetime
last_modified: datetime
file_size: ByteSize | int = Field(
default=-1, description="File size in bytes (-1 means invalid)"
file_size: UNDEFINED_SIZE_TYPE | ByteSize = Field(
default=UNDEFINED_SIZE, description="File size in bytes (-1 means invalid)"
)
entity_tag: ETag | None = Field(
default=None,
Expand All @@ -149,7 +153,7 @@ class FileMetaDataGet(BaseModel):
)

model_config = ConfigDict(
extra="forbid",
extra="ignore",
from_attributes=True,
json_schema_extra={
"examples": [
Expand Down Expand Up @@ -312,19 +316,18 @@ class FoldersBody(BaseModel):
nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict)

@model_validator(mode="after")
@classmethod
def ensure_consistent_entries(cls, values):
source_node_keys = (NodeID(n) for n in values["source"].get("workbench", {}))
if set(source_node_keys) != set(values["nodes_map"].keys()):
def ensure_consistent_entries(self) -> Self:
source_node_keys = (NodeID(n) for n in self.source.get("workbench", {}))
if set(source_node_keys) != set(self.nodes_map.keys()):
msg = "source project nodes do not fit with nodes_map entries"
raise ValueError(msg)
destination_node_keys = (
NodeID(n) for n in values["destination"].get("workbench", {})
NodeID(n) for n in self.destination.get("workbench", {})
)
if set(destination_node_keys) != set(values["nodes_map"].values()):
if set(destination_node_keys) != set(self.nodes_map.values()):
msg = "destination project nodes do not fit with nodes_map values"
raise ValueError(msg)
return values
return self


class SoftCopyBody(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import orjson
from pydantic import NameEmail, SecretBytes, SecretStr
from pydantic_core import Url
from pydantic_extra_types.color import Color


Expand Down Expand Up @@ -84,6 +85,7 @@ def decimal_encoder(dec_value: Decimal) -> int | float:
SecretBytes: str,
SecretStr: str,
set: list,
Url: str,
UUID: str,
}

Expand Down
11 changes: 5 additions & 6 deletions packages/settings-library/src/settings_library/node_ports.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from datetime import timedelta
from typing import Final
from typing import Final, Self

from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator

Expand All @@ -22,14 +22,13 @@ def auth_required(self) -> bool:
return self.STORAGE_USERNAME is not None and self.STORAGE_PASSWORD is not None

@model_validator(mode="after")
@classmethod
def _validate_auth_fields(cls, values):
username = values.STORAGE_USERNAME
password = values.STORAGE_PASSWORD
def _validate_auth_fields(self) -> Self:
username = self.STORAGE_USERNAME
password = self.STORAGE_PASSWORD
if (username is None) != (password is None):
msg = f"Both {username=} and {password=} must be either set or unset!"
raise ValueError(msg)
return values
return self


class NodePortsSettings(BaseCustomSettings):
Expand Down
4 changes: 3 additions & 1 deletion packages/settings-library/src/settings_library/utils_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import logging
import os
from collections.abc import Callable
from enum import Enum
from pprint import pformat
from typing import Any

Expand Down Expand Up @@ -64,7 +65,8 @@ def print_as_envfile(

if verbose and field.description:
typer.echo(f"# {field.description}")

if isinstance(value, Enum):
value = value.value
typer.echo(f"{name}={value}")


Expand Down
11 changes: 1 addition & 10 deletions packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from models_library.projects import ProjectID
from models_library.projects_nodes_io import NodeID, StorageFileID
from models_library.users import UserID
from pydantic import ByteSize, TypeAdapter
from pydantic import TypeAdapter
from servicelib.archiving_utils import unarchive_dir
from servicelib.logging_utils import log_context
from servicelib.progress_bar import ProgressBarData
Expand Down Expand Up @@ -184,15 +184,6 @@ async def _delete_legacy_archive(
)


async def get_remote_size(
*, user_id: UserID, project_id: ProjectID, node_uuid: NodeID, source_path: Path
) -> ByteSize:
s3_object = __create_s3_object_key(project_id, node_uuid, source_path)
return await filemanager.get_path_size(
user_id=user_id, store_id=SIMCORE_LOCATION, s3_object=s3_object
)


async def push(
user_id: UserID,
project_id: ProjectID,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from models_library.basic_types import IDStr, SHA256Str
from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
from models_library.users import UserID
from pydantic import AnyUrl, ByteSize, TypeAdapter, parse_obj_as
from pydantic import AnyUrl, ByteSize, TypeAdapter
from servicelib.file_utils import create_sha256_checksum
from servicelib.progress_bar import ProgressBarData
from settings_library.aws_s3_cli import AwsS3CliSettings
Expand Down Expand Up @@ -189,7 +189,9 @@ async def download_path_from_s3(
aws_s3_cli_settings,
progress_bar,
local_directory_path=local_path,
download_s3_link=TypeAdapter(AnyUrl).validate_python(f"{download_link}"),
download_s3_link=TypeAdapter(AnyUrl).validate_python(
f"{download_link}"
),
)
elif r_clone_settings:
await r_clone.sync_s3_to_local(
Expand Down Expand Up @@ -572,21 +574,6 @@ async def get_file_metadata(
)


async def get_path_size(
user_id: UserID,
store_id: LocationID,
s3_object: StorageFileID,
client_session: ClientSession | None = None,
) -> ByteSize:
file_metadata: FileMetaDataGet = await _get_file_meta_data(
user_id=user_id,
store_id=store_id,
s3_object=s3_object,
client_session=client_session,
)
return ByteSize(file_metadata.file_size)


async def delete_file(
user_id: UserID,
store_id: LocationID,
Expand Down
4 changes: 2 additions & 2 deletions scripts/release/monitor_release/settings.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os
from pathlib import Path
from typing import Final
from typing import Final, Self

from dotenv import load_dotenv
from pydantic import BaseModel, Field, HttpUrl, TypeAdapter, model_validator
Expand Down Expand Up @@ -47,7 +47,7 @@ class ReleaseSettings(BaseSettings):
model_config = SettingsConfigDict(extra="ignore")

@model_validator(mode="after")
def deduce_portainer_url(self):
def deduce_portainer_url(self) -> Self:
self.portainer_url = TypeAdapter(HttpUrl).validate_python(
f"https://{self.PORTAINER_DOMAIN}"
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import re
from typing import Self

from models_library.basic_regex import SIMPLE_VERSION_RE
from models_library.services import ServiceMetaDataPublished
Expand Down Expand Up @@ -49,7 +50,7 @@ class ContainerHostConfig(BaseModel):
)

@model_validator(mode="after")
def ensure_memory_swap_is_not_unlimited(self) -> "ContainerHostConfig":
def ensure_memory_swap_is_not_unlimited(self) -> Self:
if self.memory_swap is None:
self.memory_swap = self.memory

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,12 @@ async def export_service_runs(
)

# Create presigned S3 link
generated_url: str = await s3_client.create_single_presigned_download_link(
generated_url = await s3_client.create_single_presigned_download_link(
bucket=s3_bucket_name,
object_key=s3_object_key,
expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC,
)
return generated_url
return f"{generated_url}"


async def get_osparc_credits_aggregated_usages_page(
Expand Down
Loading

0 comments on commit 7410516

Please sign in to comment.