Skip to content

Commit

Permalink
Define layout parameter callback types and formatter types
Browse files Browse the repository at this point in the history
  • Loading branch information
sultaniman committed Mar 26, 2024
1 parent 8ce90d6 commit 5543cd4
Showing 1 changed file with 11 additions and 2 deletions.
13 changes: 11 additions & 2 deletions dlt/common/storages/configuration.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
from typing import TYPE_CHECKING, Any, Literal, Optional, Type, get_args, ClassVar, Dict, Union
from typing import Any, Callable, ClassVar, Dict, Literal, Optional, Type, get_args, Union
from urllib.parse import urlparse
from typing_extensions import TypeAlias

from dlt.common.configuration import configspec, resolve_type
from dlt.common.configuration.exceptions import ConfigurationValueError
Expand All @@ -13,12 +14,18 @@
AzureCredentialsWithoutDefaults,
BaseConfiguration,
)

from dlt.common.typing import DictStrAny
from dlt.common.schema import Schema
from dlt.common.utils import digest128

from pendulum import datetime

TSchemaFileFormat = Literal["json", "yaml"]
SchemaFileExtensions = get_args(TSchemaFileFormat)
TDatetimeFormatter: TypeAlias = Callable[[datetime], str]
TDatetimeFormat: TypeAlias = Union[str, TDatetimeFormatter]
TLayoutParamCallback: TypeAlias = Callable[[Schema, datetime], str]


@configspec
Expand Down Expand Up @@ -74,7 +81,9 @@ class FilesystemConfiguration(BaseConfiguration):
"adl": Union[AzureCredentialsWithoutDefaults, AzureCredentials],
}

bucket_url: str = None
bucket_url: Optional[str] = None
datetime_format: Optional[TDatetimeFormat] = None
layout_params: Optional[Dict[str, Union[str, TLayoutParamCallback]]] = None

# should be a union of all possible credentials as found in PROTOCOL_CREDENTIALS
credentials: FileSystemCredentials = None
Expand Down

0 comments on commit 5543cd4

Please sign in to comment.