Skip to content

Commit

Permalink
some more
Browse files Browse the repository at this point in the history
  • Loading branch information
cdkini committed Dec 19, 2024
1 parent 5c81607 commit 81454a3
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,9 @@ def checkpoint_store(self) -> CheckpointStore:
@property
@public_api
def data_sources(self) -> DataSourceManager:
"""
Responsible for basic CRUD operations on a context's DataSources.
"""
return self._data_sources

@property
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,16 @@ class DirectoryDataAsset(PathDataAsset[DatasourceT, ColumnPartitioner], Generic[

@public_api
def add_batch_definition_daily(self, name: str, column: str) -> BatchDefinition:
"""
Add a BatchDefinition which creates a single batch for each day in the directory.
Args:
name: Name of the batch definition.
column: Column to partition on.
Returns:
A BatchDefinition that is partitioned daily.
"""
# todo: test column
return self.add_batch_definition(
name=name,
Expand All @@ -55,6 +65,16 @@ def add_batch_definition_daily(self, name: str, column: str) -> BatchDefinition:

@public_api
def add_batch_definition_monthly(self, name: str, column: str) -> BatchDefinition:
"""
Add a BatchDefinition which creates a single batch for each month in the directory.
Args:
name: Name of the batch definition.
column: Column to partition on.
Returns:
A BatchDefinition that is partitioned monthly.
"""
# todo: test column
return self.add_batch_definition(
name=name,
Expand All @@ -65,6 +85,16 @@ def add_batch_definition_monthly(self, name: str, column: str) -> BatchDefinitio

@public_api
def add_batch_definition_yearly(self, name: str, column: str) -> BatchDefinition:
"""
Add a BatchDefinition which creates a single batch for each year in the directory.
Args:
name: Name of the batch definition.
column: Column to partition on.
Returns:
A BatchDefinition that is partitioned year.
"""
# todo: test column
return self.add_batch_definition(
name=name,
Expand Down

0 comments on commit 81454a3

Please sign in to comment.