Skip to content

Commit

Permalink
refactored getting model location function
Browse files Browse the repository at this point in the history
this was done because the bucket name in the main module config can be different for how the manifest file is compiled
  • Loading branch information
kiranvasudev committed Nov 29, 2023
1 parent eaf4286 commit ab9a2c8
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions dagger/utilities/dbt_config_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def _get_s3_task(self, node: dict) -> dict:
table = node.get("name", "")
task["name"] = f"{schema}__{table}_s3"
task["bucket"] = self._default_data_bucket
task["path"] = self._get_model_data_location(node, schema, table)
task["path"] = self._get_model_data_location(node, schema, table)[1]

return task

Expand Down Expand Up @@ -155,7 +155,7 @@ def _generate_dagger_tasks(

def _get_model_data_location(
self, node: dict, schema: str, dbt_model_name: str
) -> str:
) -> Tuple[str, str]:
"""
Gets the S3 path of the dbt model relative to the data bucket.
If external location is not specified in the DBT model config, then the default data directory from the
Expand All @@ -173,7 +173,10 @@ def _get_model_data_location(
if not location:
location = join(self._default_data_dir, schema, dbt_model_name)

return location.split(self._default_data_bucket)[1].lstrip("/")
split = location.split("//")[1].split("/")
bucket_name, data_path = split[0], "/".join(split[1:])

return bucket_name, data_path

def generate_dagger_io(self, model_name: str) -> Tuple[list, list]:
"""
Expand Down

0 comments on commit ab9a2c8

Please sign in to comment.