Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replacing string replacement with jinja in module processor #40

Merged
merged 1 commit into from
Jun 12, 2024
Merged

Replacing string replacement with jinja in module processor #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions dagger/utilities/dbt_config_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@ def __init__(self, config_parameters: dict):
self._nodes_in_manifest = self._manifest_data.get("nodes", {})
self._sources_in_manifest = self._manifest_data.get("sources", {})

@property
def nodes_in_manifest(self):
return self._nodes_in_manifest

@property
def sources_in_manifest(self):
return self._sources_in_manifest

@property
def dbt_default_schema(self):
return self._default_schema

def _get_manifest_path(self) -> str:
"""
Construct path for manifest.json file based on configuration parameters.
Expand Down
28 changes: 17 additions & 11 deletions dagger/utilities/module.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
DatabricksDBTConfigParser,
)

import jinja2

import yaml

_logger = logging.getLogger("root")
Expand Down Expand Up @@ -48,19 +50,13 @@ def read_task_config(self, task):

@staticmethod
def replace_template_parameters(_task_str, _template_parameters):
for _key, _value in _template_parameters.items():
if type(_value) == str:
try:
int_value = int(_value)
_value = f'"{_value}"'
except:
pass
locals()[_key] = _value
environment = jinja2.Environment()
template = environment.from_string(_task_str)
rendered_task = template.render(_template_parameters)

return (
_task_str.format(**locals())
.replace("{", "{{")
.replace("}", "}}")
rendered_task
# TODO Remove this hack and use Jinja escaping instead of special expression in template files
.replace("__CBS__", "{")
.replace("__CBE__", "}")
)
Expand All @@ -79,12 +75,22 @@ def generate_task_configs(self):
template_parameters = {}
template_parameters.update(self._default_parameters or {})
template_parameters.update(attrs)
template_parameters['branch_name'] = branch_name

dbt_manifest = None
if "dbt" in self._tasks.keys():
if template_parameters.get("profile_name") == "athena":
self._dbt_module = AthenaDBTConfigParser(template_parameters)
if template_parameters.get("profile_name") == "databricks":
self._dbt_module = DatabricksDBTConfigParser(template_parameters)

dbt_manifest = {}
dbt_manifest['nodes'] = self._dbt_module.nodes_in_manifest
dbt_manifest['sources'] = self._dbt_module.sources_in_manifest

template_parameters["dbt_manifest"] = dbt_manifest
template_parameters["dbt_default_schema"] = self._dbt_module.dbt_default_schema

for task, task_yaml in self._tasks.items():
task_name = f"{branch_name}_{task}"
_logger.info(f"Generating task {task_name}")
Expand Down
Loading