generate inputs for intermediate models and updated tests

chocoapp · Nov 29, 2023 · 65a07d7 · 65a07d7
1 parent 8ef7050
commit 65a07d7
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 8 deletions.
diff --git a/dagger/utilities/dbt_config_parser.py b/dagger/utilities/dbt_config_parser.py
@@ -202,11 +202,10 @@ def generate_dagger_io(self, model_name: str) -> Tuple[list, list]:
         parent_node_names = model_node.get("depends_on", {}).get("nodes", [])
 
         for index, parent_node_name in enumerate(parent_node_names):
-            if not (".int_" in parent_node_name):
-                parent_model_node = self._nodes_in_manifest.get(parent_node_name)
-                dagger_input = self._generate_dagger_tasks(parent_model_node)
+            parent_model_node = self._nodes_in_manifest.get(parent_node_name)
+            dagger_input = self._generate_dagger_tasks(parent_model_node)
 
-                inputs_list += dagger_input
+            inputs_list += dagger_input
 
         output_list = self._generate_dagger_output(model_node)
 

diff --git a/tests/fixtures/modules/dbt_config_parser_fixtures.py b/tests/fixtures/modules/dbt_config_parser_fixtures.py
@@ -37,6 +37,9 @@
             "schema": "analytics_engineering",
             "unique_id": "model.main.stg_core_schema1__table1",
             "name": "stg_core_schema1__table1",
+            "config": {
+                "materialized": "view",
+            },
             "depends_on": {
                 "macros": [],
                 "nodes": ["source.main.core_schema1.table1"],
@@ -46,6 +49,9 @@
             "schema": "analytics_engineering",
             "name": "stg_core_schema2__table2",
             "unique_id": "model.main.stg_core_schema2__table2",
+            "config": {
+                "materialized": "view",
+            },
             "depends_on": {
                 "macros": [],
                 "nodes": [
@@ -61,13 +67,17 @@
             "unique_id": "model.main.model2",
             "config": {
                 "external_location": "s3://bucket1-data-lake/path2/model2",
+                "materialized": "table",
             },
             "depends_on": {"macros": [], "nodes": []},
         },
         "model.main.int_model3": {
             "name": "int_model3",
             "unique_id": "model.main.int_model3",
             "schema": "analytics_engineering",
+            "config": {
+                "materialized": "ephemeral",
+            },
         },
         "seed.main.seed_buyer_country_overwrite": {
             "database": "awsdatacatalog",
@@ -188,6 +198,11 @@
 ]
 
 EXPECTED_MODEL_MULTIPLE_DEPENDENCIES = [
+    {
+        "type": "dummy",
+        "name": "int_model3",
+        "follow_external_dependency": True,
+    },
     {
         "type": "athena",
         "name": "analytics_engineering__model2_athena",
@@ -221,6 +236,14 @@
     },
 ]
 
+EXPECTED_EPHEMERAL_NODE = [
+    {
+        "type": "dummy",
+        "name": "int_model3",
+        "follow_external_dependency": True,
+    }
+]
+
 EXPECTED_DAGGER_INPUTS = [
     {
         "name": "core_schema2__table2_athena",
@@ -250,6 +273,11 @@
         "path": "path2/model2",
         "type": "s3",
     },
+    {
+        "type": "dummy",
+        "name": "int_model3",
+        "follow_external_dependency": True,
+    },
 ]
 
 EXPECTED_DAGGER_OUTPUTS = [

diff --git a/tests/utilities/test_dbt_config_parser.py b/tests/utilities/test_dbt_config_parser.py
@@ -14,6 +14,7 @@
     EXPECTED_STAGING_NODE_MULTIPLE_DEPENDENCIES,
     EXPECTED_SEED_NODE,
     EXPECTED_MODEL_MULTIPLE_DEPENDENCIES,
+    EXPECTED_EPHEMERAL_NODE,
 )
 
 _logger = logging.getLogger("root")
@@ -35,7 +36,7 @@ def setUp(self, mock_open, mock_json_load, mock_safe_load):
         self._dbt_config_parser = DBTConfigParser(DEFAULT_CONFIG_PARAMS)
         self._sample_dbt_node = DBT_MANIFEST_FILE_FIXTURE["nodes"]["model.main.model1"]
 
-    @skip("Run only locally")
+    # @skip("Run only locally")
     def test_generate_task_configs(self):
         module = Module(
             path_to_config="./tests/fixtures/modules/dbt_test_config.yaml",
@@ -64,6 +65,10 @@ def test_generate_dagger_inputs(self):
                 ],
                 EXPECTED_SEED_NODE,
             ),
+            (
+                DBT_MANIFEST_FILE_FIXTURE["nodes"]["model.main.int_model3"],
+                EXPECTED_EPHEMERAL_NODE,
+            ),
         ]
         for mock_input, expected_output in test_inputs:
             result = self._dbt_config_parser._generate_dagger_tasks(mock_input)
@@ -78,9 +83,7 @@ def test_generate_io_inputs(self):
             ),
         ]
         for mock_input, expected_output in fixtures:
-            result, _ = self._dbt_config_parser.generate_dagger_io(
-                mock_input
-            )
+            result, _ = self._dbt_config_parser.generate_dagger_io(mock_input)
 
             self.assertListEqual(result, expected_output)