Merge branch 'main' into rishimohan/sc-23110/web-crawler

metaphor/dbt/artifact_parser.py

@@ -1,3 +1,4 @@
+import json
 from copy import deepcopy
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 
@@ -37,6 +38,7 @@
     DbtMacroArgument,
     DbtMaterialization,
     DbtMaterializationType,
+    DbtMetadataItem,
     DbtMetric,
     DbtModel,
     DbtTest,
@@ -741,6 +743,13 @@ def get_dataset():
         if len(tag_names) > 0:
             get_dataset().tag_assignment = TagAssignment(tag_names=tag_names)
 
+        # Capture the whole "meta" field as key-value pairs
+        if len(meta) > 0:
+            virtual_view.dbt_model.meta = [
+                DbtMetadataItem(key=key, value=json.dumps(value))
+                for key, value in meta.items()
+            ]
+
     def _parse_node_materialization(
         self, node: VIRTUAL_VIEW_NODE_TYPE, dbt_model: DbtModel
     ) -> None:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.13.122"
+version = "0.13.124"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]
@@ -38,7 +38,7 @@ lkml = { version = "^1.3.1", optional = true }
 llama-hub = {version = "0.0.67", optional = true }
 looker-sdk = { version = "^23.6.0", optional = true }
 lxml = { version = "~=5.0.0", optional = true }
-metaphor-models = "0.30.16"
+metaphor-models = "0.30.17"
 more-itertools = { version = "^10.1.0", optional = true }
 msal = { version = "^1.20.0", optional = true }
 msgraph-beta-sdk = { version = "1.0.0", optional = true }

tests/dbt/data/trial_v4/expected.json

@@ -177,6 +177,16 @@
         "targetDataset": "DATASET~95F35FDE6E7C3F9E556D701A1B00CA23",
         "type": "VIEW"
       },
+      "meta": [
+        {
+          "key": "owner",
+          "value": "\"[email protected]\""
+        },
+        {
+          "key": "pii",
+          "value": "true"
+        }
+      ],
       "packageName": "trial",
       "rawSql": "/*\n    Welcome to your first dbt model!\n    Did you know that you can also configure models directly within SQL files?\n    This will override configurations stated in dbt_project.yml\n\n    Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n    select 1 as id\n    union all\n    select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n    Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null",
       "tests": [

tests/dbt/data/trial_v5/expected.json

@@ -177,6 +177,16 @@
         "targetDataset": "DATASET~95F35FDE6E7C3F9E556D701A1B00CA23",
         "type": "VIEW"
       },
+      "meta": [
+        {
+          "key": "owner",
+          "value": "\"[email protected]\""
+        },
+        {
+          "key": "pii",
+          "value": "true"
+        }
+      ],
       "packageName": "trial",
       "rawSql": "/*\n    Welcome to your first dbt model!\n    Did you know that you can also configure models directly within SQL files?\n    This will override configurations stated in dbt_project.yml\n\n    Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n    select 1 as id\n    union all\n    select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n    Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null",
       "tests": [

tests/dbt/data/trial_v6/expected.json

@@ -177,6 +177,16 @@
         "targetDataset": "DATASET~95F35FDE6E7C3F9E556D701A1B00CA23",
         "type": "VIEW"
       },
+      "meta": [
+        {
+          "key": "owner",
+          "value": "\"[email protected]\""
+        },
+        {
+          "key": "pii",
+          "value": "true"
+        }
+      ],
       "packageName": "trial",
       "rawSql": "/*\n    Welcome to your first dbt model!\n    Did you know that you can also configure models directly within SQL files?\n    This will override configurations stated in dbt_project.yml\n\n    Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n    select 1 as id\n    union all\n    select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n    Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null",
       "tests": [

tests/dbt/data/trial_v7/expected.json

@@ -177,6 +177,16 @@
         "targetDataset": "DATASET~95F35FDE6E7C3F9E556D701A1B00CA23",
         "type": "VIEW"
       },
+      "meta": [
+        {
+          "key": "owner",
+          "value": "\"[email protected]\""
+        },
+        {
+          "key": "pii",
+          "value": "true"
+        }
+      ],
       "packageName": "trial",
       "rawSql": "/*\n    Welcome to your first dbt model!\n    Did you know that you can also configure models directly within SQL files?\n    This will override configurations stated in dbt_project.yml\n\n    Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n    select 1 as id\n    union all\n    select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n    Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null",
       "tests": [

tests/dbt/data/trial_v8/expected.json

@@ -177,6 +177,16 @@
         "targetDataset": "DATASET~95F35FDE6E7C3F9E556D701A1B00CA23",
         "type": "VIEW"
       },
+      "meta": [
+        {
+          "key": "owner",
+          "value": "\"[email protected]\""
+        },
+        {
+          "key": "pii",
+          "value": "true"
+        }
+      ],
       "packageName": "trial",
       "rawSql": "/*\n    Welcome to your first dbt model!\n    Did you know that you can also configure models directly within SQL files?\n    This will override configurations stated in dbt_project.yml\n\n    Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n    select 1 as id\n    union all\n    select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n    Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null",
       "tests": [

tests/dbt/data/trial_v9/expected.json

@@ -177,6 +177,16 @@
         "targetDataset": "DATASET~95F35FDE6E7C3F9E556D701A1B00CA23",
         "type": "VIEW"
       },
+      "meta": [
+        {
+          "key": "owner",
+          "value": "\"[email protected]\""
+        },
+        {
+          "key": "pii",
+          "value": "true"
+        }
+      ],
       "packageName": "trial",
       "rawSql": "/*\n    Welcome to your first dbt model!\n    Did you know that you can also configure models directly within SQL files?\n    This will override configurations stated in dbt_project.yml\n\n    Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n    select 1 as id\n    union all\n    select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n    Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null",
       "tests": [