Exclude self lineage in snowflake by default [sc-25567] (#828)

metaphor/snowflake/lineage/README.md

@@ -17,7 +17,7 @@ enable_view_lineage: bool = True
 # (Optional) Whether to enable finding table lineage information from access history and query history, default True.
 enable_lineage_from_history: bool = True
 
-# (Optional) Whether to include self-referencing loops in lineage, default True
+# (Optional) Whether to include self-referencing loops in lineage, default False (exclude self loop)
 include_self_lineage: <boolean>
 
 # (Optional) Number of days to include in the usage analysis. Default to 7.

metaphor/snowflake/lineage/config.py

@@ -17,8 +17,8 @@ class SnowflakeLineageRunConfig(SnowflakeBaseConfig):
     # Number of days back in the query log to process
     lookback_days: int = 7
 
-    # Whether to include self loop in lineage
-    include_self_lineage: bool = True
+    # Whether to include self loop in lineage, default to exclude self loop
+    include_self_lineage: bool = False
 
     # The number of access logs fetched in a batch, default to 100000
     batch_size: int = DEFAULT_BATCH_SIZE

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.13.164"
+version = "0.13.165"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]
@@ -41,10 +41,10 @@ llama-index-readers-confluence = { version = "^0.1.4", optional = true }
 llama-index-readers-notion = { version = "^0.1.6", optional = true }
 looker-sdk = { version = "^23.6.0", optional = true }
 lxml = { version = "~=5.0.0", optional = true }
-metaphor-models = "0.32.1"
+metaphor-models = "0.33.1"
 more-itertools = { version = "^10.1.0", optional = true }
 msal = { version = "^1.28.0", optional = true }
-msgraph-beta-sdk = { version = "1.1.0", optional = true }
+msgraph-beta-sdk = { version = "1.2.0", optional = true }
 parse = { version = "^1.20.0", optional = true }
 pathvalidate = "^3.2.0"
 pyarrow = { version = "^14.0.1", extras = ["pandas"]}

tests/snowflake/lineage/test_config.py

@@ -14,7 +14,7 @@ def test_yaml_config(test_root_dir):
         password="password",
         enable_view_lineage=True,
         enable_lineage_from_history=True,
-        include_self_lineage=True,
+        include_self_lineage=False,
         filter=DatasetFilter(),
         lookback_days=7,
         output=OutputConfig(),

tests/snowflake/lineage/test_extractor.py

@@ -96,7 +96,7 @@ def test_parse_access_log(mock_connect: MagicMock, test_root_dir: str):
     )
 
     # Include self-lineage
-    extractor = SnowflakeLineageExtractor(dummy_config())
+    extractor = SnowflakeLineageExtractor(dummy_config(include_self_lineage=True))
     extractor._parse_access_log(accessed_objects, modified_objects, "query")
 
     results = {}
@@ -109,7 +109,7 @@ def test_parse_access_log(mock_connect: MagicMock, test_root_dir: str):
     )
 
     # Exclude self-lineage
-    extractor = SnowflakeLineageExtractor(dummy_config(include_self_lineage=False))
+    extractor = SnowflakeLineageExtractor(dummy_config())
     extractor._parse_access_log(accessed_objects, modified_objects, "query")
 
     results = {}