Support BigQuery table snapshots (#898)

* Support bigquery snapshot table

* Bump version

metaphor/bigquery/extractor.py

@@ -223,6 +223,15 @@ def parse_schema(bq_table: bigquery.table.Table) -> DatasetSchema:
                 materialization=MaterializationType.MATERIALIZED_VIEW,
                 table_schema=bq_table.mview_query,
             )
+        elif bq_table.table_type == "SNAPSHOT":
+            schema.sql_schema = SQLSchema(
+                materialization=MaterializationType.SNAPSHOT,
+                snapshot_time=(
+                    bq_table.snapshot_definition.snapshot_time
+                    if bq_table.snapshot_definition
+                    else None
+                ),
+            )
         else:
             raise ValueError(f"Unexpected table type {bq_table.table_type}")
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.30"
+version = "0.14.31"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]
@@ -29,7 +29,7 @@ databricks-sdk = { version = "^0.29.0", optional = true }
 databricks-sql-connector = { version = "^3.1.1", optional = true }
 fastavro = { version = "^1.9.2", optional = true }
 GitPython = "^3.1.37"
-google-cloud-bigquery = { version = "^3.1.0", optional = true }
+google-cloud-bigquery = { version = "^3.25.0", optional = true }
 google-cloud-logging = { version = "^3.5.0", optional = true }
 gql = { extras = ["requests"], version = "^3.4.1", optional = true }
 grpcio-tools = { version = "^1.59.3", optional = true }
@@ -41,7 +41,7 @@ llama-index-readers-confluence = { version = "^0.1.4", optional = true }
 llama-index-readers-notion = { version = "^0.1.6", optional = true }
 looker-sdk = { version = "^24.2.0", optional = true }
 lxml = { version = "~=5.0.0", optional = true }
-metaphor-models = "0.35.5"
+metaphor-models = "0.36.1"
 more-itertools = { version = "^10.1.0", optional = true }
 msal = { version = "^1.28.0", optional = true }
 msgraph-beta-sdk = { version = "~1.4.0", optional = true }

tests/snowflake/test_extractor.py

@@ -253,7 +253,7 @@ def should_raise(_0, _1):
     extractor._fetch_table_info({normalized_name: table_info}, False, set())
 
     assert dataset.schema.sql_schema.table_schema is None
-    assert dataset.statistics.last_updated is None
+    assert dataset.source_info.last_updated is None
 
 
 @patch("metaphor.snowflake.auth.connect")