From 7e2d07a75626d3d74c2efb9ed73c648708f86491 Mon Sep 17 00:00:00 2001
From: Scott Ssuyi Huang <scott@metaphor.io>
Date: Thu, 11 Jul 2024 22:14:32 +0800
Subject: [PATCH] Support BigQuery table snapshots (#898)

* Support bigquery snapshot table

* Bump version
---
 metaphor/bigquery/extractor.py    |  9 +++++++++
 poetry.lock                       | 10 +++++-----
 pyproject.toml                    |  6 +++---
 tests/snowflake/test_extractor.py |  2 +-
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/metaphor/bigquery/extractor.py b/metaphor/bigquery/extractor.py
index d4096e79..cc67a6d3 100644
--- a/metaphor/bigquery/extractor.py
+++ b/metaphor/bigquery/extractor.py
@@ -223,6 +223,15 @@ def parse_schema(bq_table: bigquery.table.Table) -> DatasetSchema:
                 materialization=MaterializationType.MATERIALIZED_VIEW,
                 table_schema=bq_table.mview_query,
             )
+        elif bq_table.table_type == "SNAPSHOT":
+            schema.sql_schema = SQLSchema(
+                materialization=MaterializationType.SNAPSHOT,
+                snapshot_time=(
+                    bq_table.snapshot_definition.snapshot_time
+                    if bq_table.snapshot_definition
+                    else None
+                ),
+            )
         else:
             raise ValueError(f"Unexpected table type {bq_table.table_type}")
 
diff --git a/poetry.lock b/poetry.lock
index 1eb958b4..5c4c591b 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -3095,13 +3095,13 @@ files = [
 
 [[package]]
 name = "metaphor-models"
-version = "0.35.5"
+version = "0.36.1"
 description = ""
 optional = false
 python-versions = "<4.0,>=3.8"
 files = [
-    {file = "metaphor_models-0.35.5-py3-none-any.whl", hash = "sha256:31b7853ac4fba9b6c6e6162784923e67abadc244baff9129d6fa07a612a5fa31"},
-    {file = "metaphor_models-0.35.5.tar.gz", hash = "sha256:61906d78c73b1d32d6cbe952eee2084cc69ae391a7a8b6273dc78dfebb2ae560"},
+    {file = "metaphor_models-0.36.1-py3-none-any.whl", hash = "sha256:d72c3b05fa9ad6978f4c96229c01cea543c404cc653aa634f94020120bfae72f"},
+    {file = "metaphor_models-0.36.1.tar.gz", hash = "sha256:a4f1e0a5458a695d7c8f874b688ffe1140b460398fb3b6014dfe709971f13a72"},
 ]
 
 [[package]]
@@ -6294,4 +6294,4 @@ unity-catalog = ["databricks-sdk", "databricks-sql-connector"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<3.12"
-content-hash = "10fe789c432a81bf8db11c93c65b84d89e87278ffd7c9250377231aefcc61f8e"
+content-hash = "ad8cc4619e0ba7a931364854685208ecdae94bfbd294e54d4836bca8d4b74d10"
diff --git a/pyproject.toml b/pyproject.toml
index af2c9090..011831fb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.30"
+version = "0.14.31"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <dev@metaphor.io>"]
@@ -29,7 +29,7 @@ databricks-sdk = { version = "^0.29.0", optional = true }
 databricks-sql-connector = { version = "^3.1.1", optional = true }
 fastavro = { version = "^1.9.2", optional = true }
 GitPython = "^3.1.37"
-google-cloud-bigquery = { version = "^3.1.0", optional = true }
+google-cloud-bigquery = { version = "^3.25.0", optional = true }
 google-cloud-logging = { version = "^3.5.0", optional = true }
 gql = { extras = ["requests"], version = "^3.4.1", optional = true }
 grpcio-tools = { version = "^1.59.3", optional = true }
@@ -41,7 +41,7 @@ llama-index-readers-confluence = { version = "^0.1.4", optional = true }
 llama-index-readers-notion = { version = "^0.1.6", optional = true }
 looker-sdk = { version = "^24.2.0", optional = true }
 lxml = { version = "~=5.0.0", optional = true }
-metaphor-models = "0.35.5"
+metaphor-models = "0.36.1"
 more-itertools = { version = "^10.1.0", optional = true }
 msal = { version = "^1.28.0", optional = true }
 msgraph-beta-sdk = { version = "~1.4.0", optional = true }
diff --git a/tests/snowflake/test_extractor.py b/tests/snowflake/test_extractor.py
index da15cef5..33b2266c 100644
--- a/tests/snowflake/test_extractor.py
+++ b/tests/snowflake/test_extractor.py
@@ -253,7 +253,7 @@ def should_raise(_0, _1):
     extractor._fetch_table_info({normalized_name: table_info}, False, set())
 
     assert dataset.schema.sql_schema.table_schema is None
-    assert dataset.statistics.last_updated is None
+    assert dataset.source_info.last_updated is None
 
 
 @patch("metaphor.snowflake.auth.connect")