Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add temporary try-catch to handle snapshot time parsing issue #933

Merged
merged 3 commits into from
Aug 2, 2024
Merged

Add temporary try-catch to handle snapshot time parsing issue #933

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
810c06b
Add temporary try-catch to handle snapshot time parsing issue
mars-lan Aug 1, 2024
e79b5e7
Merge branch 'main' into marslan/sc-27798/bigquery-crawler-throws-err…
mars-lan Aug 2, 2024
e42eab2
Bump version
mars-lan Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions metaphor/bigquery/extractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import re
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from typing import (
Any,
Collection,
Expand Down Expand Up @@ -224,13 +225,10 @@ def parse_schema(bq_table: bigquery.table.Table) -> DatasetSchema:
table_schema=bq_table.mview_query,
)
elif bq_table.table_type == "SNAPSHOT":

schema.sql_schema = SQLSchema(
materialization=MaterializationType.SNAPSHOT,
snapshot_time=(
bq_table.snapshot_definition.snapshot_time
if bq_table.snapshot_definition
else None
),
snapshot_time=BigQueryExtractor._get_snapshot_time(bq_table),
)
else:
raise ValueError(f"Unexpected table type {bq_table.table_type}")
Expand All @@ -239,6 +237,18 @@ def parse_schema(bq_table: bigquery.table.Table) -> DatasetSchema:

return schema

@staticmethod
def _get_snapshot_time(bq_table: bigquery.table.Table) -> Optional[datetime]:
# bigquery client fails to parse snapshot time sometimes
# See https://github.com/googleapis/python-bigquery/issues/1986
try:
if bq_table.snapshot_definition:
return bq_table.snapshot_definition.snapshot_time
except ValueError:
return None

return None

# See https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.schema.SchemaField.html#google.cloud.bigquery.schema.SchemaField
@staticmethod
def _parse_fields(
Expand Down
Loading