From 2875a5666101fffde973052fd39fea1ec130e308 Mon Sep 17 00:00:00 2001 From: Isha Tarte Date: Wed, 23 Oct 2024 12:47:42 -0700 Subject: [PATCH] Use date_trunc for DATE field in dynamic partition overwrite (#1310) --- .../connector/common/BigQueryUtil.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryUtil.java b/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryUtil.java index be1c3671f..0fa63fc27 100644 --- a/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryUtil.java +++ b/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryUtil.java @@ -735,12 +735,24 @@ static String getQueryForTimePartitionedTable( TimePartitioning timePartitioning) { TimePartitioning.Type partitionType = timePartitioning.getType(); String partitionField = timePartitioning.getField(); + FieldList allFields = destinationDefinition.getSchema().getFields(); + Optional partitionFieldType = + allFields.stream() + .filter(field -> partitionField.equals(field.getName())) + .map(field -> field.getType()) + .findFirst(); + // Using timestamp_trunc on a DATE field results in $cast_to_DATETIME which prevents pruning + // when required_partition_filter is set, as it is not considered a monotonic function + String truncFuntion = + (partitionFieldType.isPresent() && partitionFieldType.get().equals(LegacySQLTypeName.DATE)) + ? "date_trunc" + : "timestamp_trunc"; String extractedPartitionedSource = - String.format("timestamp_trunc(`%s`, %s)", partitionField, partitionType.toString()); + String.format("%s(`%s`, %s)", truncFuntion, partitionField, partitionType.toString()); String extractedPartitionedTarget = String.format( - "timestamp_trunc(`%s`.`%s`, %s)", "target", partitionField, partitionType.toString()); - FieldList allFields = destinationDefinition.getSchema().getFields(); + "%s(`target`.`%s`, %s)", truncFuntion, partitionField, partitionType.toString()); + String commaSeparatedFields = allFields.stream().map(Field::getName).collect(Collectors.joining("`,`", "`", "`"));