From 653623600488a432294b391e073ed1e2dc4864fc Mon Sep 17 00:00:00 2001 From: aastha25 Date: Tue, 5 Dec 2023 22:28:04 -0800 Subject: [PATCH 1/6] /OrdinalReturnTypeInferenceV2 to infer RexCall return type --- .../functions/OrdinalReturnTypeInferenceV2.java | 17 +++++++++++++++++ .../functions/StaticHiveFunctionRegistry.java | 2 +- .../coral/schema/avro/SchemaUtilities.java | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java new file mode 100644 index 000000000..55a5da4da --- /dev/null +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java @@ -0,0 +1,17 @@ +package com.linkedin.coral.hive.hive2rel.functions; + +import org.apache.calcite.sql.type.OrdinalReturnTypeInference; + + +public class OrdinalReturnTypeInferenceV2 extends OrdinalReturnTypeInference { + private final int ordinal; + + public OrdinalReturnTypeInferenceV2(int ordinal) { + super(ordinal); + this.ordinal = ordinal; + } + + public int getOrdinal() { + return ordinal; + } +} diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java index c882e4fe0..c94b37d70 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java @@ -665,7 +665,7 @@ public boolean isOptional(int i) { STRING_STRING_STRING); createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.HasMemberConsent", ReturnTypes.BOOLEAN, family(SqlTypeFamily.STRING, SqlTypeFamily.ANY, SqlTypeFamily.TIMESTAMP)); - createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactFieldIf", ARG1, + createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactFieldIf", new OrdinalReturnTypeInferenceV2(1), family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.STRING, SqlTypeFamily.ANY)); createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactSecondarySchemaFieldIf", ARG1, family( diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java index cde0a992d..b4f49c833 100644 --- a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java +++ b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java @@ -5,6 +5,7 @@ */ package com.linkedin.coral.schema.avro; +import com.linkedin.coral.hive.hive2rel.functions.OrdinalReturnTypeInferenceV2; import java.io.PrintWriter; import java.io.StringWriter; import java.util.*; @@ -29,6 +30,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OrdinalReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -237,6 +239,19 @@ static boolean isFieldNullable(@Nonnull RexCall rexCall, @Nonnull Schema inputSc return rexCall.getType().isNullable(); } + if (rexCall.getOperator().getReturnTypeInference() instanceof OrdinalReturnTypeInferenceV2) { + int index = ((OrdinalReturnTypeInferenceV2) rexCall.getOperator().getReturnTypeInference()) + .getOrdinal(); + RexNode operand = rexCall.operands.get(index); + + if (operand instanceof RexInputRef) { + Schema schema = inputSchema.getFields().get(((RexInputRef) operand).getIndex()).schema(); + return isNullableType(schema); + } else if (operand instanceof RexCall) { + return isFieldNullable((RexCall) operand, inputSchema); + } + } + // the field is non-nullable only if all operands are RexInputRef // and corresponding field schema type of RexInputRef index is not UNION List operands = rexCall.getOperands(); From 3a7e7af81b05e8016b03797f3ff28f922f91b35d Mon Sep 17 00:00:00 2001 From: aastha25 Date: Wed, 13 Dec 2023 10:42:15 -0800 Subject: [PATCH 2/6] add cast_nullability as an inbuilt function --- .../OrdinalReturnTypeInferenceV2.java | 5 +- .../functions/StaticHiveFunctionRegistry.java | 4 +- .../schema/avro/RelDataTypeToAvroType.java | 2 + .../schema/avro/RelToAvroSchemaConverter.java | 36 +++++++++--- .../coral/schema/avro/SchemaUtilities.java | 15 ----- .../linkedin/coral/schema/avro/TestUtils.java | 2 + .../avro/ViewToAvroSchemaConverterTests.java | 11 ++++ .../testCastNullabilityUDF-expected.avsc | 58 +++++++++++++++++++ 8 files changed, 108 insertions(+), 25 deletions(-) create mode 100644 coral-schema/src/test/resources/testCastNullabilityUDF-expected.avsc diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java index 55a5da4da..897dc83a1 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java @@ -2,7 +2,10 @@ import org.apache.calcite.sql.type.OrdinalReturnTypeInference; - +/** + * Custom implementation of {@link OrdinalReturnTypeInference} which allows inferring the return type + * based on the ordinal of a given input argument and also exposes the ordinal. + */ public class OrdinalReturnTypeInferenceV2 extends OrdinalReturnTypeInference { private final int ordinal; diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java index c94b37d70..98c5fbe5d 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java @@ -665,8 +665,10 @@ public boolean isOptional(int i) { STRING_STRING_STRING); createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.HasMemberConsent", ReturnTypes.BOOLEAN, family(SqlTypeFamily.STRING, SqlTypeFamily.ANY, SqlTypeFamily.TIMESTAMP)); - createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactFieldIf", new OrdinalReturnTypeInferenceV2(1), + createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactFieldIf", ARG1, family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.STRING, SqlTypeFamily.ANY)); + createAddUserDefinedFunction("cast_nullability", new OrdinalReturnTypeInferenceV2(1), + family(SqlTypeFamily.ANY, SqlTypeFamily.ANY)); createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactSecondarySchemaFieldIf", ARG1, family( SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER, SqlTypeFamily.ANY)); diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelDataTypeToAvroType.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelDataTypeToAvroType.java index 5b2f04406..53963c321 100644 --- a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelDataTypeToAvroType.java +++ b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelDataTypeToAvroType.java @@ -84,6 +84,8 @@ static Schema relDataTypeToAvroTypeNonNullable(@Nonnull RelDataType relDataType, private static Schema relDataTypeToAvroType(RelDataType relDataType, String recordName) { final Schema avroSchema = relDataTypeToAvroTypeNonNullable(relDataType, recordName); + // TODO: Current logic ALWAYS sets the inner fields of RelDataType record nullable. + // Modify this to be applied only when RelDataType record was generated from a HIVE_UDF RexCall return SchemaUtilities.makeNullable(avroSchema, false); } diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java index 6c94ac56c..f3b791394 100644 --- a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java +++ b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java @@ -5,6 +5,7 @@ */ package com.linkedin.coral.schema.avro; +import com.linkedin.coral.hive.hive2rel.functions.OrdinalReturnTypeInferenceV2; import java.util.Deque; import java.util.HashMap; import java.util.LinkedList; @@ -407,14 +408,7 @@ public SchemaRexShuttle(Schema inputSchema, RelNode inputNode, Queue sug @Override public RexNode visitInputRef(RexInputRef rexInputRef) { RexNode rexNode = super.visitInputRef(rexInputRef); - - Schema.Field field = inputSchema.getFields().get(rexInputRef.getIndex()); - String oldFieldName = field.name(); - String suggestNewFieldName = suggestedFieldNames.poll(); - String newFieldName = SchemaUtilities.getFieldName(oldFieldName, suggestNewFieldName); - - SchemaUtilities.appendField(newFieldName, field, fieldAssembler); - + appendRexInputRefField(rexInputRef); return rexNode; } @@ -442,6 +436,23 @@ public RexNode visitCall(RexCall rexCall) { * For SqlUserDefinedFunction and SqlOperator RexCall, no need to handle it recursively * and only return type of udf or sql operator is relevant */ + + /** + * If the return type of RexCall is based on the ordinal of its input argument + * and the corresponding input argument refers to a field from the input schema, + * use the field's schema as is. + */ + if (rexCall.getOperator().getReturnTypeInference() instanceof OrdinalReturnTypeInferenceV2) { + int index = ((OrdinalReturnTypeInferenceV2) rexCall.getOperator().getReturnTypeInference()) + .getOrdinal(); + RexNode operand = rexCall.operands.get(index); + + if (operand instanceof RexInputRef) { + appendRexInputRefField((RexInputRef) operand); + return rexCall; + } + } + RelDataType fieldType = rexCall.getType(); boolean isNullable = SchemaUtilities.isFieldNullable(rexCall, inputSchema); @@ -545,6 +556,15 @@ public RexNode visitPatternFieldRef(RexPatternFieldRef rexPatternFieldRef) { return super.visitPatternFieldRef(rexPatternFieldRef); } + private void appendRexInputRefField(RexInputRef rexInputRef) { + Schema.Field field = inputSchema.getFields().get(rexInputRef.getIndex()); + String oldFieldName = field.name(); + String suggestNewFieldName = suggestedFieldNames.poll(); + String newFieldName = SchemaUtilities.getFieldName(oldFieldName, suggestNewFieldName); + + SchemaUtilities.appendField(newFieldName, field, fieldAssembler); + } + private void appendField(RelDataType fieldType, boolean isNullable, String doc) { String fieldName = SchemaUtilities.getFieldName("", suggestedFieldNames.poll()); SchemaUtilities.appendField(fieldName, fieldType, doc, fieldAssembler, isNullable); diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java index b4f49c833..cde0a992d 100644 --- a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java +++ b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/SchemaUtilities.java @@ -5,7 +5,6 @@ */ package com.linkedin.coral.schema.avro; -import com.linkedin.coral.hive.hive2rel.functions.OrdinalReturnTypeInferenceV2; import java.io.PrintWriter; import java.io.StringWriter; import java.util.*; @@ -30,7 +29,6 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.type.OrdinalReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -239,19 +237,6 @@ static boolean isFieldNullable(@Nonnull RexCall rexCall, @Nonnull Schema inputSc return rexCall.getType().isNullable(); } - if (rexCall.getOperator().getReturnTypeInference() instanceof OrdinalReturnTypeInferenceV2) { - int index = ((OrdinalReturnTypeInferenceV2) rexCall.getOperator().getReturnTypeInference()) - .getOrdinal(); - RexNode operand = rexCall.operands.get(index); - - if (operand instanceof RexInputRef) { - Schema schema = inputSchema.getFields().get(((RexInputRef) operand).getIndex()).schema(); - return isNullableType(schema); - } else if (operand instanceof RexCall) { - return isFieldNullable((RexCall) operand, inputSchema); - } - } - // the field is non-nullable only if all operands are RexInputRef // and corresponding field schema type of RexInputRef index is not UNION List operands = rexCall.getOperands(); diff --git a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/TestUtils.java b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/TestUtils.java index 73340ec98..cb79495f4 100644 --- a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/TestUtils.java +++ b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/TestUtils.java @@ -99,6 +99,7 @@ private static void initializeTables() { String baseComplexUnionTypeSchema = loadSchema("base-complex-union-type.avsc"); String baseNestedUnionSchema = loadSchema("base-nested-union.avsc"); String baseComplexLowercase = loadSchema("base-complex-lowercase.avsc"); + String baseComplexNonNullable = loadSchema("base-complex-non-nullable.avsc"); String baseComplexNullableWithDefaults = loadSchema("base-complex-nullable-with-defaults.avsc"); String basePrimitive = loadSchema("base-primitive.avsc"); String baseComplexNestedStructSameName = loadSchema("base-complex-nested-struct-same-name.avsc"); @@ -121,6 +122,7 @@ private static void initializeTables() { executeCreateTableWithPartitionFieldSchemaQuery("default", "basecomplexfieldschema", baseComplexFieldSchema); executeCreateTableWithPartitionQuery("default", "basenestedcomplex", baseNestedComplexSchema); executeCreateTableWithPartitionQuery("default", "basecomplexnullablewithdefaults", baseComplexNullableWithDefaults); + executeCreateTableWithPartitionQuery("default", "basecomplexnonnullable", baseComplexNonNullable); String baseComplexSchemaWithDoc = loadSchema("docTestResources/base-complex-with-doc.avsc"); String baseEnumSchemaWithDoc = loadSchema("docTestResources/base-enum-with-doc.avsc"); diff --git a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java index 69d28eef3..7b9d121dc 100644 --- a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java +++ b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java @@ -1102,5 +1102,16 @@ public void testDivideReturnType() { Assert.assertEquals(actualSchema.toString(true), TestUtils.loadSchema("testDivideReturnType-expected.avsc")); } + @Test + public void testCastNullabilityUDF() { + ViewToAvroSchemaConverter viewToAvroSchemaConverter = ViewToAvroSchemaConverter.create(hiveMetastoreClient); + + Schema schemaWithUDF = viewToAvroSchemaConverter.toAvroSchema("SELECT cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); + Schema schemaWithField = viewToAvroSchemaConverter.toAvroSchema("SELECT Struct_Col AS modCol FROM basecomplexnonnullable"); + + Assert.assertEquals(schemaWithUDF.toString(true), TestUtils.loadSchema("testCastNullabilityUDF-expected.avsc")); + Assert.assertEquals(schemaWithField.toString(true), TestUtils.loadSchema("testCastNullabilityUDF-expected.avsc")); + } + // TODO: add more unit tests } diff --git a/coral-schema/src/test/resources/testCastNullabilityUDF-expected.avsc b/coral-schema/src/test/resources/testCastNullabilityUDF-expected.avsc new file mode 100644 index 000000000..c5b5f18ca --- /dev/null +++ b/coral-schema/src/test/resources/testCastNullabilityUDF-expected.avsc @@ -0,0 +1,58 @@ +{ + "type" : "record", + "name" : "basecomplexnonnullable", + "namespace" : "coral.schema.avro.base.complex.nonnullable", + "fields" : [ { + "name" : "modCol", + "type" : { + "type" : "record", + "name" : "Struct_col", + "namespace" : "coral.schema.avro.base.complex.nonnullable.basecomplexnonnullable", + "fields" : [ { + "name" : "Bool_Field", + "type" : "boolean" + }, { + "name" : "Int_Field", + "type" : "int" + }, { + "name" : "Bigint_Field", + "type" : "long" + }, { + "name" : "Float_Field", + "type" : "float" + }, { + "name" : "Double_Field", + "type" : "double" + }, { + "name" : "Date_String_Field", + "type" : "string" + }, { + "name" : "String_Field", + "type" : "string" + }, { + "name" : "Array_Col", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "Struct_col", + "namespace" : "coral.schema.avro.base.complex.nonnullable.basecomplexnonnullable.basecomplexnonnullable", + "fields" : [ { + "name" : "key", + "type" : "string" + }, { + "name" : "value", + "type" : "string" + } ] + } + } + }, { + "name" : "Map_Col", + "type" : { + "type" : "map", + "values" : "string" + } + } ] + } + } ] +} \ No newline at end of file From 5288845d8ef95eadab8cc7f41a281378c45b3548 Mon Sep 17 00:00:00 2001 From: aastha25 Date: Wed, 13 Dec 2023 11:55:02 -0800 Subject: [PATCH 3/6] spotless fix --- .../coral/schema/avro/RelToAvroSchemaConverter.java | 5 ++--- .../coral/schema/avro/ViewToAvroSchemaConverterTests.java | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java index f3b791394..01109a372 100644 --- a/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java +++ b/coral-schema/src/main/java/com/linkedin/coral/schema/avro/RelToAvroSchemaConverter.java @@ -5,7 +5,6 @@ */ package com.linkedin.coral.schema.avro; -import com.linkedin.coral.hive.hive2rel.functions.OrdinalReturnTypeInferenceV2; import java.util.Deque; import java.util.HashMap; import java.util.LinkedList; @@ -60,6 +59,7 @@ import com.linkedin.coral.com.google.common.base.Preconditions; import com.linkedin.coral.common.HiveMetastoreClient; import com.linkedin.coral.common.HiveUncollect; +import com.linkedin.coral.hive.hive2rel.functions.OrdinalReturnTypeInferenceV2; /** @@ -443,8 +443,7 @@ public RexNode visitCall(RexCall rexCall) { * use the field's schema as is. */ if (rexCall.getOperator().getReturnTypeInference() instanceof OrdinalReturnTypeInferenceV2) { - int index = ((OrdinalReturnTypeInferenceV2) rexCall.getOperator().getReturnTypeInference()) - .getOrdinal(); + int index = ((OrdinalReturnTypeInferenceV2) rexCall.getOperator().getReturnTypeInference()).getOrdinal(); RexNode operand = rexCall.operands.get(index); if (operand instanceof RexInputRef) { diff --git a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java index 7b9d121dc..867214e33 100644 --- a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java +++ b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java @@ -1106,8 +1106,10 @@ public void testDivideReturnType() { public void testCastNullabilityUDF() { ViewToAvroSchemaConverter viewToAvroSchemaConverter = ViewToAvroSchemaConverter.create(hiveMetastoreClient); - Schema schemaWithUDF = viewToAvroSchemaConverter.toAvroSchema("SELECT cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); - Schema schemaWithField = viewToAvroSchemaConverter.toAvroSchema("SELECT Struct_Col AS modCol FROM basecomplexnonnullable"); + Schema schemaWithUDF = viewToAvroSchemaConverter + .toAvroSchema("SELECT cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); + Schema schemaWithField = + viewToAvroSchemaConverter.toAvroSchema("SELECT Struct_Col AS modCol FROM basecomplexnonnullable"); Assert.assertEquals(schemaWithUDF.toString(true), TestUtils.loadSchema("testCastNullabilityUDF-expected.avsc")); Assert.assertEquals(schemaWithField.toString(true), TestUtils.loadSchema("testCastNullabilityUDF-expected.avsc")); From 03a29bbfbbc082b5293fbcd85b3c763c553cc8c1 Mon Sep 17 00:00:00 2001 From: aastha25 Date: Wed, 13 Dec 2023 11:58:27 -0800 Subject: [PATCH 4/6] coral-hive spotlesscheck --- .../hive2rel/functions/OrdinalReturnTypeInferenceV2.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java index 897dc83a1..f41079ab1 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/OrdinalReturnTypeInferenceV2.java @@ -1,7 +1,13 @@ +/** + * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ package com.linkedin.coral.hive.hive2rel.functions; import org.apache.calcite.sql.type.OrdinalReturnTypeInference; + /** * Custom implementation of {@link OrdinalReturnTypeInference} which allows inferring the return type * based on the ordinal of a given input argument and also exposes the ordinal. From 079d01dbf92d7181a04ef747c1e65344de6a4076 Mon Sep 17 00:00:00 2001 From: aastha25 Date: Fri, 15 Dec 2023 10:00:31 -0800 Subject: [PATCH 5/6] modify UDF name --- .../functions/StaticHiveFunctionRegistry.java | 2 +- .../schema/avro/ViewToAvroSchemaConverterTests.java | 11 ++++++----- ....avsc => testLiGrootCastNullability-expected.avsc} | 0 3 files changed, 7 insertions(+), 6 deletions(-) rename coral-schema/src/test/resources/{testCastNullabilityUDF-expected.avsc => testLiGrootCastNullability-expected.avsc} (100%) diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java index 98c5fbe5d..c9e9fd67b 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/StaticHiveFunctionRegistry.java @@ -667,7 +667,7 @@ public boolean isOptional(int i) { family(SqlTypeFamily.STRING, SqlTypeFamily.ANY, SqlTypeFamily.TIMESTAMP)); createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactFieldIf", ARG1, family(SqlTypeFamily.BOOLEAN, SqlTypeFamily.ANY, SqlTypeFamily.STRING, SqlTypeFamily.ANY)); - createAddUserDefinedFunction("cast_nullability", new OrdinalReturnTypeInferenceV2(1), + createAddUserDefinedFunction("li_groot_cast_nullability", new OrdinalReturnTypeInferenceV2(1), family(SqlTypeFamily.ANY, SqlTypeFamily.ANY)); createAddUserDefinedFunction("com.linkedin.policy.decoration.udfs.RedactSecondarySchemaFieldIf", ARG1, family( diff --git a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java index 867214e33..84b468389 100644 --- a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java +++ b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java @@ -1103,16 +1103,17 @@ public void testDivideReturnType() { } @Test - public void testCastNullabilityUDF() { + public void testLiGrootCastNullability() { ViewToAvroSchemaConverter viewToAvroSchemaConverter = ViewToAvroSchemaConverter.create(hiveMetastoreClient); - Schema schemaWithUDF = viewToAvroSchemaConverter - .toAvroSchema("SELECT cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); + Schema schemaWithUDF = viewToAvroSchemaConverter.toAvroSchema( + "SELECT li_groot_cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); Schema schemaWithField = viewToAvroSchemaConverter.toAvroSchema("SELECT Struct_Col AS modCol FROM basecomplexnonnullable"); - Assert.assertEquals(schemaWithUDF.toString(true), TestUtils.loadSchema("testCastNullabilityUDF-expected.avsc")); - Assert.assertEquals(schemaWithField.toString(true), TestUtils.loadSchema("testCastNullabilityUDF-expected.avsc")); + Assert.assertEquals(schemaWithUDF.toString(true), TestUtils.loadSchema("testLiGrootCastNullability-expected.avsc")); + Assert.assertEquals(schemaWithField.toString(true), + TestUtils.loadSchema("testLiGrootCastNullability-expected.avsc")); } // TODO: add more unit tests diff --git a/coral-schema/src/test/resources/testCastNullabilityUDF-expected.avsc b/coral-schema/src/test/resources/testLiGrootCastNullability-expected.avsc similarity index 100% rename from coral-schema/src/test/resources/testCastNullabilityUDF-expected.avsc rename to coral-schema/src/test/resources/testLiGrootCastNullability-expected.avsc From ca1402ddbbf14866daa58e9daec6c1fd194d50d8 Mon Sep 17 00:00:00 2001 From: aastha25 Date: Fri, 15 Dec 2023 10:12:46 -0800 Subject: [PATCH 6/6] coral-schema spotlessApply --- .../coral/schema/avro/ViewToAvroSchemaConverterTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java index 84b468389..4bf8130c5 100644 --- a/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java +++ b/coral-schema/src/test/java/com/linkedin/coral/schema/avro/ViewToAvroSchemaConverterTests.java @@ -1106,8 +1106,8 @@ public void testDivideReturnType() { public void testLiGrootCastNullability() { ViewToAvroSchemaConverter viewToAvroSchemaConverter = ViewToAvroSchemaConverter.create(hiveMetastoreClient); - Schema schemaWithUDF = viewToAvroSchemaConverter.toAvroSchema( - "SELECT li_groot_cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); + Schema schemaWithUDF = viewToAvroSchemaConverter + .toAvroSchema("SELECT li_groot_cast_nullability(Struct_Col, Struct_Col) AS modCol FROM basecomplexnonnullable"); Schema schemaWithField = viewToAvroSchemaConverter.toAvroSchema("SELECT Struct_Col AS modCol FROM basecomplexnonnullable");