From 8c4b054bed0fab60e7ab4b62424ffa2abed3d30c Mon Sep 17 00:00:00 2001 From: Aastha Agrrawal Date: Fri, 14 Jul 2023 09:09:13 -0700 Subject: [PATCH] [Coral=Spark] Remove CastToNamedStructTransformer (#435) --- .../spark/CoralToSparkSqlCallConverter.java | 4 -- .../CastToNamedStructTransformer.java | 53 ------------------- 2 files changed, 57 deletions(-) delete mode 100644 coral-spark/src/main/java/com/linkedin/coral/spark/transformers/CastToNamedStructTransformer.java diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/CoralToSparkSqlCallConverter.java b/coral-spark/src/main/java/com/linkedin/coral/spark/CoralToSparkSqlCallConverter.java index f8bcba5ad..c8a86d35c 100644 --- a/coral-spark/src/main/java/com/linkedin/coral/spark/CoralToSparkSqlCallConverter.java +++ b/coral-spark/src/main/java/com/linkedin/coral/spark/CoralToSparkSqlCallConverter.java @@ -15,7 +15,6 @@ import com.linkedin.coral.common.transformers.OperatorRenameSqlCallTransformer; import com.linkedin.coral.common.transformers.SqlCallTransformers; import com.linkedin.coral.spark.containers.SparkUDFInfo; -import com.linkedin.coral.spark.transformers.CastToNamedStructTransformer; import com.linkedin.coral.spark.transformers.ExtractUnionFunctionTransformer; import com.linkedin.coral.spark.transformers.FallBackToLinkedInHiveUDFTransformer; import com.linkedin.coral.spark.transformers.FuzzyUnionGenericProjectTransformer; @@ -158,9 +157,6 @@ public CoralToSparkSqlCallConverter(Set sparkUDFInfos) { // Fall back to the original Hive UDF defined in StaticHiveFunctionRegistry after failing to apply transformers above new FallBackToLinkedInHiveUDFTransformer(sparkUDFInfos), - // Transform `CAST(ROW: RECORD_TYPE)` to `named_struct` - new CastToNamedStructTransformer(), - // Transform `extract_union` to `coalesce_struct` new ExtractUnionFunctionTransformer(sparkUDFInfos), diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/CastToNamedStructTransformer.java b/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/CastToNamedStructTransformer.java deleted file mode 100644 index 324b7b3a6..000000000 --- a/coral-spark/src/main/java/com/linkedin/coral/spark/transformers/CastToNamedStructTransformer.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2023 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.coral.spark.transformers; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlRowTypeNameSpec; -import org.apache.calcite.sql.SqlRowTypeSpec; -import org.apache.calcite.sql.parser.SqlParserPos; - -import com.linkedin.coral.common.transformers.SqlCallTransformer; -import com.linkedin.coral.hive.hive2rel.functions.HiveNamedStructFunction; - - -/** - * This transformer transforms Coral IR function `CAST(ROW: RECORD_TYPE)` to Spark compatible function `named_struct`. - * For example, the SqlCall `CAST(ROW(123, 'xyz') AS ROW(`abc` INTEGER, `def` CHAR(3) CHARACTER SET `ISO-8859-1`))` - * will be transformed to `named_struct('abc', 123, 'def', 'xyz')` - */ -public class CastToNamedStructTransformer extends SqlCallTransformer { - @Override - protected boolean condition(SqlCall sqlCall) { - if (sqlCall.getOperator().getKind() == SqlKind.CAST) { - final SqlNode firstOperand = sqlCall.getOperandList().get(0); - final SqlNode secondOperand = sqlCall.getOperandList().get(1); - return firstOperand instanceof SqlCall && ((SqlCall) firstOperand).getOperator().getKind() == SqlKind.ROW - && secondOperand instanceof SqlRowTypeSpec; - } - return false; - } - - @Override - protected SqlCall transform(SqlCall sqlCall) { - List newOperands = new ArrayList<>(); - final SqlCall rowCall = (SqlCall) sqlCall.getOperandList().get(0); // like `ROW(123, 'xyz')` in above example - final SqlRowTypeSpec sqlRowTypeSpec = (SqlRowTypeSpec) sqlCall.getOperandList().get(1); // like `ROW(`abc` INTEGER, `def` CHAR(3) CHARACTER SET `ISO-8859-1`))` in above example - for (int i = 0; i < rowCall.getOperandList().size(); ++i) { - final String fieldName = - ((SqlRowTypeNameSpec) sqlRowTypeSpec.getTypeNameSpec()).getFieldNames().get(i).names.get(0); - newOperands.add(new SqlIdentifier("'" + fieldName + "'", SqlParserPos.ZERO)); // need to single-quote the field name - newOperands.add(rowCall.getOperandList().get(i)); - } - return HiveNamedStructFunction.NAMED_STRUCT.createCall(sqlCall.getParserPosition(), newOperands); - } -}