From 14cce88475fe21f30fa63172d17ae0fda760146c Mon Sep 17 00:00:00 2001 From: Zouxxyy Date: Tue, 6 Feb 2024 14:01:18 +0800 Subject: [PATCH] [spark] Fix PaimonParseException compatibility with spark3.3 (#2849) --- .../spark/procedure/ProcedureTest.scala | 20 +++++++++++++++++++ .../PaimonSparkSqlExtensionsParser.scala | 19 +++++------------- .../PaimonSqlExtensionsAstBuilder.scala | 2 -- .../spark/procedure/ProcedureTestBase.scala | 14 ++++++++++++- 4 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala new file mode 100644 index 000000000000..2d582c5ac8e1 --- /dev/null +++ b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.paimon.spark.procedure + +class ProcedureTest extends ProcedureTestBase {} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala index f42cbab5570d..188917330611 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala @@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} import org.apache.spark.sql.catalyst.parser.extensions.PaimonSqlExtensionsParser.{NonReservedContext, QuotedIdentifierContext} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.internal.VariableSubstitution import org.apache.spark.sql.types.{DataType, StructType} @@ -220,23 +219,15 @@ case object PaimonParseErrorListener extends BaseErrorListener { } /** - * Copied from Apache Spark A [[ParseException]] is an [[AnalysisException]] that is thrown during - * the parse process. It contains fields and an extended error message that make reporting and - * diagnosing errors easier. + * Copied from Apache Spark [[ParseException]], it contains fields and an extended error message + * that make reporting and diagnosing errors easier. */ class PaimonParseException( val command: Option[String], message: String, - val start: Origin, - val stop: Origin) - extends AnalysisException(message, start.line, start.startPosition) { - - def this(message: String, ctx: ParserRuleContext) = - this( - Option(PaimonParserUtils.command(ctx)), - message, - PaimonParserUtils.position(ctx.getStart), - PaimonParserUtils.position(ctx.getStop)) + start: Origin, + stop: Origin) + extends Exception { override def getMessage: String = { val builder = new StringBuilder diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala index bb8fd9a334b1..ff6b15d2ca8f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala @@ -23,14 +23,12 @@ import org.apache.paimon.spark.catalyst.plans.logical.{PaimonCallArgument, Paimo import org.antlr.v4.runtime._ import org.antlr.v4.runtime.misc.Interval import org.antlr.v4.runtime.tree.{ParseTree, TerminalNode} -import org.apache.spark.QueryContext import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.parser.extensions.PaimonParserUtils.withOrigin import org.apache.spark.sql.catalyst.parser.extensions.PaimonSqlExtensionsParser._ import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.trees.SQLQueryContext import scala.collection.JavaConverters._ diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala index 9dd7124b9a2d..8d1717cb0a98 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala @@ -20,6 +20,7 @@ package org.apache.paimon.spark.procedure import org.apache.paimon.spark.PaimonSparkTestBase import org.apache.paimon.spark.analysis.NoSuchProcedureException +import org.apache.spark.sql.catalyst.parser.extensions.PaimonParseException import org.assertj.core.api.Assertions.assertThatThrownBy abstract class ProcedureTestBase extends PaimonSparkTestBase { @@ -29,7 +30,18 @@ abstract class ProcedureTestBase extends PaimonSparkTestBase { |CREATE TABLE T (id INT, name STRING, dt STRING) |""".stripMargin) - assertThatThrownBy(() => spark.sql("CALL unknown_procedure(table => 'test.T')")) + assertThatThrownBy(() => spark.sql("CALL sys.unknown_procedure(table => 'test.T')")) .isInstanceOf(classOf[NoSuchProcedureException]) } + + test(s"test parse exception") { + spark.sql(s""" + |CREATE TABLE T (id INT, name STRING, dt STRING) + |""".stripMargin) + + // Using Chinese comma to simulate parser exception + assertThatThrownBy( + () => spark.sql("CALL sys.expire_snapshots(table => 'test.T'，retain_max => 1)")) + .isInstanceOf(classOf[PaimonParseException]) + } }