From 2402a0abff5bf0240cb1f4e5426a59175525947e Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Mon, 3 Jun 2024 19:22:46 +0800 Subject: [PATCH] add more test --- .../optimizer/EvalSubqueriesForDeleteTable.scala | 5 ++++- .../spark/sql/PaimonOptimizationTestBase.scala | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala index e54c13281355..aaaed10f1c9f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala @@ -36,6 +36,9 @@ import scala.collection.JavaConverters._ * For those delete conditions with subqueries that only contain partition columns, we can eval them * in advance. So that when running [[DeleteFromPaimonTableCommand]], we can directly call * dropPartitions to achieve fast deletion. + * + * Note: this rule must be placed before [[MergePaimonScalarSubqueriers]], because + * [[MergePaimonScalarSubqueriers]] will merge subqueries. */ object EvalSubqueriesForDeleteTable extends Rule[LogicalPlan] with ExpressionHelper with Logging { @@ -77,7 +80,7 @@ object EvalSubqueriesForDeleteTable extends Rule[LogicalPlan] with ExpressionHel evalPhysicalSubquery(physicalSubquery) physicalSubquery.values() match { - case Some(l) if l.nonEmpty => In(expr, l.map(Literal(_, expr.dataType))) + case Some(l) if l.length > 0 => In(expr, l.map(Literal(_, expr.dataType))) case _ => Literal(false, BooleanType) } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala index 6ab248022844..9ab5551dd362 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala +++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala @@ -146,6 +146,14 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase { checkAnswer( spark.sql("SELECT * FROM t1 ORDER BY id"), Row(1, "a", 1) :: Row(5, "e", 4) :: Nil) + + // subquery eval nothing + spark.sql(s"""DELETE FROM t1 WHERE + |pt >= (SELECT min(id) FROM t2 WHERE n > 10)""".stripMargin) + + checkAnswer( + spark.sql("SELECT * FROM t1 ORDER BY id"), + Row(1, "a", 1) :: Row(5, "e", 4) :: Nil) } }) } @@ -182,6 +190,14 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase { checkAnswer( spark.sql("SELECT * FROM t1 ORDER BY id"), Row(1, "a", 1) :: Row(5, "e", 4) :: Nil) + + // subquery eval nothing + spark.sql(s"""DELETE FROM t1 WHERE + |pt in (SELECT id FROM t2 WHERE n > 10)""".stripMargin) + + checkAnswer( + spark.sql("SELECT * FROM t1 ORDER BY id"), + Row(1, "a", 1) :: Row(5, "e", 4) :: Nil) } }) }