From 3c08d2bb4d504c2502b0e569bd0fbbcad0b7cc93 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 15 Aug 2024 14:41:29 +0800 Subject: [PATCH] update --- .../execution/CHHashJoinExecTransformer.scala | 31 +++---------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala index 252b9bc03fd68..193edb346ddf1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala @@ -115,9 +115,6 @@ case class CHShuffledHashJoinExecTransformer( override def genJoinParameters(): Any = { val (isBHJ, isNullAwareAntiJoin, buildHashTableId): (Int, Int, String) = (0, 0, "") - // Don't use lef/right directly, they may be reordered in `HashJoinLikeExecTransformer` - val leftStats = getShuffleStageStatistics(streamedPlan) - val rightStats = getShuffleStageStatistics(buildPlan) // Start with "JoinParameters:" val joinParametersStr = new StringBuffer("JoinParameters:") // isBHJ: 0 for SHJ, 1 for BHJ @@ -139,13 +136,13 @@ case class CHShuffledHashJoinExecTransformer( logicalLink match { case Some(join: Join) => val leftRowCount = - if (needSwitchChildren) join.left.stats.rowCount else join.right.stats.rowCount + if (!needSwitchChildren) join.left.stats.rowCount else join.right.stats.rowCount val rightRowCount = - if (needSwitchChildren) join.right.stats.rowCount else join.left.stats.rowCount + if (!needSwitchChildren) join.right.stats.rowCount else join.left.stats.rowCount val leftSizeInBytes = - if (needSwitchChildren) join.left.stats.sizeInBytes else join.right.stats.sizeInBytes + if (!needSwitchChildren) join.left.stats.sizeInBytes else join.right.stats.sizeInBytes val rightSizeInBytes = - if (needSwitchChildren) join.right.stats.sizeInBytes else join.left.stats.sizeInBytes + if (!needSwitchChildren) join.right.stats.sizeInBytes else join.left.stats.sizeInBytes val numPartitions = outputPartitioning.numPartitions joinParametersStr .append("leftRowCount=") @@ -171,26 +168,6 @@ case class CHShuffledHashJoinExecTransformer( .build() BackendsApiManager.getTransformerApiInstance.packPBMessage(message) } - - private def getShuffleStageStatistics(plan: SparkPlan): ShuffleStageStaticstics = { - plan match { - case queryStage: ShuffleQueryStageExec => - ShuffleStageStaticstics( - queryStage.shuffle.numPartitions, - queryStage.shuffle.numMappers, - queryStage.getRuntimeStatistics.rowCount) - case shuffle: ColumnarShuffleExchangeExec => - // FIXEME: We cannot access shuffle.numPartitions and shuffle.numMappers here. - // Otherwise it will cause an exception `ProjectExecTransformer has column support mismatch` - ShuffleStageStaticstics(-1, -1, None) - case _ => - if (plan.children.length == 1) { - getShuffleStageStatistics(plan.children.head) - } else { - ShuffleStageStaticstics(-1, -1, None) - } - } - } } case class CHBroadcastBuildSideRDD(