From 203d25d435245b70f746999419a6892f2bf758ec Mon Sep 17 00:00:00 2001 From: minghong Date: Sun, 29 Sep 2024 10:45:50 +0800 Subject: [PATCH] [feat](nereids) disable join reorder if any table row count is not available #40349 (2.0) (#41425) ## Proposed changes pick #40349 Issue Number: close #xxx --- .../org/apache/doris/catalog/OlapTable.java | 3 ++ .../apache/doris/nereids/NereidsPlanner.java | 13 +++++ .../doris/nereids/stats/StatsCalculator.java | 48 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index ffff643a8619cd..c3f8600b214c1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1277,6 +1277,9 @@ public long fetchRowCount() { return getRowCountForIndex(baseIndexId, false); } + /** + * @return -1 if there are some tablets whose row count is not reported to FE + */ public long getRowCountForIndex(long indexId, boolean strict) { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index c17d3fa545ba30..150780d95e1ad8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.NereidsException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugUtil; @@ -43,12 +44,14 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.pre.PlanPreprocessors; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.stats.StatsCalculator; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -258,6 +261,16 @@ private Plan planWithoutLock( } } + // if we cannot get table row count, skip join reorder + // except: + // 1. user set leading hint + // 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is false or FeConstants.runningUnitTest is true + if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest && cascadesContext.isLeadingJoin()) { + List scans = cascadesContext.getRewritePlan() + .collectToList(LogicalOlapScan.class::isInstance); + StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); + } + optimize(); if (statementContext.getConnectContext().getExecutor() != null) { statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 1b67b80a62d5d5..9458407295d903 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.stats; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; @@ -179,6 +180,11 @@ private StatsCalculator(GroupExpression groupExpression, boolean forbidUnknownCo this.cascadesContext = context; } + private StatsCalculator(CascadesContext context) { + this.groupExpression = null; + this.cascadesContext = context; + } + public Map getTotalHistogramMap() { return totalHistogramMap; } @@ -1132,4 +1138,46 @@ public Statistics visitPhysicalCTEAnchor( return groupExpression.childStatistics(1); } + /** + * if the table is not analyzed and BE does not report row count, return -1 + */ + private double getOlapTableRowCount(OlapScan olapScan) { + OlapTable olapTable = olapScan.getTable(); + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId()); + double rowCount = -1; + if (tableMeta != null && tableMeta.userInjected) { + rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); + } else { + rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true); + if (rowCount == -1) { + if (tableMeta != null) { + rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); + } + } + } + return rowCount; + } + + /** + * disable join reorder if any table row count is not available. + */ + public static void disableJoinReorderIfTableRowCountNotAvailable( + List scans, + CascadesContext context) { + StatsCalculator calculator = new StatsCalculator(context); + for (LogicalOlapScan scan : scans) { + double rowCount = calculator.getOlapTableRowCount(scan); + if (rowCount == -1 && ConnectContext.get() != null) { + try { + ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce(); + LOG.info("disable join reorder since row count not available: " + + scan.getTable().getNameWithFullQualifiers()); + } catch (Exception e) { + LOG.info("disableNereidsJoinReorderOnce failed"); + } + return; + } + } + } }