Skip to content

Commit

Permalink
pick 40529_41464_40349
Browse files Browse the repository at this point in the history
  • Loading branch information
@englefly
englefly committed Oct 18, 2024
1 parent cec0458 commit 15d2f1e
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 25 deletions.
22 changes: 4 additions & 18 deletions fe/.idea/vcs.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 13 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
Expand All @@ -46,11 +47,13 @@
import org.apache.doris.nereids.processor.pre.PlanPreprocessors;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext;
import org.apache.doris.nereids.stats.StatsCalculator;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.ComputeResultSet;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
Expand Down Expand Up @@ -242,7 +245,16 @@ private Plan planWithoutLock(
return rewrittenPlan;
}
}

// if we cannot get table row count, skip join reorder
// except:
// 1. user set leading hint
// 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is false or FeConstants.runningUnitTest is true
if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest
&& !cascadesContext.isLeadingDisableJoinReorder()) {
List<LogicalOlapScan> scans = cascadesContext.getRewritePlan()
.collectToList(LogicalOlapScan.class::isInstance);
StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext);
}
optimize();
if (statementContext.getConnectContext().getExecutor() != null) {
statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime();
Expand Down
74 changes: 68 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
Expand Down Expand Up @@ -187,6 +188,11 @@ private StatsCalculator(GroupExpression groupExpression, boolean forbidUnknownCo
this.cascadesContext = context;
}

private StatsCalculator(CascadesContext context) {
this.groupExpression = null;
this.cascadesContext = context;
}

public Map<String, Histogram> getTotalHistogramMap() {
return totalHistogramMap;
}
Expand Down Expand Up @@ -256,6 +262,29 @@ private void estimate() {
groupExpression.setStatDerived(true);
}

/**
* disable join reorder if any table row count is not available.
*/
public static void disableJoinReorderIfTableRowCountNotAvailable(
List<LogicalOlapScan> scans, CascadesContext context) {
StatsCalculator calculator = new StatsCalculator(context);
for (LogicalOlapScan scan : scans) {
double rowCount = calculator.getOlapTableRowCount(scan);
// analyzed rowCount may be zero, but BE-reported rowCount could be positive.
// check ndv validation when reported rowCount > 0
if (rowCount == -1 && ConnectContext.get() != null) {
try {
ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce();
LOG.info("disable join reorder since row count not available: "
+ scan.getTable().getNameWithFullQualifiers());
} catch (Exception e) {
LOG.info("disableNereidsJoinReorderOnce failed");
}
return;
}
}
}

@Override
public Statistics visitLogicalSink(LogicalSink<? extends Plan> logicalSink, Void context) {
return groupExpression.childStatistics(0);
Expand Down Expand Up @@ -762,6 +791,43 @@ private ColumnStatistic getColumnStatistic(TableIf table, String colName, long i
}
}

private long computeDeltaRowCount(CatalogRelation scan) {
TableIf table = scan.getTable();
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(table.getId());
return tableMeta == null ? 0 : tableMeta.updatedRows.get();
}

/**
* if the table is not analyzed and BE does not report row count, return -1
*/
private double getOlapTableRowCount(OlapScan olapScan) {
OlapTable olapTable = olapScan.getTable();
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());
double rowCount = -1;
if (tableMeta != null && tableMeta.userInjected) {
rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId());
} else {
rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
if (rowCount == -1) {
if (tableMeta != null) {
rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId())
+ computeDeltaRowCount((CatalogRelation) olapScan);
}
}
}
return rowCount;
}

private double getTableRowCount(CatalogRelation relation) {
if (relation instanceof OlapScan) {
return getOlapTableRowCount((OlapScan) relation);
} else {
return relation.getTable().getRowCountForNereids();
}
}

// TODO: 1. Subtract the pruned partition
// 2. Consider the influence of runtime filter
// 3. Get NDV and column data size from StatisticManger, StatisticManager doesn't support it now.
Expand Down Expand Up @@ -791,11 +857,9 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) {
Set<SlotReference> slotSet = slotSetBuilder.build();
Map<Expression, ColumnStatisticBuilder> columnStatisticBuilderMap = new HashMap<>();
TableIf table = catalogRelation.getTable();
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(table.getId());
// rows newly updated after last analyze
long deltaRowCount = tableMeta == null ? 0 : tableMeta.updatedRows.get();
double rowCount = catalogRelation.getTable().getRowCountForNereids();
long deltaRowCount = computeDeltaRowCount(catalogRelation);
double rowCount = getTableRowCount(catalogRelation);
boolean hasUnknownCol = false;
long idxId = -1;
if (catalogRelation instanceof OlapScan) {
Expand Down Expand Up @@ -829,8 +893,6 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) {
colStatsBuilder.setAvgSizeByte(slotReference.getColumn().get().getType().getSlotSize());
}
if (!cache.isUnKnown) {
rowCount = Math.max(rowCount, cache.count + deltaRowCount);
} else {
hasUnknownCol = true;
}
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableStats) {
Expand Down
9 changes: 9 additions & 0 deletions regression-test/suites/nereids_p0/delta_row/delta_row.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
suite("delta_row") {
String database = context.config.getDbNameByFile(context.file)
sql """
set global enable_auto_analyze=false;
drop database if exists ${database};
create database ${database};
use ${database};
drop table if exists t;
CREATE TABLE IF NOT EXISTS t (
k int(11) null comment "",
v string replace null comment "",
Expand Down Expand Up @@ -52,4 +54,11 @@ suite("delta_row") {
// +--PhysicalFilter[72]@1 ( stats=0.5, predicates=(k#0 > 6) )
// +--PhysicalOlapScan[t]@0 ( stats=5(1) )
}
sql "alter table t modify column v set stats ('row_count'='200000', 'ndv'='197960', 'num_nulls'='0', 'min_value'=' 2MrUy', 'max_value'='zzzqXhTdKxT0RAR8yxbc', 'data_size'='2998285')"
// verify that inject rowCount has highest priority
explain{
sql "physical plan select * from t"
contains("PhysicalOlapScan[t]@0 ( stats=200,000")
}
sql "set global enable_auto_analyze=true;"
}

0 comments on commit 15d2f1e

Please sign in to comment.