Skip to content

Commit

Permalink
[opt](nereids) tabe row count priority: user injected > BE report > a…
Browse files Browse the repository at this point in the history
…nalyzed (apache#40529)

## Proposed changes
there are 2 sources for table row count, one is analyzed result, another
is from BE report.
But neither of them is accurate. the priorities of them from high to
low:
user injected > BE report > analyzed

Issue Number: close #xxx

<!--Describe your changes.-->
  • Loading branch information
englefly authored Sep 9, 2024
1 parent 5392cb1 commit 0d3374b
Showing 1 changed file with 27 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -364,19 +364,28 @@ private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) {
}
}

private Statistics computeOlapScan(OlapScan olapScan) {
private double getOlapTableRowCount(OlapScan olapScan) {
OlapTable olapTable = olapScan.getTable();
double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
if (tableRowCount <= 0) {
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());
if (tableMeta != null) {
// create-view after analyzing, we may get -1 for this view row count
tableRowCount = Math.max(1, tableMeta.getRowCount(olapScan.getSelectedIndexId()));
} else {
tableRowCount = 1;
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());
double rowCount = -1;
if (tableMeta != null && tableMeta.userInjected) {
rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId());
} else {
rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
if (rowCount == -1) {
if (tableMeta != null) {
rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId());
}
}
}
return rowCount;
}

private Statistics computeOlapScan(OlapScan olapScan) {
OlapTable olapTable = olapScan.getTable();
double tableRowCount = getOlapTableRowCount(olapScan);
tableRowCount = Math.max(1, tableRowCount);

if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId() || olapTable instanceof MTMV) {
// mv is selected, return its estimated stats
Expand Down Expand Up @@ -441,10 +450,13 @@ private Statistics computeOlapScan(OlapScan olapScan) {
}
}

boolean useTableLevelStats = true;
if (olapScan.getSelectedPartitionIds().size() < olapScan.getTable().getPartitionNum()) {
// partition pruned
// try to use selected partition stats, if failed, fall back to table stats
double selectedPartitionsRowCount = getSelectedPartitionRowCount(olapScan);
if (selectedPartitionsRowCount > 0) {
if (selectedPartitionsRowCount >= 0) {
useTableLevelStats = false;
List<String> selectedPartitionNames = new ArrayList<>(olapScan.getSelectedPartitionIds().size());
olapScan.getSelectedPartitionIds().forEach(id -> {
selectedPartitionNames.add(olapScan.getTable().getPartition(id).getName());
Expand All @@ -458,19 +470,11 @@ private Statistics computeOlapScan(OlapScan olapScan) {
}
checkIfUnknownStatsUsedAsKey(builder);
builder.setRowCount(selectedPartitionsRowCount + deltaRowCount);
} else {
// if partition row count is invalid (-1), fallback to table stats
for (SlotReference slot : visibleOutputSlots) {
ColumnStatistic cache = getColumnStatsFromTableCache((CatalogRelation) olapScan, slot);
ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache);
colStatsBuilder.setCount(tableRowCount);
colStatsBuilder.normalizeAvgSizeByte(slot);
builder.putColumnStatistics(slot, colStatsBuilder.build());
}
checkIfUnknownStatsUsedAsKey(builder);
builder.setRowCount(tableRowCount + deltaRowCount);
}
} else {
}
// 1. no partition is pruned, or
// 2. fall back to table stats
if (useTableLevelStats) {
// get table level stats
for (SlotReference slot : visibleOutputSlots) {
ColumnStatistic cache = getColumnStatsFromTableCache((CatalogRelation) olapScan, slot);
Expand Down

0 comments on commit 0d3374b

Please sign in to comment.