[opt](nereids) refine operator estimation

xzj7019 · Sep 24, 2024 · d17fe57 · d17fe57
1 parent e31bdd4
commit d17fe57
Showing 1 changed file with 8 additions and 9 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -120,8 +120,7 @@ public Statistics visitCompoundPredicate(CompoundPredicate predicate, Estimation
         leftStats.normalizeColumnStatistics(context.statistics.getRowCount(), true);
         Statistics andStats = rightExpr.accept(this, new EstimationContext(leftStats));
         if (predicate instanceof And) {
-            // TODO: this will cause estimation change
-            //andStats.normalizeColumnStatistics();
+            andStats.normalizeColumnStatistics(context.statistics.getRowCount(), true);
             return andStats;
         } else if (predicate instanceof Or) {
             Statistics rightStats = rightExpr.accept(this, context);
@@ -309,8 +308,7 @@ private Optional<DateLiteral> tryConvertStrLiteralToDateLiteral(LiteralExpr lite
     }
 
     private Statistics estimateColumnEqualToConstant(ComparisonPredicate cp, ColumnStatistic statsForLeft,
-            ColumnStatistic statsForRight,
-            EstimationContext context) {
+            ColumnStatistic statsForRight, EstimationContext context) {
         double selectivity;
         if (statsForLeft.isUnKnown) {
             selectivity = DEFAULT_INEQUALITY_COEFFICIENT;
@@ -327,8 +325,8 @@ private Statistics estimateColumnEqualToConstant(ComparisonPredicate cp, ColumnS
             } else {
                 double val = statsForRight.maxValue;
                 if (val > statsForLeft.maxValue || val < statsForLeft.minValue) {
-                    // TODO: make sure left's stats is RangeScalable whose min/max is trustable
-                    // The equal to constant which don't rely on the range, so maybe safe here.
+                    // TODO: make sure left's stats is RangeScalable whose min/max is trustable.
+                    // The equal to constant doesn't rely on the range, so maybe safe here.
                     selectivity = 0.0;
                 } else if (ndv >= 1.0) {
                     selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv);
@@ -345,8 +343,8 @@ private Statistics estimateColumnEqualToConstant(ComparisonPredicate cp, ColumnS
         if (!(left instanceof SlotReference)) {
             left.accept(new ColumnStatsAdjustVisitor(), equalStats);
         }
-        // TODO: normalizeColumnStatistics() will have problem after ColumnStatsAdjustVisitor
-        //equalStats.normalizeColumnStatistics();
+        // normalize column statistics here to sync numNulls by proportion and ndv by current row count
+        equalStats.normalizeColumnStatistics(context.statistics.getRowCount(), true);
         return equalStats;
     }
 
@@ -645,7 +643,8 @@ private Statistics estimateColumnToConstantRange(Expression leftExpr, DataType d
         updatedStatistics.addColumnStats(leftExpr, leftColumnStatisticBuilder.build());
         context.addKeyIfSlot(leftExpr);
         leftExpr.accept(new ColumnStatsAdjustVisitor(), updatedStatistics);
-        updatedStatistics.normalizeColumnStatistics();
+        // normalize column statistics here to sync numNulls by proportion and ndv by current row count
+        updatedStatistics.normalizeColumnStatistics(context.statistics.getRowCount(), true);
 
         return updatedStatistics;
     }