diff --git a/be/src/vec/exprs/vruntimefilter_wrapper.h b/be/src/vec/exprs/vruntimefilter_wrapper.h index 79e9361e854719..060f33e01a1863 100644 --- a/be/src/vec/exprs/vruntimefilter_wrapper.h +++ b/be/src/vec/exprs/vruntimefilter_wrapper.h @@ -77,7 +77,8 @@ class VRuntimeFilterWrapper final : public VExpr { template static void judge_selectivity(double ignore_threshold, int64_t filter_rows, int64_t input_rows, T& always_true) { - always_true = filter_rows / (input_rows * 1.0L) < ignore_threshold; + always_true = static_cast(filter_rows) / static_cast(input_rows) < + ignore_threshold; } bool is_rf_wrapper() const override { return true; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java index 8c9011bcbad18f..7408e2888cc3a5 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java @@ -17,6 +17,8 @@ package org.apache.doris.common; +import org.apache.commons.io.IOUtils; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -40,7 +42,7 @@ public static byte[] compress(byte[] data) throws IOException { public static byte[] decompress(byte[] data) throws IOException { ByteArrayInputStream bytesStream = new ByteArrayInputStream(data); try (GZIPInputStream gzipStream = new GZIPInputStream(bytesStream)) { - return gzipStream.readAllBytes(); + return IOUtils.toByteArray(gzipStream); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java index 7280463b0f2bba..bdb8b7bd8d33b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java @@ -308,7 +308,6 @@ public void setMutableState(String key, Object value) { @Override public Expression withChildren(List children) { throw new AnalysisException("could not call withChildren on UdfSignatureSearcher"); - } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 8a9ce4a2a2a545..9f1ee80508166e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -56,6 +56,7 @@ import org.apache.doris.mtmv.MTMVVersionSnapshot; import org.apache.doris.nereids.hint.Hint; import org.apache.doris.nereids.hint.UseMvHint; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; @@ -1384,12 +1385,7 @@ public List getEqualPartitionNames(List partitionIds1, List } public List getPartitionIds() { - readLock(); - try { - return new ArrayList<>(idToPartition.keySet()); - } finally { - readUnlock(); - } + return Utils.fastToImmutableList(idToPartition.keySet()); } public Set getCopiedBfColumns() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 4777a14741e192..021698ea747e39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -549,6 +549,9 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical } private long getGarbageCollectionTime() { + if (!ConnectContext.get().getSessionVariable().enableProfile()) { + return 0; + } List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); long initialGCTime = 0; for (GarbageCollectorMXBean gcBean : gcMxBeans) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 69848ca8f04da2..627e1145f4b872 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -174,6 +174,8 @@ public class StatementContext implements Closeable { private Backend groupCommitMergeBackend; + private boolean checkedPrivileges; + public StatementContext() { this(ConnectContext.get(), null, 0); } @@ -580,4 +582,12 @@ public void setGroupCommitMergeBackend( Backend groupCommitMergeBackend) { this.groupCommitMergeBackend = groupCommitMergeBackend; } + + public boolean isCheckedPrivileges() { + return checkedPrivileges; + } + + public void setCheckedPrivileges(boolean checkedPrivileges) { + this.checkedPrivileges = checkedPrivileges; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 654ccc8ca1155a..3435c05658ae3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1890,12 +1890,15 @@ public PlanFragment visitPhysicalProject(PhysicalProject project int layerCount = project.getMultiLayerProjects().size(); for (int i = 0; i < layerCount; i++) { List layer = project.getMultiLayerProjects().get(i); - projectionExprs = layer.stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(Collectors.toList()); - slots = layer.stream() - .map(NamedExpression::toSlot) - .collect(Collectors.toList()); + + projectionExprs = new ArrayList<>(layer.size()); + slots = new ArrayList<>(layer.size()); + for (int j = 0; j < layer.size(); j++) { + NamedExpression layerExpr = layer.get(j); + projectionExprs.add(ExpressionTranslator.translate(layerExpr, context)); + slots.add(layerExpr.toSlot()); + } + if (i < layerCount - 1) { inputPlanNode.addIntermediateProjectList(projectionExprs); TupleDescriptor projectionTuple = generateTupleDesc(slots, null, context); @@ -1904,14 +1907,15 @@ public PlanFragment visitPhysicalProject(PhysicalProject project allProjectionExprs.addAll(projectionExprs); } } else { - projectionExprs = project.getProjects() - .stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(Collectors.toList()); - slots = project.getProjects() - .stream() - .map(NamedExpression::toSlot) - .collect(Collectors.toList()); + List projects = project.getProjects(); + int projectNum = projects.size(); + projectionExprs = new ArrayList<>(projectNum); + slots = new ArrayList<>(projectNum); + for (int j = 0; j < projectNum; j++) { + NamedExpression layerExpr = projects.get(j); + projectionExprs.add(ExpressionTranslator.translate(layerExpr, context)); + slots.add(layerExpr.toSlot()); + } allProjectionExprs.addAll(projectionExprs); } // process multicast sink diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java index 4eebf6ffc05f4b..2d6268bbb80ee2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java @@ -26,9 +26,10 @@ import org.apache.doris.nereids.jobs.rewrite.RewriteJob; import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob; import org.apache.doris.nereids.jobs.rewrite.TopicRewriteJob; -import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.FilteredRules; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.Rules; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; @@ -94,10 +95,10 @@ public static RewriteJob bottomUp(RuleFactory... ruleFactories) { } public static RewriteJob bottomUp(List ruleFactories) { - List rules = ruleFactories.stream() + Rules rules = new FilteredRules(ruleFactories.stream() .map(RuleFactory::buildRules) .flatMap(List::stream) - .collect(ImmutableList.toImmutableList()); + .collect(ImmutableList.toImmutableList())); return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteBottomUpJob::new, getTraversePredicate(), true); } @@ -110,10 +111,10 @@ public static RewriteJob topDown(List ruleFactories) { } public static RewriteJob topDown(List ruleFactories, boolean once) { - List rules = ruleFactories.stream() + Rules rules = new FilteredRules(ruleFactories.stream() .map(RuleFactory::buildRules) .flatMap(List::stream) - .collect(ImmutableList.toImmutableList()); + .collect(ImmutableList.toImmutableList())); return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteTopDownJob::new, getTraversePredicate(), once); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 894d4264201533..9e960d8a722d1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.analysis.VariableToLiteral; +import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; @@ -177,7 +178,11 @@ private static List buildAnalyzerJobs(Optional topDown(new LeadingJoin()), bottomUp(new NormalizeGenerate()), bottomUp(new SubqueryToApply()), - topDown(new MergeProjects()) + topDown( + new MergeProjects(), + // merge normal filter and hidden column filter + new MergeFilters() + ) ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index f4ca9a972a6814..61b6863f654917 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -45,7 +45,6 @@ import org.apache.doris.nereids.rules.rewrite.CheckDataTypes; import org.apache.doris.nereids.rules.rewrite.CheckMatchExpression; import org.apache.doris.nereids.rules.rewrite.CheckMultiDistinct; -import org.apache.doris.nereids.rules.rewrite.CheckPrivileges; import org.apache.doris.nereids.rules.rewrite.CheckRestorePartition; import org.apache.doris.nereids.rules.rewrite.ClearContextStatus; import org.apache.doris.nereids.rules.rewrite.CollectCteConsumerOutput; @@ -84,7 +83,6 @@ import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull; import org.apache.doris.nereids.rules.rewrite.InferPredicates; import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct; -import org.apache.doris.nereids.rules.rewrite.InlineLogicalView; import org.apache.doris.nereids.rules.rewrite.LimitAggToTopNAgg; import org.apache.doris.nereids.rules.rewrite.LimitSortToTopN; import org.apache.doris.nereids.rules.rewrite.LogicalResultSinkToShortCircuitPointQuery; @@ -220,13 +218,6 @@ public class Rewriter extends AbstractBatchJobExecutor { // but it appeared at LogicalApply.right. After the `Subquery unnesting` topic, all slots is placed in a // normal position, then we can check column privileges by these steps // - // 1. use ColumnPruning rule to derive the used slots in LogicalView - // 2. and then check the column privileges - // 3. finally, we can eliminate the LogicalView - topic("Inline view and check column privileges", - custom(RuleType.CHECK_PRIVILEGES, CheckPrivileges::new), - bottomUp(new InlineLogicalView()) - ), topic("Eliminate optimization", bottomUp( new EliminateLimit(), @@ -438,8 +429,7 @@ public class Rewriter extends AbstractBatchJobExecutor { new CollectCteConsumerOutput() ) ), - topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new) - ) + topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new)) ) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java index 656fde3024e79f..7db761d007828a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; -import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.Rules; import org.apache.doris.nereids.trees.plans.Plan; import java.util.List; @@ -40,7 +40,7 @@ public class PlanTreeRewriteBottomUpJob extends PlanTreeRewriteJob { // so we will do specified action for each node based on their 'RewriteState'. private static final String REWRITE_STATE_KEY = "rewrite_state"; private final RewriteJobContext rewriteJobContext; - private final List rules; + private final Rules rules; private final int batchId; enum RewriteState { @@ -57,7 +57,7 @@ enum RewriteState { public PlanTreeRewriteBottomUpJob( RewriteJobContext rewriteJobContext, JobContext context, - Predicate isTraverseChildren, List rules) { + Predicate isTraverseChildren, Rules rules) { super(JobType.BOTTOM_UP_REWRITE, context, isTraverseChildren); this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext, "rewriteContext cannot be null"); this.rules = Objects.requireNonNull(rules, "rules cannot be null"); @@ -88,6 +88,13 @@ public void execute() { private void rewriteThis() { // Link the current node with the sub-plan to get the current plan which is used in the rewrite phase later. Plan plan = linkChildren(rewriteJobContext.plan, rewriteJobContext.childrenContext); + if (rules.getCurrentAndChildrenRules(plan).isEmpty()) { + // No new plan is generated, so just set the state of the current plan to 'REWRITTEN'. + setState(plan, RewriteState.REWRITTEN, batchId); + rewriteJobContext.setResult(plan); + return; + } + RewriteResult rewriteResult = rewrite(plan, rules, rewriteJobContext); if (rewriteResult.hasNewPlan) { RewriteJobContext newJobContext = rewriteJobContext.withPlan(rewriteResult.plan); @@ -110,6 +117,13 @@ private void rewriteThis() { private void ensureChildrenRewritten() { Plan plan = rewriteJobContext.plan; + if (rules.getCurrentAndChildrenRules(plan).isEmpty()) { + // No new plan is generated, so just set the state of the current plan to 'REWRITTEN'. + setState(plan, RewriteState.REWRITTEN, batchId); + rewriteJobContext.setResult(plan); + return; + } + int batchId = rewriteJobContext.batchId; setState(plan, RewriteState.REWRITE_THIS, batchId); pushJob(new PlanTreeRewriteBottomUpJob(rewriteJobContext, context, isTraverseChildren, rules)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java index 60969cdf6e701d..3967c1a3df8dc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.minidump.NereidsTracer; import org.apache.doris.nereids.pattern.Pattern; import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.Rules; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; @@ -43,12 +44,12 @@ public PlanTreeRewriteJob(JobType type, JobContext context, Predicate isTr this.isTraverseChildren = Objects.requireNonNull(isTraverseChildren, "isTraverseChildren can not be null"); } - protected final RewriteResult rewrite(Plan plan, List rules, RewriteJobContext rewriteJobContext) { + protected final RewriteResult rewrite(Plan plan, Rules rules, RewriteJobContext rewriteJobContext) { CascadesContext cascadesContext = context.getCascadesContext(); cascadesContext.setIsRewriteRoot(rewriteJobContext.isRewriteRoot()); boolean showPlanProcess = cascadesContext.showPlanProcess(); - for (Rule rule : rules) { + for (Rule rule : rules.getCurrentRules(plan)) { if (disableRules.get(rule.getRuleType().type())) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java index 806b2cf61ea454..e504955595ff74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java @@ -19,7 +19,7 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; -import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.Rules; import org.apache.doris.nereids.trees.plans.Plan; import java.util.List; @@ -34,11 +34,11 @@ public class PlanTreeRewriteTopDownJob extends PlanTreeRewriteJob { private final RewriteJobContext rewriteJobContext; - private final List rules; + private final Rules rules; public PlanTreeRewriteTopDownJob( RewriteJobContext rewriteJobContext, JobContext context, - Predicate isTraverseChildren, List rules) { + Predicate isTraverseChildren, Rules rules) { super(JobType.TOP_DOWN_REWRITE, context, isTraverseChildren); this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext, "rewriteContext cannot be null"); this.rules = Objects.requireNonNull(rules, "rules cannot be null"); @@ -47,6 +47,14 @@ public PlanTreeRewriteTopDownJob( @Override public void execute() { if (!rewriteJobContext.childrenVisited) { + if (rules.getCurrentAndChildrenRules(rewriteJobContext.plan).isEmpty()) { + rewriteJobContext.setResult(rewriteJobContext.plan); + if (rewriteJobContext.parentContext == null) { + context.getCascadesContext().setRewritePlan(rewriteJobContext.plan); + } + return; + } + RewriteResult rewriteResult = rewrite(rewriteJobContext.plan, rules, rewriteJobContext); if (rewriteResult.hasNewPlan) { RewriteJobContext newContext = rewriteJobContext diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java index c0f2f784da7c8d..15f5139053b46d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java @@ -29,8 +29,10 @@ import org.apache.doris.nereids.metrics.consumer.LogConsumer; import org.apache.doris.nereids.metrics.event.TransformEvent; import org.apache.doris.nereids.pattern.GroupExpressionMatching; +import org.apache.doris.nereids.rules.FilteredRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.Rules; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.base.Preconditions; @@ -50,15 +52,15 @@ public class RewriteTopDownJob extends Job { EventChannel.getDefaultChannel().addConsumers(new LogConsumer(TransformEvent.class, NereidsPlanner.LOG))); private final Group group; - private final List rules; + private final Rules rules; public RewriteTopDownJob(Group group, JobContext context, List factories) { - this(group, factories.stream() + this(group, new FilteredRules(factories.stream() .flatMap(factory -> factory.buildRules().stream()) - .collect(Collectors.toList()), context, true); + .collect(Collectors.toList())), context, true); } - public RewriteTopDownJob(Group group, List rules, JobContext context) { + public RewriteTopDownJob(Group group, Rules rules, JobContext context) { this(group, rules, context, true); } @@ -69,7 +71,7 @@ public RewriteTopDownJob(Group group, List rules, JobContext context) { * @param rules rewrite rules * @param context planner context */ - public RewriteTopDownJob(Group group, List rules, JobContext context, boolean once) { + public RewriteTopDownJob(Group group, Rules rules, JobContext context, boolean once) { super(JobType.TOP_DOWN_REWRITE, context, once); this.group = Objects.requireNonNull(group, "group cannot be null"); this.rules = Objects.requireNonNull(rules, "rules cannot be null"); @@ -84,7 +86,7 @@ public EventProducer getEventTracer() { public void execute() { GroupExpression logicalExpression = group.getLogicalExpression(); countJobExecutionTimesOfGroupExpressions(logicalExpression); - for (Rule rule : rules) { + for (Rule rule : rules.getCurrentAndChildrenRules()) { if (rule.isInvalid(disableRules, logicalExpression)) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java index 551efa1026d695..b8e7e49c905da7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.jobs.scheduler.JobStack; import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.Rules; import org.apache.doris.nereids.trees.plans.Plan; import java.util.List; @@ -34,17 +35,17 @@ public class RootPlanTreeRewriteJob implements RewriteJob { private static final AtomicInteger BATCH_ID = new AtomicInteger(); - private final List rules; + private final Rules rules; private final RewriteJobBuilder rewriteJobBuilder; private final boolean once; private final Predicate isTraverseChildren; - public RootPlanTreeRewriteJob(List rules, RewriteJobBuilder rewriteJobBuilder, boolean once) { + public RootPlanTreeRewriteJob(Rules rules, RewriteJobBuilder rewriteJobBuilder, boolean once) { this(rules, rewriteJobBuilder, plan -> true, once); } public RootPlanTreeRewriteJob( - List rules, RewriteJobBuilder rewriteJobBuilder, Predicate isTraverseChildren, boolean once) { + Rules rules, RewriteJobBuilder rewriteJobBuilder, Predicate isTraverseChildren, boolean once) { this.rules = Objects.requireNonNull(rules, "rules cannot be null"); this.rewriteJobBuilder = Objects.requireNonNull(rewriteJobBuilder, "rewriteJobBuilder cannot be null"); this.once = once; @@ -76,7 +77,7 @@ public boolean isOnce() { /** RewriteJobBuilder */ public interface RewriteJobBuilder { Job build(RewriteJobContext rewriteJobContext, JobContext jobContext, - Predicate isTraverseChildren, List rules); + Predicate isTraverseChildren, Rules rules); } /** RootRewriteJobContext */ @@ -130,7 +131,12 @@ public Plan getNewestPlan() { } public List getRules() { - return rules; + return rules.getCurrentAndChildrenRules(); + } + + @Override + public String toString() { + return rules.toString(); } /** use to assemble the rewriting plan */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternRules.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternRules.java index 523540e6435d89..57cce469a62a71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternRules.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternRules.java @@ -20,12 +20,15 @@ import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext; import org.apache.doris.nereids.rules.expression.ExpressionPatternMatchRule; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.SuperClassId; +import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.lang.reflect.Field; +import java.util.BitSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -33,9 +36,18 @@ /** ExpressionPatternMapping */ public class ExpressionPatternRules extends TypeMappings { private static final Logger LOG = LogManager.getLogger(ExpressionPatternRules.class); + private BitSet typePatternIds; + /** ExpressionPatternRules */ public ExpressionPatternRules(List typeMappings) { super(typeMappings); + + BitSet typePatternIds = new BitSet(); + for (ExpressionPatternMatchRule typeMapping : typeMappings) { + Class topType = typeMapping.getType(); + typePatternIds.set(SuperClassId.getClassId(topType)); + } + this.typePatternIds = typePatternIds; } @Override @@ -43,9 +55,17 @@ protected Set> getChildrenClasses(Class treeNode) { + BitSet classTypes = treeNode.getAllChildrenTypes(); + if (!typePatternIds.intersects(classTypes)) { + return false; + } + return true; + } + /** matchesAndApply */ public Optional matchesAndApply(Expression expr, ExpressionRewriteContext context, Expression parent) { - List rules = singleMappings.get(expr.getClass()); + List rules = getSingleMapping(expr.getClass()); ExpressionMatchingContext matchingContext = new ExpressionMatchingContext<>(expr, parent, context); switch (rules.size()) { @@ -55,9 +75,11 @@ public Optional matchesAndApply(Expression expr, ExpressionRewriteCo Expression newExpr = multiMatchRule.apply(matchingContext); if (!newExpr.equals(expr)) { if (context.cascadesContext.isEnableExprTrace()) { - traceExprChanged(multiMatchRule, expr, newExpr); + traceExprChanged(multiMatchRule, expr, newExpr, true); } return Optional.of(newExpr); + } else if (context.cascadesContext.isEnableExprTrace()) { + traceExprChanged(multiMatchRule, expr, newExpr, false); } } } @@ -69,9 +91,11 @@ public Optional matchesAndApply(Expression expr, ExpressionRewriteCo Expression newExpr = rule.apply(matchingContext); if (!newExpr.equals(expr)) { if (context.cascadesContext.isEnableExprTrace()) { - traceExprChanged(rule, expr, newExpr); + traceExprChanged(rule, expr, newExpr, true); } return Optional.of(newExpr); + } else if (context.cascadesContext.isEnableExprTrace()) { + traceExprChanged(rule, expr, newExpr, false); } } return Optional.empty(); @@ -82,9 +106,11 @@ public Optional matchesAndApply(Expression expr, ExpressionRewriteCo Expression newExpr = rule.apply(matchingContext); if (!expr.equals(newExpr)) { if (context.cascadesContext.isEnableExprTrace()) { - traceExprChanged(rule, expr, newExpr); + traceExprChanged(rule, expr, newExpr, true); } return Optional.of(newExpr); + } else if (context.cascadesContext.isEnableExprTrace()) { + traceExprChanged(rule, expr, newExpr, false); } } } @@ -93,7 +119,8 @@ public Optional matchesAndApply(Expression expr, ExpressionRewriteCo } } - private static void traceExprChanged(ExpressionPatternMatchRule rule, Expression expr, Expression newExpr) { + private static void traceExprChanged( + ExpressionPatternMatchRule rule, Expression expr, Expression newExpr, boolean changed) { try { Field[] declaredFields = (rule.matchingAction).getClass().getDeclaredFields(); Class ruleClass; @@ -104,7 +131,11 @@ private static void traceExprChanged(ExpressionPatternMatchRule rule, Expression field.setAccessible(true); ruleClass = field.get(rule.matchingAction).getClass(); } - LOG.info("RULE: " + ruleClass + "\nbefore: " + expr + "\nafter: " + newExpr); + if (changed) { + LOG.info("RULE: " + ruleClass + "\nbefore: " + expr + "\nafter: " + newExpr); + } else { + LOG.info("RULE: " + ruleClass + " not changed\nbefore: " + expr); + } } catch (Throwable t) { LOG.error(t.getMessage(), t); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternTraverseListeners.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternTraverseListeners.java index 3f3640a43bf8b2..53213a26e747d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternTraverseListeners.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternTraverseListeners.java @@ -45,7 +45,7 @@ protected Set> getChildrenClasses(Class listenerSingleMappings = singleMappings.get(expr.getClass()); + List listenerSingleMappings = getSingleMapping(expr.getClass()); ExpressionMatchingContext matchingContext = new ExpressionMatchingContext<>(expr, parent, context); switch (listenerSingleMappings.size()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java index 91dd87ba457837..4b28a93fa93b10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java @@ -239,4 +239,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(predicates, patternType, planType); } + + // for TypePattern + public Class getMatchedType() { + return null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java index 5e2eb5757aac81..a3a2baee282821 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java @@ -58,12 +58,12 @@ public PatternDescriptor whenNot(Predicate predicate) { public PatternMatcher then( Function matchedAction) { MatchedAction adaptMatchedAction = ctx -> matchedAction.apply(ctx.root); - return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedAction); + return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedAction, matchedAction.getClass().getName()); } public PatternMatcher thenApply( MatchedAction matchedAction) { - return new PatternMatcher<>(pattern, defaultPromise, matchedAction); + return new PatternMatcher<>(pattern, defaultPromise, matchedAction, matchedAction.getClass().getName()); } /** @@ -79,18 +79,18 @@ public PatternMatcher thenAp return null; } }; - return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedAction); + return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedAction, matchedAction.getClass().getName()); } public PatternMatcher thenMulti( Function> matchedAction) { MatchedMultiAction adaptMatchedAction = ctx -> matchedAction.apply(ctx.root); - return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedAction); + return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedAction, matchedAction.getClass().getName()); } public PatternMatcher thenApplyMulti( MatchedMultiAction matchedAction) { - return new PatternMatcher<>(pattern, defaultPromise, matchedAction); + return new PatternMatcher<>(pattern, defaultPromise, matchedAction, matchedAction.getClass().getName()); } /** @@ -106,7 +106,8 @@ public PatternMatcher thenAp return null; } }; - return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedMultiAction); + return new PatternMatcher<>(pattern, defaultPromise, + adaptMatchedMultiAction, matchedMultiAction.getClass().getName()); } public Pattern getPattern() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternMatcher.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternMatcher.java index 2a6e1505bd1abc..5fa8cb86db56a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternMatcher.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternMatcher.java @@ -38,6 +38,7 @@ public class PatternMatcher { public final RulePromise defaultRulePromise; public final MatchedAction matchedAction; public final MatchedMultiAction matchedMultiAction; + public final String ruleName; /** * PatternMatcher wrap a pattern, defaultRulePromise and matchedAction. @@ -47,21 +48,24 @@ public class PatternMatcher { * @param matchedAction matched callback function */ public PatternMatcher(Pattern pattern, RulePromise defaultRulePromise, - MatchedAction matchedAction) { + MatchedAction matchedAction, String ruleName) { this.pattern = Objects.requireNonNull(pattern, "pattern can not be null"); this.defaultRulePromise = Objects.requireNonNull( defaultRulePromise, "defaultRulePromise can not be null"); this.matchedAction = Objects.requireNonNull(matchedAction, "matchedAction can not be null"); this.matchedMultiAction = null; + this.ruleName = Objects.requireNonNull(ruleName, "ruleName can not be null"); } + /** PatternMatcher */ public PatternMatcher(Pattern pattern, RulePromise defaultRulePromise, - MatchedMultiAction matchedAction) { + MatchedMultiAction matchedAction, String ruleName) { this.pattern = Objects.requireNonNull(pattern, "pattern can not be null"); this.defaultRulePromise = Objects.requireNonNull( defaultRulePromise, "defaultRulePromise can not be null"); this.matchedMultiAction = Objects.requireNonNull(matchedAction, "matchedMultiAction can not be null"); this.matchedAction = null; + this.ruleName = Objects.requireNonNull(ruleName, "ruleName can not be null"); } public Rule toRule(RuleType ruleType) { @@ -93,6 +97,11 @@ public List transform(Plan originPlan, CascadesContext context) { return ImmutableList.of(replacePlan == null ? originPlan : replacePlan); } } + + @Override + public String ruleName() { + return ruleName; + } }; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypeMappings.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypeMappings.java index 4eb5ffc76d22a2..2b0ff7d21a6a7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypeMappings.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypeMappings.java @@ -20,23 +20,25 @@ import org.apache.doris.nereids.pattern.TypeMappings.TypeMapping; import org.apache.doris.nereids.util.Utils; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import javax.annotation.Nullable; /** ExpressionPatternMappings */ public abstract class TypeMappings> { - protected final ListMultimap, T> singleMappings; + protected final Map, List> singleMappings; protected final List multiMappings; /** ExpressionPatternMappings */ public TypeMappings(List typeMappings) { - this.singleMappings = ArrayListMultimap.create(); + this.singleMappings = Maps.newHashMap(); this.multiMappings = Lists.newArrayList(); for (T mapping : typeMappings) { @@ -75,8 +77,9 @@ public TypeMappings(List typeMappings) { } } - public @Nullable List get(Class clazz) { - return singleMappings.get(clazz); + public @Nullable List getSingleMapping(Class clazz) { + List ts = singleMappings.get(clazz); + return ts == null ? ImmutableList.of() : ts; } private void addSimpleMapping(T typeMapping) { @@ -108,7 +111,7 @@ private void addMultiMapping(T multiMapping) { for (Class existSingleType : existSingleMappingTypes) { Class type = multiMapping.getType(); if (type.isAssignableFrom(existSingleType)) { - singleMappings.put(existSingleType, multiMapping); + putMultimap(existSingleType, multiMapping); } } } @@ -117,11 +120,16 @@ private void addSingleMapping(Class clazz, T singleMapping) { if (!singleMappings.containsKey(clazz) && !multiMappings.isEmpty()) { for (T multiMapping : multiMappings) { if (multiMapping.getType().isAssignableFrom(clazz)) { - singleMappings.put(clazz, multiMapping); + putMultimap(clazz, multiMapping); } } } - singleMappings.put(clazz, singleMapping); + putMultimap(clazz, singleMapping); + } + + private void putMultimap(Class clazz, T singleMapping) { + List ts = singleMappings.computeIfAbsent(clazz, c -> new ArrayList<>()); + ts.add(singleMapping); } protected abstract Set> getChildrenClasses(Class clazz); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java index 0de972113c24c9..59573c1e0d9f0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypePattern.java @@ -48,4 +48,9 @@ public TypePattern withPredicates(List> predicates) { public boolean matchRoot(Plan plan) { return type.isInstance(plan); } + + @Override + public Class getMatchedType() { + return type; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java index 8f7ea106236b5d..d3d46ca4f99dd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java @@ -62,6 +62,11 @@ public void acceptPlan(Plan plan) { appliedRules.set(ruleTypeIndex); } + @Override + public String ruleName() { + return rule.ruleName(); + } + /** * AppliedAwareRuleCondition: convert one rule to AppliedAwareRule, so that the rule can add * some condition depends on whether this rule is applied to some plan @@ -104,5 +109,10 @@ public boolean matchPlanTree(Plan plan) { public boolean matchRoot(Plan plan) { return matchRootPredicate.test(plan) && super.matchRoot(plan); } + + @Override + public Class getMatchedType() { + return pattern.getMatchedType(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/FilteredRules.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/FilteredRules.java new file mode 100644 index 00000000000000..0729309d4d67e4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/FilteredRules.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules; + +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.trees.SuperClassId; +import org.apache.doris.nereids.trees.TreeNode; +import org.apache.doris.nereids.trees.plans.Plan; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** FilteredRules */ +public class FilteredRules extends Rules { + private BitSet typePatternIds; + private Map, List> classToRules = new ConcurrentHashMap<>(); + private List typeFilterRules; + private List otherRules; + + /** FilteredRules */ + public FilteredRules(List rules) { + super(rules); + + this.typeFilterRules = new ArrayList<>(); + this.otherRules = new ArrayList<>(); + + BitSet typePatternIds = new BitSet(); + for (Rule rule : rules) { + Pattern pattern = rule.getPattern(); + Class topType = pattern.getMatchedType(); + if (topType != null) { + typePatternIds.set(SuperClassId.getClassId(topType)); + typeFilterRules.add(rule); + } else { + otherRules.add(rule); + } + } + this.typePatternIds = typePatternIds; + } + + @Override + public List getCurrentAndChildrenRules(TreeNode treeNode) { + if (otherRules.isEmpty()) { + BitSet classTypes = treeNode.getAllChildrenTypes(); + if (!typePatternIds.intersects(classTypes)) { + return ImmutableList.of(); + } + } + return rules; + } + + @Override + public List getCurrentRules(TreeNode treeNode) { + if (otherRules.isEmpty()) { + BitSet classTypes = treeNode.getSuperClassTypes(); + if (!typePatternIds.intersects(classTypes)) { + return ImmutableList.of(); + } else { + Class treeClass = treeNode.getClass(); + List rulesCache = classToRules.get(treeClass); + if (rulesCache != null) { + return rulesCache; + } + return classToRules.computeIfAbsent(treeClass, c -> { + ImmutableList.Builder matchedTopTypeRules = ImmutableList.builder(); + for (Rule rule : this.rules) { + Class type = rule.getPattern().getMatchedType(); + if (type.isAssignableFrom(c)) { + matchedTopTypeRules.add(rule); + } + } + return matchedTopTypeRules.build(); + }); + } + } + return this.rules; + } + + @Override + public String toString() { + return "rules: " + rules.stream() + .map(Rule::ruleName) + .collect(Collectors.joining(", ", "[", "]")); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java index 7d5b4001d9ae8c..c103c388e65a50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java @@ -88,4 +88,6 @@ public boolean isInvalid(BitSet disableRules, GroupExpression groupExpression) { || !groupExpression.notApplied(this) || !this.getPattern().matchRoot(groupExpression.getPlan()); } + + public abstract String ruleName(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InlineLogicalView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rules.java similarity index 62% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InlineLogicalView.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rules.java index 4d3395a609fdcd..c75b10077f7892 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InlineLogicalView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rules.java @@ -15,16 +15,25 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.rewrite; +package org.apache.doris.nereids.rules; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalView; +import org.apache.doris.nereids.trees.TreeNode; -/** InlineLogicalView */ -public class InlineLogicalView extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalView().then(LogicalView::child).toRule(RuleType.INLINE_VIEW); +import java.util.List; + +/** Rules */ +public abstract class Rules { + protected List rules; + + public Rules(List rules) { + this.rules = rules; + } + + public abstract List getCurrentAndChildrenRules(TreeNode treeNode); + + public List getCurrentAndChildrenRules() { + return rules; } + + public abstract List getCurrentRules(TreeNode treeNode); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index c776e73c240f93..b88b358e27be3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -364,7 +364,7 @@ public static LogicalPlan checkAndAddDeleteSignFilter(LogicalOlapScan scan, Conn } } Preconditions.checkArgument(deleteSlot != null); - Expression conjunct = new EqualTo(new TinyIntLiteral((byte) 0), deleteSlot); + Expression conjunct = new EqualTo(deleteSlot, new TinyIntLiteral((byte) 0)); if (!olapTable.getEnableUniqueKeyMergeOnWrite()) { scan = scan.withPreAggStatus(PreAggStatus.off( Column.DELETE_SIGN + " is used as conjuncts.")); @@ -400,8 +400,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio return logicalPlan.get(); } - List qualifierWithoutTableName = Lists.newArrayList(); - qualifierWithoutTableName.addAll(qualifiedTableName.subList(0, qualifiedTableName.size() - 1)); + List qualifierWithoutTableName = qualifiedTableName.subList(0, qualifiedTableName.size() - 1); boolean isView = false; try { switch (table.getType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java index c7ba1bfe6a7af3..3f9be8ed67714e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.ExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; import org.apache.doris.nereids.rules.expression.rules.ReplaceVariableByLiteral; @@ -30,7 +31,7 @@ * replace Variable To Literal */ public class VariableToLiteral extends ExpressionRewrite { - public static final List NORMALIZE_REWRITE_RULES = + public static final List> NORMALIZE_REWRITE_RULES = ImmutableList.of(bottomUp(ReplaceVariableByLiteral.INSTANCE)); public VariableToLiteral() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/AbstractExpressionRewriteRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/AbstractExpressionRewriteRule.java index 5abd9228ba60e7..9d8b67b923b692 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/AbstractExpressionRewriteRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/AbstractExpressionRewriteRule.java @@ -26,6 +26,13 @@ public abstract class AbstractExpressionRewriteRule extends DefaultExpressionRewriter implements ExpressionRewriteRule { + public final String name = "Rewrite_" + this; + + @Override + public String getRewriteStateKey() { + return name; + } + @Override public Expression rewrite(Expression expr, ExpressionRewriteContext ctx) { return expr.accept(this, ctx); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionBottomUpRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionBottomUpRewriter.java index 932446ce48b16d..f5b0830c6bec76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionBottomUpRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionBottomUpRewriter.java @@ -35,6 +35,9 @@ public class ExpressionBottomUpRewriter implements ExpressionRewriteRule rewriteState = expression.getMutableState(BATCH_ID_KEY); if (!rewriteState.isPresent() || rewriteState.get() != currentBatch) { @@ -68,7 +80,7 @@ private static Expression rewriteBottomUp( Expression afterRewrite = expression; try { Expression beforeRewrite; - afterRewrite = rewriteChildren(expression, context, currentBatch, rules, listeners); + afterRewrite = rewriteChildren(expression, context, currentBatch, rules, listeners, name); // use rewriteTimes to avoid dead loop int rewriteTimes = 0; boolean changed; @@ -82,13 +94,14 @@ private static Expression rewriteBottomUp( if (changed) { afterRewrite = applied.get(); // ensure children are rewritten - afterRewrite = rewriteChildren(afterRewrite, context, currentBatch, rules, listeners); + afterRewrite = rewriteChildren(afterRewrite, context, currentBatch, rules, listeners, name); } rewriteTimes++; - } while (changed && rewriteTimes < 100); + } while (changed && rewriteTimes < 100 && rules.hasCurrentAndChildrenRules(beforeRewrite)); // set rewritten afterRewrite.setMutableState(BATCH_ID_KEY, currentBatch); + afterRewrite.setMutableState(name, true); } finally { if (hasChildren && listener != null) { listener.onExit(afterRewrite); @@ -103,11 +116,11 @@ private static Expression rewriteBottomUp( } private static Expression rewriteChildren(Expression parent, ExpressionRewriteContext context, int currentBatch, - ExpressionPatternRules rules, ExpressionPatternTraverseListeners listeners) { + ExpressionPatternRules rules, ExpressionPatternTraverseListeners listeners, String name) { boolean changed = false; ImmutableList.Builder newChildren = ImmutableList.builderWithExpectedSize(parent.arity()); for (Expression child : parent.children()) { - Expression newChild = rewriteBottomUp(child, context, currentBatch, parent, rules, listeners); + Expression newChild = rewriteBottomUp(child, context, currentBatch, parent, rules, listeners, name); changed |= !child.equals(newChild); newChildren.add(newChild); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java index 0e52f2aaaf6527..a262aab473de87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.rules.expression.rules.SimplifyCastRule; import org.apache.doris.nereids.rules.expression.rules.SimplifyNotExprRule; import org.apache.doris.nereids.rules.expression.rules.SupportJavaDateFormatter; +import org.apache.doris.nereids.trees.expressions.Expression; import com.google.common.collect.ImmutableList; @@ -41,25 +42,31 @@ public class ExpressionNormalization extends ExpressionRewrite { // we should run supportJavaDateFormatter before foldConstantRule or be will fold // from_unixtime(timestamp, 'yyyyMMdd') to 'yyyyMMdd' - public static final List NORMALIZE_REWRITE_RULES = ImmutableList.of( - bottomUp( - SupportJavaDateFormatter.INSTANCE, - NormalizeBinaryPredicatesRule.INSTANCE, - InPredicateDedup.INSTANCE, - InPredicateToEqualToRule.INSTANCE, - SimplifyNotExprRule.INSTANCE, - SimplifyArithmeticRule.INSTANCE, - FoldConstantRule.INSTANCE, - SimplifyCastRule.INSTANCE, - DigitalMaskingConvert.INSTANCE, - SimplifyArithmeticComparisonRule.INSTANCE, - ConvertAggStateCast.INSTANCE, - MergeDateTrunc.INSTANCE, - CheckCast.INSTANCE - ) + public static final List> NORMALIZE_REWRITE_RULES + = ImmutableList.of( + bottomUp( + SupportJavaDateFormatter.INSTANCE, + NormalizeBinaryPredicatesRule.INSTANCE, + InPredicateDedup.INSTANCE, + InPredicateToEqualToRule.INSTANCE, + SimplifyNotExprRule.INSTANCE, + SimplifyArithmeticRule.INSTANCE, + FoldConstantRule.INSTANCE, + SimplifyCastRule.INSTANCE, + DigitalMaskingConvert.INSTANCE, + SimplifyArithmeticComparisonRule.INSTANCE, + ConvertAggStateCast.INSTANCE, + MergeDateTrunc.INSTANCE, + CheckCast.INSTANCE + ) ); public ExpressionNormalization() { super(new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES)); } + + @Override + public Expression rewrite(Expression expression, ExpressionRewriteContext expressionRewriteContext) { + return super.rewrite(expression, expressionRewriteContext); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java index d694062ef1f049..409465e00ec251 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalizationAndOptimization.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.rules.expression; +import org.apache.doris.nereids.trees.expressions.Expression; + import com.google.common.collect.ImmutableList; /** ExpressionNormalizationAndOptimization */ @@ -24,10 +26,15 @@ public class ExpressionNormalizationAndOptimization extends ExpressionRewrite { /** ExpressionNormalizationAndOptimization */ public ExpressionNormalizationAndOptimization() { super(new ExpressionRuleExecutor( - ImmutableList.builder() + ImmutableList.>builder() .addAll(ExpressionNormalization.NORMALIZE_REWRITE_RULES) .addAll(ExpressionOptimization.OPTIMIZE_REWRITE_RULES) .build() )); } + + @Override + public Expression rewrite(Expression expression, ExpressionRewriteContext expressionRewriteContext) { + return super.rewrite(expression, expressionRewriteContext); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java index 0ae179489aceb1..abd58ad7923997 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java @@ -39,7 +39,7 @@ * optimize expression of plan rule set. */ public class ExpressionOptimization extends ExpressionRewrite { - public static final List OPTIMIZE_REWRITE_RULES = ImmutableList.of( + public static final List> OPTIMIZE_REWRITE_RULES = ImmutableList.of( bottomUp( ExtractCommonFactorRule.INSTANCE, DistinctPredicatesRule.INSTANCE, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index e79dd6705c0513..ac4c2dca35bd9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -58,7 +59,7 @@ public class ExpressionRewrite implements RewriteRuleFactory { private final ExpressionRuleExecutor rewriter; - public ExpressionRewrite(ExpressionRewriteRule... rules) { + public ExpressionRewrite(ExpressionRewriteRule... rules) { this.rewriter = new ExpressionRuleExecutor(ImmutableList.copyOf(rules)); } @@ -145,12 +146,18 @@ public Rule build() { return logicalFilter().thenApply(ctx -> { LogicalFilter filter = ctx.root; ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - Set newConjuncts = ImmutableSet.copyOf(ExpressionUtils.extractConjunction( - rewriter.rewrite(filter.getPredicate(), context))); - if (newConjuncts.equals(filter.getConjuncts())) { + Expression originPredicate = filter.getPredicate(); + Expression predicate = rewriter.rewrite(originPredicate, context); + if (predicate == originPredicate) { + return filter; + } + Set newConjuncts = Utils.fastToImmutableSet( + ExpressionUtils.extractConjunction(predicate) + ); + if (predicate.equals(originPredicate)) { return filter; } - return new LogicalFilter<>(newConjuncts, filter.child()); + return new LogicalFilter<>(newConjuncts, predicate, filter.child()); }).toRule(RuleType.REWRITE_FILTER_EXPRESSION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteRule.java index fa80d56d0c8438..43c5e8eae631c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteRule.java @@ -23,5 +23,7 @@ * The interface of expression rewrite rule. */ public interface ExpressionRewriteRule { + String getRewriteStateKey(); + Expression rewrite(Expression expr, T ctx); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleExecutor.java index 0f951448dd2582..84add61ae86018 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleExecutor.java @@ -30,9 +30,9 @@ * Expression rewrite entry, which contains all rewrite rules. */ public class ExpressionRuleExecutor { - private final List rules; + private final List> rules; - public ExpressionRuleExecutor(List rules) { + public ExpressionRuleExecutor(List> rules) { this.rules = rules; } @@ -49,7 +49,7 @@ public List rewrite(List exprs, ExpressionRewriteContext */ public Expression rewrite(Expression root, ExpressionRewriteContext ctx) { Expression result = root; - for (ExpressionRewriteRule rule : rules) { + for (ExpressionRewriteRule rule : rules) { result = applyRule(result, rule, ctx); } return result; @@ -62,8 +62,15 @@ public Optional rewrite(Optional root, ExpressionRewrite return root.map(r -> this.rewrite(r, ctx)); } - private Expression applyRule(Expression expr, ExpressionRewriteRule rule, ExpressionRewriteContext ctx) { - return rule.rewrite(expr, ctx); + private Expression applyRule( + Expression expr, ExpressionRewriteRule rule, ExpressionRewriteContext ctx) { + String rewriteStateKey = rule.getRewriteStateKey(); + if (!expr.getMutableState(rewriteStateKey).isPresent()) { + Expression result = rule.rewrite(expr, ctx); + result.setMutableState(rewriteStateKey, true); + return result; + } + return expr; } /** normalize */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/NullableDependentExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/NullableDependentExpressionRewrite.java index 558e3884739e75..89179203cb6aaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/NullableDependentExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/NullableDependentExpressionRewrite.java @@ -32,10 +32,11 @@ AdjustAggregateNullableForEmptySet.class }) public class NullableDependentExpressionRewrite extends ExpressionRewrite { - public static final List OPTIMIZE_REWRITE_RULES = ImmutableList.of( - bottomUp( - SimplifyConditionalFunction.INSTANCE - ) + public static final List> OPTIMIZE_REWRITE_RULES + = ImmutableList.of( + bottomUp( + SimplifyConditionalFunction.INSTANCE + ) ); private static final ExpressionRuleExecutor EXECUTOR = new ExpressionRuleExecutor(OPTIMIZE_REWRITE_RULES); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java index 1aa81c36d9a731..eca3fe78cf7e05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java @@ -58,17 +58,16 @@ public class QueryColumnCollector extends DefaultPlanRewriter @Override public Plan rewriteRoot(Plan plan, JobContext jobContext) { ConnectContext connectContext = ConnectContext.get(); - if (connectContext != null && connectContext.getSessionVariable().internalSession) { + if (connectContext != null && connectContext.getSessionVariable().internalSession + || !StatisticsUtil.enableAutoAnalyze()) { return plan; } CollectorContext context = new CollectorContext(); plan.accept(this, context); - if (StatisticsUtil.enableAutoAnalyze()) { - context.midPriority.removeAll(context.highPriority); - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - analysisManager.updateHighPriorityColumn(context.highPriority); - analysisManager.updateMidPriorityColumn(context.midPriority); - } + context.midPriority.removeAll(context.highPriority); + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + analysisManager.updateHighPriorityColumn(context.highPriority); + analysisManager.updateMidPriorityColumn(context.midPriority); return plan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java index 4032db4aadf550..8afe94cbfff8a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ExtractCommonFactorRule.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.annotation.Developing; import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; +import org.apache.doris.nereids.rules.rewrite.SkipSimpleExprs; import org.apache.doris.nereids.trees.expressions.CompoundPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -63,6 +64,8 @@ private static Expression extractCommonFactor(CompoundPredicate originExpr) { if (!(originExpr.left() instanceof CompoundPredicate || originExpr.left() instanceof BooleanLiteral) && !(originExpr.right() instanceof CompoundPredicate || originExpr.right() instanceof BooleanLiteral)) { return originExpr; + } else if (SkipSimpleExprs.isSimpleExpr(originExpr)) { + return originExpr; } // flatten same type to a list diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 1e01e544a2510d..f68700da46c499 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -377,15 +377,17 @@ public Expression visitConnectionId(ConnectionId connectionId, ExpressionRewrite public Expression visitAnd(And and, ExpressionRewriteContext context) { List nonTrueLiteral = Lists.newArrayList(); int nullCount = 0; + boolean changed = false; for (Expression e : and.children()) { - e = deepRewrite ? e.accept(this, context) : e; - if (BooleanLiteral.FALSE.equals(e)) { + Expression newExpr = deepRewrite ? e.accept(this, context) : e; + if (BooleanLiteral.FALSE.equals(newExpr)) { return BooleanLiteral.FALSE; - } else if (e instanceof NullLiteral) { + } else if (newExpr instanceof NullLiteral) { nullCount++; - nonTrueLiteral.add(e); - } else if (!BooleanLiteral.TRUE.equals(e)) { - nonTrueLiteral.add(e); + nonTrueLiteral.add(newExpr); + } else if (!BooleanLiteral.TRUE.equals(newExpr)) { + changed |= !e.equals(newExpr); + nonTrueLiteral.add(newExpr); } } @@ -399,7 +401,7 @@ public Expression visitAnd(And and, ExpressionRewriteContext context) { return nonTrueLiteral.get(0); default: // x and y - return and.withChildren(nonTrueLiteral); + return changed ? and.withChildren(nonTrueLiteral) : and; } } else if (nullCount == 1) { if (nonTrueLiteral.size() == 1) { @@ -407,7 +409,7 @@ public Expression visitAnd(And and, ExpressionRewriteContext context) { return new NullLiteral(BooleanType.INSTANCE); } // null and x - return and.withChildren(nonTrueLiteral); + return changed ? and.withChildren(nonTrueLiteral) : and; } else { // null and null return new NullLiteral(BooleanType.INSTANCE); @@ -418,15 +420,17 @@ public Expression visitAnd(And and, ExpressionRewriteContext context) { public Expression visitOr(Or or, ExpressionRewriteContext context) { List nonFalseLiteral = Lists.newArrayList(); int nullCount = 0; + boolean changed = false; for (Expression e : or.children()) { - e = deepRewrite ? e.accept(this, context) : e; - if (BooleanLiteral.TRUE.equals(e)) { + Expression newExpr = deepRewrite ? e.accept(this, context) : e; + if (BooleanLiteral.TRUE.equals(newExpr)) { return BooleanLiteral.TRUE; - } else if (e instanceof NullLiteral) { + } else if (newExpr instanceof NullLiteral) { nullCount++; - nonFalseLiteral.add(e); - } else if (!BooleanLiteral.FALSE.equals(e)) { - nonFalseLiteral.add(e); + nonFalseLiteral.add(newExpr); + } else if (!BooleanLiteral.FALSE.equals(newExpr)) { + changed |= !newExpr.equals(newExpr); + nonFalseLiteral.add(newExpr); } } @@ -440,7 +444,7 @@ public Expression visitOr(Or or, ExpressionRewriteContext context) { return nonFalseLiteral.get(0); default: // x or y - return or.withChildren(nonFalseLiteral); + return changed ? or.withChildren(nonFalseLiteral) : or; } } else if (nullCount == 1) { if (nonFalseLiteral.size() == 1) { @@ -448,7 +452,7 @@ public Expression visitOr(Or or, ExpressionRewriteContext context) { return new NullLiteral(BooleanType.INSTANCE); } // null or x - return or.withChildren(nonFalseLiteral); + return changed ? or.withChildren(nonFalseLiteral) : or; } else { // null or null return new NullLiteral(BooleanType.INSTANCE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java index af61bbdaddf974..63e73956afff22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java @@ -64,7 +64,7 @@ public List> buildRules() { public Expression rewriteTree(Expression expr, ExpressionRewriteContext context) { if (expr instanceof CompoundPredicate) { - expr = SimplifyRange.rewrite((CompoundPredicate) expr, context); + expr = SimplifyRange.INSTANCE.rewrite((CompoundPredicate) expr, context); } ExpressionBottomUpRewriter bottomUpRewriter = ExpressionRewrite.bottomUp(this); return bottomUpRewriter.rewrite(expr, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java index 297f11abd0e00e..d41f4b5b653ac8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.rewrite.SkipSimpleExprs; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.CompoundPredicate; @@ -91,6 +92,9 @@ public List> buildRules() { /** rewrite */ public static Expression rewrite(CompoundPredicate expr, ExpressionRewriteContext context) { + if (SkipSimpleExprs.isSimpleExpr(expr)) { + return expr; + } ValueDesc valueDesc = expr.accept(new RangeInference(), context); Expression exprForNonNull = valueDesc.toExpressionForNonNull(); if (exprForNonNull == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java index 808288b8fe3cab..b0d2f334a7f062 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java @@ -102,7 +102,7 @@ public Plan visitLogicalAggregate(LogicalAggregate aggregate, Ma public Plan visitLogicalFilter(LogicalFilter filter, Map replaceMap) { filter = (LogicalFilter) super.visit(filter, replaceMap); Set conjuncts = updateExpressions(filter.getConjuncts(), replaceMap); - return filter.withConjuncts(conjuncts).recomputeLogicalProperties(); + return filter.withConjunctsAndChild(conjuncts, filter.child()); } @Override @@ -227,10 +227,12 @@ public Plan visitLogicalSetOperation(LogicalSetOperation setOperation, Map sort, Map replaceMap) { sort = (LogicalSort) super.visit(sort, replaceMap); - List newKeys = sort.getOrderKeys().stream() - .map(old -> old.withExpression(updateExpression(old.getExpr(), replaceMap))) - .collect(ImmutableList.toImmutableList()); - return sort.withOrderKeys(newKeys).recomputeLogicalProperties(); + + ImmutableList.Builder newOrderKeys = ImmutableList.builder(); + for (OrderKey orderKey : sort.getOrderKeys()) { + newOrderKeys.add(orderKey.withExpression(updateExpression(orderKey.getExpr(), replaceMap))); + } + return sort.withOrderKeysAndChild(newOrderKeys.build(), sort.child()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java deleted file mode 100644 index 74609694431e33..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckPrivileges.java +++ /dev/null @@ -1,119 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.UserException; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.SqlCacheContext; -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.rules.analysis.UserAuthentication; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalView; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.Sets; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** CheckPrivileges */ -public class CheckPrivileges extends ColumnPruning { - private JobContext jobContext; - - @Override - public Plan rewriteRoot(Plan plan, JobContext jobContext) { - this.jobContext = jobContext; - super.rewriteRoot(plan, jobContext); - - // don't rewrite plan - return plan; - } - - @Override - public Plan visitLogicalView(LogicalView view, PruneContext context) { - checkColumnPrivileges(view.getView(), computeUsedColumns(view, context.requiredSlots)); - - // stop check privilege in the view - return view; - } - - @Override - public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, PruneContext context) { - TableValuedFunction tvf = tvfRelation.getFunction(); - tvf.checkAuth(jobContext.getCascadesContext().getConnectContext()); - return super.visitLogicalTVFRelation(tvfRelation, context); - } - - @Override - public Plan visitLogicalRelation(LogicalRelation relation, PruneContext context) { - if (relation instanceof LogicalCatalogRelation) { - TableIf table = ((LogicalCatalogRelation) relation).getTable(); - checkColumnPrivileges(table, computeUsedColumns(relation, context.requiredSlots)); - } - return super.visitLogicalRelation(relation, context); - } - - private Set computeUsedColumns(Plan plan, Set requiredSlots) { - List outputs = plan.getOutput(); - Map idToSlot = new LinkedHashMap<>(outputs.size()); - for (Slot output : outputs) { - idToSlot.putIfAbsent(output.getExprId().asInt(), output); - } - - Set usedColumns = Sets.newLinkedHashSetWithExpectedSize(requiredSlots.size()); - for (Slot requiredSlot : requiredSlots) { - Slot slot = idToSlot.get(requiredSlot.getExprId().asInt()); - if (slot != null) { - // don't check privilege for hidden column, e.g. __DORIS_DELETE_SIGN__ - if (slot instanceof SlotReference && ((SlotReference) slot).getColumn().isPresent() - && !((SlotReference) slot).getColumn().get().isVisible()) { - continue; - } - usedColumns.add(slot.getName()); - } - } - return usedColumns; - } - - private void checkColumnPrivileges(TableIf table, Set usedColumns) { - CascadesContext cascadesContext = jobContext.getCascadesContext(); - ConnectContext connectContext = cascadesContext.getConnectContext(); - try { - UserAuthentication.checkPermission(table, connectContext, usedColumns); - } catch (UserException e) { - throw new AnalysisException(e.getMessage(), e); - } - StatementContext statementContext = cascadesContext.getStatementContext(); - Optional sqlCacheContext = statementContext.getSqlCacheContext(); - if (sqlCacheContext.isPresent()) { - sqlCacheContext.get().addCheckPrivilegeTablesOrViews(table, usedColumns); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java index 4fc02581ca4c77..7bacb92b726275 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java @@ -17,8 +17,14 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.SqlCacheContext; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.analysis.UserAuthentication; import org.apache.doris.nereids.rules.rewrite.ColumnPruning.PruneContext; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; @@ -26,6 +32,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; @@ -33,12 +40,16 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.logical.OutputPrunable; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; @@ -48,12 +59,15 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -85,6 +99,9 @@ */ public class ColumnPruning extends DefaultPlanRewriter implements CustomRewriter { private Set keys; + private JobContext jobContext; + private boolean skipInnerCheck; + private boolean alreadyCheckedPrivileges; /** * collect all columns used in expressions, which should not be pruned @@ -131,22 +148,31 @@ public LogicalAggregate visitLogicalAggregate(LogicalAggregate view, PruneContext context) { + return withOuterCheck( + () -> checkColumnPrivileges(view.getView(), computeUsedColumns(view, context.requiredSlots)), + () -> { + Plan plan = super.visitLogicalView(view, context); + while (plan instanceof LogicalView) { + plan = plan.child(0); + } + return plan; + } + ); + } + + @Override + public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, PruneContext context) { + TableValuedFunction tvf = tvfRelation.getFunction(); + + return withOuterCheck( + () -> tvf.checkAuth(jobContext.getCascadesContext().getConnectContext()), + () -> super.visitLogicalTVFRelation(tvfRelation, context) + ); + } + + @Override + public Plan visitLogicalRelation(LogicalRelation relation, PruneContext context) { + return withOuterCheck( + () -> { + if (relation instanceof LogicalCatalogRelation) { + TableIf table = ((LogicalCatalogRelation) relation).getTable(); + checkColumnPrivileges(table, computeUsedColumns(relation, context.requiredSlots)); + } + }, + () -> super.visitLogicalRelation(relation, context) + ); + } + // union can not prune children by the common logic, we must override visit method to write special code. @Override public Plan visitLogicalUnion(LogicalUnion union, PruneContext context) { @@ -459,4 +522,55 @@ public PruneContext(Set requiredSlots, Plan parent) { this.parent = Optional.ofNullable(parent); } } + + private Set computeUsedColumns(Plan plan, Set requiredSlots) { + List outputs = plan.getOutput(); + Map idToSlot = new LinkedHashMap<>(outputs.size()); + for (Slot output : outputs) { + idToSlot.putIfAbsent(output.getExprId().asInt(), output); + } + + Set usedColumns = Sets.newLinkedHashSetWithExpectedSize(requiredSlots.size()); + for (Slot requiredSlot : requiredSlots) { + Slot slot = idToSlot.get(requiredSlot.getExprId().asInt()); + if (slot != null) { + // don't check privilege for hidden column, e.g. __DORIS_DELETE_SIGN__ + if (slot instanceof SlotReference && ((SlotReference) slot).getColumn().isPresent() + && !((SlotReference) slot).getColumn().get().isVisible()) { + continue; + } + usedColumns.add(slot.getName()); + } + } + return usedColumns; + } + + private void checkColumnPrivileges(TableIf table, Set usedColumns) { + CascadesContext cascadesContext = jobContext.getCascadesContext(); + ConnectContext connectContext = cascadesContext.getConnectContext(); + try { + UserAuthentication.checkPermission(table, connectContext, usedColumns); + } catch (UserException e) { + throw new AnalysisException(e.getMessage(), e); + } + StatementContext statementContext = cascadesContext.getStatementContext(); + Optional sqlCacheContext = statementContext.getSqlCacheContext(); + if (sqlCacheContext.isPresent()) { + sqlCacheContext.get().addCheckPrivilegeTablesOrViews(table, usedColumns); + } + } + + private T withOuterCheck(Runnable check, Supplier traverse) { + if (alreadyCheckedPrivileges || skipInnerCheck) { + return traverse.get(); + } else { + try { + skipInnerCheck = true; + check.run(); + return traverse.get(); + } finally { + skipInnerCheck = false; + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferFilterNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferFilterNotNull.java index f19976cb753550..9656376f625286 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferFilterNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferFilterNotNull.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.util.PlanUtils; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSet.Builder; import com.google.common.collect.Streams; import java.util.Set; @@ -44,15 +45,20 @@ public class InferFilterNotNull extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter() - .when(filter -> filter.getConjuncts().stream() - .filter(Not.class::isInstance) - .map(Not.class::cast) - .noneMatch(Not::isGeneratedIsNotNull)) + .when(filter -> { + for (Expression conjunct : filter.getConjuncts()) { + if (conjunct.containsType(Not.class) + && conjunct.anyMatch(n -> n instanceof Not && ((Not) n).isGeneratedIsNotNull())) { + return false; + } + } + return true; + }) .thenApply(ctx -> { LogicalFilter filter = ctx.root; Set predicates = filter.getConjuncts(); Set isNotNulls = ExpressionUtils.inferNotNull(predicates, ctx.cascadesContext); - ImmutableSet.Builder needGenerateNotNullsBuilder = ImmutableSet.builder(); + Builder needGenerateNotNullsBuilder = ImmutableSet.builder(); for (Expression isNotNull : isNotNulls) { if (!predicates.contains(isNotNull)) { needGenerateNotNullsBuilder.add(((Not) isNotNull).withGeneratedIsNotNull(true)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkipSimpleExprs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkipSimpleExprs.java new file mode 100644 index 00000000000000..3ea785c949846a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkipSimpleExprs.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Slot; + +import com.google.common.collect.Maps; + +import java.util.Map; +import java.util.Map.Entry; + +/** SkipSimpleExprs */ +public class SkipSimpleExprs { + /** isSimpleExpr */ + public static boolean isSimpleExpr(Expression expression) { + ExprFeature exprFeature = computeExprFeature(expression); + for (Entry kv : exprFeature.slotCount.entrySet()) { + Integer slotId = kv.getKey(); + Integer count = kv.getValue(); + if (count > 1) { + Integer isNullCount = exprFeature.slotIsNullCount.get(slotId); + if (isNullCount == null || isNullCount > 1) { + return false; + } + } + } + return true; + } + + /** computeExprFeature */ + public static ExprFeature computeExprFeature(Expression expr) { + Map slotCount = Maps.newHashMap(); + Map slotIsNullCount = Maps.newHashMap(); + computeExprFeature(expr, slotCount, slotIsNullCount); + return new ExprFeature(slotCount, slotIsNullCount); + } + + private static void computeExprFeature( + Expression e, Map slotCount, Map slotIsNullCount) { + if (e instanceof Slot) { + int slotId = ((Slot) e).getExprId().asInt(); + Integer count = slotCount.get(slotId); + slotCount.put(slotId, count == null ? 1 : count + 1); + } else if (e instanceof IsNull && e.child(0) instanceof Slot) { + int slotId = ((Slot) e.child(0)).getExprId().asInt(); + Integer count = slotIsNullCount.get(slotId); + slotIsNullCount.put(slotId, count == null ? 1 : count + 1); + } else { + for (Expression child : e.children()) { + computeExprFeature(child, slotCount, slotIsNullCount); + } + } + } + + private static class ExprFeature { + private Map slotCount; + private Map slotIsNullCount; + + public ExprFeature(Map slotCount, Map slotIsNullCount) { + this.slotCount = slotCount; + this.slotIsNullCount = slotIsNullCount; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java index 92bbcdb9b38fd0..a79eac63396a7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.util.MutableState.EmptyMutableState; import org.apache.doris.nereids.util.Utils; +import java.util.BitSet; import java.util.List; import java.util.Optional; @@ -32,6 +33,9 @@ */ public abstract class AbstractTreeNode> implements TreeNode { + + protected final BitSet containsTypes; + protected final List children; // this field is special, because other fields in tree node is immutable, but in some scenes, mutable @@ -45,12 +49,26 @@ protected AbstractTreeNode(NODE_TYPE... children) { // NOTE: ImmutableList.copyOf has additional clone of the list, so here we // direct generate a ImmutableList this.children = Utils.fastToImmutableList(children); + + this.containsTypes = new BitSet(); + for (NODE_TYPE child : children) { + BitSet childTypes = child.getAllChildrenTypes(); + containsTypes.or(childTypes); + } + containsTypes.or(getSuperClassTypes()); } protected AbstractTreeNode(List children) { // NOTE: ImmutableList.copyOf has additional clone of the list, so here we // direct generate a ImmutableList this.children = Utils.fastToImmutableList(children); + + this.containsTypes = new BitSet(); + for (NODE_TYPE child : children) { + BitSet childTypes = child.getAllChildrenTypes(); + containsTypes.or(childTypes); + } + containsTypes.or(getSuperClassTypes()); } @Override @@ -73,7 +91,18 @@ public void setMutableState(String key, Object state) { this.mutableState = this.mutableState.set(key, state); } + @Override + public BitSet getAllChildrenTypes() { + return containsTypes; + } + + @Override public int arity() { return children.size(); } + + @Override + public BitSet getSuperClassTypes() { + return SuperClassId.getSuperClassIds(getClass()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/SuperClassId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/SuperClassId.java new file mode 100644 index 00000000000000..ae36ce12c11cd8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/SuperClassId.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees; + +import java.util.BitSet; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** SuperClassId */ +// NOTE: static method let jvm do more aggressive inline, so we not make the instance +public class SuperClassId { + private static final AtomicInteger idGenerator = new AtomicInteger(); + private static final Map, Integer> classIdCache = new ConcurrentHashMap<>(1000); + private static final Map, BitSet> superClassIdsCache = new ConcurrentHashMap<>(1000); + + private SuperClassId() {} + + /** getSuperClassIds */ + public static BitSet getSuperClassIds(Class clazz) { + // get is more efficiency than computeIfAbsent, is lots of cases, we not need to put into the map + BitSet ids = superClassIdsCache.get(clazz); + if (ids != null) { + return ids; + } + return superClassIdsCache.computeIfAbsent(clazz, c -> { + BitSet bitSet = new BitSet(); + fillSuperClassIds(bitSet, clazz, true); + return bitSet; + }); + } + + /** getClassId */ + public static int getClassId(Class clazz) { + // get is more efficiency than computeIfAbsent, is lots of cases, we not need to put into the map + Integer id = classIdCache.get(clazz); + if (id != null) { + return id; + } + return classIdCache.computeIfAbsent(clazz, c -> idGenerator.incrementAndGet()); + } + + private static void fillSuperClassIds(BitSet superClassIds, Class clazz, boolean isTop) { + BitSet cache = isTop ? null : superClassIdsCache.get(clazz); + if (cache != null) { + superClassIds.or(cache); + return; + } + + Class superclass = clazz.getSuperclass(); + if (superclass != null) { + fillSuperClassIds(superClassIds, superclass, false); + } + + for (Class trait : clazz.getInterfaces()) { + fillSuperClassIds(superClassIds, trait, false); + } + + superClassIds.set(getClassId(clazz)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 2c5decb9f9a1fc..36fdc1e55f52b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -25,6 +25,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.BitSet; import java.util.Deque; import java.util.LinkedList; import java.util.List; @@ -65,6 +66,22 @@ default T getOrInitMutableState(String key, Supplier initState) { void setMutableState(String key, Object value); + /** getAllChildrenTypes */ + default BitSet getAllChildrenTypes() { + BitSet bitSet = new BitSet(); + for (TreeNode child : children()) { + bitSet.or(child.getAllChildrenTypes()); + } + bitSet.or(getSuperClassTypes()); + return bitSet; + } + + default BitSet getSuperClassTypes() { + BitSet bitSet = new BitSet(); + SuperClassId.getSuperClassIds(getClass()); + return bitSet; + } + default NODE_TYPE withChildren(NODE_TYPE... children) { return withChildren(Utils.fastToImmutableList(children)); } @@ -305,14 +322,14 @@ default Optional collectFirst(Predicate> predicate) { * @return true if it has any instance of the types */ default boolean containsType(Class... types) { - return anyMatch(node -> { - for (Class type : types) { - if (type.isInstance(node)) { - return true; - } + BitSet allChildrenTypes = getAllChildrenTypes(); + for (Class type : types) { + int classId = SuperClassId.getClassId(type); + if (allChildrenTypes.get(classId)) { + return true; } - return false; - }); + } + return false; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 5eb9d868cdb521..4c2c65fbed552c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -67,7 +67,7 @@ public LogicalCatalogRelation(RelationId relationId, PlanType type, TableIf tabl Optional groupExpression, Optional logicalProperties) { super(relationId, type, groupExpression, logicalProperties); this.table = Objects.requireNonNull(table, "table can not be null"); - this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + this.qualifier = Utils.fastToImmutableList(Objects.requireNonNull(qualifier, "qualifier can not be null")); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index d23ea3d2395f05..e0d3483c67286e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -40,6 +41,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -49,15 +51,22 @@ public class LogicalFilter extends LogicalUnary implements Filter { private final Set conjuncts; + private final Supplier predicate; public LogicalFilter(Set conjuncts, CHILD_TYPE child) { - this(conjuncts, Optional.empty(), Optional.empty(), child); + this(conjuncts, null, Optional.empty(), Optional.empty(), child); } - private LogicalFilter(Set conjuncts, Optional groupExpression, + public LogicalFilter(Set conjuncts, Expression andConjuncts, CHILD_TYPE child) { + this(conjuncts, () -> andConjuncts, Optional.empty(), Optional.empty(), child); + } + + private LogicalFilter(Set conjuncts, + Supplier predicate, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_FILTER, groupExpression, logicalProperties, child); this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts can not be null")); + this.predicate = predicate == null ? Suppliers.memoize(Filter.super::getPredicate) : predicate; } @Override @@ -65,6 +74,11 @@ public Set getConjuncts() { return conjuncts; } + @Override + public Expression getPredicate() { + return predicate.get(); + } + public List getExpressions() { return ImmutableList.copyOf(conjuncts); } @@ -115,25 +129,25 @@ public R accept(PlanVisitor visitor, C context) { } public LogicalFilter withConjuncts(Set conjuncts) { - return new LogicalFilter<>(conjuncts, Optional.empty(), Optional.of(getLogicalProperties()), child()); + return new LogicalFilter<>(conjuncts, null, Optional.empty(), Optional.of(getLogicalProperties()), child()); } @Override public LogicalFilter withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalFilter<>(conjuncts, children.get(0)); + return new LogicalFilter<>(conjuncts, predicate, Optional.empty(), Optional.empty(), children.get(0)); } @Override public LogicalFilter withGroupExpression(Optional groupExpression) { - return new LogicalFilter<>(conjuncts, groupExpression, Optional.of(getLogicalProperties()), child()); + return new LogicalFilter<>(conjuncts, predicate, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalFilter<>(conjuncts, groupExpression, logicalProperties, children.get(0)); + return new LogicalFilter<>(conjuncts, predicate, groupExpression, logicalProperties, children.get(0)); } public LogicalFilter withConjunctsAndChild(Set conjuncts, Plan child) { @@ -143,7 +157,7 @@ public LogicalFilter withConjunctsAndChild(Set conjuncts, Plan public LogicalFilter withConjunctsAndProps(Set conjuncts, Optional groupExpression, Optional logicalProperties, Plan child) { - return new LogicalFilter<>(conjuncts, groupExpression, logicalProperties, child); + return new LogicalFilter<>(conjuncts, null, groupExpression, logicalProperties, child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index bf4d6e084795f1..e9167e2cb82d86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; import org.apache.doris.nereids.rules.expression.rules.ReplaceVariableByLiteral; +import org.apache.doris.nereids.trees.SuperClassId; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; @@ -663,8 +664,9 @@ public static List flatExpressions(List> expre /** containsType */ public static boolean containsType(Collection expressions, Class type) { + int classId = SuperClassId.getClassId(type); for (Expression expression : expressions) { - if (expression.anyMatch(expr -> expr.anyMatch(type::isInstance))) { + if (expression.getAllChildrenTypes().get(classId)) { return true; } } @@ -929,7 +931,7 @@ public static List distinctSlotByName(List slots) { /** containsWindowExpression */ public static boolean containsWindowExpression(List expressions) { for (NamedExpression expression : expressions) { - if (expression.anyMatch(WindowExpression.class::isInstance)) { + if (expression.containsType(WindowExpression.class)) { return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index afdffc748c01a6..6d50c48cd49bf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1179,8 +1179,13 @@ private void computeTabletInfo() throws UserException { Preconditions.checkState(scanTabletIds.size() == 0); Map> backendAlivePathHashs = Maps.newHashMap(); for (Backend backend : Env.getCurrentSystemInfo().getAllClusterBackendsNoException().values()) { - backendAlivePathHashs.put(backend.getId(), backend.getDisks().values().stream() - .filter(DiskInfo::isAlive).map(DiskInfo::getPathHash).collect(Collectors.toSet())); + Set hashSet = Sets.newLinkedHashSet(); + for (DiskInfo diskInfo : backend.getDisks().values()) { + if (diskInfo.isAlive()) { + hashSet.add(diskInfo.getPathHash()); + } + } + backendAlivePathHashs.put(backend.getId(), hashSet); } for (Long partitionId : selectedPartitionIds) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java index 7d2cb54d33ecd3..87e337a661ee1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java @@ -24,7 +24,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; -import org.apache.doris.nereids.rules.rewrite.InlineLogicalView; +import org.apache.doris.nereids.rules.rewrite.ColumnPruning; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.util.MemoPatternMatchSupported; @@ -142,7 +142,7 @@ public void testNestedView() { + "ON X.ID1 = Y.ID3" ) .applyTopDown(new LogicalSubQueryAliasToLogicalProject()) - .applyBottomUp(new InlineLogicalView()) + .applyCustom(new ColumnPruning()) .applyTopDown(new MergeProjects()) .matches( logicalProject( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index b95027a1385009..8b6c857df8590f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.jobs.executor.Optimizer; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteTopDownJob; import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob; @@ -45,6 +46,7 @@ import org.apache.doris.nereids.processor.post.Validator; import org.apache.doris.nereids.properties.DistributionSpecGather; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.FilteredRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; @@ -192,13 +194,21 @@ public PlanChecker applyTopDown(RuleFactory ruleFactory) { public PlanChecker applyTopDown(List rule) { Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, - ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) + ImmutableList.of(new RootPlanTreeRewriteJob(new FilteredRules(rule), PlanTreeRewriteTopDownJob::new, true))) .execute(); cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); return this; } + public PlanChecker applyCustom(CustomRewriter customRewriter) { + CustomRewriteJob customRewriteJob = new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE); + customRewriteJob.execute(cascadesContext.getCurrentJobContext()); + cascadesContext.toMemo(); + MemoValidator.validate(cascadesContext.getMemo()); + return this; + } + /** * apply a top down rewrite rule if you not care the ruleId * @@ -227,7 +237,7 @@ public PlanChecker applyBottomUp(RuleFactory rule) { public PlanChecker applyBottomUp(List rule) { Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, - ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteBottomUpJob::new, true))) + ImmutableList.of(new RootPlanTreeRewriteJob(new FilteredRules(rule), PlanTreeRewriteBottomUpJob::new, true))) .execute(); cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo());