Skip to content

Commit

Permalink
[improvement] (nereids) Get partition related table disable nullable …
Browse files Browse the repository at this point in the history
…field and modify regression test, complete agg mv rules.
  • Loading branch information
seawinde committed Dec 25, 2023
1 parent 29d3d5e commit 6074634
Show file tree
Hide file tree
Showing 14 changed files with 257 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,13 @@
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose;
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterJoinRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterJoinRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectJoinRule;
import org.apache.doris.nereids.rules.implementation.AggregateStrategies;
Expand Down Expand Up @@ -233,6 +236,9 @@ public class RuleSet {
.add(MaterializedViewProjectFilterJoinRule.INSTANCE)
.add(MaterializedViewAggregateRule.INSTANCE)
.add(MaterializedViewProjectAggregateRule.INSTANCE)
.add(MaterializedViewFilterAggregateRule.INSTANCE)
.add(MaterializedViewProjectFilterAggregateRule.INSTANCE)
.add(MaterializedViewFilterProjectAggregateRule.INSTANCE)
.build();

public List<Rule> getDPHypReorderRules() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.rules.exploration.mv;

import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;

import com.google.common.collect.ImmutableList;

import java.util.List;

/**
* MaterializedViewFilterAggregateRule
*/
public class MaterializedViewFilterAggregateRule extends AbstractMaterializedViewAggregateRule {

public static final MaterializedViewFilterAggregateRule INSTANCE = new MaterializedViewFilterAggregateRule();

@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalFilter(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalAggregate<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.rules.exploration.mv;

import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;

import com.google.common.collect.ImmutableList;

import java.util.List;

/**
* MaterializedViewFilterProjectAggregateRule
*/
public class MaterializedViewFilterProjectAggregateRule extends AbstractMaterializedViewAggregateRule {

public static final MaterializedViewFilterProjectAggregateRule
INSTANCE = new MaterializedViewFilterProjectAggregateRule();

@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalFilter(logicalProject(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.rules.exploration.mv;

import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;

import com.google.common.collect.ImmutableList;

import java.util.List;

/**
* MaterializedViewProjectFilterAggregateRule
*/
public class MaterializedViewProjectFilterAggregateRule extends AbstractMaterializedViewAggregateRule {

public static final MaterializedViewProjectFilterAggregateRule
INSTANCE = new MaterializedViewProjectFilterAggregateRule();

@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalProject(logicalFilter(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalFilter<LogicalAggregate<Plan>>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte
if (partitionColumnSet.contains(mvReferenceColumn)) {
context.setRelatedTable(table);
context.setRelatedTableColumn(mvReferenceColumn);
context.setPctPossible(!mvReferenceColumn.isAllowNull());
}
return visit(relation, context);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,31 @@ protected void runBeforeAll() throws Exception {
+ "PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
+ ")");

createTable("CREATE TABLE IF NOT EXISTS lineitem_null (\n"
+ " L_ORDERKEY INTEGER NOT NULL,\n"
+ " L_PARTKEY INTEGER NOT NULL,\n"
+ " L_SUPPKEY INTEGER NOT NULL,\n"
+ " L_LINENUMBER INTEGER NOT NULL,\n"
+ " L_QUANTITY DECIMALV3(15,2) NOT NULL,\n"
+ " L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL,\n"
+ " L_DISCOUNT DECIMALV3(15,2) NOT NULL,\n"
+ " L_TAX DECIMALV3(15,2) NOT NULL,\n"
+ " L_RETURNFLAG CHAR(1) NOT NULL,\n"
+ " L_LINESTATUS CHAR(1) NOT NULL,\n"
+ " L_SHIPDATE DATE NULL,\n"
+ " L_COMMITDATE DATE NULL,\n"
+ " L_RECEIPTDATE DATE NULL,\n"
+ " L_SHIPINSTRUCT CHAR(25) NOT NULL,\n"
+ " L_SHIPMODE CHAR(10) NOT NULL,\n"
+ " L_COMMENT VARCHAR(44) NOT NULL\n"
+ ")\n"
+ "DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)\n"
+ "PARTITION BY RANGE(L_SHIPDATE) (PARTITION `day_1` VALUES LESS THAN ('2017-02-01'))\n"
+ "DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3\n"
+ "PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
+ ")");
}

@Test
Expand Down Expand Up @@ -121,6 +146,29 @@ public void getRelatedTableInfoTestWithoutGroupTest() {
});
}

@Test
public void getRelatedTableInfoTestWithoutGroupNullTest() {
PlanChecker.from(connectContext)
.checkExplain("SELECT (o.c1_abs + ps.c2_abs) as add_alias, l.L_SHIPDATE, l.L_ORDERKEY, o.O_ORDERDATE, "
+ "ps.PS_AVAILQTY "
+ "FROM "
+ "lineitem_null as l "
+ "LEFT JOIN "
+ "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERDATE, O_ORDERKEY "
+ "FROM orders) as o "
+ "ON l.L_ORDERKEY = o.O_ORDERKEY "
+ "JOIN "
+ "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY "
+ "FROM partsupp) as ps "
+ "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY",
nereidsPlanner -> {
Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
Optional<RelatedTableInfo> relatedTableInfo =
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
Assertions.assertFalse(relatedTableInfo.isPresent());
});
}

@Test
public void getRelatedTableInfoTestWithSubqueryTest() {
PlanChecker.from(connectContext)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,10 @@
3 3 2023-12-11 43.20 43.20 43.20 1 0

-- !query18_0_before --
3 2023-12-11 43.20 43.20 43.20 1 0

-- !query18_0_after --
3 2023-12-11 43.20 43.20 43.20 1 0

-- !query19_0_before --
2 3 2023-12-08 20.00
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !query1_0_before --
1 yy 0 0 77.50 33.50 9.50 5
2 mi 0 0 57.40 56.20 1.20 2
2 mm 0 0 43.20 43.20 43.20 1

-- !query1_0_after --
1 yy 0 0 77.50 33.50 9.50 5
2 mi 0 0 57.40 56.20 1.20 2
2 mm 0 0 43.20 43.20 43.20 1

-- !query1_1_before --
2023-12-08 1 yy 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Expand Down Expand Up @@ -114,16 +120,24 @@
5 2 mi 0 0 0 0 0 0 0 0 0 0 0 0 0

-- !query18_0_before --
3 2023-12-11 43.20 43.20 43.20 1 0

-- !query18_0_after --
3 2023-12-11 43.20 43.20 43.20 1 0

-- !query18_1_before --
4 43.20

-- !query18_1_after --
4 43.20

-- !query18_2_before --
4 43.20
6 57.40

-- !query18_2_after --
4 43.20
6 57.40

-- !query19_0_before --
2 3 2023-12-08 20.00 10.50 9.50 2
Expand Down
20 changes: 20 additions & 0 deletions regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,24 @@
6

-- !query2_0_before --
4
4
4
4
4
4
6
6

-- !query2_0_after --
4
4
4
4
4
4
6
6

-- !query2_1_before --
4
Expand Down Expand Up @@ -268,8 +284,12 @@
2 3 2023-12-08

-- !query7_0_before --
2 3 2023-12-08
2 3 2023-12-08

-- !query7_0_after --
2 3 2023-12-08
2 3 2023-12-08

-- !query10_0_before --

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,24 @@
2 2

-- !query2_0_before --
4
4
4
4
4
4
6
6

-- !query2_0_after --
4
4
4
4
4
4
6
6

-- !query2_1_before --
4
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ suite("aggregate_with_roll_up") {
"count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) " +
"from lineitem t1 " +
"left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate " +
"where o_orderdate = '2023-12-11' and l_partkey = 2 " +
"where o_orderdate = '2023-12-11' and l_partkey = 3 " +
"group by " +
"l_shipdate, " +
"l_suppkey"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ suite("aggregate_without_roll_up") {
"min(o_totalprice), " +
"count(*) " +
"from orders " +
"where o_shippriority in (9.5, 10.5)" +
"where o_shippriority in (1, 2)" +
"group by " +
"o_shippriority, " +
"o_comment "
Expand Down Expand Up @@ -645,7 +645,7 @@ suite("aggregate_without_roll_up") {
"count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) " +
"from lineitem t1 " +
"left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate " +
"where l_shipdate = '2023-12-11' and l_suppkey = 2 " +
"where l_shipdate = '2023-12-11' and l_suppkey = 3 " +
"group by " +
"l_shipdate, " +
"l_suppkey"
Expand All @@ -662,7 +662,7 @@ suite("aggregate_without_roll_up") {
def query18_1 = "select l_linenumber, sum(o_totalprice) as sum_alias " +
"from lineitem " +
"inner join orders on l_orderkey = o_orderkey " +
"where o_custkey = 2 and l_linenumber = 3 " +
"where o_custkey = 2 and l_linenumber = 4 " +
"group by l_linenumber, o_custkey "
order_qt_query18_1_before "${query18_1}"
check_rewrite(mv18_1, query18_1, "mv18_1")
Expand All @@ -677,7 +677,7 @@ suite("aggregate_without_roll_up") {
def query18_2 = "select lineitem.l_linenumber, sum(o_totalprice) as sum_alias " +
"from lineitem " +
"inner join orders on lineitem.l_orderkey = orders.o_orderkey " +
"where o_custkey = 2 and l_suppkey= 4 " +
"where o_custkey = 2 and l_suppkey= 3 " +
"group by lineitem.l_linenumber, orders.o_custkey "
order_qt_query18_2_before "${query18_2}"
check_not_match(mv18_2, query18_2, "mv18_2")
Expand Down
Loading

0 comments on commit 6074634

Please sign in to comment.