From bc8258a936835eb968bfe032d581ec8f8af74021 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 5 Dec 2024 12:07:50 +0800 Subject: [PATCH] [fix](mtmv) Fix refresh materialized view fail when mv def contains cte (#44857) ### What problem does this PR solve? Related PR: #33988 Problem Summary: if mv def contain cte and is partition mv, such as following: CREATE MATERIALIZED VIEW mv_name BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (l_shipdate) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS WITH scan_data_cte as ( select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey from (select * from lineitem where L_LINENUMBER > 1) t1 left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY ) SELECT * FROM scan_data_cte; if run refresh cmd as following, this would fail, throw exception `no partition for this tuple`, this pr fix this refresh materialized view mv_name partitions(p_20231210_20231211); ### Release note Fix refresh materialized view fail when mv def contains cte --- .../commands/UpdateMvByPartitionCommand.java | 9 +- .../mv/micro_test/micro_test_when_cte.out | 128 +++++++++++ .../mv/micro_test/micro_test_when_cte.groovy | 204 ++++++++++++++++++ 3 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out create mode 100644 regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index de284bd837748f..36cc0f95a77a8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -246,11 +246,16 @@ public Plan visitLogicalCTE(LogicalCTE cte, PredicateAddContext if (predicates.isEmpty()) { return cte; } + List> rewrittenSubQueryAlias = new ArrayList<>(); for (LogicalSubQueryAlias subQueryAlias : cte.getAliasQueries()) { + List subQueryAliasChildren = new ArrayList<>(); this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier()); - subQueryAlias.children().forEach(subQuery -> subQuery.accept(this, predicates)); + subQueryAlias.children().forEach(subQuery -> + subQueryAliasChildren.add(subQuery.accept(this, predicates)) + ); + rewrittenSubQueryAlias.add(subQueryAlias.withChildren(subQueryAliasChildren)); } - return super.visitLogicalCTE(cte, predicates); + return super.visitLogicalCTE(new LogicalCTE<>(rewrittenSubQueryAlias, cte.child()), predicates); } @Override diff --git a/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out new file mode 100644 index 00000000000000..8abaaf9adec0a8 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out @@ -0,0 +1,128 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_0_after -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_0 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_1 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_0_after -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_2 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_3 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + diff --git a/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy new file mode 100644 index 00000000000000..9e80ea966fc1a2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy @@ -0,0 +1,204 @@ +package mv.micro_test +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("micro_test_when_cte") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate)( + FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) + (FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='18');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + + def query_sql = """ + WITH scan_data_cte as ( + select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey + from (select * from lineitem where L_LINENUMBER > 1) t1 + left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY + ) + SELECT * FROM scan_data_cte; + """ + def mv_sql = """ + WITH scan_data_cte as ( + select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey + from (select * from lineitem where L_LINENUMBER > 1) t1 + left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY + ) + SELECT * FROM scan_data_cte; + """ + def mv_name = """mv_with_cte_test""" + + // query directly + order_qt_query_0_after "${query_sql}" + + // create and build complete mv + create_async_mv(db, mv_name, mv_sql) + // refresh mv complete + sql """refresh materialized view ${mv_name} complete""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_0 "select * from ${mv_name}" + + // create and build partition mv + create_async_partition_mv(db, mv_name, mv_sql, "(l_shipdate)") + + // refresh mv partly + sql """refresh materialized view ${mv_name} partitions(p_20231208_20231209)""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_1 "select * from ${mv_name}" + + // query rewrite + mv_rewrite_success(mv_sql, mv_name) + order_qt_query_0_after "${query_sql}" + + // DML + // base table insert into data when not partition table + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'); + """ + sql """refresh materialized view ${mv_name} complete""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_2 "select * from ${mv_name}" + + // base table insert into data when partition table + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'); + """ + sql """refresh materialized view ${mv_name} partitions(p_20231210_20231211)""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_3 "select * from ${mv_name}" +}