From 198e290fb09cd5fca764c32b011297c0e6e5df83 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 15 Oct 2024 23:24:31 +0800 Subject: [PATCH] add partition check --- .../doris/mtmv/MTMVPartitionCheckUtil.java | 2 +- .../exploration/mv/MaterializedViewUtils.java | 76 +++++++++++++--- .../mv/PartitionColumnTraceTest.java | 87 ++++++++++++++++++- 3 files changed, 146 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java index 1f65880583c327..769731155ed839 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java @@ -120,7 +120,7 @@ public static Pair compareDynamicPartition(OlapTable originalTa @VisibleForTesting public static Pair compareAutoPartition(OlapTable originalTable, OlapTable relatedTable) throws AnalysisException { - if (!isDynamicPartition(relatedTable)) { + if (!isAutoPartition(relatedTable)) { return Pair.of(false, "relatedTable is not dynamic partition."); } FunctionIntervalInfo originalFunctionIntervalInfo = PartitionExprUtil.getFunctionIntervalInfo( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 7f65eeea06e3e8..ac05a1dbbadc8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -23,9 +23,11 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVPartitionCheckUtil; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; @@ -199,32 +201,74 @@ public static RelatedTableInfo getRelatedTableInfos(String column, String timeUn "partition rollup expressions is not consistent, partition rollup expressions map is %s", checkContext.getPartitionAndRollupExpressionChecked())); } - List tableColumnInfos = new ArrayList<>(); + List derivedRelatedTables = new ArrayList<>(); + TableIf originalRelatedTable = null; + Map tableAndPartitionMap = new HashMap<>(); Set dataTypeSet = new HashSet<>(); for (Map.Entry, Boolean>> entry : checkContext.getPartitionAndRollupExpressionChecked().entrySet()) { SlotReference partitionColumn = entry.getKey(); + Column relatedColumn = extractColumn(partitionColumn); + if (relatedColumn == null) { + // Partition column is not from table + continue; + } dataTypeSet.add(partitionColumn.getDataType()); if (dataTypeSet.size() > 1) { return RelatedTableInfo.failWith(String.format( "multi partition column data types are different, data type are %s", dataTypeSet)); } - if (!partitionColumn.isColumnFromTable()) { - return RelatedTableInfo.failWith(String.format( - "partition checked is not from table, partition rollup expressions map is %s", - checkContext.getPartitionAndRollupExpressionChecked())); + TableIf relatedTable = partitionColumn.getTable().get(); + tableAndPartitionMap.put(relatedTable, + new TableColumnInfo(partitionColumn.getTable().map(BaseTableInfo::new).get(), + relatedColumn.getName(), + entry.getValue().key().orElse(null), + entry.getValue().value())); + if (entry.getValue().value()) { + originalRelatedTable = relatedTable; + } else { + derivedRelatedTables.add(relatedTable); } - Column relatedColumn = extractColumn(partitionColumn); - tableColumnInfos.add(new TableColumnInfo(partitionColumn.getTable().map(BaseTableInfo::new).get(), - relatedColumn.getName(), - entry.getValue().key().orElse(null), - entry.getValue().value())); } - if (!tableColumnInfos.isEmpty()) { - return RelatedTableInfo.successWith(tableColumnInfos); + if (originalRelatedTable == null) { + return RelatedTableInfo.failWith(String.format( + "original related table is null, err info is %s, partition checked expressions map is %s", + checkContext.getFailReasons(), checkContext.getPartitionAndRollupExpressionChecked())); + } + // If derivedRelatedTables is not empty, check partition between original and derived table is valid or not + Pair checkResult = MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh( + (MTMVRelatedTableIf) originalRelatedTable); + // TODO: 2024/10/15 Support external table in the future + TableColumnInfo originalTableColumnInfo = tableAndPartitionMap.get(originalRelatedTable); + if (!checkResult.key() || !(originalRelatedTable instanceof OlapTable)) { + // If original related table is invalid or is external table, degrade into single input trigger refresh + return RelatedTableInfo.successWith(ImmutableList.of(originalTableColumnInfo)); + } + // TODO: 2024/10/15 Support external table in the future + derivedRelatedTables = derivedRelatedTables.stream() + .filter(tableIf -> MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh( + (MTMVRelatedTableIf) tableIf).key() && tableIf instanceof OlapTable) + .collect(Collectors.toList()); + if (derivedRelatedTables.isEmpty()) { + // If valid derivedRelatedTables is empty, degrade into single input trigger refresh + return RelatedTableInfo.successWith(ImmutableList.of(originalTableColumnInfo)); + } + List checkedTableColumnInfos = new ArrayList<>(); + checkedTableColumnInfos.add(originalTableColumnInfo); + for (TableIf derivedTable : derivedRelatedTables) { + try { + // check the original related table partition is same with each derived related table + Pair derivedCheckResult = MTMVPartitionCheckUtil.compareOriginalTableAndRelatedTable( + (OlapTable) originalRelatedTable, + (OlapTable) derivedTable); + if (derivedCheckResult.key() && tableAndPartitionMap.get(derivedTable) != null) { + checkedTableColumnInfos.add(tableAndPartitionMap.get(derivedTable)); + } + } catch (AnalysisException e) { + // noting + } } - return RelatedTableInfo.failWith("can't not find valid partition track column finally, " - + checkContext.getFailReasons()); + return RelatedTableInfo.successWith(checkedTableColumnInfos); } private static boolean checkRollupExpression(Collection, Boolean>> rollupExpressions) { @@ -839,6 +883,10 @@ private static Column extractColumn(SlotReference slotReference) { if (!slotReferenceColumn.isPresent()) { return null; } + if (!slotReference.isColumnFromTable()) { + // Column is not from table + return null; + } Expr definExpr = slotReferenceColumn.get().getDefineExpr(); if (definExpr instanceof SlotRef) { // If slotReference is from sync mv when rbo, should get actual column diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionColumnTraceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionColumnTraceTest.java index a8569e9d556232..39f3cce4fa35c0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionColumnTraceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionColumnTraceTest.java @@ -84,6 +84,49 @@ protected void runBeforeAll() throws Exception { + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ")"); + + createTable("CREATE TABLE IF NOT EXISTS lineitem_auto (\n" + + " L_ORDERKEY INTEGER NOT NULL,\n" + + " L_PARTKEY INTEGER NOT NULL,\n" + + " L_SUPPKEY INTEGER NOT NULL,\n" + + " L_LINENUMBER INTEGER NOT NULL,\n" + + " L_QUANTITY DECIMALV3(15,2) NOT NULL,\n" + + " L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL,\n" + + " L_DISCOUNT DECIMALV3(15,2) NOT NULL,\n" + + " L_TAX DECIMALV3(15,2) NOT NULL,\n" + + " L_RETURNFLAG CHAR(1) NOT NULL,\n" + + " L_LINESTATUS CHAR(1) NOT NULL,\n" + + " L_SHIPDATE DATE NOT NULL,\n" + + " L_COMMITDATE DATE NOT NULL,\n" + + " L_RECEIPTDATE DATE NOT NULL,\n" + + " L_SHIPINSTRUCT CHAR(25) NOT NULL,\n" + + " L_SHIPMODE CHAR(10) NOT NULL,\n" + + " L_COMMENT VARCHAR(44) NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)\n" + + "auto partition by range (date_trunc(`L_SHIPDATE`, 'day')) ()\n" + + "DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); + + createTable("CREATE TABLE IF NOT EXISTS orders_auto (\n" + + " O_ORDERKEY INTEGER NOT NULL,\n" + + " O_CUSTKEY INTEGER NOT NULL,\n" + + " O_ORDERSTATUS CHAR(1) NOT NULL,\n" + + " O_TOTALPRICE DECIMALV3(15,2) NOT NULL,\n" + + " O_ORDERDATE DATE NOT NULL,\n" + + " O_ORDERPRIORITY CHAR(15) NOT NULL, \n" + + " O_CLERK CHAR(15) NOT NULL, \n" + + " O_SHIPPRIORITY INTEGER NOT NULL,\n" + + " O_COMMENT VARCHAR(79) NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY)\n" + + "auto partition by range (date_trunc(`O_ORDERDATE`, 'day')) ()\n" + + "DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")"); // Should not make scan to empty relation when the table used by materialized view has no data connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION"); } @@ -198,8 +241,26 @@ public void test5() { MaterializedViewUtils.getRelatedTableInfos("l_shipdate", null, rewrittenPlan, nereidsPlanner.getCascadesContext()); successWith(relatedTableInfo, - ImmutableSet.of(Pair.of("lineitem", "l_shipdate"), - Pair.of("orders", "o_orderdate")), ""); + ImmutableSet.of(Pair.of("lineitem", "l_shipdate")), ""); + }); + } + + @Test + public void test502() { + PlanChecker.from(connectContext) + .checkExplain(" select l_shipdate, o_orderdate, count(l_shipdate) \n" + + " from lineitem_auto\n" + + " inner join orders_auto\n" + + " on l_shipdate = o_orderdate\n" + + " group by l_shipdate, o_orderdate", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfos("l_shipdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + successWith(relatedTableInfo, + ImmutableSet.of(Pair.of("lineitem_auto", "l_shipdate"), + Pair.of("orders_auto", "o_orderdate")), ""); }); } @@ -253,8 +314,26 @@ public void test6() { MaterializedViewUtils.getRelatedTableInfos("o_orderdate", null, rewrittenPlan, nereidsPlanner.getCascadesContext()); successWith(relatedTableInfo, - ImmutableSet.of(Pair.of("lineitem", "l_shipdate"), - Pair.of("orders", "o_orderdate")), ""); + ImmutableSet.of(Pair.of("orders", "o_orderdate")), ""); + }); + } + + @Test + public void test601() { + PlanChecker.from(connectContext) + .checkExplain(" select l_shipdate, o_orderdate, count(l_shipdate) \n" + + " from lineitem_auto\n" + + " inner join orders_auto\n" + + " on l_shipdate = o_orderdate\n" + + " group by l_shipdate, o_orderdate", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfos("o_orderdate", null, + rewrittenPlan, nereidsPlanner.getCascadesContext()); + successWith(relatedTableInfo, + ImmutableSet.of(Pair.of("lineitem_auto", "l_shipdate"), + Pair.of("orders_auto", "o_orderdate")), ""); }); }