diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 8e9ef1eaa97b7a..18f521b71b2a86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -461,18 +461,14 @@ protected Pair>, Map>> return Pair.of(ImmutableMap.of(), ImmutableMap.of()); } // Collect the mv related base table partitions which query used - Map> queryUsedBaseTablePartitions = new LinkedHashMap<>(); + Map> queryUsedBaseTablePartitions = new LinkedHashMap<>(); queryUsedBaseTablePartitions.put(relatedPartitionTable, new HashSet<>()); queryPlan.accept(new StructInfo.QueryScanPartitionsCollector(), queryUsedBaseTablePartitions); // Bail out, not check invalid partition if not olap scan, support later if (queryUsedBaseTablePartitions.isEmpty()) { return Pair.of(ImmutableMap.of(), ImmutableMap.of()); } - Set queryUsedBaseTablePartitionNameSet = queryUsedBaseTablePartitions.get(relatedPartitionTable) - .stream() - .map(Partition::getName) - .collect(Collectors.toSet()); - + Set queryUsedBaseTablePartitionNameSet = queryUsedBaseTablePartitions.get(relatedPartitionTable); Collection mvValidPartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, cascadesContext.getConnectContext(), System.currentTimeMillis(), false); Set mvValidPartitionNameSet = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 5c7925834845c2..7e88daf22529ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -18,11 +18,9 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; -import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreCache; @@ -76,7 +74,6 @@ import java.util.ArrayList; import java.util.BitSet; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -735,21 +732,21 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, * Collect partitions on base table */ public static class QueryScanPartitionsCollector extends DefaultPlanVisitor>> { + Map>> { @Override public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, - Map> targetTablePartitionMap) { + Map> targetTablePartitionMap) { TableIf table = catalogRelation.getTable(); BaseTableInfo relatedPartitionTable = new BaseTableInfo(table); if (!targetTablePartitionMap.containsKey(relatedPartitionTable)) { return catalogRelation; } + Set tablePartitions = targetTablePartitionMap.get(relatedPartitionTable); if (catalogRelation instanceof LogicalOlapScan) { // Handle olap table LogicalOlapScan logicalOlapScan = (LogicalOlapScan) catalogRelation; - Set tablePartitions = targetTablePartitionMap.get(relatedPartitionTable); for (Long partitionId : logicalOlapScan.getSelectedPartitionIds()) { - tablePartitions.add(logicalOlapScan.getTable().getPartition(partitionId)); + tablePartitions.add(logicalOlapScan.getTable().getPartition(partitionId).getName()); } } else if (catalogRelation instanceof LogicalFileScan && catalogRelation.getTable() instanceof HMSExternalTable) { @@ -762,9 +759,8 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( externalTable.getDbName(), externalTable.getName(), externalTable.getPartitionColumnTypes()); BiMap idToName = hivePartitionValues.getPartitionNameToIdMap().inverse(); - Set hiveTablePartitions = new HashSet<>(); for (Entry partitionEntry : selectedPartitions.selectedPartitions.entrySet()) { - hiveTablePartitions.add(idToName.get(partitionEntry.getKey())); + tablePartitions.add(idToName.get(partitionEntry.getKey())); } } else { // Not support to partition check now, doesn't try to compensate when part partition become invalid diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index de284bd837748f..f7020f75787afd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -301,16 +302,17 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, MTMVRelatedTableIf targetTable = (MTMVRelatedTableIf) table; for (String partitionName : filterTableEntry.getValue()) { Partition partition = targetTable.getPartition(partitionName); - if (!(targetTable instanceof OlapTable)) { - // check partition is have data or not, only support olap table - break; - } - if (!((OlapTable) targetTable).selectNonEmptyPartitionIds( + if (targetTable instanceof OlapTable && !((OlapTable) targetTable).selectNonEmptyPartitionIds( Lists.newArrayList(partition.getId())).isEmpty()) { - // Add filter only when partition has data + // Add filter only when partition has data when olap table partitionHasDataItems.add( ((OlapTable) targetTable).getPartitionInfo().getItem(partition.getId())); } + if (targetTable instanceof HMSExternalTable) { + // Add filter only when partition has data when hms external table + partitionHasDataItems.add( + ((HMSExternalTable) targetTable).getAndCopyPartitionItems().get(partitionName)); + } } if (partitionHasDataItems.isEmpty()) { predicates.setNeedAddFilter(false);