From 4c121d931a96e0bca4d397b65b5a8e8e355abe52 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Wed, 11 Dec 2024 16:59:17 +0800 Subject: [PATCH] fix --- .../datasource/hive/HMSExternalTable.java | 23 +++++++++++++++++++ .../hive/HiveMetaStoreClientHelper.java | 6 +---- .../datasource/hudi/source/HudiScanNode.java | 7 ++---- .../trees/plans/logical/LogicalHudiScan.java | 2 +- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index a24a47f542d165b..9ff431f577eba83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -33,6 +33,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; +import org.apache.doris.datasource.hudi.source.HudiLocalEngineContext; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.mtmv.MTMVBaseTableIf; @@ -71,7 +72,10 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -119,6 +123,8 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI private HoodieTableMetaClient hudiClient = null; private final byte[] hudiClientLock = new byte[0]; + private volatile HoodieTableFileSystemView hudiFsView = null; + private final byte[] hudiFsViewLock = new byte[0]; static { SUPPORTED_HIVE_FILE_FORMATS = Sets.newHashSet(); @@ -1008,4 +1014,21 @@ public HoodieTableMetaClient getHudiClient() { return hudiClient; } } + + public HoodieTableFileSystemView getHudiFsView(HoodieTableMetaClient hudiClient) { + if (hudiFsView != null) { + return hudiFsView; + } + synchronized (hudiFsViewLock) { + if (hudiFsView != null) { + return hudiFsView; + } + // If we need to cache fsView later, + // we can use `FileSystemViewManager.createViewManagerWithTableMetadata` to get view manager + HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build(); + HudiLocalEngineContext ctx = new HudiLocalEngineContext(hudiClient.getStorageConf()); + hudiFsView = FileSystemViewManager.createInMemoryFileSystemView(ctx, hudiClient, metadataConfig); + return hudiFsView; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java index 980c84f19e64f1b..7a1a9640fb8d665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java @@ -804,7 +804,7 @@ public static String showCreateTable(org.apache.hadoop.hive.metastore.api.Table } public static Schema getHudiTableSchema(HMSExternalTable table) { - HoodieTableMetaClient metaClient = getHudiClient(table); + HoodieTableMetaClient metaClient = table.getHudiClient(); TableSchemaResolver schemaUtil = new TableSchemaResolver(metaClient); Schema hudiSchema; try { @@ -828,10 +828,6 @@ public static T ugiDoAs(Configuration conf, PrivilegedExceptionAction act return HadoopUGI.ugiDoAs(krbConfig, action); } - public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) { - return table.getHudiClient(); - } - public static Configuration getConfiguration(HMSExternalTable table) { Configuration conf = DFSFileSystem.getHdfsConf(table.getCatalog().ifNotSetFallbackToSimpleAuth()); for (Map.Entry entry : table.getHadoopProperties().entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 4d6c64a0e21234c..06bde4a46d29e46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -60,7 +60,6 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.table.view.FileSystemViewManager; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.Option; import org.apache.hudi.metadata.FileSystemBackedTableMetadata; @@ -170,7 +169,7 @@ protected void doInitialize() throws UserException { initBackendPolicy(); initSchemaParams(); - hudiClient = HiveMetaStoreClientHelper.getHudiClient(hmsTable); + hudiClient = hmsTable.getHudiClient(); hudiClient.reloadActiveTimeline(); basePath = hmsTable.getRemoteTable().getSd().getLocation(); inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); @@ -232,9 +231,7 @@ protected void doInitialize() throws UserException { LOG.info("Can't find metadataTable or metadataTable is not initialized for table:{}", hmsTable.getName()); } - // If we need to cache fsView later, - // we can use `FileSystemViewManager.createViewManagerWithTableMetadata` to get view manager - inMemoryFileSystemView = FileSystemViewManager.createInMemoryFileSystemView(ctx, hudiClient, metadataConfig); + inMemoryFileSystemView = hmsTable.getHudiFsView(hudiClient); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java index 629690889432b37..caf96e6b268a072 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java @@ -187,7 +187,7 @@ public LogicalHudiScan withScanParams(HMSExternalTable table, TableScanParams sc optParams.put(k, v); } }); - HoodieTableMetaClient hudiClient = HiveMetaStoreClientHelper.getHudiClient(table); + HoodieTableMetaClient hudiClient = table.getHudiClient(); try { boolean isCowOrRoTable = table.isHoodieCowTable(); if (isCowOrRoTable) {