5

morningman · Jan 10, 2025 · cd560f4 · cd560f4
1 parent dcbee58
commit cd560f4
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 17 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/CatalogProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/CatalogProperties.java
@@ -28,22 +28,28 @@
 import java.util.Map;
 
 public class CatalogProperties {
+    protected Map<String, String> origProps;
 
-    protected void normalizedAndCheckProps(Map<String, String> origProps) {
+    protected CatalogProperties(Map<String, String> origProps) {
+        this.origProps = origProps;
+        normalizedAndCheckProps();
+    }
+
+    protected void normalizedAndCheckProps() {
         // 1. prepare phase
-        Map<String, String> resultProps = loadConfigFromFile(getResouceConfigPropName());
+        Map<String, String> allProps = loadConfigFromFile(getResourceConfigPropName());
         // 2. overwrite result properties with original properties
-        resultProps.putAll(origProps);
+        allProps.putAll(origProps);
         // 3. set fields from resultProps
         List<Field> supportedProps = PropertyUtils.getConnectorProperties(this.getClass());
         for (Field field : supportedProps) {
             field.setAccessible(true);
             ConnectorProperty anno = field.getAnnotation(ConnectorProperty.class);
             String[] names = anno.names();
             for (String name : names) {
-                if (origProps.containsKey(name)) {
+                if (allProps.containsKey(name)) {
                     try {
-                        field.set(this, origProps.get(name));
+                        field.set(this, allProps.get(name));
                     } catch (IllegalAccessException e) {
                         throw new RuntimeException("Failed to set property " + name + ", " + e.getMessage(), e);
                     }
@@ -58,7 +64,7 @@ protected void normalizedAndCheckProps(Map<String, String> origProps) {
     // Some properties may be loaded from file
     // Subclass can override this method to load properties from file.
     // The return value is the properties loaded from file, not include original properties
-    private Map<String, String> loadConfigFromFile(String resourceConfig) {
+    protected Map<String, String> loadConfigFromFile(String resourceConfig) {
         if (Strings.isNullOrEmpty(resourceConfig)) {
             return Maps.newHashMap();
         }
@@ -71,7 +77,7 @@ private Map<String, String> loadConfigFromFile(String resourceConfig) {
     }
 
     // Subclass can override this method to return the property name of resource config.
-    protected String getResouceConfigPropName() {
+    protected String getResourceConfigPropName() {
         return "";
     }
 

diff --git a/...e/src/main/java/org/apache/doris/datasource/property/catalog/PaimonCatalogProperties.java b/...e/src/main/java/org/apache/doris/datasource/property/catalog/PaimonCatalogProperties.java
@@ -122,7 +122,7 @@ private void initHadoopFileIOProps(StorageProperties storeProps) throws UserExce
                     + storeProps.getType());
         }
         HDFSProperties hdfsProps = (HDFSProperties) storeProps;
-
+        hdfsProps.toHadoopConfiguration(hadoopConf);
     }
 
     private void initS3FileIOProps(StorageProperties storeProps) throws UserException {

diff --git a/...ore/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java b/...ore/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
@@ -55,7 +55,7 @@ public class AliyunDLFProperties extends MetastoreProperties {
 
     private static final String DLF_PREFIX = "dlf.";
 
-    private Map<String, String> otherDlfProps = Maps.newHashMap();
+    private final Map<String, String> otherDlfProps = Maps.newHashMap();
 
     public AliyunDLFProperties(Map<String, String> origProps) {
         super(Type.DLF, origProps);
@@ -98,7 +98,7 @@ private String getEndpointOrFromRegion(String endpoint, String region, String dl
     }
 
     @Override
-    protected String getResouceConfigPropName() {
+    protected String getResourceConfigPropName() {
         return "dlf.resouce_config";
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
@@ -20,11 +20,13 @@
 import org.apache.doris.datasource.property.ConnectorProperty;
 
 import com.google.common.base.Strings;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.paimon.options.Options;
 
 import java.util.Map;
 
+@Slf4j
 public class HMSProperties extends MetastoreProperties {
     @ConnectorProperty(names = {"hive.metastore.uri"},
             description = "The uri of the hive metastore.")
@@ -55,7 +57,7 @@ public HMSProperties(Map<String, String> origProps) {
     }
 
     @Override
-    protected String getResouceConfigPropName() {
+    protected String getResourceConfigPropName() {
         return "hive.resource_config";
     }
 
@@ -74,6 +76,8 @@ protected void checkRequiredProperties() {
 
     public void toPaimonOptionsAndConf(Options options, Configuration conf) {
         options.set("uri", hiveMetastoreUri);
+        Map<String, String> allProps = loadConfigFromFile(getResourceConfigPropName());
+        allProps.forEach(conf::set);
         conf.set("hive.metastore.authentication.type", hiveMetastoreAuthenticationType);
         if ("kerberos".equalsIgnoreCase(hiveMetastoreAuthenticationType)) {
             conf.set("hive.metastore.service.principal", hiveMetastoreServicePrincipal);

diff --git a/...ore/src/main/java/org/apache/doris/datasource/property/metastore/MetastoreProperties.java b/...ore/src/main/java/org/apache/doris/datasource/property/metastore/MetastoreProperties.java
@@ -41,7 +41,7 @@ public enum Type {
     protected MetastoreProperties.Type type;
 
     public MetastoreProperties(Type type, Map<String, String> origProps) {
+        super(origProps);
         this.type = type;
-        normalizedAndCheckProps(origProps);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HDFSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HDFSProperties.java
@@ -20,6 +20,7 @@
 import org.apache.doris.datasource.property.ConnectorProperty;
 
 import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
 
 import java.util.Map;
 
@@ -59,13 +60,10 @@ public HDFSProperties(Map<String, String> origProps) {
     }
 
     @Override
-    protected String getResouceConfigPropName() {
+    protected String getResourceConfigPropName() {
         return "hdfs.resource_config";
     }
 
-    s
-    @Override
-
     protected void checkRequiredProperties() {
         super.checkRequiredProperties();
         if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType)) {
@@ -76,4 +74,14 @@ protected void checkRequiredProperties() {
             }
         }
     }
+
+    public void toHadoopConfiguration(Configuration conf) {
+        Map<String, String> allProps = loadConfigFromFile(getResourceConfigPropName());
+        allProps.forEach(conf::set);
+        conf.set("hdfs.authentication.type", hdfsAuthenticationType);
+        if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType)) {
+            conf.set("hdfs.authentication.kerberos.principal", hdfsKerberosPrincipal);
+            conf.set("hdfs.authentication.kerberos.keytab", hdfsKerberosKeytab);
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
@@ -35,7 +35,7 @@ public enum Type {
     protected Type type;
 
     public StorageProperties(Type type, Map<String, String> origProps) {
+        super(origProps);
         this.type = type;
-        normalizedAndCheckProps(origProps);
     }
 }