From a035d20af96ba0f530697e81b9e6d312ba9524a4 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 28 Dec 2024 18:00:55 +0800 Subject: [PATCH] 6 --- .../apache/doris/datasource/CatalogMgr.java | 10 +- .../doris/datasource/CatalogProperty.java | 6 + .../paimon/PaimonExternalCatalog.java | 3 +- .../paimon/PaimonHMSExternalCatalog.java | 2 + .../catalog/IcebergCatalogProperties.java | 126 ++++++++++++++++++ .../property/metastore/AWSGlueProperties.java | 34 ++--- .../property/metastore/HMSProperties.java | 12 ++ .../metastore/IcebergRestProperties.java | 11 ++ .../property/storage/S3Properties.java | 55 +++++++- .../property/storage/StorageProperties.java | 28 +++- .../doris/datasource/CatalogAPITest.java | 31 ++++- 11 files changed, 280 insertions(+), 38 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/catalog/IcebergCatalogProperties.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 2fcec2ddd5b638b..82c761548f52098 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -110,7 +110,7 @@ public CatalogMgr() { public static CatalogMgr read(DataInput in) throws IOException { String json = Text.readString(in); if (LOG.isDebugEnabled()) { - LOG.debug("debug: read json: {}", json); + LOG.debug("Reading catalog configuration from JSON: {}", json); } return GsonUtils.GSON.fromJson(json, CatalogMgr.class); } @@ -137,7 +137,7 @@ private void addCatalog(CatalogIf catalog) { private CatalogIf removeCatalog(long catalogId) { CatalogIf catalog = idToCatalog.remove(catalogId); - LOG.info("Removed catalog with id {}, name {}", catalogId, catalog == null ? "N/A" : catalog.getName()); + LOG.info("Removed catalog [id: {}, name: {}]", catalogId, catalog == null ? "N/A" : catalog.getName()); if (catalog != null) { catalog.onClose(); nameToCatalog.remove(catalog.getName()); @@ -243,7 +243,7 @@ private void createCatalogImpl(CatalogIf catalog, String catalogName, try { if (nameToCatalog.containsKey(catalog.getName())) { if (ifNotExists) { - LOG.warn("Catalog {} is already exist.", catalogName); + LOG.warn("Catalog '{}' already exists", catalogName); return; } throw new DdlException("Catalog had already exist with name: " + catalogName); @@ -280,7 +280,7 @@ public void dropCatalog(String catalogName, boolean ifExists) throws UserExcepti writeLock(); try { if (ifExists && !nameToCatalog.containsKey(catalogName)) { - LOG.warn("Non catalog {} is found.", catalogName); + LOG.warn("No catalog found with name '{}'", catalogName); return; } CatalogIf> catalog = nameToCatalog.get(catalogName); @@ -815,7 +815,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl return; } if (!(table instanceof HMSExternalTable)) { - LOG.warn("only support HMSTable"); + LOG.warn("Operation only supported for HMS (Hive Metastore) tables"); return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java index 68957e69daa183a..a734766d1fc7336 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java @@ -24,6 +24,7 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties.Type; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.persist.gson.GsonUtils; @@ -127,4 +128,9 @@ public static CatalogProperty read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, CatalogProperty.class); } + + public void initialize(Type type) { + metastoreProperties = new MetastoreProperties(type, properties); + storageProperties = new StorageProperties(properties); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 42b8eae02409192..f07f8ef599ecbe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -24,7 +24,6 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.datasource.property.constants.PaimonProperties; import org.apache.doris.fs.remote.dfs.DFSFileSystem; @@ -63,7 +62,7 @@ public abstract class PaimonExternalCatalog extends ExternalCatalog { public PaimonExternalCatalog(long catalogId, String name, String resource, Map props, String comment) { super(catalogId, name, InitCatalogLog.Type.PAIMON, comment); - props = PropertyConverter.convertToMetaProperties(props); + // props = PropertyConverter.convertToMetaProperties(props); catalogProperty = new CatalogProperty(resource, props); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java index fb27bb56696edc2..8ee0a399244fe3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java @@ -20,6 +20,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.datasource.property.constants.PaimonProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties.Type; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; @@ -37,6 +38,7 @@ public class PaimonHMSExternalCatalog extends PaimonExternalCatalog { public PaimonHMSExternalCatalog(long catalogId, String name, String resource, Map props, String comment) { super(catalogId, name, resource, props, comment); + catalogProperty.initialize(Type.HMS); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/catalog/IcebergCatalogProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/catalog/IcebergCatalogProperties.java new file mode 100644 index 000000000000000..3f32b617a8fbc7c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/catalog/IcebergCatalogProperties.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at + +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.catalog; + +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.metastore.AWSGlueProperties; +import org.apache.doris.datasource.property.metastore.HMSProperties; +import org.apache.doris.datasource.property.metastore.IcebergRestProperties; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.storage.HDFSProperties; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.datasource.property.storage.StorageProperties.Type; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; +import java.nio.file.Paths; +import java.util.Map; + +public class IcebergCatalogProperties { + + private Map catalogProps = Maps.newHashMap(); + private Configuration hadoopConf = new Configuration(false); + + public IcebergCatalogProperties(String warehouse, MetastoreProperties metaProps, StorageProperties storeProps) + throws UserException { + init(warehouse, metaProps, storeProps); + } + + private void init(String warehouse, MetastoreProperties metaProps, StorageProperties storeProps) + throws UserException { + initMetastoreProperties(metaProps); + initFileIOProperties(warehouse, storeProps); + } + + private void initMetastoreProperties(MetastoreProperties metaProps) + throws UserException { + switch (metaProps.getType()) { + case HMS: + HMSProperties hmsProperties = (HMSProperties) metaProps; + hmsProperties.toIcebergHiveCatalogProperties(catalogProps); + break; + case GLUE: + AWSGlueProperties glueProperties = (AWSGlueProperties) metaProps; + glueProperties.toIcebergGlueCatalogProperties(catalogProps); + break; + case ICEBERG_REST: + IcebergRestProperties restProperties = (IcebergRestProperties) metaProps; + restProperties.toIcebergRestCatalogProperties(catalogProps); + break; + case FILE_SYSTEM: + break; + default: + throw new UserException("Unsupported metastore type: " + metaProps.getType()); + } + } + + private void initFileIOProperties(String warehouse, StorageProperties storeProps) + throws UserException { + String finalWarehouse = warehouse; + // init file io properties + URI uri = Paths.get(warehouse).toUri(); + // need to set file io properties based on the warehouse scheme. + String scheme = Strings.nullToEmpty(uri.getScheme()); + switch (scheme) { + case "": + case "file": + break; + case "hdfs": + initHadoopFileIOProps(storeProps); + break; + case "s3": + case "oss": + case "cos": + case "obs": + case "tos": + case "bos": + case "gcs": + // Use S3FileIO for all S3-like storage, + // replace the scheme with s3. + finalWarehouse = "s3://" + uri.getAuthority() + uri.getPath(); + initS3FileIOProps(storeProps); + break; + default: + throw new UserException("Unsupported warehouse type: " + scheme); + } + catalogProps.put("warehouse", finalWarehouse); + } + + private void initHadoopFileIOProps(StorageProperties storeProps) throws UserException { + if (storeProps.getType() != Type.HDFS) { + throw new UserException("The warehouse is on HDFS-like storage, but the storage property is not for HDFS: " + + storeProps.getType()); + } + HDFSProperties hdfsProps = (HDFSProperties) storeProps; + hdfsProps.toHadoopConfiguration(hadoopConf); + } + + private void initS3FileIOProps(StorageProperties storeProps) throws UserException { + if (storeProps.getType() != Type.S3) { + throw new UserException("The warehouse is on S3-like storage, but the storage property is not for S3: " + + storeProps.getType()); + } + S3Properties s3Props = (S3Properties) storeProps; + s3Props.toIcebergS3FileIOProperties(catalogProps); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueProperties.java index 2a9f1b4c61bf8a8..f71a8e9be724039 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueProperties.java @@ -27,7 +27,7 @@ public class AWSGlueProperties extends MetastoreProperties { - @ConnectorProperty(names = {"glue.endpoint", "aws.region"}, + @ConnectorProperty(names = {"glue.endpoint", "aws.endpoint"}, description = "The endpoint of the AWS Glue.") private String glueEndpoint = ""; @@ -70,33 +70,23 @@ protected void checkRequiredProperties() { } } - public GlueCatalogCredentials getGlueCatalogCredentials() { - return new GlueCatalogCredentials(glueEndpoint, glueAccessKey, glueSecretKey); - } - public AWSCatalogMetastoreClientCredentials getAWSCatalogMetastoreClientCredentials() { return new AWSCatalogMetastoreClientCredentials(glueEndpoint, glueAccessKey, glueSecretKey); } - @Getter - public static class GlueCatalogCredentials { - private Map credentials = Maps.newHashMap(); - - // Used for GlueCatalog in IcebergGlueExternalCatalog + public void toIcebergGlueCatalogProperties(Map catalogProps) { // See AwsClientProperties.java for property keys - public GlueCatalogCredentials(String endpoint, String ak, String sk) { - credentials.put("client.credentials-provider", - "com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x"); - credentials.put("client.credentials-provider.glue.access_key", ak); - credentials.put("client.credentials-provider.glue.secret_key", sk); - credentials.put("client.region", getRegionFromGlueEndpoint(endpoint)); - } + catalogProps.put("client.credentials-provider", + "com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x"); + catalogProps.put("client.credentials-provider.glue.access_key", glueAccessKey); + catalogProps.put("client.credentials-provider.glue.secret_key", glueSecretKey); + catalogProps.put("client.region", getRegionFromGlueEndpoint()); + } - private String getRegionFromGlueEndpoint(String endpoint) { - // https://glue.ap-northeast-1.amazonaws.com - // -> ap-northeast-1 - return endpoint.split("\\.")[1]; - } + private String getRegionFromGlueEndpoint() { + // https://glue.ap-northeast-1.amazonaws.com + // -> ap-northeast-1 + return glueEndpoint.split("\\.")[1]; } @Getter diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java index 438f730b17ebcfe..d304d7e731934d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java @@ -85,4 +85,16 @@ public void toPaimonOptionsAndConf(Options options, Configuration conf) { conf.set("hive.metastore.client.keytab", hiveMetastoreClientKeytab); } } + + public void toIcebergHiveCatalogProperties(Map catalogProps) { + catalogProps.put("uri", hiveMetastoreUri); + Map allProps = loadConfigFromFile(getResourceConfigPropName()); + allProps.forEach(catalogProps::put); + catalogProps.put("hive.metastore.authentication.type", hiveMetastoreAuthenticationType); + if ("catalogProps".equalsIgnoreCase(hiveMetastoreAuthenticationType)) { + catalogProps.put("hive.metastore.service.principal", hiveMetastoreServicePrincipal); + catalogProps.put("hive.metastore.client.principal", hiveMetastoreClientPrincipal); + catalogProps.put("hive.metastore.client.keytab", hiveMetastoreClientKeytab); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java index 0cf61a209292428..987aa0d519b2715 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java @@ -28,10 +28,14 @@ public class IcebergRestProperties extends MetastoreProperties { private String icebergRestUri = ""; @ConnectorProperty(names = {"iceberg.rest.security.type"}, + required = false, + supported = false, description = "The security type of the iceberg rest catalog service.") private String icebergRestSecurityType = "none"; @ConnectorProperty(names = {"iceberg.rest.prefix"}, + required = false, + supported = false, description = "The prefix of the iceberg rest catalog service.") private String icebergRestPrefix = ""; @@ -42,4 +46,11 @@ public IcebergRestProperties(Map origProps) { @Override protected void checkRequiredProperties() { } + + public void toIcebergRestCatalogProperties(Map catalogProps) { + // See CatalogUtil.java + catalogProps.put("type", "rest"); + // See CatalogProperties.java + catalogProps.put("uri", icebergRestUri); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 79a3a5dc2195068..e92a461b86bdd30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -19,6 +19,7 @@ import org.apache.doris.datasource.property.ConnectorProperty; +import com.google.common.base.Strings; import org.apache.paimon.options.Options; import java.util.Map; @@ -27,11 +28,13 @@ public class S3Properties extends StorageProperties { @ConnectorProperty(names = {"s3.endpoint", "oss.endpoint", "cos.endpoint", "obs.endpoint", "gcs.endpoint"}, + required = false, description = "The endpoint of S3.") protected String s3Endpoint = ""; @ConnectorProperty(names = {"s3.region", "oss.region", "cos.region", "obs.region", "gcs.region"}, + required = false, description = "The region of S3.") protected String s3Region = ""; @@ -45,36 +48,58 @@ public class S3Properties extends StorageProperties { description = "The secret key of S3.") protected String s3SecretKey = ""; + @ConnectorProperty(names = {"use_path_style", + "s3.path-style-access"}, + required = false, + description = "Whether to use path style access.") + protected String usePathStyle = "false"; + @ConnectorProperty(names = {"s3.connection.maximum"}, + required = false, description = "The maximum number of connections to S3.") - protected String s3ConnectionMaximum = ""; + protected String s3ConnectionMaximum = "50"; @ConnectorProperty(names = {"s3.connection.request.timeout"}, - description = "The request timeout of S3 in second,") - protected String s3ConnectionRequestTimeoutS = ""; + required = false, + description = "The request timeout of S3 in milliseconds,") + protected String s3ConnectionRequestTimeoutS = "3000"; @ConnectorProperty(names = {"s3.connection.timeout"}, - description = "The connection timeout of S3 in second,") - protected String s3ConnectionTimeoutS = ""; + required = false, + description = "The connection timeout of S3 in milliseconds,") + protected String s3ConnectionTimeoutS = "1000"; @ConnectorProperty(names = {"s3.sts_endpoint"}, + supported = false, + required = false, description = "The sts endpoint of S3.") protected String s3StsEndpoint = ""; @ConnectorProperty(names = {"s3.sts_region"}, + supported = false, + required = false, description = "The sts region of S3.") protected String s3StsRegion = ""; @ConnectorProperty(names = {"s3.iam_role"}, + supported = false, + required = false, description = "The iam role of S3.") protected String s3IAMRole = ""; @ConnectorProperty(names = {"s3.external_id"}, + supported = false, + required = false, description = "The external id of S3.") protected String s3ExternalId = ""; public S3Properties(Map origProps) { super(Type.S3, origProps); + if (Strings.isNullOrEmpty(s3Region)) { + // Some object storage services do not have region concept, eg: minio. + // Use a default one. + s3Endpoint = "us-east-1"; + } } public void toPaimonOSSFileIOProperties(Options options) { @@ -88,4 +113,24 @@ public void toPaimonS3FileIOProperties(Options options) { options.set("s3.access-key", s3AccessKey); options.set("s3.secret-key", s3SecretKey); } + + public void toIcebergS3FileIOProperties(Map catalogProps) { + // See S3FileIOProperties.java + catalogProps.put("s3.endpoint", s3Endpoint); + catalogProps.put("s3.access-key-id", s3AccessKey); + catalogProps.put("s3.secret-access-key", s3SecretKey); + catalogProps.put("client.region", s3Region); + catalogProps.put("s3.path-style-access", usePathStyle); + } + + public void toBackendS3ClientProperties(Map s3Props) { + s3Props.put("AWS_ENDPOINT", s3Endpoint); + s3Props.put("AWS_REGION", s3Region); + s3Props.put("AWS_ACCESS_KEY", s3AccessKey); + s3Props.put("AWS_SECRET_KEY", s3SecretKey); + s3Props.put("AWS_MAX_CONNECTIONS", s3ConnectionMaximum); + s3Props.put("AWS_REQUEST_TIMEOUT_MS", s3ConnectionRequestTimeoutS); + s3Props.put("AWS_CONNECTION_TIMEOUT_MS", s3ConnectionTimeoutS); + s3Props.put("use_path_style", usePathStyle); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index 55d64862b67316d..1063c00956cd168 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.CatalogProperties; import lombok.Getter; @@ -25,6 +26,11 @@ public class StorageProperties extends CatalogProperties { + public static final String FS_HDFS_SUPPORT = "fs.hdfs.support"; + public static final String FS_S3_SUPPORT = "fs.s3.support"; + public static final String FS_GCS_SUPPORT = "fs.gcs.support"; + public static final String FS_AZURE_SUPPORT = "fs.azure.support"; + public enum Type { HDFS, S3, @@ -34,8 +40,28 @@ public enum Type { @Getter protected Type type; - public StorageProperties(Type type, Map origProps) { + public StorageProperties create(Map origProps) throws UserException { + if (isFsSupport(origProps, FS_HDFS_SUPPORT)) { + return new HDFSProperties(origProps); + } else if (isFsSupport(origProps, FS_S3_SUPPORT)) { + return new S3Properties(origProps); + } else if (isFsSupport(origProps, FS_GCS_SUPPORT)) { + throw new UserException("Unsupported native GCS filesystem"); + } else if (isFsSupport(origProps, FS_AZURE_SUPPORT)) { + throw new UserException("Unsupported native AZURE filesystem"); + } + + // In previous version, we don't support fs.xxx.support properties. + // So we need to "guess" this info from the properties. + + } + + protected StorageProperties(Type type, Map origProps) { super(origProps); this.type = type; } + + private boolean isFsSupport(Map origProps, String fsEnable) { + return origProps.getOrDefault(fsEnable, "false").equalsIgnoreCase("true"); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogAPITest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogAPITest.java index d80f536f0eedd03..5544ca72139cc10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogAPITest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogAPITest.java @@ -11,8 +11,10 @@ import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; +import org.apache.iceberg.Table; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; @@ -46,18 +48,41 @@ public void testAWSCatalogMetastoreClient() throws Exception { @Test public void testGlueCatalog() { + String glueAk = ""; + String glueSk = ""; String region = "ap-northeast-1"; GlueCatalog glueCatalog = new GlueCatalog(); Map properties = Maps.newHashMap(); // See AwsClientProperties.java for property keys properties.put("client.credentials-provider", "com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x"); - properties.put("client.credentials-provider.glue.access_key", ak); - properties.put("client.credentials-provider.glue.secret_key", sk); + properties.put("client.credentials-provider.glue.access_key", glueAk); + properties.put("client.credentials-provider.glue.secret_key", glueSk); properties.put("client.region", region); + properties.put("s3.access-key-id", glueAk); + properties.put("s3.secret-access-key", glueSk); + // properties.put("s3.endpoint", "https://s3.ap-northeast-1.amazonaws.com"); + glueCatalog.initialize("glue", properties); List dbs = glueCatalog.listNamespaces(); - System.out.println(dbs); + for (Namespace db : dbs) { + List tables = glueCatalog.listTables(db); + System.out.println(db + ": " + tables); + for (TableIdentifier table : tables) { + if (table.name().contains("iceberg_table")) { + continue; + } + try { + Table icebergTable = glueCatalog.loadTable(table); + Map tableProperties = icebergTable.properties(); + for (Map.Entry entry : tableProperties.entrySet()) { + System.out.println("table:" + table + ": " + entry.getKey() + ": " + entry.getValue()); + } + } catch (Throwable t) { + t.printStackTrace(); + } + } + } } @Test