diff --git a/README.md b/README.md index d5bf8506aa..49bef1fe52 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-format-mixed/am * Build and skip tests: `mvn clean package -DskipTests` * Build and skip dashboard: `mvn clean package -Pskip-dashboard-build` * Build and disable disk storage, RocksDB will NOT be introduced to avoid memory overflow: `mvn clean package -DskipTests -Pno-extented-disk-storage` +* Build and enable aliyun-oss-sdk: `mvn clean package -DskipTests -Paliyun-oss-sdk` * Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Phadoop2` * Specify Flink version for Flink optimizer(the default is 1.20.0): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.20.0` * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6` diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java index c85111a806..6772d87c54 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java @@ -40,12 +40,14 @@ import static org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HIVE; import static org.apache.amoro.properties.CatalogMetaProperties.KEY_WAREHOUSE; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE; -import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_ENDPOINT; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_HIVE_SITE; +import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_OSS_ENDPOINT; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_REGION; +import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_S3_ENDPOINT; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP; +import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_OSS; import static org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3; import static org.apache.amoro.properties.CatalogMetaProperties.TABLE_FORMATS; @@ -75,6 +77,7 @@ import org.apache.amoro.utils.CatalogUtil; import org.apache.commons.lang.StringUtils; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.aliyun.AliyunProperties; import org.apache.iceberg.aws.AwsClientProperties; import org.apache.iceberg.aws.glue.GlueCatalog; import org.apache.iceberg.aws.s3.S3FileIOProperties; @@ -114,6 +117,10 @@ public class CatalogController { CatalogDescriptor.of(CATALOG_TYPE_AMS, STORAGE_CONFIGS_VALUE_TYPE_S3, ICEBERG)); VALIDATE_CATALOGS.add( CatalogDescriptor.of(CATALOG_TYPE_AMS, STORAGE_CONFIGS_VALUE_TYPE_S3, MIXED_ICEBERG)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_AMS, STORAGE_CONFIGS_VALUE_TYPE_OSS, ICEBERG)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_AMS, STORAGE_CONFIGS_VALUE_TYPE_OSS, MIXED_ICEBERG)); VALIDATE_CATALOGS.add( CatalogDescriptor.of(CATALOG_TYPE_AMS, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, ICEBERG)); VALIDATE_CATALOGS.add( @@ -145,6 +152,14 @@ public class CatalogController { CatalogDescriptor.of(CATALOG_TYPE_CUSTOM, STORAGE_CONFIGS_VALUE_TYPE_S3, ICEBERG)); VALIDATE_CATALOGS.add( CatalogDescriptor.of(CATALOG_TYPE_CUSTOM, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, ICEBERG)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_OSS, PAIMON)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_OSS, ICEBERG)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_OSS, MIXED_ICEBERG)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_CUSTOM, STORAGE_CONFIGS_VALUE_TYPE_OSS, ICEBERG)); VALIDATE_CATALOGS.add( CatalogDescriptor.of( CATALOG_TYPE_CUSTOM, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, MIXED_ICEBERG)); @@ -182,11 +197,16 @@ private static Set getHiddenCatalogProperties( String.valueOf(authConfig.get(AUTH_CONFIGS_KEY_TYPE)))) { hiddenProperties.add(S3FileIOProperties.ACCESS_KEY_ID); hiddenProperties.add(S3FileIOProperties.SECRET_ACCESS_KEY); + hiddenProperties.add(AliyunProperties.CLIENT_ACCESS_KEY_ID); + hiddenProperties.add(AliyunProperties.CLIENT_ACCESS_KEY_SECRET); } if (STORAGE_CONFIGS_VALUE_TYPE_S3.equals(storageConfig.get(STORAGE_CONFIGS_KEY_TYPE))) { hiddenProperties.add(AwsClientProperties.CLIENT_REGION); hiddenProperties.add(S3FileIOProperties.ENDPOINT); } + if (STORAGE_CONFIGS_VALUE_TYPE_OSS.equals(storageConfig.get(STORAGE_CONFIGS_KEY_TYPE))) { + hiddenProperties.add(AliyunProperties.OSS_ENDPOINT); + } return hiddenProperties; } @@ -205,7 +225,10 @@ public void getCatalogTypeList(Context ctx) { } private void fillAuthConfigs2CatalogMeta( - CatalogMeta catalogMeta, Map serverAuthConfig, CatalogMeta oldCatalogMeta) { + CatalogMeta catalogMeta, + Map serverAuthConfig, + CatalogMeta oldCatalogMeta, + String storageType) { Map metaAuthConfig = new HashMap<>(); String authType = serverAuthConfig @@ -253,19 +276,19 @@ private void fillAuthConfigs2CatalogMeta( serverAuthConfig, catalogMeta.getCatalogProperties(), AUTH_CONFIGS_KEY_ACCESS_KEY, - S3FileIOProperties.ACCESS_KEY_ID); + getStorageAccessKey(storageType)); CatalogUtil.copyProperty( serverAuthConfig, catalogMeta.getCatalogProperties(), AUTH_CONFIGS_KEY_SECRET_KEY, - S3FileIOProperties.SECRET_ACCESS_KEY); + getStorageSecretKey(storageType)); break; } catalogMeta.setAuthConfigs(metaAuthConfig); } private Map extractAuthConfigsFromCatalogMeta( - String catalogName, CatalogMeta catalogMeta) { + String catalogName, CatalogMeta catalogMeta, String storageType) { Map serverAuthConfig = new HashMap<>(); Map metaAuthConfig = catalogMeta.getAuthConfigs(); String authType = @@ -298,12 +321,12 @@ private Map extractAuthConfigsFromCatalogMeta( CatalogUtil.copyProperty( catalogMeta.getCatalogProperties(), serverAuthConfig, - S3FileIOProperties.ACCESS_KEY_ID, + getStorageAccessKey(storageType), AUTH_CONFIGS_KEY_ACCESS_KEY); CatalogUtil.copyProperty( catalogMeta.getCatalogProperties(), serverAuthConfig, - S3FileIOProperties.SECRET_ACCESS_KEY, + getStorageSecretKey(storageType), AUTH_CONFIGS_KEY_SECRET_KEY); break; } @@ -311,11 +334,25 @@ private Map extractAuthConfigsFromCatalogMeta( return serverAuthConfig; } + private String getStorageAccessKey(String storageType) { + if (STORAGE_CONFIGS_VALUE_TYPE_OSS.equals(storageType)) { + return AliyunProperties.CLIENT_ACCESS_KEY_ID; + } + // default s3 + return S3FileIOProperties.ACCESS_KEY_ID; + } + + private String getStorageSecretKey(String storageType) { + if (STORAGE_CONFIGS_VALUE_TYPE_OSS.equals(storageType)) { + return AliyunProperties.CLIENT_ACCESS_KEY_SECRET; + } + // default s3 + return S3FileIOProperties.SECRET_ACCESS_KEY; + } + private Map extractStorageConfigsFromCatalogMeta( - String catalogName, CatalogMeta catalogMeta) { + String catalogName, CatalogMeta catalogMeta, String storageType) { Map storageConfig = new HashMap<>(); - Map config = catalogMeta.getStorageConfigs(); - String storageType = CatalogUtil.getCompatibleStorageType(config); storageConfig.put(STORAGE_CONFIGS_KEY_TYPE, storageType); if (STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equals(storageType)) { storageConfig.put( @@ -354,7 +391,13 @@ private Map extractStorageConfigsFromCatalogMeta( catalogMeta.getCatalogProperties(), storageConfig, S3FileIOProperties.ENDPOINT, - STORAGE_CONFIGS_KEY_ENDPOINT); + STORAGE_CONFIGS_KEY_S3_ENDPOINT); + } else if (STORAGE_CONFIGS_VALUE_TYPE_OSS.equals(storageType)) { + CatalogUtil.copyProperty( + catalogMeta.getCatalogProperties(), + storageConfig, + AliyunProperties.OSS_ENDPOINT, + STORAGE_CONFIGS_KEY_OSS_ENDPOINT); } return storageConfig; @@ -387,12 +430,12 @@ private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info, CatalogMeta o "Invalid table format list, " + String.join(",", info.getTableFormatList())); } catalogMeta.getCatalogProperties().put(CatalogMetaProperties.TABLE_FORMATS, tableFormats); - fillAuthConfigs2CatalogMeta(catalogMeta, info.getAuthConfig(), oldCatalogMeta); - // change fileId to base64Code - Map metaStorageConfig = new HashMap<>(); String storageType = info.getStorageConfig() .getOrDefault(STORAGE_CONFIGS_KEY_TYPE, STORAGE_CONFIGS_VALUE_TYPE_HADOOP); + fillAuthConfigs2CatalogMeta(catalogMeta, info.getAuthConfig(), oldCatalogMeta, storageType); + // change fileId to base64Code + Map metaStorageConfig = new HashMap<>(); metaStorageConfig.put(STORAGE_CONFIGS_KEY_TYPE, storageType); if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_HADOOP)) { List metaKeyList = @@ -429,8 +472,14 @@ private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info, CatalogMeta o CatalogUtil.copyProperty( info.getStorageConfig(), catalogMeta.getCatalogProperties(), - STORAGE_CONFIGS_KEY_ENDPOINT, + STORAGE_CONFIGS_KEY_S3_ENDPOINT, S3FileIOProperties.ENDPOINT); + } else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_OSS)) { + CatalogUtil.copyProperty( + info.getStorageConfig(), + catalogMeta.getCatalogProperties(), + STORAGE_CONFIGS_KEY_OSS_ENDPOINT, + AliyunProperties.OSS_ENDPOINT); } else { throw new RuntimeException("Invalid storage type " + storageType); } @@ -541,8 +590,10 @@ public void getCatalogDetail(Context ctx) { } else { info.setType(catalogMeta.getCatalogType()); } - info.setAuthConfig(extractAuthConfigsFromCatalogMeta(catalogName, catalogMeta)); - info.setStorageConfig(extractStorageConfigsFromCatalogMeta(catalogName, catalogMeta)); + String storageType = CatalogUtil.getCompatibleStorageType(catalogMeta.getStorageConfigs()); + info.setAuthConfig(extractAuthConfigsFromCatalogMeta(catalogName, catalogMeta, storageType)); + info.setStorageConfig( + extractStorageConfigsFromCatalogMeta(catalogName, catalogMeta, storageType)); // we put the table format single String tableFormat = catalogMeta.getCatalogProperties().get(CatalogMetaProperties.TABLE_FORMATS); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/internal/InternalTableConstants.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/internal/InternalTableConstants.java index dcec9de4b4..b2afee8bef 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/internal/InternalTableConstants.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/internal/InternalTableConstants.java @@ -33,6 +33,8 @@ public class InternalTableConstants { public static final String HADOOP_FILE_IO_IMPL = "org.apache.iceberg.hadoop.HadoopFileIO"; public static final String S3_FILE_IO_IMPL = "org.apache.iceberg.aws.s3.S3FileIO"; public static final String S3_PROTOCOL_PREFIX = "s3://"; + public static final String OSS_FILE_IO_IMPL = "org.apache.iceberg.aliyun.oss.OSSFileIO"; + public static final String OSS_PROTOCOL_PREFIX = "oss://"; public static final String CHANGE_STORE_TABLE_NAME_SUFFIX = InternalMixedIcebergCatalog.CHANGE_STORE_SEPARATOR diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/utils/InternalTableUtil.java b/amoro-ams/src/main/java/org/apache/amoro/server/utils/InternalTableUtil.java index 5d5ab39d0b..87306dd561 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/utils/InternalTableUtil.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/utils/InternalTableUtil.java @@ -21,6 +21,8 @@ import static org.apache.amoro.server.table.internal.InternalTableConstants.HADOOP_FILE_IO_IMPL; import static org.apache.amoro.server.table.internal.InternalTableConstants.METADATA_FOLDER_NAME; import static org.apache.amoro.server.table.internal.InternalTableConstants.MIXED_ICEBERG_BASED_REST; +import static org.apache.amoro.server.table.internal.InternalTableConstants.OSS_FILE_IO_IMPL; +import static org.apache.amoro.server.table.internal.InternalTableConstants.OSS_PROTOCOL_PREFIX; import static org.apache.amoro.server.table.internal.InternalTableConstants.S3_FILE_IO_IMPL; import static org.apache.amoro.server.table.internal.InternalTableConstants.S3_PROTOCOL_PREFIX; @@ -88,6 +90,8 @@ public static AuthenticatedFileIO newIcebergFileIo(CatalogMeta meta) { String defaultImpl = HADOOP_FILE_IO_IMPL; if (warehouse.toLowerCase().startsWith(S3_PROTOCOL_PREFIX)) { defaultImpl = S3_FILE_IO_IMPL; + } else if (warehouse.toLowerCase().startsWith(OSS_PROTOCOL_PREFIX)) { + defaultImpl = OSS_FILE_IO_IMPL; } String ioImpl = catalogProperties.getOrDefault(CatalogProperties.FILE_IO_IMPL, defaultImpl); FileIO fileIO = org.apache.iceberg.CatalogUtil.loadFileIO(ioImpl, catalogProperties, conf); diff --git a/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java b/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java index 8748190fb5..e94f88961e 100644 --- a/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java +++ b/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java @@ -26,11 +26,13 @@ public class CatalogMetaProperties { public static final String STORAGE_CONFIGS_KEY_CORE_SITE = "hadoop.core.site"; public static final String STORAGE_CONFIGS_KEY_HIVE_SITE = "hive.site"; public static final String STORAGE_CONFIGS_KEY_REGION = "storage.s3.region"; - public static final String STORAGE_CONFIGS_KEY_ENDPOINT = "storage.s3.endpoint"; + public static final String STORAGE_CONFIGS_KEY_S3_ENDPOINT = "storage.s3.endpoint"; + public static final String STORAGE_CONFIGS_KEY_OSS_ENDPOINT = "storage.oss.endpoint"; public static final String STORAGE_CONFIGS_VALUE_TYPE_HDFS_LEGACY = "hdfs"; public static final String STORAGE_CONFIGS_VALUE_TYPE_HADOOP = "Hadoop"; public static final String STORAGE_CONFIGS_VALUE_TYPE_S3 = "S3"; + public static final String STORAGE_CONFIGS_VALUE_TYPE_OSS = "OSS"; public static final String AUTH_CONFIGS_KEY_TYPE = "auth.type"; public static final String AUTH_CONFIGS_KEY_PRINCIPAL = "auth.kerberos.principal"; diff --git a/amoro-format-iceberg/pom.xml b/amoro-format-iceberg/pom.xml index f05f58c2f6..47919fce4e 100644 --- a/amoro-format-iceberg/pom.xml +++ b/amoro-format-iceberg/pom.xml @@ -107,6 +107,22 @@ iceberg-aws + + org.apache.iceberg + iceberg-aliyun + + + + com.aliyun.oss + aliyun-sdk-oss + + + httpclient + org.apache.httpcomponents + + + + org.apache.parquet parquet-avro @@ -163,7 +179,7 @@ org.apache.amoro amoro-common - ${parent.version} + ${project.parent.version} tests test diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml index acce293f81..f243bd1d4b 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-runtime-3.2/pom.xml @@ -75,6 +75,7 @@ org.apache.iceberg:iceberg-orc org.apache.iceberg:iceberg-parquet org.apache.iceberg:iceberg-aws + org.apache.iceberg:iceberg-aliyun org.apache.parquet:parquet-column org.apache.parquet:parquet-hadoop org.apache.parquet:parquet-common diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml index 6e54618f58..8d6cbff8b4 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml @@ -277,7 +277,7 @@ org.apache.amoro amoro-format-paimon - ${parent.version} + ${project.parent.version} test diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-runtime-3.3/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-runtime-3.3/pom.xml index 3af0cf6de2..09e898abe0 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-runtime-3.3/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-runtime-3.3/pom.xml @@ -80,6 +80,7 @@ org.apache.iceberg:iceberg-orc org.apache.iceberg:iceberg-parquet org.apache.iceberg:iceberg-aws + org.apache.iceberg:iceberg-aliyun org.apache.parquet:parquet-column org.apache.parquet:parquet-hadoop org.apache.parquet:parquet-common diff --git a/amoro-web/src/views/catalogs/Detail.vue b/amoro-web/src/views/catalogs/Detail.vue index 9cbad77638..4f3d530426 100644 --- a/amoro-web/src/views/catalogs/Detail.vue +++ b/amoro-web/src/views/catalogs/Detail.vue @@ -172,6 +172,14 @@ const s3ConfigTypeOps = reactive([{ value: 'CUSTOM', }]) +const ossConfigTypeOps = reactive([{ + label: 'AK/SK', + value: 'AK/SK', +}, { + label: 'CUSTOM', + value: 'CUSTOM', +}]) + const storageConfigMap = { 'hadoop.core.site': 'Hadoop core-site', 'hadoop.hdfs.site': 'Hadoop hdfs-site', @@ -366,9 +374,17 @@ async function changeProperties() { formState.properties = properties } -const storageConfigTypeS3 = reactive([{ +const storageConfigTypeS3Oss = reactive([{ label: 'S3', value: 'S3', +}, { + label: 'OSS', + value: 'OSS', +}]) + +const storageConfigTypeOSS = reactive([{ + label: 'OSS', + value: 'OSS', }]) const storageConfigTypeHadoop = reactive([{ @@ -376,27 +392,30 @@ const storageConfigTypeHadoop = reactive([{ value: 'Hadoop', }]) -const storageConfigTypeHadoopS3 = reactive([{ +const storageConfigTypeHadoopS3Oss = reactive([{ label: 'Hadoop', value: 'Hadoop', }, { label: 'S3', value: 'S3', +}, { + label: 'OSS', + value: 'OSS', }]) const storageConfigTypeOps = computed(() => { const type = formState.catalog.type if (type === 'ams' || type === 'custom') { - return storageConfigTypeHadoopS3 + return storageConfigTypeHadoopS3Oss } else if (type === 'glue') { - return storageConfigTypeS3 + return storageConfigTypeS3Oss } else if (type === 'hive') { return storageConfigTypeHadoop } else if (type === 'hadoop') { - return storageConfigTypeHadoopS3 + return storageConfigTypeHadoopS3Oss } else { return null @@ -411,6 +430,9 @@ const authTypeOptions = computed(() => { else if (type === 'S3') { return s3ConfigTypeOps } + else if (type === 'OSS') { + return ossConfigTypeOps + } return null }) @@ -677,6 +699,13 @@ onMounted(() => { {{ formState.storageConfig['storage.s3.region'] }} + + + {{ formState.storageConfig['storage.oss.endpoint'] }} +
+ +# HTTP Client Configuration for Local Testing + +This file is designed for local testing of HTTP interfaces using the **HTTP Client** plugin in IntelliJ IDEA. It enables quick testing of REST APIs directly within the IDE without requiring additional external tools or scripts. + +## Prerequisites +1. Install IntelliJ IDEA (Community or Ultimate Edition). +2. Install the **HTTP Client** plugin if it is not already installed. + + For more information on the HTTP Client plugin, refer to the official documentation: + [HTTP Client in IntelliJ IDEA](https://www.jetbrains.com/help/idea/http-client-in-product-code-editor.html) + +## How to Use +1. Open the `.http` or `.rest` file in IntelliJ IDEA. +2. Use the provided HTTP requests to test your API endpoints. +3. Select an HTTP request and click the **Run** button that appears next to it in the editor, or press `Ctrl+Enter` (Windows/Linux) or `Command+Enter` (Mac). +4. View the response in the dedicated response panel. + +### Example +```http +GET http://localhost:8080/api/example +Content-Type: application/json +Authorization: Bearer +``` + +### Features + +- Supports HTTP methods like GET, POST, PUT, DELETE, etc. +- Easily add headers, query parameters, and body content. +- View responses interactively with support for formats like JSON, XML, etc. + +## Scope of Use + +This file is intended **only for local development and testing purposes**. It is not meant for production or CI/CD pipelines. + +## Recommendations for Automated Testing + +While the HTTP Client plugin is great for manual testing, we recommend integrating REST API tests into your automated test suite for better coverage and reliability. + +### Other Suggested Tools + +- **JUnit** with [RestAssured](https://rest-assured.io/) for Java-based integration tests. +- **Postman** with [Newman](https://www.npmjs.com/package/newman) for automated API testing. + diff --git a/http/RestCatalogLocalTest.http b/http/RestCatalogLocalTest.http new file mode 100644 index 0000000000..6c59ebfc68 --- /dev/null +++ b/http/RestCatalogLocalTest.http @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### get catalog config +GET http://localhost:1630/api/iceberg/rest/v1/config?warehouse=iceberg +Content-Type: application/json + +### list namespace +GET http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces +Content-Type: application/json + +### create namespace +POST http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces +Content-Type: application/json + +{ + "namespace": [ + "test_db" + ] +} + +### query namespace +GET http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces/test_db +Content-Type: application/json + +### set namespace properties throws UnsupportedOperationException +POST http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces/test_db +Content-Type: application/json + +### delete namespace +DELETE http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces/test_db +Content-Type: application/json + +### list tables in namespace +GET http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces/sandbox/tables +Content-Type: application/json + +### load table +GET http://localhost:1630/api/iceberg/rest/v1/catalogs/iceberg/namespaces/sandbox/tables/test_tbl +Content-Type: application/json diff --git a/pom.xml b/pom.xml index 9132f4e598..97a882b187 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ 8.0.33 1.8.3 2.24.12 + 3.10.2 3.3.2 3.3 4.2.19 @@ -153,6 +154,7 @@ compile compile + provided @@ -350,6 +352,19 @@ ${iceberg.version} + + org.apache.iceberg + iceberg-aliyun + ${iceberg.version} + + + + com.aliyun.oss + aliyun-sdk-oss + ${aliyun-sdk-oss.version} + ${aliyun-sdk-dependency-scope} + + org.apache.parquet parquet-avro @@ -1356,5 +1371,11 @@ provided + + aliyun-oss-sdk + + compile + +