From def0bc24d475b742bc52bc073643aaa9514d3d8d Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 19 Nov 2024 14:38:30 +0800 Subject: [PATCH] [fix](s3) do not replace https scheme if specified (#44242) ### What problem does this PR solve? Problem Summary: When creating s3 resource with endpoint, it user already specify `https://` schema in endpoint url, we should not replace it with `http://`. ### Release note [fix](s3) fix bug that s3 resource do not support `https://` --- .../org/apache/doris/catalog/S3Resource.java | 2 +- .../org/apache/doris/catalog/S3ResourceTest.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index e1cde40c4ad40f..a40e91f47d46d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -95,7 +95,7 @@ protected void setProperties(Map properties) throws DdlException // the endpoint for ping need add uri scheme. String pingEndpoint = properties.get(S3Properties.ENDPOINT); - if (!pingEndpoint.startsWith("http://")) { + if (!pingEndpoint.startsWith("http://") && !pingEndpoint.startsWith("https://")) { pingEndpoint = "http://" + properties.get(S3Properties.ENDPOINT); properties.put(S3Properties.ENDPOINT, pingEndpoint); properties.put(S3Properties.Env.ENDPOINT, pingEndpoint); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java index 720e2690c05fa4..4e620d569031f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java @@ -222,4 +222,20 @@ public void testModifyProperties() throws Exception { modify.put("s3.access_key", "aaa"); s3Resource.modifyProperties(modify); } + + @Test + public void testHttpScheme() throws DdlException { + // if https:// is set, it should be replaced with http:// + Map properties = new HashMap<>(); + properties.put("AWS_ENDPOINT", "https://aaa"); + properties.put("AWS_REGION", "bbb"); + properties.put("AWS_ROOT_PATH", "/path/to/root"); + properties.put("AWS_ACCESS_KEY", "xxx"); + properties.put("AWS_SECRET_KEY", "yyy"); + properties.put("AWS_BUCKET", "test-bucket"); + properties.put("s3_validity_check", "false"); + S3Resource s3Resource = new S3Resource("s3_2"); + s3Resource.setProperties(properties); + Assert.assertEquals(s3Resource.getProperty(S3Properties.ENDPOINT), "https://aaa"); + } }