From 6366ecd80aec7539667a9326caf5e6db05550b74 Mon Sep 17 00:00:00 2001 From: gaoyan Date: Sun, 17 Dec 2023 14:40:55 +0800 Subject: [PATCH] Fixed the S3 configuration error in Resource (#2670) * Fixed the S3 configuration error in Resource * fix i81n --- .../src/main/java/org/dinky/init/SystemInit.java | 6 +++++- dinky-common/pom.xml | 1 - .../src/main/java/org/dinky/data/enums/Status.java | 2 ++ .../org/dinky/data/model/SystemConfiguration.java | 4 ++++ .../main/resources/i18n/messages_en_US.properties | 9 ++++++--- .../main/resources/i18n/messages_zh_CN.properties | 10 +++++++--- .../components/ResourcesUploadModal/index.tsx | 12 ++++++++---- pom.xml | 13 +++++++++++++ 8 files changed, 45 insertions(+), 12 deletions(-) diff --git a/dinky-admin/src/main/java/org/dinky/init/SystemInit.java b/dinky-admin/src/main/java/org/dinky/init/SystemInit.java index 8f2a62c021..7356dcd559 100644 --- a/dinky-admin/src/main/java/org/dinky/init/SystemInit.java +++ b/dinky-admin/src/main/java/org/dinky/init/SystemInit.java @@ -134,7 +134,8 @@ private void initResources() { systemConfiguration.getResourcesHdfsUser(), systemConfiguration.getResourcesHdfsDefaultFS(), systemConfiguration.getResourcesOssAccessKey(), - systemConfiguration.getResourcesOssRegion()) + systemConfiguration.getResourcesOssRegion(), + systemConfiguration.getResourcesPathStyleAccess()) .forEach(x -> x.addParameterCheck(y -> { if (Boolean.TRUE.equals( systemConfiguration.getResourcesEnable().getValue())) { @@ -156,6 +157,9 @@ private void initResources() { ossProperties.setRegion(systemConfiguration .getResourcesOssRegion() .getValue()); + ossProperties.setPathStyleAccess(systemConfiguration + .getResourcesPathStyleAccess() + .getValue()); Singleton.get(OssResourceManager.class).setOssTemplate(new OssTemplate(ossProperties)); break; case HDFS: diff --git a/dinky-common/pom.xml b/dinky-common/pom.xml index e4815e50a8..fd785853db 100644 --- a/dinky-common/pom.xml +++ b/dinky-common/pom.xml @@ -34,7 +34,6 @@ com.amazonaws aws-java-sdk-s3 - 1.12.589 com.github.xiaoymin diff --git a/dinky-common/src/main/java/org/dinky/data/enums/Status.java b/dinky-common/src/main/java/org/dinky/data/enums/Status.java index c7c1fe12e9..ac3f2185d9 100644 --- a/dinky-common/src/main/java/org/dinky/data/enums/Status.java +++ b/dinky-common/src/main/java/org/dinky/data/enums/Status.java @@ -367,6 +367,8 @@ public enum Status { SYS_RESOURCE_SETTINGS_HDFS_ROOT_USER_NOTE(173, "sys.resource.settings.hdfs.root.user.note"), SYS_RESOURCE_SETTINGS_HDFS_FS_DEFAULTFS(174, "sys.resource.settings.hdfs.fs.defaultFS"), SYS_RESOURCE_SETTINGS_HDFS_FS_DEFAULTFS_NOTE(175, "sys.resource.settings.hdfs.fs.defaultFS.note"), + SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS(176, "sys.resource.settings.oss.path.style.access"), + SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS_NOTE(177, "sys.resource.settings.oss.path.style.access.note"), /** * gateway config diff --git a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java index 48d4bab13c..56aa8bdf4c 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java +++ b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java @@ -251,6 +251,10 @@ public static Configuration.OptionBuilder key(Status status) { .stringType() .defaultValue("file:///") .note(Status.SYS_RESOURCE_SETTINGS_HDFS_FS_DEFAULTFS_NOTE); + private final Configuration resourcesPathStyleAccess = key(Status.SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS) + .booleanType() + .defaultValue(true) + .note(Status.SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS_NOTE); /** * Initialize after spring bean startup diff --git a/dinky-common/src/main/resources/i18n/messages_en_US.properties b/dinky-common/src/main/resources/i18n/messages_en_US.properties index 89b5f25ba4..f9e16b4975 100644 --- a/dinky-common/src/main/resources/i18n/messages_en_US.properties +++ b/dinky-common/src/main/resources/i18n/messages_en_US.properties @@ -222,10 +222,10 @@ sys.metrics.settings.flink.gatherTimeout.note=Flink Metrics collection timeout p sys.resource.settings.base.enable=Whether to enable Resource sys.resource.settings.base.enable.note=Enable resource management function. If you switch storage mode, you need to turn off this switch. After the relevant configuration is completed, turn it on again. sys.resource.settings.base.upload.base.path=Root path of the upload directory -sys.resource.settings.base.upload.base.path.note=Resources are stored on the HDFS/OSS path. Resource files will be stored in this base path. Configure it by yourself. Please ensure that the directory exists on the relevant storage system and has read capabilities. Write permission. recommend +sys.resource.settings.base.upload.base.path.note=Resources are stored on the HDFS/OSS(S3) path. Resource files will be stored in this base path. Configure it by yourself. Please ensure that the directory exists on the relevant storage system and has read capabilities. Write permission. recommend sys.resource.settings.base.model=Storage model -sys.resource.settings.base.model.note=Supports HDFS and OSS, it will take effect after switching the option, and resource files will be migrated at the same time. -sys.resource.settings.oss.endpoint=URL of the object storage service +sys.resource.settings.base.model.note=Supports HDFS and S3(Minio,OSS,COS ant etc...), it will take effect after switching the option, and resource files will be migrated at the same time. +sys.resource.settings.oss.endpoint=URL of the object storage service(Endpoint) sys.resource.settings.oss.endpoint.note=For example: https://oss-cn-hangzhou.aliyuncs.com sys.resource.settings.oss.accessKey=Access key sys.resource.settings.oss.accessKey.note=Access key is like a user ID that uniquely identifies your account @@ -235,6 +235,9 @@ sys.resource.settings.oss.bucketName=Bucket name sys.resource.settings.oss.bucketName.note=Default bucket name sys.resource.settings.oss.region=region sys.resource.settings.oss.region.note=region +sys.resource.settings.oss.path.style.access=Path Style +sys.resource.settings.oss.path.style.access.note=Whether to enable Path Style, Different providers (such as Alibaba Cloud OSS and Tencent Cloud COS) have different support conditions, please read the description of the provider + sys.resource.settings.hdfs.root.user=HDFS operation user name sys.resource.settings.hdfs.root.user.note=HDFS operation user name sys.resource.settings.hdfs.fs.defaultFS=HDFS defaultFS diff --git a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties index 5b4f9f668d..dff82a03a5 100644 --- a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties +++ b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties @@ -222,10 +222,11 @@ sys.metrics.settings.flink.gatherTimeout.note=Flink Metrics 采集超时时长 sys.resource.settings.base.enable=是否启用Resource sys.resource.settings.base.enable.note=启用资源管理功能，如果切换存储模式时，需关闭此开关，相关配置完成后，再开启 sys.resource.settings.base.upload.base.path=上传目录的根路径 -sys.resource.settings.base.upload.base.path.note=资源存储在HDFS/OSS路径上，资源文件将存储到此基本路径，自行配置，请确保该目录存在于相关存储系统上并具有读写权限。 +sys.resource.settings.base.upload.base.path.note=资源存储在HDFS/OSS (S3)路径上，资源文件将存储到此基本路径，自行配置，请确保该目录存在于相关存储系统上并具有读写权限。 sys.resource.settings.base.model=存储模式 -sys.resource.settings.base.model.note=支持HDFS、OSS，切换选项后即可生效，同时并迁移资源文件 -sys.resource.settings.oss.endpoint=对象存储服务的 URL +sys.resource.settings.base.model.note=支持HDFS、S3(Minio、阿里云OSS、腾讯云COS等..)，切换选项后即可生效，同时并迁移资源文件 + +sys.resource.settings.oss.endpoint=对象存储服务的 URL（Endpoint） sys.resource.settings.oss.endpoint.note=例如：https://oss-cn-hangzhou.aliyuncs.com sys.resource.settings.oss.accessKey=Access key sys.resource.settings.oss.accessKey.note=Access key就像用户ID，可以唯一标识你的账户 @@ -235,6 +236,9 @@ sys.resource.settings.oss.bucketName=存储桶名称 sys.resource.settings.oss.bucketName.note=默认的存储桶名称 sys.resource.settings.oss.region=区域 sys.resource.settings.oss.region.note=区域,例如：oss-cn-hangzhou +sys.resource.settings.oss.path.style.access=Path Style +sys.resource.settings.oss.path.style.access.note=是否开启 path style, 不同的提供方（如阿里云oss，腾讯云cos）支持情况不同，请阅读提供方文档说明进行填写 + sys.resource.settings.hdfs.root.user=HDFS 操作用户名 sys.resource.settings.hdfs.root.user.note=HDFS 操作用户名 sys.resource.settings.hdfs.fs.defaultFS=HDFS defaultFS diff --git a/dinky-web/src/pages/RegCenter/Resource/components/ResourcesUploadModal/index.tsx b/dinky-web/src/pages/RegCenter/Resource/components/ResourcesUploadModal/index.tsx index 7af51aac08..4c42dfea59 100644 --- a/dinky-web/src/pages/RegCenter/Resource/components/ResourcesUploadModal/index.tsx +++ b/dinky-web/src/pages/RegCenter/Resource/components/ResourcesUploadModal/index.tsx @@ -42,14 +42,18 @@ const ResourcesUploadModal: React.FC = (props) => { multiple: true, action: url + '?pid=' + pid, onChange: async (info) => { - const { status } = info.file; + const { status, response } = info.file; if (status !== 'uploading') { console.log(info.file, info.fileList); } if (status === 'done') { - await SuccessMessageAsync( - l('rc.resource.upload.success', '', { fileName: info.file.name }) - ); + if (response.success) { + await SuccessMessageAsync( + l('rc.resource.upload.success', '', { fileName: info.file.name }) + ); + } else { + await ErrorMessageAsync(response.msg); + } } else if (status === 'error') { await ErrorMessageAsync(l('rc.resource.upload.fail', '', { fileName: info.file.name })); } diff --git a/pom.xml b/pom.xml index 4290850f6c..e2a81a8471 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,7 @@ 3.21.0 + 1.12.589 1.7 0.2.6 1.5 @@ -124,6 +125,11 @@ + + com.amazonaws + aws-java-sdk-s3 + ${aws-java-sdk-s3.version} + org.commonmark commonmark @@ -275,6 +281,7 @@ protobuf-java ${protobuf-java.version} + cn.hutool @@ -490,6 +497,12 @@ org.dinky dinky-metadata-phoenix ${project.version} + + + joda-time + joda-time + + org.dinky