From 5b1354a2b5894ded7242d28ae17ff4b6a23441d6 Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Tue, 3 Dec 2024 13:38:46 +0800 Subject: [PATCH 1/6] add_redis --- dinky-admin/pom.xml | 19 +++++++ .../dinky/configure/RedisConfiguration.java | 50 +++++++++++++++++++ .../properties/DinkyRedisProperties.java | 19 +++++++ .../src/main/resources/application-redis.yml | 43 ++++++++++++++++ .../src/main/resources/application.yml | 1 + pom.xml | 6 +++ 6 files changed, 138 insertions(+) create mode 100644 dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java create mode 100644 dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java create mode 100644 dinky-admin/src/main/resources/application-redis.yml diff --git a/dinky-admin/pom.xml b/dinky-admin/pom.xml index 8181ec7da1..b446bbd7c6 100644 --- a/dinky-admin/pom.xml +++ b/dinky-admin/pom.xml @@ -379,6 +379,25 @@ org.dinky dinky-alert-http + + org.signal + embedded-redis + 0.9.0 + + + org.redisson + redisson-spring-boot-starter + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-web + + + diff --git a/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java b/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java new file mode 100644 index 0000000000..caa10245b9 --- /dev/null +++ b/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java @@ -0,0 +1,50 @@ +package org.dinky.configure; + +import org.dinky.configure.properties.DinkyRedisProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.annotation.Order; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import redis.embedded.RedisServer; + +@Configuration +@EnableConfigurationProperties(DinkyRedisProperties.class) +public class RedisConfiguration { + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty( + prefix = "spring.redis", + name = "mode", + havingValue = "embedded", + matchIfMissing = true) + @Order(-1) + public RedisServer redisServer(RedisProperties redisProperties){ + RedisServer redisServer = new RedisServer(redisProperties.getPort()); + redisServer.start(); + return redisServer; + } + @Bean + @Order(1) + public RedisTemplate redisTemplate(RedisConnectionFactory redisConnectionFactory) { + RedisTemplate template = new RedisTemplate<>(); + //设置Redis链接工厂对象 + template.setConnectionFactory(redisConnectionFactory); + + // 设置键的序列化方式为 String + template.setKeySerializer(new StringRedisSerializer()); + template.setHashKeySerializer(new StringRedisSerializer()); + + // 设置值的序列化方式为 JSON + template.setValueSerializer(new GenericJackson2JsonRedisSerializer()); + template.setHashValueSerializer(new GenericJackson2JsonRedisSerializer()); + + return template; + } +} diff --git a/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java b/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java new file mode 100644 index 0000000000..2afbdc7423 --- /dev/null +++ b/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java @@ -0,0 +1,19 @@ +package org.dinky.configure.properties; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "spring.redis") +@Getter +@Setter +public class DinkyRedisProperties { + private RedisRunMode mode; + + public enum RedisRunMode { + Standalone, Embedded + } + public boolean isEmbedded(){ + return RedisRunMode.Embedded.equals(mode); + } +} diff --git a/dinky-admin/src/main/resources/application-redis.yml b/dinky-admin/src/main/resources/application-redis.yml new file mode 100644 index 0000000000..945b9f5dba --- /dev/null +++ b/dinky-admin/src/main/resources/application-redis.yml @@ -0,0 +1,43 @@ +spring: + redis: + host: 127.0.0.1 + port: 6379 + # redis 运行模式，standalone ，embedded + mode: embedded + database: 8 + redisson: + # 单节点配置 + singleServerConfig: + # 连接空闲超时，单位：毫秒 + idleConnectionTimeout: 10000 + # 连接超时，单位：毫秒 + connectTimeout: 10000 + # 命令等待超时，单位：毫秒 + timeout: 3000 + # 命令失败重试次数,如果尝试达到 retryAttempts（命令失败重试次数） 仍然不能将命令发送至某个指定的节点时，将抛出错误。 + # 如果尝试在此限制之内发送成功，则开始启用 timeout（命令等待超时） 计时。 + retryAttempts: 3 + # 命令重试发送时间间隔，单位：毫秒 + retryInterval: 1500 + # 密码 + password: + # 客户端名称 + clientName: dinky-redis + # 单个连接最大订阅数量 + subscriptionsPerConnection: 5 + # 节点地址 + address: redis://${spring.redis.host}:${spring.redis.port} + # 发布和订阅连接的最小空闲连接数 + subscriptionConnectionMinimumIdleSize: 1 + # 发布和订阅连接池大小 + subscriptionConnectionPoolSize: 50 + # 最小空闲连接数 + connectionMinimumIdleSize: 5 + # 连接池大小 + connectionPoolSize: 10 + # 数据库编号 + database: ${spring.redis.database} + # DNS监测时间间隔，单位：毫秒 + dnsMonitoringInterval: 5000 + # 传输模式 + transportMode: "NIO" diff --git a/dinky-admin/src/main/resources/application.yml b/dinky-admin/src/main/resources/application.yml index 0855ab1b26..404cbff885 100644 --- a/dinky-admin/src/main/resources/application.yml +++ b/dinky-admin/src/main/resources/application.yml @@ -20,6 +20,7 @@ spring: include: - jmx - flyway + - redis lifecycle: timeout-per-shutdown-phase: 30s diff --git a/pom.xml b/pom.xml index cd73757a55..0794299a2f 100644 --- a/pom.xml +++ b/pom.xml @@ -125,6 +125,7 @@ 3.46.0.0 8 1.16.2 + 3.39.0 @@ -734,6 +735,11 @@ auto-service 1.1.1 + + org.redisson + redisson-spring-boot-starter + ${redisson.version} + From d28b8329e2fea2198477bc7270ba5b7ca1b11843 Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Tue, 3 Dec 2024 13:41:42 +0800 Subject: [PATCH 2/6] add_redis --- .../src/main/resources/application-redis.yml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/dinky-admin/src/main/resources/application-redis.yml b/dinky-admin/src/main/resources/application-redis.yml index 945b9f5dba..81a8d10786 100644 --- a/dinky-admin/src/main/resources/application-redis.yml +++ b/dinky-admin/src/main/resources/application-redis.yml @@ -2,42 +2,42 @@ spring: redis: host: 127.0.0.1 port: 6379 - # redis 运行模式，standalone ，embedded + # redis operating mode,standalone\embedded mode: embedded database: 8 redisson: - # 单节点配置 + # Single-node configuration singleServerConfig: - # 连接空闲超时，单位：毫秒 + # Connection idle timeout, in milliseconds idleConnectionTimeout: 10000 - # 连接超时，单位：毫秒 + # Connection timeout in milliseconds connectTimeout: 10000 - # 命令等待超时，单位：毫秒 + # Command wait timeout, in milliseconds timeout: 3000 - # 命令失败重试次数,如果尝试达到 retryAttempts（命令失败重试次数） 仍然不能将命令发送至某个指定的节点时，将抛出错误。 - # 如果尝试在此限制之内发送成功，则开始启用 timeout（命令等待超时） 计时。 + # Command Failed Retries (an error is thrown if the attempt reaches retryAttempts and the command still fails to send the command to a specified node). + # If the attempt to send within this limit is successful, timeout timing is enabled. retryAttempts: 3 - # 命令重试发送时间间隔，单位：毫秒 + # The command retry send interval in milliseconds retryInterval: 1500 - # 密码 + # password password: - # 客户端名称 + # clientName clientName: dinky-redis - # 单个连接最大订阅数量 + # The maximum number of subscriptions for a single connection subscriptionsPerConnection: 5 - # 节点地址 + # Node address address: redis://${spring.redis.host}:${spring.redis.port} - # 发布和订阅连接的最小空闲连接数 + # The minimum number of idle connections for publish and subscribe connections subscriptionConnectionMinimumIdleSize: 1 - # 发布和订阅连接池大小 + # Pub and sub connection pool size subscriptionConnectionPoolSize: 50 - # 最小空闲连接数 + # Minimum number of idle connections connectionMinimumIdleSize: 5 - # 连接池大小 + # Connection pool size connectionPoolSize: 10 - # 数据库编号 + # Database number database: ${spring.redis.database} - # DNS监测时间间隔，单位：毫秒 + # DNS monitoring interval, in milliseconds dnsMonitoringInterval: 5000 - # 传输模式 + # Transmission mode transportMode: "NIO" From a63ef373a479ed135ae1b5bf55ffd2ddc576571f Mon Sep 17 00:00:00 2001 From: zackyoungh Date: Tue, 3 Dec 2024 05:46:31 +0000 Subject: [PATCH 3/6] Spotless Apply --- .../dinky/configure/RedisConfiguration.java | 34 ++++++++++++++----- .../properties/DinkyRedisProperties.java | 28 +++++++++++++-- pom.xml | 2 +- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java b/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java index caa10245b9..8c2a0fd7f3 100644 --- a/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java +++ b/dinky-admin/src/main/java/org/dinky/configure/RedisConfiguration.java @@ -1,6 +1,26 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.dinky.configure; import org.dinky.configure.properties.DinkyRedisProperties; + import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.data.redis.RedisProperties; @@ -12,6 +32,7 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; + import redis.embedded.RedisServer; @Configuration @@ -19,22 +40,19 @@ public class RedisConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty( - prefix = "spring.redis", - name = "mode", - havingValue = "embedded", - matchIfMissing = true) + @ConditionalOnProperty(prefix = "spring.redis", name = "mode", havingValue = "embedded", matchIfMissing = true) @Order(-1) - public RedisServer redisServer(RedisProperties redisProperties){ + public RedisServer redisServer(RedisProperties redisProperties) { RedisServer redisServer = new RedisServer(redisProperties.getPort()); redisServer.start(); - return redisServer; + return redisServer; } + @Bean @Order(1) public RedisTemplate redisTemplate(RedisConnectionFactory redisConnectionFactory) { RedisTemplate template = new RedisTemplate<>(); - //设置Redis链接工厂对象 + // 设置Redis链接工厂对象 template.setConnectionFactory(redisConnectionFactory); // 设置键的序列化方式为 String diff --git a/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java b/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java index 2afbdc7423..981e111541 100644 --- a/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java +++ b/dinky-admin/src/main/java/org/dinky/configure/properties/DinkyRedisProperties.java @@ -1,8 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.dinky.configure.properties; +import org.springframework.boot.context.properties.ConfigurationProperties; + import lombok.Getter; import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "spring.redis") @Getter @@ -11,9 +31,11 @@ public class DinkyRedisProperties { private RedisRunMode mode; public enum RedisRunMode { - Standalone, Embedded + Standalone, + Embedded } - public boolean isEmbedded(){ + + public boolean isEmbedded() { return RedisRunMode.Embedded.equals(mode); } } diff --git a/pom.xml b/pom.xml index 0794299a2f..7bcbf68800 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ UTF-8 UTF-8 2.5.0 + 3.39.0 0.10.2 1.2.0-rc4 1.37.0 @@ -125,7 +126,6 @@ 3.46.0.0 8 1.16.2 - 3.39.0 From d3a90c025b2f389f2af6a32d3a4c0f560a5a8263 Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Tue, 3 Dec 2024 14:01:13 +0800 Subject: [PATCH 4/6] add_redis --- dinky-admin/src/main/resources/application-redis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dinky-admin/src/main/resources/application-redis.yml b/dinky-admin/src/main/resources/application-redis.yml index 81a8d10786..6320e2a203 100644 --- a/dinky-admin/src/main/resources/application-redis.yml +++ b/dinky-admin/src/main/resources/application-redis.yml @@ -20,7 +20,7 @@ spring: # The command retry send interval in milliseconds retryInterval: 1500 # password - password: + password: ${spring.redis.password} # clientName clientName: dinky-redis # The maximum number of subscriptions for a single connection From 179278f4154f4c5ca35a140bbde0dee0120bc6b8 Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Tue, 3 Dec 2024 17:00:27 +0800 Subject: [PATCH 5/6] add_redis --- .../main/java/org/dinky/job/ClearJobHistoryTask.java | 1 + .../java/org/dinky/job/handler/JobMetricsHandler.java | 10 +++------- .../main/java/org/dinky/daemon/task/DaemonTask.java | 5 +---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java b/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java index 2641785796..f695783c57 100644 --- a/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java +++ b/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java @@ -31,6 +31,7 @@ import org.dinky.service.JobInstanceService; import org.dinky.service.impl.ClusterInstanceServiceImpl; +import org.redisson.Redisson; import org.springframework.context.annotation.DependsOn; import lombok.Data; diff --git a/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java b/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java index 2916c25fdd..503a828bde 100644 --- a/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java +++ b/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java @@ -29,6 +29,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -89,13 +90,8 @@ private static void fetchFlinkMetrics(String v, Map m, String[] } String metricsName = String.join(",", m.keySet()); - String urlParam = null; - try { - urlParam = String.format( - "/jobs/%s/vertices/%s/metrics?get=%s", jid, v, URLEncoder.encode(metricsName, "UTF-8")); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } + String urlParam = String.format( + "/jobs/%s/vertices/%s/metrics?get=%s", jid, v, URLEncoder.encode(metricsName, StandardCharsets.UTF_8)); HttpUtils.request(new ArrayList<>(Arrays.asList(urlList)), urlParam, NetConstant.READ_TIME_OUT, x -> { List array = JsonUtils.toList(x.body(), Dict.class); diff --git a/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java b/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java index db56f8cc2b..c1db4b7de4 100644 --- a/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java +++ b/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java @@ -41,10 +41,7 @@ static Optional get(DaemonTaskConfig config) { static DaemonTask build(DaemonTaskConfig config) { Optional optionalDaemonTask = DaemonTask.get(config); - if (!optionalDaemonTask.isPresent()) { - throw new DaemonTaskException(Status.DAEMON_TASK_NOT_SUPPORT.getMessage() + config.getType()); - } - return optionalDaemonTask.get(); + return optionalDaemonTask.orElseThrow(()-> new DaemonTaskException(Status.DAEMON_TASK_NOT_SUPPORT.getMessage() + config.getType())); } DaemonTask setConfig(DaemonTaskConfig config); From ac6eb2419b061fdae725e97bd3c475d574e73b3a Mon Sep 17 00:00:00 2001 From: zackyoungh Date: Tue, 3 Dec 2024 09:02:19 +0000 Subject: [PATCH 6/6] Spotless Apply --- .../src/main/java/org/dinky/job/ClearJobHistoryTask.java | 1 - .../src/main/java/org/dinky/job/handler/JobMetricsHandler.java | 1 - .../src/main/java/org/dinky/daemon/task/DaemonTask.java | 3 ++- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java b/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java index f695783c57..2641785796 100644 --- a/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java +++ b/dinky-admin/src/main/java/org/dinky/job/ClearJobHistoryTask.java @@ -31,7 +31,6 @@ import org.dinky.service.JobInstanceService; import org.dinky.service.impl.ClusterInstanceServiceImpl; -import org.redisson.Redisson; import org.springframework.context.annotation.DependsOn; import lombok.Data; diff --git a/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java b/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java index 503a828bde..b09851d3d2 100644 --- a/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java +++ b/dinky-admin/src/main/java/org/dinky/job/handler/JobMetricsHandler.java @@ -27,7 +27,6 @@ import org.dinky.utils.JsonUtils; import org.dinky.utils.TimeUtil; -import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; diff --git a/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java b/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java index c1db4b7de4..a8bcfd743c 100644 --- a/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java +++ b/dinky-daemon/src/main/java/org/dinky/daemon/task/DaemonTask.java @@ -41,7 +41,8 @@ static Optional get(DaemonTaskConfig config) { static DaemonTask build(DaemonTaskConfig config) { Optional optionalDaemonTask = DaemonTask.get(config); - return optionalDaemonTask.orElseThrow(()-> new DaemonTaskException(Status.DAEMON_TASK_NOT_SUPPORT.getMessage() + config.getType())); + return optionalDaemonTask.orElseThrow( + () -> new DaemonTaskException(Status.DAEMON_TASK_NOT_SUPPORT.getMessage() + config.getType())); } DaemonTask setConfig(DaemonTaskConfig config);