From d652df1207add7f238ddfe927da3cdf0f327d6d3 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Thu, 31 Aug 2023 20:39:37 +0800 Subject: [PATCH 1/9] update bean name (#267) --- .../common/lock/CommonLockSpringConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/CommonLockSpringConfiguration.java b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/CommonLockSpringConfiguration.java index 07ec34dbb8..92f11730bc 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/CommonLockSpringConfiguration.java +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/CommonLockSpringConfiguration.java @@ -30,7 +30,7 @@ public class CommonLockSpringConfiguration { @Bean @ConditionalOnMissingBean - public CommonLockService getDefaultLockManagerPersistence(CommonLockMapper commonLockMapper) { + public CommonLockService getCommonLockService(CommonLockMapper commonLockMapper) { DefaultCommonLockService defaultCommonLockService = new DefaultCommonLockService(); defaultCommonLockService.setLockManagerMapper(commonLockMapper); return defaultCommonLockService; From 685e6186f1daa08f7d70d065eec4ae1ccc9b956b Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 3 Sep 2023 22:48:30 +0800 Subject: [PATCH 2/9] update lock owner field (#273) --- .../service/DefaultEngineConnResourceService.scala | 8 ++++---- .../am/service/heartbeat/AMHeartbeatService.scala | 6 +++--- linkis-dist/package/db/linkis_ddl.sql | 2 +- .../db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql | 2 +- .../cs/server/label/CSInstanceLabelClient.java | 6 +++--- .../common/lock/dao/CommonLockMapper.java | 2 +- .../common/lock/entity/CommonLock.java | 10 +++++----- .../service/impl/DefaultCommonLockService.java | 2 +- .../resources/mapper/common/CommonLockMapper.xml | 10 +++++----- .../common/lock/dao/CommonLockMapperTest.java | 14 +++++++------- .../src/test/resources/create.sql | 2 +- .../src/test/resources/data.sql | 2 +- .../linkis/udf/service/impl/UDFServiceImpl.java | 4 ++-- 13 files changed, 35 insertions(+), 35 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala index 3feb880bf2..1aba512f4c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala @@ -64,7 +64,7 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo private val bmlClient = BmlClientFactory.createBmlClient() private var isRefreshing: Boolean = false - private val _LOCK = "_LOAD_ENGINE_CONN_LOCK" + private val _LOCK = "_MASTER_LOAD_ENGINE_CONN_LOCK" val commonLock = new CommonLock private var lock = false @@ -74,13 +74,13 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo commonLock.setCreateTime(new Date) commonLock.setUpdateTime(new Date) commonLock.setCreator(Utils.getJvmUser) - commonLock.setHost(Utils.getLocalHostname) + commonLock.setLocker(Utils.getLocalHostname) commonLock.setUpdator(Utils.getJvmUser) lock = commonLockService.reentrantLock(commonLock, -1) if (lock) { logger.info( "The master LinkisManager node get lock by {}. And start to refresh all engineconn plugins when inited.", - _LOCK + "-" + commonLock.getHost + _LOCK + "-" + commonLock.getLocker ) refreshAll(false) } @@ -92,7 +92,7 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo logger.info( "The master LinkisManager node hase released lock {}.", commonLock.getLockObject + "-" + - commonLock.getHost + commonLock.getLocker ) commonLockService.unlock(commonLock) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala index b5d1ac55b7..c951e01052 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala @@ -64,13 +64,13 @@ class AMHeartbeatService extends HeartbeatService with Logging { commonLock.setCreateTime(new Date) commonLock.setUpdateTime(new Date) commonLock.setCreator(Utils.getJvmUser) - commonLock.setHost(Utils.getLocalHostname) + commonLock.setLocker(Utils.getLocalHostname) commonLock.setUpdator(Utils.getJvmUser) lock = commonLockService.reentrantLock(commonLock, -1) if (null != managerMonitor && lock) { logger.info( "The master am get lock by {}. And start to init AMHeartbeatService monitor.", - _LOCK + "-" + commonLock.getHost + _LOCK + "-" + commonLock.getLocker ) Utils.defaultScheduler.scheduleAtFixedRate( managerMonitor, @@ -87,7 +87,7 @@ class AMHeartbeatService extends HeartbeatService with Logging { commonLockService.unlock(commonLock) logger.info( "The master am has released lock {}.", - commonLock.getLockObject + "-" + commonLock.getHost + commonLock.getLockObject + "-" + commonLock.getLocker ); } } diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 795cccea21..45a14a34c7 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -136,7 +136,7 @@ DROP TABLE IF EXISTS `linkis_ps_common_lock`; CREATE TABLE `linkis_ps_common_lock` ( `id` int(11) NOT NULL AUTO_INCREMENT, `lock_object` varchar(255) COLLATE utf8_bin DEFAULT NULL, - `host` varchar(255) COLLATE utf8_bin NOT NULL, + `locker` varchar(255) COLLATE utf8_bin NOT NULL, `time_out` longtext COLLATE utf8_bin, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, diff --git a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql index 8611a41e8d..93e5edb384 100644 --- a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql +++ b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql @@ -25,7 +25,7 @@ ALTER TABLE `linkis_ps_configuration_config_key` ADD UNIQUE `uniq_key_ectype` (` ALTER TABLE `linkis_ps_configuration_config_key` modify column `engine_conn_type` varchar(50) DEFAULT '' COMMENT 'engine type,such as spark,hive etc'; -ALTER TABLE linkis_ps_common_lock ADD COLUMN `host` VARCHAR(255) NOT NULL COMMENT 'host'; +ALTER TABLE linkis_ps_common_lock ADD COLUMN locker VARCHAR(255) NOT NULL COMMENT 'locker'; ALTER TABLE `linkis_ps_configuration_config_key` ADD column `template_required` tinyint(1) DEFAULT 0 COMMENT 'template required 0 none / 1 must' diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/label/CSInstanceLabelClient.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/label/CSInstanceLabelClient.java index 6a3681dfce..7e3b671385 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/label/CSInstanceLabelClient.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/label/CSInstanceLabelClient.java @@ -65,7 +65,7 @@ public void init(AvailabilityChangeEvent availabilityChangeEv commonLock.setCreateTime(new Date()); commonLock.setUpdateTime(new Date()); commonLock.setCreator(Utils.getJvmUser()); - commonLock.setHost(Utils.getLocalHostname()); + commonLock.setLocker(Utils.getLocalHostname()); commonLock.setUpdator(Utils.getJvmUser()); lock = commonLockService.reentrantLock(commonLock, -1L); String suffix = ContextServerConf.CS_LABEL_SUFFIX; @@ -73,7 +73,7 @@ public void init(AvailabilityChangeEvent availabilityChangeEv if (lock) { // master node set cs_1_xxx label - logger.info("The master ps-cs node get lock by {}", _LOCK + "-" + commonLock.getHost()); + logger.info("The master ps-cs node get lock by {}", _LOCK + "-" + commonLock.getLocker()); confLabel = "cs_1_" + suffix; } else { confLabel = "cs_2_" + suffix; @@ -98,7 +98,7 @@ public void shutdown(ContextClosedEvent contextClosedEvent) { commonLockService.unlock(commonLock); logger.info( "The master ps-cs node has released lock {}.", - commonLock.getLockObject() + "-" + commonLock.getHost()); + commonLock.getLockObject() + "-" + commonLock.getLocker()); } } } diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapper.java b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapper.java index dbf9e8acde..162341efd4 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapper.java +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapper.java @@ -32,5 +32,5 @@ public interface CommonLockMapper { List getAll(); - CommonLock getLockByHost(@Param("lockObject") String lockObject, @Param("host") String host); + CommonLock getLockByLocker(@Param("lockObject") String lockObject, @Param("locker") String host); } diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/entity/CommonLock.java b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/entity/CommonLock.java index 680d9daff0..49db1a710e 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/entity/CommonLock.java +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/entity/CommonLock.java @@ -28,7 +28,7 @@ public class CommonLock { private Date createTime; private String updator; private String creator; - private String host; + private String locker; public Integer getId() { return id; @@ -86,11 +86,11 @@ public void setCreator(String creator) { this.creator = creator; } - public String getHost() { - return host; + public String getLocker() { + return locker; } - public void setHost(String host) { - this.host = host; + public void setLocker(String locker) { + this.locker = locker; } } diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/service/impl/DefaultCommonLockService.java b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/service/impl/DefaultCommonLockService.java index 283c434ed6..1fc6a293cc 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/service/impl/DefaultCommonLockService.java +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/main/java/org/apache/linkis/publicservice/common/lock/service/impl/DefaultCommonLockService.java @@ -60,7 +60,7 @@ public Boolean lock(CommonLock commonLock, Long timeOut) { @Override public Boolean reentrantLock(CommonLock commonLock, Long timeOut) { CommonLock oldLock = - commonLockMapper.getLockByHost(commonLock.getLockObject(), commonLock.getHost()); + commonLockMapper.getLockByLocker(commonLock.getLockObject(), commonLock.getLocker()); if (oldLock != null) { return true; } diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml b/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml index 67cb626a16..a21818fc14 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml @@ -21,18 +21,18 @@ - insert into linkis_ps_common_lock (lock_object, `host`, time_out, update_time, create_time) - values(#{commonLock.lockObject}, #{commonLock.host}, #{timeOut}, now(), now()) + insert into linkis_ps_common_lock (lock_object, locker, time_out, update_time, create_time) + values(#{commonLock.lockObject}, #{commonLock.locker}, #{timeOut}, now(), now()) - delete from linkis_ps_common_lock where lock_object = #{commonLock.lockObject} and `host` = #{commonLock.host} + delete from linkis_ps_common_lock where lock_object = #{commonLock.lockObject} and locker = #{commonLock.locker} - + select * from linkis_ps_common_lock where lock_object = #{lockObject} and locker = #{locker} diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/test/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapperTest.java b/linkis-public-enhancements/linkis-ps-common-lock/src/test/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapperTest.java index 6ea4935edd..7535b38ab6 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/test/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapperTest.java +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/test/java/org/apache/linkis/publicservice/common/lock/dao/CommonLockMapperTest.java @@ -40,7 +40,7 @@ public void getAllTest() { public Boolean reentrantLock(CommonLock commonLock) { CommonLock oldLock = - commonLockMapper.getLockByHost(commonLock.getLockObject(), commonLock.getHost()); + commonLockMapper.getLockByLocker(commonLock.getLockObject(), commonLock.getLocker()); if (oldLock != null) { return true; } @@ -59,12 +59,12 @@ public void reentrantLockTest() { String lockObject = "hadoop-warehouse4"; CommonLock commonLock = new CommonLock(); commonLock.setLockObject(lockObject); - commonLock.setHost("test"); + commonLock.setLocker("test"); Boolean lock = reentrantLock(commonLock); Assertions.assertTrue(lock); lock = reentrantLock(commonLock); Assertions.assertTrue(lock); - commonLock.setHost("test1"); + commonLock.setLocker("test1"); lock = reentrantLock(commonLock); Assertions.assertFalse(lock); } @@ -75,7 +75,7 @@ public void unlockTest() { String lockObject = "hadoop-warehouse"; CommonLock commonLock = new CommonLock(); commonLock.setLockObject(lockObject); - commonLock.setHost("test"); + commonLock.setLocker("test"); commonLockMapper.unlock(commonLock); List locks = commonLockMapper.getAll(); @@ -93,7 +93,7 @@ public void lockTest() { Assertions.assertThrows( RuntimeException.class, () -> commonLockMapper.lock(commonLock, timeOut)); - commonLock.setHost("test"); + commonLock.setLocker("test"); commonLockMapper.lock(commonLock, timeOut); List locks = commonLockMapper.getAll(); Assertions.assertTrue(locks.size() == 2); @@ -105,10 +105,10 @@ public void getLockByHostTest() { String lockObject = "hadoop-warehouse3"; CommonLock commonLock = new CommonLock(); commonLock.setLockObject(lockObject); - commonLock.setHost("test"); + commonLock.setLocker("test"); commonLockMapper.lock(commonLock, -1L); CommonLock lock = - commonLockMapper.getLockByHost(commonLock.getLockObject(), commonLock.getHost()); + commonLockMapper.getLockByLocker(commonLock.getLockObject(), commonLock.getLocker()); Assertions.assertTrue(lock != null); } } diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/create.sql b/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/create.sql index dbee34d50e..3f6a9992aa 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/create.sql +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/create.sql @@ -23,7 +23,7 @@ DROP TABLE IF EXISTS `linkis_ps_common_lock`; CREATE TABLE `linkis_ps_common_lock` ( `id` int(11) NOT NULL AUTO_INCREMENT, `lock_object` varchar(255) DEFAULT NULL, - `host` varchar(512) NOT NULL, + `locker` varchar(512) NOT NULL, `time_out` longtext , `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/data.sql b/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/data.sql index 3dc5a76193..16635299f8 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/data.sql +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/data.sql @@ -17,6 +17,6 @@ DELETE FROM linkis_ps_common_lock; -insert into linkis_ps_common_lock(`id`,`lock_object`, `host`, `time_out`,`update_time`,`create_time`) values (1,'hadoop-warehouse', 'test', 1000000,now(),now()); +insert into linkis_ps_common_lock(`id`,`lock_object`, `locker`, `time_out`,`update_time`,`create_time`) values (1,'hadoop-warehouse', 'test', 1000000,now(),now()); diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java index b5ae3c8340..4e41ea28b3 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java @@ -114,7 +114,7 @@ public long addUDF(UDFAddVo udfVo, String userName) throws Exception { CommonLock commonLock = new CommonLock(); commonLock.setLockObject(userName + _LOCK); commonLock.setCreator(userName); - commonLock.setHost(Utils.getLocalHostname()); + commonLock.setLocker(Utils.getLocalHostname()); commonLock.setCreateTime(new Date()); commonLock.setUpdateTime(new Date()); try { @@ -318,7 +318,7 @@ public void updateUDF(UDFUpdateVo udfUpdateVo, String userName) throws Exception CommonLock persistenceLock = new CommonLock(); persistenceLock.setLockObject(userName + _LOCK); persistenceLock.setCreator(userName); - persistenceLock.setHost(Utils.getLocalHostname()); + persistenceLock.setLocker(Utils.getLocalHostname()); persistenceLock.setCreateTime(new Date()); persistenceLock.setUpdateTime(new Date()); try { From aec3a158ff92c1edc2c6ef633c50b39e7bc56962 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 3 Sep 2023 22:50:20 +0800 Subject: [PATCH 3/9] service merge (#271) Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> From d541e97c46496a9c6b0b863bdbc8fca1ffc4960a Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 3 Sep 2023 22:51:05 +0800 Subject: [PATCH 4/9] add txt file type (#270) Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- docs/configuration/linkis-storage.md | 2 +- .../org/apache/linkis/storage/conf/LinkisStorageConf.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/linkis-storage.md b/docs/configuration/linkis-storage.md index 45b07afc1d..8e04b073be 100644 --- a/docs/configuration/linkis-storage.md +++ b/docs/configuration/linkis-storage.md @@ -29,4 +29,4 @@ |linkis-storage|wds.linkis.fs.hdfs.impl.disable.cache| false |disable.cache |true| |linkis-storage|wds.linkis.hdfs.rest.errs| |rest.errs|true| |linkis-storage|wds.linkis.resultset.row.max.str | 2m | max.str |true| -|linkis-storage|wds.linkis.storage.file.type | dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql | file.type |true| +|linkis-storage|wds.linkis.storage.file.type | dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql,txt | file.type |true| diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala index d66f6b691f..a76fe82fcb 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala @@ -39,7 +39,7 @@ object LinkisStorageConf { val FILE_TYPE = CommonVars( "wds.linkis.storage.file.type", - "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql" + "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql,txt" ).getValue private var fileTypeArr: Array[String] = null From 89a082b8c0b88eb9495436ce573aa8a223cf569f Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 3 Sep 2023 22:52:57 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E3=80=901.1.15=E3=80=91fix=20spark=20conf?= =?UTF-8?q?=20error=20(#269)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix TemplateRequired error --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- .../restful/api/ConfigurationRestfulApi.java | 26 ++++++++++++------- .../impl/TemplateConfigKeyServiceImpl.java | 5 ++-- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index fcc94d8f2c..3d996793f9 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -46,7 +46,6 @@ import java.text.MessageFormat; import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -358,22 +357,31 @@ public Message saveFullTree(HttpServletRequest req, @RequestBody JsonNode json) private void sparkConfCheck(List settings) throws ConfigurationException { for (ConfigKeyValue setting : settings) { - if (setting.getKey().equals("spark.conf")) { + if (setting.getKey().equals("spark.conf") + && StringUtils.isNotBlank(setting.getConfigValue())) { // Check if there are any duplicates in spark. conf String[] split = setting.getConfigValue().split(";"); - Stream stringStream = Arrays.stream(split).map(s -> s.split("=")[0].trim()); - int setSize = stringStream.collect(Collectors.toSet()).size(); - int listSize = (int) stringStream.count(); + int setSize = + Arrays.stream(split) + .map(s -> s.split("=")[0].trim()) + .collect(Collectors.toSet()) + .size(); + int listSize = + Arrays.stream(split) + .map(s -> s.split("=")[0].trim()) + .collect(Collectors.toList()) + .size(); if (listSize != setSize) { - throw new ConfigurationException("Key has duplicate entries"); + throw new ConfigurationException("Spark.conf contains duplicate keys"); } // Check if there are any duplicates in the spark.conf configuration and other individual - // configurations - for (String key : split) { + for (String keyValue : split) { + String key = keyValue.split("=")[0].trim(); boolean matchResult = settings.stream().anyMatch(settingKey -> key.equals(settingKey.getKey())); if (matchResult) { - throw new ConfigurationException("Key has duplicate entries,key :" + key); + throw new ConfigurationException( + "Saved key is duplicated with the spark conf key , key :" + key); } } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index 08dc39812f..5cbd22287e 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -259,9 +259,8 @@ public List queryKeyInfoList(List uuidList) throws Configuration temp.put("validateRange", info.getValidateRange()); temp.put("boundaryType", info.getBoundaryType()); temp.put("defaultValue", info.getDefaultValue()); - // for front-end to judge whether input is required - if (StringUtils.isNotEmpty(info.getDefaultValue())) { - temp.put("require", "true"); + if (StringUtils.isNotBlank(info.getTemplateRequired())) { + temp.put("require", info.getTemplateRequired().equals("1")); } else { temp.put("require", "false"); } From 835706414c933508dad1a83b140b607e409a3b74 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 3 Sep 2023 22:53:48 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E3=80=901.1.15=E3=80=91Fix=20Datatype=20De?= =?UTF-8?q?cimal=20Error=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix datatype decimal error * update datatype --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- .../scala/org/apache/linkis/storage/domain/DataType.scala | 3 ++- .../org/apache/linkis/storage/domain/DataTypeTest.scala | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index 61d5fb7b3d..57b454e0ed 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -28,6 +28,7 @@ object DataType extends Logging { val LOWCASE_NULL_VALUE = "null" val DECIMAL_REGEX = "^decimal\\(\\d*\\,\\d*\\)".r.unanchored + val DECIMAL_REGEX_BLANK = "^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)".r.unanchored val SHORT_REGEX = "^short.*".r.unanchored val INT_REGEX = "^int.*".r.unanchored @@ -62,7 +63,7 @@ object DataType extends Logging { case "date" => DateType case "timestamp" => TimestampType case "binary" => BinaryType - case "decimal" | DECIMAL_REGEX() => DecimalType + case "decimal" | DECIMAL_REGEX() | DECIMAL_REGEX_BLANK() => DecimalType case ARRAY_REGEX() => ArrayType case MAP_REGEX() => MapType case LIST_REGEX() => ListType diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala index 5537794840..1636ffce17 100644 --- a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala @@ -68,4 +68,12 @@ class DataTypeTest { Assertions.assertNotNull(str) } + @Test + @DisplayName("decimalTest") + def decimalTest(): Unit = { + val dateType = DataType.toDataType("decimal") + val str = DataType.toValue(dateType, "decimal(10,8)") + Assertions.assertNotNull(str) + } + } From 16896282820cee1fdfa1649e7decdc0d56d3a653 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 3 Sep 2023 23:06:38 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E3=80=901.1.15=E3=80=91Security=20Work=20O?= =?UTF-8?q?rder=20Upgrade=20(#256)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * version update * update version * remove avro --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- linkis-engineconn-plugins/flink/pom.xml | 17 +++++++++++++++++ linkis-engineconn-plugins/sqoop/pom.xml | 4 ++++ .../linkis-metadata-query/service/hive/pom.xml | 10 ++++++++++ 3 files changed, 31 insertions(+) diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml index 3ceca971e2..55355d9f39 100644 --- a/linkis-engineconn-plugins/flink/pom.xml +++ b/linkis-engineconn-plugins/flink/pom.xml @@ -48,6 +48,12 @@ org.apache.flink flink-runtime_${scala.binary.version} ${flink.version} + + + org.xerial.snappy + snappy-java + + org.apache.flink @@ -188,6 +194,17 @@ org.apache.flink flink-connector-kafka_${scala.binary.version} ${flink.version} + + + org.apache.kafka + kafka-clients + + + + + org.apache.kafka + kafka-clients + 3.4.0 org.apache.flink diff --git a/linkis-engineconn-plugins/sqoop/pom.xml b/linkis-engineconn-plugins/sqoop/pom.xml index 8435ca1705..00dd52bdd3 100644 --- a/linkis-engineconn-plugins/sqoop/pom.xml +++ b/linkis-engineconn-plugins/sqoop/pom.xml @@ -87,6 +87,10 @@ javax.servlet servlet-api + + org.apache.avro + avro + diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml index 1279139446..4247acd8f4 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml @@ -78,6 +78,12 @@ + + org.apache.avro + avro + 1.11.0 + provided + org.apache.hadoop hadoop-mapreduce-client-core @@ -91,6 +97,10 @@ io.netty netty + + org.apache.avro + avro + From d5535865ae1637367904430fe1234adc87ad94c0 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:10:11 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E3=80=901.1.15=E3=80=91fix=20=20query=20?= =?UTF-8?q?=20config=20key=20error=20(#274)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update error code dml * fix configkey bug --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- .../src/main/resources/mapper/common/ConfigMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml index 0fb9611279..a7ea542e20 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml @@ -225,7 +225,7 @@