[Improvement][datavines] Upgrade spring-boot version to 2.7.18

datavines-dist/src/main/assembly/datavines-bin.xml

@@ -100,7 +100,7 @@
                 <include>io.datavines:datavines-engine-common</include>
                 <include>io.datavines:datavines-engine-spark-transform-sql</include>
                 <include>io.datavines:datavines-engine-api</include>
-                <include>mysql:mysql-connector-java</include>
+                <include>com.mysql:mysql-connector-j</include>
                 <include>org.apache.httpcomponents:httpclient</include>
                 <include>org.apache.httpcomponents:httpcore</include>
                 <include>ru.yandex.clickhouse:clickhouse-jdbc</include>

datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/SqlUtils.java

@@ -169,16 +169,16 @@ public static Set<String> getQueryFromsAndJoins(String sql) {
         try {
             net.sf.jsqlparser.statement.Statement parse = CCJSqlParserUtil.parse(sql);
             Select select = (Select) parse;
-            SelectBody selectBody = select.getSelectBody();
+            Select selectBody = select.getSelectBody();
             if (selectBody instanceof PlainSelect) {
                 PlainSelect plainSelect = (PlainSelect) selectBody;
                 columnPrefixExtractor(columnPrefixes, plainSelect);
             }
 
             if (selectBody instanceof SetOperationList) {
                 SetOperationList setOperationList = (SetOperationList) selectBody;
-                List<SelectBody> selects = setOperationList.getSelects();
-                for (SelectBody optSelectBody : selects) {
+                List<Select> selects = setOperationList.getSelects();
+                for (Select optSelectBody : selects) {
                     PlainSelect plainSelect = (PlainSelect) optSelectBody;
                     columnPrefixExtractor(columnPrefixes, plainSelect);
                 }
@@ -223,27 +223,21 @@ public void visit(Table tableName) {
             }
 
             @Override
-            public void visit(SubSelect subSelect) {
-            }
+            public void visit(ParenthesedSelect parenthesedSelect) {
 
-            @Override
-            public void visit(SubJoin subjoin) {
             }
 
             @Override
             public void visit(LateralSubSelect lateralSubSelect) {
             }
 
-            @Override
-            public void visit(ValuesList valuesList) {
-            }
-
             @Override
             public void visit(TableFunction tableFunction) {
             }
 
             @Override
-            public void visit(ParenthesisFromItem aThis) {
+            public void visit(ParenthesedFromItem parenthesedFromItem) {
+
             }
         };
     }

datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlMutex.java

@@ -70,7 +70,7 @@ public boolean acquire(String lockKey, long time) {
             RegistryLock registryLock = null;
             int count = 1;
             if (time > 0) {
-                count  = Math.max(1, (int) (time * 1000 / LOCK_ACQUIRE_INTERVAL));
+                count = Math.max(1, (int) (time * 1000 / LOCK_ACQUIRE_INTERVAL));
             }
             while (count > 0) {
                 try {
@@ -82,7 +82,7 @@ public boolean acquire(String lockKey, long time) {
                     try {
                         clearExpireLock();
                     } catch (SQLException ex) {
-                       log.error("clear expire lock error : ", ex);
+                        log.error("clear expire lock error : ", ex);
                     }
                     ThreadUtils.sleep(LOCK_ACQUIRE_INTERVAL);
                     count--;
@@ -151,7 +151,7 @@ private void executeDelete(String key) throws SQLException {
 
     private boolean isExists(String key, ServerInfo serverInfo) throws SQLException {
         checkConnection();
-        PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_registry_lock where lock_key=?");
+        PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_registry_lock where lock_key=?", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
         preparedStatement.setString(1, key);
         ResultSet resultSet = preparedStatement.executeQuery();
 
@@ -172,7 +172,7 @@ private void clearExpireLock() throws SQLException {
         preparedStatement.executeUpdate();
         preparedStatement.close();
         // 将超时的lockKey移除掉
-        lockHoldMap.values().removeIf((v -> v.getUpdateTime().getTime() < (System.currentTimeMillis()- expireTimeWindow)));
+        lockHoldMap.values().removeIf((v -> v.getUpdateTime().getTime() < (System.currentTimeMillis() - expireTimeWindow)));
     }
 
     private void checkConnection() throws SQLException {

datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlServerStateManager.java

@@ -47,10 +47,10 @@ public class MysqlServerStateManager {
     public MysqlServerStateManager(Connection connection, Properties properties) throws SQLException {
         this.connection = connection;
         this.properties = properties;
-        serverInfo = new ServerInfo(NetUtils.getHost(), Integer.valueOf((String) properties.get("server.port")), new Timestamp(System.currentTimeMillis()),new Timestamp(System.currentTimeMillis()));
+        serverInfo = new ServerInfo(NetUtils.getHost(), Integer.valueOf((String) properties.get("server.port")), new Timestamp(System.currentTimeMillis()), new Timestamp(System.currentTimeMillis()));
         ScheduledExecutorService executorService = Executors.newScheduledThreadPool(2);
-        executorService.scheduleAtFixedRate(new HeartBeater(),2,2, TimeUnit.SECONDS);
-        executorService.scheduleAtFixedRate(new ServerChecker(),5,10, TimeUnit.SECONDS);
+        executorService.scheduleAtFixedRate(new HeartBeater(), 2, 2, TimeUnit.SECONDS);
+        executorService.scheduleAtFixedRate(new ServerChecker(), 5, 10, TimeUnit.SECONDS);
     }
 
     public void registry(SubscribeListener subscribeListener) throws SQLException {
@@ -71,14 +71,14 @@ public void unRegistry() throws SQLException {
     }
 
     public void refreshServer() throws SQLException {
-        ConcurrentHashMap<String,ServerInfo> newServers = fetchServers();
+        ConcurrentHashMap<String, ServerInfo> newServers = fetchServers();
         Set<String> offlineServer = new HashSet<>();
         if (newServers == null) {
             //do nothing
             return;
         }
         Set<String> onlineServer = new HashSet<>();
-        newServers.forEach((k, v) ->{
+        newServers.forEach((k, v) -> {
             long updateTime = v.getUpdateTime().getTime();
             long now = System.currentTimeMillis();
             if (now - updateTime > 20000) {
@@ -100,7 +100,7 @@ public void refreshServer() throws SQLException {
             if (!deadServers.contains(x) && !x.equals(serverInfo.getAddr())) {
                 String[] values = x.split(":");
                 try {
-                    executeDelete(new ServerInfo(values[0],Integer.valueOf(values[1])));
+                    executeDelete(new ServerInfo(values[0], Integer.valueOf(values[1])));
                     liveServerMap.remove(x);
                 } catch (SQLException e) {
                     log.error("delete server info error", e);
@@ -131,7 +131,7 @@ public void refreshServer() throws SQLException {
 
     private void executeInsert(ServerInfo serverInfo) throws SQLException {
         checkConnection();
-        PreparedStatement preparedStatement = connection.prepareStatement("insert into dv_server (host,port) values (?,?)");
+        PreparedStatement preparedStatement = connection.prepareStatement("insert into dv_server (host, port) values (?, ?)");
         preparedStatement.setString(1, serverInfo.getHost());
         preparedStatement.setInt(2, serverInfo.getServerPort());
         preparedStatement.executeUpdate();
@@ -159,7 +159,7 @@ private void executeDelete(ServerInfo serverInfo) throws SQLException {
 
     private boolean isExists(ServerInfo serverInfo) throws SQLException {
         checkConnection();
-        PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_server where host=? and port=?");
+        PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_server where host= ? and port= ?", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
         preparedStatement.setString(1, serverInfo.getHost());
         preparedStatement.setInt(2, serverInfo.getServerPort());
         ResultSet resultSet = preparedStatement.executeQuery();
@@ -197,9 +197,9 @@ private ConcurrentHashMap<String, ServerInfo> fetchServers() throws SQLException
         return map;
     }
 
-    public List<ServerInfo> getActiveServerList(){
+    public List<ServerInfo> getActiveServerList() {
         List<ServerInfo> activeServerList = new ArrayList<>();
-        liveServerMap.forEach((k,v)-> {
+        liveServerMap.forEach((k, v) -> {
             String[] values = k.split(":");
             if (values.length == 2) {
                 activeServerList.add(v);

datavines-runner/pom.xml

@@ -106,8 +106,8 @@
         </dependency>
 
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
 
         <dependency>
@@ -145,4 +145,4 @@
         </dependency>
     </dependencies>
 
-</project>
+</project>

datavines-server/pom.xml

@@ -206,8 +206,8 @@
         </dependency>
 
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
 
         <dependency>
@@ -301,7 +301,7 @@
             <artifactId>datavines-engine-executor</artifactId>
             <version>${project.version}</version>
         </dependency>
-        
+
         <dependency>
             <groupId>io.datavines</groupId>
             <artifactId>datavines-engine-spark-executor</artifactId>

datavines-server/src/main/resources/application.yaml

@@ -14,16 +14,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+server:
+  port: 5600
+
 spring:
+  config:
+    activate:
+      on-profile: mysql
+
   main:
     banner-mode: off
     allow-circular-references: true
   application:
     name: datavines-server
+
+  #datasource:
+  #  driver-class-name: org.postgresql.Driver
+  #  url: jdbc:postgresql://127.0.0.1:5432/datavines
+  #  username: postgres
+  #  password: 123456
+  #quartz:
+  #  properties:
+  #    org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
   datasource:
-    driver-class-name: org.postgresql.Driver
-    url: jdbc:postgresql://127.0.0.1:5432/datavines
-    username: postgres
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/datavines?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai
+    username: root
     password: 123456
     hikari:
       connection-test-query: select 1
@@ -55,7 +71,7 @@ spring:
       org.quartz.jobStore.misfireThreshold: 60000
       org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
       org.quartz.scheduler.makeSchedulerThreadDaemon: true
-      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
       org.quartz.jobStore.clusterCheckinInterval: 5000
   mvc:
     pathmatch:
@@ -66,9 +82,6 @@ mybatis-plus:
 #  configuration:
 #    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
 
-server:
-  port: 5600
-
 management:
   endpoints:
     web:
@@ -80,16 +93,3 @@ management:
 
 logging:
   config: classpath:server-logback.xml
----
-spring:
-  config:
-    activate:
-      on-profile: mysql
-  datasource:
-    driver-class-name: com.mysql.cj.jdbc.Driver
-    url: jdbc:mysql://127.0.0.1:3306/datavines?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai
-    username: root
-    password: 123456
-  quartz:
-    properties:
-      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate

pom.xml

@@ -47,8 +47,8 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <commons.version>1.0.0</commons.version>
-        <slf4j.api.version>1.7.5</slf4j.api.version>
-        <slf4j.log4j12.version>1.7.5</slf4j.log4j12.version>
+        <slf4j.api.version>1.7.25</slf4j.api.version>
+        <slf4j.log4j12.version>1.7.25</slf4j.log4j12.version>
         <jackson.version>2.12.5</jackson.version>
         <oshi.core.version>5.8.7</oshi.core.version>
         <commons.collections4.version>4.1</commons.collections4.version>
@@ -57,23 +57,23 @@
         <scala.version>2.11.8</scala.version>
         <scala.binary.version>2.11</scala.binary.version>
         <hadoop.version>2.7.5</hadoop.version>
-        <mysql.version>8.0.16</mysql.version>
+        <mysql.version>8.4.0</mysql.version>
         <skip.pmd.check>false</skip.pmd.check>
         <maven.deploy.skip>false</maven.deploy.skip>
         <maven.javadoc.skip>false</maven.javadoc.skip>
         <maven.compiler.encoding>UTF-8</maven.compiler.encoding>
         <main.basedir>${project.basedir}</main.basedir>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <druid.version>1.1.13</druid.version>
+        <druid.version>1.2.23</druid.version>
         <swagger.version>1.9.3</swagger.version>
         <springfox.version>2.9.2</springfox.version>
-        <swagger-models.version>1.5.24</swagger-models.version>
+        <swagger-models.version>1.6.14</swagger-models.version>
         <groovy.version>3.0.7</groovy.version>
         <netty.version>4.1.53.Final</netty.version>
         <guice.version>4.2.0</guice.version>
         <logback.version>1.2.3</logback.version>
-        <httpclient.version>4.4.1</httpclient.version>
-        <httpcore.version>4.4.1</httpcore.version>
+        <httpclient.version>4.5.14</httpclient.version>
+        <httpcore.version>4.4.16</httpcore.version>
         <quartz.version>2.3.2</quartz.version>
         <commons.lang.version>2.3</commons.lang.version>
         <commons.lang3.version>3.5</commons.lang3.version>
@@ -84,14 +84,14 @@
         <jcommander.version>1.81</jcommander.version>
         <commons-validator.version>1.6</commons-validator.version>
         <javax-inject.version>1</javax-inject.version>
-        <spring.boot.version>2.6.1</spring.boot.version>
-        <mybatis-plus.version>3.5.0</mybatis-plus.version>
+        <spring.boot.version>2.7.18</spring.boot.version>
+        <mybatis-plus.version>3.5.8</mybatis-plus.version>
         <swagger.version>3.0.0</swagger.version>
-        <lombok.version>1.18.22</lombok.version>
+        <lombok.version>1.18.34</lombok.version>
         <postgresql.version>42.2.6</postgresql.version>
         <javax-validation.version>2.0.1.Final</javax-validation.version>
-        <jsqlparser.version>4.2</jsqlparser.version>
-        <jjwt.version>0.7.0</jjwt.version>
+        <jsqlparser.version>4.8</jsqlparser.version>
+        <jjwt.version>0.9.1</jjwt.version>
         <jbcrypt.version>0.3m</jbcrypt.version>
         <curator.version>4.3.0</curator.version>
         <zookeeper.version>3.4.14</zookeeper.version>
@@ -102,7 +102,7 @@
         <sqlserver.version>6.1.0.jre8</sqlserver.version>
         <dameng.version>8.1.2.141</dameng.version>
         <hikari.version>4.0.3</hikari.version>
-        <spring.version>5.3.12</spring.version>
+        <spring.version>5.3.39</spring.version>
         <kerberos.version>1.0.1.RELEASE</kerberos.version>
         <jodd.version>3.5.2</jodd.version>
         <kaptcha.version>2.3.2</kaptcha.version>
@@ -191,7 +191,7 @@
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>
-                <version>1.7.25</version>
+                <version>${slf4j.api.version}</version>
             </dependency>
 
             <dependency>
@@ -279,16 +279,16 @@
             </dependency>
 
             <dependency>
-                <groupId>mysql</groupId>
-                <artifactId>mysql-connector-java</artifactId>
+                <groupId>com.mysql</groupId>
+                <artifactId>mysql-connector-j</artifactId>
                 <version>${mysql.version}</version>
 <!--                <scope>test</scope>-->
             </dependency>
 
             <dependency>
                 <groupId>junit</groupId>
                 <artifactId>junit</artifactId>
-                <version>4.11</version>
+                <version>4.13.2</version>
                 <scope>test</scope>
             </dependency>
 

scripts/sql/datavines-mysql.sql

@@ -873,7 +873,7 @@ INSERT INTO `dv_config` VALUES ('21', '-1', 'livy.execution.threshold', '1000',
 INSERT INTO `dv_config` VALUES ('22', '-1', 'livy.task.jars', CONCAT('datavines-common-1.0.0-SNAPSHOT.jar,datavines-spi-1.0.0-SNAPSHOT.jar,'
                                                                     'datavines-engine-spark-api-1.0.0-SNAPSHOT.jar,datavines-engine-spark-connector-jdbc-1.0.0-SNAPSHOT.jar,'
                                                                     'datavines-engine-core-1.0.0-SNAPSHOT.jar,datavines-engine-common-1.0.0-SNAPSHOT.jar,datavines-engine-spark-transform-sql-1.0.0-SNAPSHOT.jar,'
-                                                                    'datavines-engine-api-1.0.0-SNAPSHOT.jar,mysql-connector-java-8.0.16.jar,httpclient-4.4.1.jar,'
+                                                                    'datavines-engine-api-1.0.0-SNAPSHOT.jar,mysql-connector-j-8.4.0.jar,httpclient-4.4.1.jar,'
                                                                     'httpcore-4.4.1.jar,postgresql-42.2.6.jar,presto-jdbc-0.283.jar,trino-jdbc-407.jar,clickhouse-jdbc-0.1.53.jar,'
                                                                     'mongo-java-driver-3.9.0.jar,mongo-spark-connector_2.11-2.4.0.jar,datavines-engine-spark-connector-mongodb-1.0.0-SNAPSHOT.jar'),
                                 '1', '1', '2023-09-05 21:02:38', '1', '2023-09-05 21:02:38');