Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improvement][datavines] Upgrade spring-boot version to 2.7.18 #482

Merged
merged 1 commit into from
Dec 26, 2024
Merged

[Improvement][datavines] Upgrade spring-boot version to 2.7.18 #482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datavines-dist/src/main/assembly/datavines-bin.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
<include>io.datavines:datavines-engine-common</include>
<include>io.datavines:datavines-engine-spark-transform-sql</include>
<include>io.datavines:datavines-engine-api</include>
<include>mysql:mysql-connector-java</include>
<include>com.mysql:mysql-connector-j</include>
<include>org.apache.httpcomponents:httpclient</include>
<include>org.apache.httpcomponents:httpcore</include>
<include>ru.yandex.clickhouse:clickhouse-jdbc</include>
Expand Down
18 changes: 6 additions & 12 deletions ...atavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/SqlUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,16 @@ public static Set<String> getQueryFromsAndJoins(String sql) {
try {
net.sf.jsqlparser.statement.Statement parse = CCJSqlParserUtil.parse(sql);
Select select = (Select) parse;
SelectBody selectBody = select.getSelectBody();
Select selectBody = select.getSelectBody();
if (selectBody instanceof PlainSelect) {
PlainSelect plainSelect = (PlainSelect) selectBody;
columnPrefixExtractor(columnPrefixes, plainSelect);
}

if (selectBody instanceof SetOperationList) {
SetOperationList setOperationList = (SetOperationList) selectBody;
List<SelectBody> selects = setOperationList.getSelects();
for (SelectBody optSelectBody : selects) {
List<Select> selects = setOperationList.getSelects();
for (Select optSelectBody : selects) {
PlainSelect plainSelect = (PlainSelect) optSelectBody;
columnPrefixExtractor(columnPrefixes, plainSelect);
}
Expand Down Expand Up @@ -223,27 +223,21 @@ public void visit(Table tableName) {
}

@Override
public void visit(SubSelect subSelect) {
}
public void visit(ParenthesedSelect parenthesedSelect) {

@Override
public void visit(SubJoin subjoin) {
}

@Override
public void visit(LateralSubSelect lateralSubSelect) {
}

@Override
public void visit(ValuesList valuesList) {
}

@Override
public void visit(TableFunction tableFunction) {
}

@Override
public void visit(ParenthesisFromItem aThis) {
public void visit(ParenthesedFromItem parenthesedFromItem) {

}
};
}
Expand Down
8 changes: 4 additions & 4 deletions ...ugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlMutex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public boolean acquire(String lockKey, long time) {
RegistryLock registryLock = null;
int count = 1;
if (time > 0) {
count = Math.max(1, (int) (time * 1000 / LOCK_ACQUIRE_INTERVAL));
count = Math.max(1, (int) (time * 1000 / LOCK_ACQUIRE_INTERVAL));
}
while (count > 0) {
try {
Expand All @@ -82,7 +82,7 @@ public boolean acquire(String lockKey, long time) {
try {
clearExpireLock();
} catch (SQLException ex) {
log.error("clear expire lock error : ", ex);
log.error("clear expire lock error : ", ex);
}
ThreadUtils.sleep(LOCK_ACQUIRE_INTERVAL);
count--;
Expand Down Expand Up @@ -151,7 +151,7 @@ private void executeDelete(String key) throws SQLException {

private boolean isExists(String key, ServerInfo serverInfo) throws SQLException {
checkConnection();
PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_registry_lock where lock_key=?");
PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_registry_lock where lock_key=?", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
preparedStatement.setString(1, key);
ResultSet resultSet = preparedStatement.executeQuery();

Expand All @@ -172,7 +172,7 @@ private void clearExpireLock() throws SQLException {
preparedStatement.executeUpdate();
preparedStatement.close();
// 将超时的lockKey移除掉
lockHoldMap.values().removeIf((v -> v.getUpdateTime().getTime() < (System.currentTimeMillis()- expireTimeWindow)));
lockHoldMap.values().removeIf((v -> v.getUpdateTime().getTime() < (System.currentTimeMillis() - expireTimeWindow)));
}

private void checkConnection() throws SQLException {
Expand Down
20 changes: 10 additions & 10 deletions ...es-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlServerStateManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ public class MysqlServerStateManager {
public MysqlServerStateManager(Connection connection, Properties properties) throws SQLException {
this.connection = connection;
this.properties = properties;
serverInfo = new ServerInfo(NetUtils.getHost(), Integer.valueOf((String) properties.get("server.port")), new Timestamp(System.currentTimeMillis()),new Timestamp(System.currentTimeMillis()));
serverInfo = new ServerInfo(NetUtils.getHost(), Integer.valueOf((String) properties.get("server.port")), new Timestamp(System.currentTimeMillis()), new Timestamp(System.currentTimeMillis()));
ScheduledExecutorService executorService = Executors.newScheduledThreadPool(2);
executorService.scheduleAtFixedRate(new HeartBeater(),2,2, TimeUnit.SECONDS);
executorService.scheduleAtFixedRate(new ServerChecker(),5,10, TimeUnit.SECONDS);
executorService.scheduleAtFixedRate(new HeartBeater(), 2, 2, TimeUnit.SECONDS);
executorService.scheduleAtFixedRate(new ServerChecker(), 5, 10, TimeUnit.SECONDS);
}

public void registry(SubscribeListener subscribeListener) throws SQLException {
Expand All @@ -71,14 +71,14 @@ public void unRegistry() throws SQLException {
}

public void refreshServer() throws SQLException {
ConcurrentHashMap<String,ServerInfo> newServers = fetchServers();
ConcurrentHashMap<String, ServerInfo> newServers = fetchServers();
Set<String> offlineServer = new HashSet<>();
if (newServers == null) {
//do nothing
return;
}
Set<String> onlineServer = new HashSet<>();
newServers.forEach((k, v) ->{
newServers.forEach((k, v) -> {
long updateTime = v.getUpdateTime().getTime();
long now = System.currentTimeMillis();
if (now - updateTime > 20000) {
Expand All @@ -100,7 +100,7 @@ public void refreshServer() throws SQLException {
if (!deadServers.contains(x) && !x.equals(serverInfo.getAddr())) {
String[] values = x.split(":");
try {
executeDelete(new ServerInfo(values[0],Integer.valueOf(values[1])));
executeDelete(new ServerInfo(values[0], Integer.valueOf(values[1])));
liveServerMap.remove(x);
} catch (SQLException e) {
log.error("delete server info error", e);
Expand Down Expand Up @@ -131,7 +131,7 @@ public void refreshServer() throws SQLException {

private void executeInsert(ServerInfo serverInfo) throws SQLException {
checkConnection();
PreparedStatement preparedStatement = connection.prepareStatement("insert into dv_server (host,port) values (?,?)");
PreparedStatement preparedStatement = connection.prepareStatement("insert into dv_server (host, port) values (?, ?)");
preparedStatement.setString(1, serverInfo.getHost());
preparedStatement.setInt(2, serverInfo.getServerPort());
preparedStatement.executeUpdate();
Expand Down Expand Up @@ -159,7 +159,7 @@ private void executeDelete(ServerInfo serverInfo) throws SQLException {

private boolean isExists(ServerInfo serverInfo) throws SQLException {
checkConnection();
PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_server where host=? and port=?");
PreparedStatement preparedStatement = connection.prepareStatement("select * from dv_server where host= ? and port= ?", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
preparedStatement.setString(1, serverInfo.getHost());
preparedStatement.setInt(2, serverInfo.getServerPort());
ResultSet resultSet = preparedStatement.executeQuery();
Expand Down Expand Up @@ -197,9 +197,9 @@ private ConcurrentHashMap<String, ServerInfo> fetchServers() throws SQLException
return map;
}

public List<ServerInfo> getActiveServerList(){
public List<ServerInfo> getActiveServerList() {
List<ServerInfo> activeServerList = new ArrayList<>();
liveServerMap.forEach((k,v)-> {
liveServerMap.forEach((k, v) -> {
String[] values = k.split(":");
if (values.length == 2) {
activeServerList.add(v);
Expand Down
6 changes: 3 additions & 3 deletions datavines-runner/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@
</dependency>

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>

<dependency>
Expand Down Expand Up @@ -145,4 +145,4 @@
</dependency>
</dependencies>

</project>
</project>
6 changes: 3 additions & 3 deletions datavines-server/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,8 +206,8 @@
</dependency>

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>

<dependency>
Expand Down Expand Up @@ -301,7 +301,7 @@
<artifactId>datavines-engine-executor</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>io.datavines</groupId>
<artifactId>datavines-engine-spark-executor</artifactId>
Expand Down
34 changes: 17 additions & 17 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<commons.version>1.0.0</commons.version>
<slf4j.api.version>1.7.5</slf4j.api.version>
<slf4j.log4j12.version>1.7.5</slf4j.log4j12.version>
<slf4j.api.version>1.7.25</slf4j.api.version>
<slf4j.log4j12.version>1.7.25</slf4j.log4j12.version>
<jackson.version>2.12.5</jackson.version>
<oshi.core.version>5.8.7</oshi.core.version>
<commons.collections4.version>4.1</commons.collections4.version>
Expand All @@ -57,23 +57,23 @@
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<hadoop.version>2.7.5</hadoop.version>
<mysql.version>8.0.16</mysql.version>
<mysql.version>8.4.0</mysql.version>
<skip.pmd.check>false</skip.pmd.check>
<maven.deploy.skip>false</maven.deploy.skip>
<maven.javadoc.skip>false</maven.javadoc.skip>
<maven.compiler.encoding>UTF-8</maven.compiler.encoding>
<main.basedir>${project.basedir}</main.basedir>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<druid.version>1.1.13</druid.version>
<druid.version>1.2.23</druid.version>
<swagger.version>1.9.3</swagger.version>
<springfox.version>2.9.2</springfox.version>
<swagger-models.version>1.5.24</swagger-models.version>
<swagger-models.version>1.6.14</swagger-models.version>
<groovy.version>3.0.7</groovy.version>
<netty.version>4.1.53.Final</netty.version>
<guice.version>4.2.0</guice.version>
<logback.version>1.2.3</logback.version>
<httpclient.version>4.4.1</httpclient.version>
<httpcore.version>4.4.1</httpcore.version>
<httpclient.version>4.5.14</httpclient.version>
<httpcore.version>4.4.16</httpcore.version>
<quartz.version>2.3.2</quartz.version>
<commons.lang.version>2.3</commons.lang.version>
<commons.lang3.version>3.5</commons.lang3.version>
Expand All @@ -84,14 +84,14 @@
<jcommander.version>1.81</jcommander.version>
<commons-validator.version>1.6</commons-validator.version>
<javax-inject.version>1</javax-inject.version>
<spring.boot.version>2.6.1</spring.boot.version>
<mybatis-plus.version>3.5.0</mybatis-plus.version>
<spring.boot.version>2.7.18</spring.boot.version>
<mybatis-plus.version>3.5.8</mybatis-plus.version>
<swagger.version>3.0.0</swagger.version>
<lombok.version>1.18.22</lombok.version>
<lombok.version>1.18.34</lombok.version>
<postgresql.version>42.2.6</postgresql.version>
<javax-validation.version>2.0.1.Final</javax-validation.version>
<jsqlparser.version>4.2</jsqlparser.version>
<jjwt.version>0.7.0</jjwt.version>
<jsqlparser.version>4.8</jsqlparser.version>
<jjwt.version>0.9.1</jjwt.version>
<jbcrypt.version>0.3m</jbcrypt.version>
<curator.version>4.3.0</curator.version>
<zookeeper.version>3.4.14</zookeeper.version>
Expand All @@ -102,7 +102,7 @@
<sqlserver.version>6.1.0.jre8</sqlserver.version>
<dameng.version>8.1.2.141</dameng.version>
<hikari.version>4.0.3</hikari.version>
<spring.version>5.3.12</spring.version>
<spring.version>5.3.39</spring.version>
<kerberos.version>1.0.1.RELEASE</kerberos.version>
<jodd.version>3.5.2</jodd.version>
<kaptcha.version>2.3.2</kaptcha.version>
Expand Down Expand Up @@ -191,7 +191,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
<version>${slf4j.api.version}</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -279,16 +279,16 @@
</dependency>

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<version>${mysql.version}</version>
<!-- <scope>test</scope>-->
</dependency>

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.2</version>
<scope>test</scope>
</dependency>

Expand Down
2 changes: 1 addition & 1 deletion scripts/sql/datavines-mysql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,7 @@ INSERT INTO `dv_config` VALUES ('21', '-1', 'livy.execution.threshold', '1000',
INSERT INTO `dv_config` VALUES ('22', '-1', 'livy.task.jars', CONCAT('datavines-common-1.0.0-SNAPSHOT.jar,datavines-spi-1.0.0-SNAPSHOT.jar,'
'datavines-engine-spark-api-1.0.0-SNAPSHOT.jar,datavines-engine-spark-connector-jdbc-1.0.0-SNAPSHOT.jar,'
'datavines-engine-core-1.0.0-SNAPSHOT.jar,datavines-engine-common-1.0.0-SNAPSHOT.jar,datavines-engine-spark-transform-sql-1.0.0-SNAPSHOT.jar,'
'datavines-engine-api-1.0.0-SNAPSHOT.jar,mysql-connector-java-8.0.16.jar,httpclient-4.4.1.jar,'
'datavines-engine-api-1.0.0-SNAPSHOT.jar,mysql-connector-j-8.4.0.jar,httpclient-4.4.1.jar,'
'httpcore-4.4.1.jar,postgresql-42.2.6.jar,presto-jdbc-0.283.jar,trino-jdbc-407.jar,clickhouse-jdbc-0.1.53.jar,'
'mongo-java-driver-3.9.0.jar,mongo-spark-connector_2.11-2.4.0.jar,datavines-engine-spark-connector-mongodb-1.0.0-SNAPSHOT.jar'),
'1', '1', '2023-09-05 21:02:38', '1', '2023-09-05 21:02:38');
Expand Down
Loading