From 4569d877dad938992834c9eaf73e39a14b5844a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 13:56:18 +0800 Subject: [PATCH 1/6] Override AbstractJdbcDriver method getTable The default method takes longer to filter after obtaining listTables --- .../main/java/org/dinky/data/model/Table.java | 28 ++++++-- .../metadata/driver/AbstractJdbcDriver.java | 72 ++++++++++--------- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java index 1aba079b15..051f61a81a 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/Table.java +++ b/dinky-common/src/main/java/org/dinky/data/model/Table.java @@ -19,12 +19,15 @@ package org.dinky.data.model; +import lombok.Getter; +import lombok.Setter; import org.dinky.assertion.Asserts; import org.dinky.data.enums.TableType; import org.dinky.utils.SqlUtil; import java.beans.Transient; import java.io.Serializable; +import java.sql.SQLException; import java.text.MessageFormat; import java.util.Date; import java.util.HashMap; @@ -32,9 +35,6 @@ import java.util.Map; import java.util.stream.Collectors; -import lombok.Getter; -import lombok.Setter; - /** * Table * @@ -56,14 +56,26 @@ public class Table implements Serializable, Comparable, Cloneable { private Long rows; private Date createTime; private Date updateTime; - /** 表类型 */ + /** + * 表类型 + */ private TableType tableType = TableType.SINGLE_DATABASE_AND_TABLE; - /** 分库或分表对应的表名 */ + /** + * 分库或分表对应的表名 + */ private List schemaTableNameList; private List columns; - public Table() {} + public Table() { + } + + public Table(List columns, String databaseName, String tableName, String tableComment) throws SQLException { + this.name = tableName; + this.schema = databaseName; + this.comment = tableComment; + this.columns = columns; + } public Table(String name, String schema, List columns) { this.name = name; @@ -144,12 +156,14 @@ public String getFlinkTableSql(String catalogName, String flinkConfig) { return String.format("DROP TABLE IF EXISTS %s;\n%s", name, createSql); } + @Override public Object clone() { Table table = null; try { table = (Table) super.clone(); - } catch (CloneNotSupportedException e) { + } catch ( + CloneNotSupportedException e) { e.printStackTrace(); } return table; diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index c8f6e62ca3..608f43b1d9 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -19,10 +19,12 @@ package org.dinky.metadata.driver; -import static org.dinky.utils.SplitUtil.contains; -import static org.dinky.utils.SplitUtil.getReValue; -import static org.dinky.utils.SplitUtil.isSplit; - +import cn.hutool.core.text.CharSequenceUtil; +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidPooledConnection; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import lombok.extern.slf4j.Slf4j; import org.dinky.assertion.Asserts; import org.dinky.data.constant.CommonConstant; import org.dinky.data.enums.TableType; @@ -40,35 +42,14 @@ import org.dinky.utils.LogUtil; import org.dinky.utils.TextUtil; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidPooledConnection; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLStatement; - -import cn.hutool.core.text.CharSequenceUtil; -import lombok.extern.slf4j.Slf4j; +import static org.dinky.utils.SplitUtil.*; /** * AbstractJdbcDriver @@ -145,7 +126,9 @@ public Driver connect() { Class.forName(getDriverClass()); DruidPooledConnection connection = createDataSource().getConnection(); conn.set(connection); - } catch (ClassNotFoundException | SQLException e) { + } catch ( + ClassNotFoundException | + SQLException e) { throw new RuntimeException(e); } } @@ -292,6 +275,22 @@ public List
listTables(String schemaName) { return tableList; } + @Override + public Table getTable(String schemaName, String tableName) { + try { + DatabaseMetaData metaData = conn.get().getMetaData(); + ResultSet tables = metaData.getTables(schemaName, null, tableName, new String[]{"TABLE"}); + while (tables.next()) { + String tableComment = tables.getString("REMARKS"); + List columns = listColumns(schemaName, tableName); + return new Table(columns, schemaName, tableName, tableComment); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return null; + } + @Override public List listColumns(String schemaName, String tableName) { List columns = new ArrayList<>(); @@ -516,8 +515,11 @@ public int executeUpdate(String sql) throws Exception { } /** - * 标准sql where与order语法都是相同的 不同数据库limit语句不一样，需要单独交由driver去处理，例如oracle 通过{@query(String sql, - * Integer limit)}去截断返回数据，但是在大量数据情况下会导致数据库负载过高。 + * 标准sql where与order语法都是相同的 + * 不同数据库limit语句不一样，需要单独交由driver去处理，例如oracle + * 通过{@query(String sql, + * Integer + * limit)}去截断返回数据，但是在大量数据情况下会导致数据库负载过高。 */ @Override public StringBuilder genQueryOption(QueryData queryData) { @@ -614,7 +616,6 @@ public JdbcSelectResult query(String sql, Integer limit) { /** * 如果执行多条语句返回最后一条语句执行结果 - * */ @Override public JdbcSelectResult executeSql(String sql, Integer limit) { @@ -822,7 +823,8 @@ && contains(tableName, x.get(dbQuery.tableName()))) tableInfo.setUpdateTime( SimpleDateFormat.getDateInstance().parse(updateTime)); } - } catch (ParseException ignored) { + } catch ( + ParseException ignored) { log.warn("set date fail"); } TableType tableType = TableType.type( @@ -836,8 +838,8 @@ && contains(tableName, x.get(dbQuery.tableName()))) + getReValue(x.get(dbQuery.tableName()), splitConfig); List schemaTableNameList = mapList.stream() .filter(y -> (getReValue(y.get(dbQuery.schemaName()), splitConfig) - + "." - + getReValue(y.get(dbQuery.tableName()), splitConfig)) + + "." + + getReValue(y.get(dbQuery.tableName()), splitConfig)) .equals(currentSchemaName)) .map(y -> y.get(dbQuery.schemaName()) + "." + y.get(dbQuery.tableName())) .collect(Collectors.toList()); From 6982f84199b4cc75015717a91d0f07b5d528d21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 13:58:22 +0800 Subject: [PATCH 2/6] Override AbstractJdbcDriver method getTable The default method takes longer to filter after obtaining listTables --- .../main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 608f43b1d9..3a70bfb85e 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -286,6 +286,7 @@ public Table getTable(String schemaName, String tableName) { return new Table(columns, schemaName, tableName, tableComment); } } catch (SQLException e) { + log.error("GetTable error:", e); throw new RuntimeException(e); } return null; From 0df416dba7cf9216aa758d2ec00d3a25a8510c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 14:33:52 +0800 Subject: [PATCH 3/6] 1 --- .../src/main/java/org/dinky/data/model/Table.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java index 051f61a81a..ee47b21189 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/Table.java +++ b/dinky-common/src/main/java/org/dinky/data/model/Table.java @@ -19,8 +19,6 @@ package org.dinky.data.model; -import lombok.Getter; -import lombok.Setter; import org.dinky.assertion.Asserts; import org.dinky.data.enums.TableType; import org.dinky.utils.SqlUtil; @@ -35,6 +33,9 @@ import java.util.Map; import java.util.stream.Collectors; +import lombok.Getter; +import lombok.Setter; + /** * Table * @@ -67,8 +68,7 @@ public class Table implements Serializable, Comparable
, Cloneable { private List columns; - public Table() { - } + public Table() {} public Table(List columns, String databaseName, String tableName, String tableComment) throws SQLException { this.name = tableName; @@ -156,14 +156,12 @@ public String getFlinkTableSql(String catalogName, String flinkConfig) { return String.format("DROP TABLE IF EXISTS %s;\n%s", name, createSql); } - @Override public Object clone() { Table table = null; try { table = (Table) super.clone(); - } catch ( - CloneNotSupportedException e) { + } catch (CloneNotSupportedException e) { e.printStackTrace(); } return table; From df20badba1303c499fdc7a53812f88f033aec8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 15:01:45 +0800 Subject: [PATCH 4/6] mvn spotless:apply --- .../java/org/dinky/metadata/driver/AbstractJdbcDriver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 3a70bfb85e..aa3c31f112 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -51,6 +51,7 @@ import static org.dinky.utils.SplitUtil.*; + /** * AbstractJdbcDriver * @@ -60,9 +61,8 @@ public abstract class AbstractJdbcDriver extends AbstractDriver { protected ThreadLocal conn = new ThreadLocal<>(); - - private DruidDataSource dataSource; protected String validationQuery = "select 1"; + private DruidDataSource dataSource; abstract String getDriverClass(); From 06206f837c38a91dcd67de40b38e9b77f553257c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 15:35:06 +0800 Subject: [PATCH 5/6] mvn spotless:apply --- .../metadata/driver/AbstractJdbcDriver.java | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index aa3c31f112..b9dbec4355 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -19,12 +19,8 @@ package org.dinky.metadata.driver; -import cn.hutool.core.text.CharSequenceUtil; -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidPooledConnection; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLStatement; -import lombok.extern.slf4j.Slf4j; +import static org.dinky.utils.SplitUtil.*; + import org.dinky.assertion.Asserts; import org.dinky.data.constant.CommonConstant; import org.dinky.data.enums.TableType; @@ -42,15 +38,36 @@ import org.dinky.utils.LogUtil; import org.dinky.utils.TextUtil; -import java.sql.*; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.dinky.utils.SplitUtil.*; +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidPooledConnection; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import cn.hutool.core.text.CharSequenceUtil; +import lombok.extern.slf4j.Slf4j; /** * AbstractJdbcDriver @@ -126,9 +143,7 @@ public Driver connect() { Class.forName(getDriverClass()); DruidPooledConnection connection = createDataSource().getConnection(); conn.set(connection); - } catch ( - ClassNotFoundException | - SQLException e) { + } catch (ClassNotFoundException | SQLException e) { throw new RuntimeException(e); } } @@ -279,7 +294,7 @@ public List
listTables(String schemaName) { public Table getTable(String schemaName, String tableName) { try { DatabaseMetaData metaData = conn.get().getMetaData(); - ResultSet tables = metaData.getTables(schemaName, null, tableName, new String[]{"TABLE"}); + ResultSet tables = metaData.getTables(schemaName, null, tableName, new String[] {"TABLE"}); while (tables.next()) { String tableComment = tables.getString("REMARKS"); List columns = listColumns(schemaName, tableName); @@ -824,8 +839,7 @@ && contains(tableName, x.get(dbQuery.tableName()))) tableInfo.setUpdateTime( SimpleDateFormat.getDateInstance().parse(updateTime)); } - } catch ( - ParseException ignored) { + } catch (ParseException ignored) { log.warn("set date fail"); } TableType tableType = TableType.type( @@ -839,8 +853,8 @@ && contains(tableName, x.get(dbQuery.tableName()))) + getReValue(x.get(dbQuery.tableName()), splitConfig); List schemaTableNameList = mapList.stream() .filter(y -> (getReValue(y.get(dbQuery.schemaName()), splitConfig) - + "." - + getReValue(y.get(dbQuery.tableName()), splitConfig)) + + "." + + getReValue(y.get(dbQuery.tableName()), splitConfig)) .equals(currentSchemaName)) .map(y -> y.get(dbQuery.schemaName()) + "." + y.get(dbQuery.tableName())) .collect(Collectors.toList()); From 06e5b4069a7c7ef3fb0a7eebdbd26be021547117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 21:06:50 +0800 Subject: [PATCH 6/6] Fix code style --- .../java/org/dinky/metadata/driver/AbstractJdbcDriver.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index b9dbec4355..9b80cdab8f 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -19,7 +19,9 @@ package org.dinky.metadata.driver; -import static org.dinky.utils.SplitUtil.*; +import static org.dinky.utils.SplitUtil.contains; +import static org.dinky.utils.SplitUtil.getReValue; +import static org.dinky.utils.SplitUtil.isSplit; import org.dinky.assertion.Asserts; import org.dinky.data.constant.CommonConstant;