Skip to content

Commit

Permalink
[Optimization] Improve get table info of the schema (DataLinkDC#3772)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhuxt2015 authored Sep 8, 2024
1 parent 885061a commit e1d344f
Show file tree
Hide file tree
Showing 25 changed files with 212 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public List<Table> getTablesAndColumns(String schema) {

@Override
public Table getTable(String schemaName, String tableName) {
List<Table> tables = listTables(schemaName);
List<Table> tables = listTables(schemaName, tableName);
Table table = null;
for (Table item : tables) {
if (Asserts.isEquals(item.getName(), tableName)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,64 @@ public List<Table> listTables(String schemaName) {
return tableList;
}

@Override
public List<Table> listTables(String schemaName, String tableName) {
List<Table> tableList = new ArrayList<>();
PreparedStatement preparedStatement = null;
ResultSet results = null;
IDBQuery dbQuery = getDBQuery();
String sql = dbQuery.tablesSql(schemaName, tableName);
try {
preparedStatement = conn.get().prepareStatement(sql);
results = preparedStatement.executeQuery();
ResultSetMetaData metaData = results.getMetaData();
List<String> columnList = new ArrayList<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
columnList.add(metaData.getColumnLabel(i));
}
while (results.next()) {
String tableNameResult = results.getString(dbQuery.tableName());
if (Asserts.isNotNullString(tableNameResult)) {
Table tableInfo = new Table();
tableInfo.setDriverType(getType());
tableInfo.setName(tableNameResult);
if (columnList.contains(dbQuery.tableComment())) {
tableInfo.setComment(results.getString(dbQuery.tableComment()));
}
tableInfo.setSchema(schemaName);
if (columnList.contains(dbQuery.tableType())) {
tableInfo.setType(results.getString(dbQuery.tableType()));
}
if (columnList.contains(dbQuery.catalogName())) {
tableInfo.setCatalog(results.getString(dbQuery.catalogName()));
}
if (columnList.contains(dbQuery.engine())) {
tableInfo.setEngine(results.getString(dbQuery.engine()));
}
if (columnList.contains(dbQuery.options())) {
tableInfo.setOptions(results.getString(dbQuery.options()));
}
if (columnList.contains(dbQuery.rows())) {
tableInfo.setRows(results.getLong(dbQuery.rows()));
}
if (columnList.contains(dbQuery.createTime())) {
tableInfo.setCreateTime(results.getTimestamp(dbQuery.createTime()));
}
if (columnList.contains(dbQuery.updateTime())) {
tableInfo.setUpdateTime(results.getTimestamp(dbQuery.updateTime()));
}
tableList.add(tableInfo);
}
}
} catch (SQLException e) {
log.error("ListTables error:", e);
throw new BusException(e.getMessage());
} finally {
close(preparedStatement, results);
}
return tableList;
}

@Override
public List<Column> listColumns(String schemaName, String tableName) {
List<Column> columns = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ static Driver build(String connector, String url, String username, String passwo

List<Table> listTables(String schemaName);

List<Table> listTables(String schemaName, String tableName);

List<Column> listColumns(String schemaName, String tableName);

List<Column> listColumnsSortByPK(String schemaName, String tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ public interface IDBQuery {
/** 表信息查询 SQL */
String tablesSql(String schemaName);

/** 表信息查询 SQL */
String tablesSql(String schemaName, String tableName);

/** 表字段信息查询 SQL */
String columnsSql(String schemaName, String tableName);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,11 @@ public List<Table> listTables(String schemaName) {
return null;
}

@Override
public List<Table> listTables(String schemaName, String tableName) {
return null;
}

@Override
public List<Column> listColumns(String schemaName, String tableName) {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,19 @@ public String tablesSql(String schemaName) {
+ "from system.tables where 1=1 and database='" + schemaName + "'";
}

/**
* 获取指定数据库下指定表的元数据
*
* @param schemaName 模式名称
* @param tableName 表名
* @return String
*/
@Override
public String tablesSql(String schemaName, String tableName) {
return "select name, total_rows, engine, metadata_modification_time, comment "
+ "from system.tables where 1=1 and database='" + schemaName + "' and name='" + tableName + "'";
}

/**
* 从元数据表中获取表字段信息
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ public interface DorisConstant {
+ "from information_schema.tables"
+ " where"
+ " TABLE_SCHEMA = '%s' ";

/** 查询schema下的指定表 */
String QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME =
"select TABLE_NAME AS `NAME`,\n" + " TABLE_SCHEMA AS `SCHEMA`,\n"
+ " TABLE_COMMENT AS COMMENT,\n"
+ " TABLE_TYPE as TYPE,\n"
+ " TABLE_CATALOG as CATALOG,\n"
+ " ENGINE as ENGINE,\n"
+ " CREATE_OPTIONS as OPTIONS,\n"
+ " TABLE_ROWS as `ROWS`,\n"
+ " CREATE_TIME as CREATE_TIME,\n"
+ " UPDATE_TIME as UPDATE_TIME\n"
+ "from information_schema.tables"
+ " where"
+ " TABLE_SCHEMA = '%s' and TABLE_NAME = '%s'";
/** 查询指定schema.table下的所有列信息 */
String QUERY_COLUMNS_BY_TABLE_AND_SCHEMA = " show full columns from `%s`.`%s` ";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ public String tablesSql(String schemaName) {
return String.format(DorisConstant.QUERY_TABLE_BY_SCHEMA, schemaName);
}

@Override
public String tablesSql(String schemaName, String tableName) {
return String.format(DorisConstant.QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME, schemaName, tableName);
}

@Override
public String columnsSql(String schemaName, String tableName) {
return String.format(DorisConstant.QUERY_COLUMNS_BY_TABLE_AND_SCHEMA, schemaName, tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ public interface HiveConstant {
String QUERY_ALL_DATABASE = " show databases";
/** 查询所有schema下的所有表 */
String QUERY_ALL_TABLES_BY_SCHEMA = "show tables";
/** 查询所有schema下的所有表 */
String QUERY_ALL_TABLES_BY_SCHEMA_NAME_AND_TABLE_NAME = "show tables in %s like '%s'";
/** 扩展信息Key */
String DETAILED_TABLE_INFO = "Detailed Table Information";
/** 查询指定schema.table的扩展信息 */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class HiveDriver extends AbstractJdbcDriver implements Driver {

@Override
public Table getTable(String schemaName, String tableName) {
List<Table> tables = listTables(schemaName);
List<Table> tables = listTables(schemaName, tableName);
Table table = null;
for (Table item : tables) {
if (Asserts.isEquals(item.getName(), tableName)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ public String tablesSql(String schemaName) {
return HiveConstant.QUERY_ALL_TABLES_BY_SCHEMA;
}

@Override
public String tablesSql(String schemaName, String tableName) {
return String.format(HiveConstant.QUERY_ALL_TABLES_BY_SCHEMA_NAME_AND_TABLE_NAME, tableName);
}

@Override
public String columnsSql(String schemaName, String tableName) {
return String.format(HiveConstant.QUERY_TABLE_SCHEMA, schemaName, tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,18 @@ public String tablesSql(String schemaName) {
+ "'";
}

@Override
public String tablesSql(String schemaName, String tableName) {
return "select TABLE_NAME AS `NAME`,TABLE_SCHEMA AS `Database`,TABLE_COMMENT AS"
+ " COMMENT,TABLE_CATALOG AS `CATALOG`,TABLE_TYPE AS `TYPE`,ENGINE AS"
+ " `ENGINE`,CREATE_OPTIONS AS `OPTIONS`,TABLE_ROWS AS"
+ " `ROWS`,CREATE_TIME,UPDATE_TIME from information_schema.tables where"
+ " TABLE_SCHEMA = '"
+ schemaName + "' and TABLE_NAME ='"
+ tableName
+ "'";
}

@Override
public String columnsSql(String schemaName, String tableName) {
return "select COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT,COLUMN_KEY,EXTRA AS AUTO_INCREMENT"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ public String tablesSql(String schemaName) {
return "SELECT * FROM ALL_TAB_COMMENTS WHERE OWNER='" + schemaName + "'";
}

public String tablesSql(String schemaName, String tableName) {
return "SELECT * FROM ALL_TAB_COMMENTS WHERE OWNER='" + schemaName + "' AND TABLE_NAME='" + tableName + "'";
}

@Override
public String columnsSql(String schemaName, String tableName) {
return "SELECT A.COLUMN_NAME, CASE WHEN A.DATA_TYPE='NUMBER' THEN (CASE WHEN"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,18 @@ public List<Table> listTables(String schemaName) {
}
}

@Override
public List<Table> listTables(String schemaName, String tableName) {
try {
return catalog.listTables(schemaName).stream()
.filter(t -> t.equalsIgnoreCase(tableName))
.map(Table::new)
.collect(Collectors.toList());
} catch (Catalog.DatabaseNotExistException e) {
throw new RuntimeException(e);
}
}

@Override
public Table getTable(String schemaName, String tableName) {
Identifier identifier = Identifier.create(schemaName, tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,18 @@ public String tablesSql(String schemaName) {
+ "'";
}

@Override
public String tablesSql(String schemaName, String tableName) {
return "select TABLE_NAME AS `NAME`,TABLE_SCHEMA AS `Database`,TABLE_COMMENT AS"
+ " COMMENT,TABLE_CATALOG AS `CATALOG`,TABLE_TYPE AS `TYPE`,ENGINE AS"
+ " `ENGINE`,CREATE_OPTIONS AS `OPTIONS`,TABLE_ROWS AS"
+ " `ROWS`,CREATE_TIME,UPDATE_TIME from information_schema.tables where"
+ " TABLE_SCHEMA = '"
+ schemaName + "' and TABLE_NAME = '"
+ tableName
+ "'";
}

@Override
public String columnsSql(String schemaName, String tableName) {
return "select COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT,COLUMN_KEY,EXTRA AS AUTO_INCREMENT"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ public interface PhoenixConstant {
/** 根据schema查询table信息模板SQL */
String QUERY_TABLE_BY_SCHEMA_SQL = QUERY_TABLE_BY_SCHEMA_SQL_DEFAULT + " AND TABLE_SCHEM = '%s' ";

String QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME_SQL =
QUERY_TABLE_BY_SCHEMA_SQL_DEFAULT + " AND TABLE_SCHEM = '%s'" + " AND TABLE_NAME = '%s'";

/** Phoenix的driver */
String PHOENIX_DRIVER = "org.apache.phoenix.jdbc.PhoenixDriver";
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ public String tablesSql(String schemaName) {
return String.format(PhoenixConstant.QUERY_TABLE_BY_SCHEMA_SQL, schemaName);
}

@Override
public String tablesSql(String schemaName, String tableName) {
if (schemaName == null || schemaName.isEmpty()) {
return PhoenixConstant.QUERY_TABLE_BY_SCHEMA_SQL_DEFAULT;
}
return String.format(PhoenixConstant.QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME_SQL, schemaName, tableName);
}

@Override
public String columnsSql(String schemaName, String tableName) {
if (schemaName == null || schemaName.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,24 @@ public String tablesSql(String schemaName) {
+ "ORDER BY n.nspname, tablename";
}

@Override
public String tablesSql(String schemaName, String tableName) {
return "SELECT n.nspname AS schema_name\n"
+ " , c.relname AS tablename\n"
+ " , obj_description(c.oid) AS comments\n"
+ " , c.reltuples as rows\n"
+ "FROM pg_class c\n"
+ " LEFT JOIN pg_namespace n ON n.oid = c.relnamespace\n"
+ "WHERE ((c.relkind = 'r'::\"char\") OR (c.relkind = 'f'::\"char\") OR"
+ " (c.relkind = 'p'::\"char\"))\n"
+ " AND n.nspname = '"
+ schemaName + "'"
+ " AND c.relname = '"
+ tableName
+ "'\n"
+ "ORDER BY n.nspname, tablename";
}

@Override
public String columnsSql(String schemaName, String tableName) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ public interface PrestoConstant {
String QUERY_ALL_DATABASE = "show catalogs";
/** 查询某个schema下的所有表 */
String QUERY_ALL_TABLES_BY_SCHEMA = "show tables from %s";
/** 查询某个schema下的指定表 */
String QUERY_SPECIFIED_TABLES_BY_SCHEMA = "show tables from %s like '%s'";
/** 查询指定schema.table的信息 列 列类型 列注释 */
String QUERY_TABLE_SCHEMA = " describe %s.%s";
/** 只查询指定schema.table的列名 */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public class PrestoDriver extends AbstractJdbcDriver implements Driver {

@Override
public Table getTable(String schemaName, String tableName) {
List<Table> tables = listTables(schemaName);
List<Table> tables = listTables(schemaName, tableName);
Table table = null;
for (Table item : tables) {
if (Asserts.isEquals(item.getName(), tableName)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,12 @@ public String schemaAllSql() {

@Override
public String tablesSql(String schemaName) {
return PrestoConstant.QUERY_ALL_TABLES_BY_SCHEMA;
return String.format(PrestoConstant.QUERY_ALL_TABLES_BY_SCHEMA, schemaName);
}

@Override
public String tablesSql(String schemaName, String tableName) {
return String.format(PrestoConstant.QUERY_SPECIFIED_TABLES_BY_SCHEMA, schemaName, tableName);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,10 @@ public interface SqlServerConstant {
" SELECT table_name ,table_schema, '' as type, '' as CATALOG, '' as ENGINE , '' as"
+ " OPTIONS ,0 as rows , null as CREATE_TIME, null as UPDATE_TIME,null AS COMMENTS"
+ " FROM INFORMATION_SCHEMA.tables WHERE TABLE_SCHEMA = '%s' ";

/** 根据schema查询table信息模板SQL */
String QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME_SQL =
" SELECT table_name ,table_schema, '' as type, '' as CATALOG, '' as ENGINE , '' as"
+ " OPTIONS ,0 as rows , null as CREATE_TIME, null as UPDATE_TIME,null AS COMMENTS"
+ " FROM INFORMATION_SCHEMA.tables WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'";
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ public String tablesSql(String schemaName) {
return String.format(SqlServerConstant.QUERY_TABLE_BY_SCHEMA_SQL, schemaName);
}

@Override
public String tablesSql(String schemaName, String tableName) {
return String.format(SqlServerConstant.QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME_SQL, schemaName, tableName);
}

@Override
public String columnsSql(String schemaName, String tableName) {
return String.format(SqlServerConstant.QUERY_COLUMNS_SQL, tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@ public interface StarRocksConstant {
+ " TYPE, '' as CATALOG, '' as ENGINE , '' as OPTIONS , 0 as `ROWS`, null as"
+ " CREATE_TIME, null as UPDATE_TIME from information_schema.tables where"
+ " TABLE_SCHEMA = '%s' ";

/** 查询schema下的指定表 */
String QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME =
" select TABLE_NAME AS `NAME`,TABLE_SCHEMA AS `SCHEMA`,TABLE_COMMENT AS COMMENT, '' as"
+ " TYPE, '' as CATALOG, '' as ENGINE , '' as OPTIONS , 0 as `ROWS`, null as"
+ " CREATE_TIME, null as UPDATE_TIME from information_schema.tables where"
+ " TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'";

/** 查询指定schema.table下的所有列信息 */
String QUERY_COLUMNS_BY_TABLE_AND_SCHEMA = " show full columns from `%s`.`%s` ";
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ public String tablesSql(String schemaName) {
return String.format(StarRocksConstant.QUERY_TABLE_BY_SCHEMA, schemaName);
}

@Override
public String tablesSql(String schemaName, String tableName) {
return String.format(StarRocksConstant.QUERY_TABLE_BY_SCHEMA_NAME_AND_TABLE_NAME, schemaName, tableName);
}

@Override
public String columnsSql(String schemaName, String tableName) {
return String.format(StarRocksConstant.QUERY_COLUMNS_BY_TABLE_AND_SCHEMA, schemaName, tableName);
Expand Down

0 comments on commit e1d344f

Please sign in to comment.