diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 91273736ebfeccb..78bdaaf66605d5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -48,6 +48,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.OperationType; @@ -687,7 +688,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName, db.writeLock(); try { - HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog); + HMSExternalTable namedTable = ((HMSExternalDatabase) db) + .buildTableForInit(tableName, tblId, hmsCatalog, (HMSExternalDatabase) db); namedTable.setUpdateTime(updateTime); db.registerTable(namedTable); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 345254108345068..f893d92fef9c6be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -48,6 +48,7 @@ import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase; +import org.apache.doris.datasource.mapping.IdentifierMapping; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; @@ -149,6 +150,7 @@ public abstract class ExternalCatalog protected Optional useMetaCache = Optional.empty(); protected MetaCache> metaCache; + protected IdentifierMapping identifierMapping; public ExternalCatalog() { } @@ -171,6 +173,7 @@ public Configuration getConfiguration() { /** * set some default properties when creating catalog + * * @return list of database names in this catalog */ protected List listDatabaseNames() { @@ -182,6 +185,10 @@ protected List listDatabaseNames() { } } + public String fromRemoteDatabaseName(String remoteDatabaseName) { + return remoteDatabaseName; + } + // Will be called when creating catalog(so when as replaying) // to add some default properties if missing. public void setDefaultPropsIfMissing(boolean isReplay) { @@ -219,6 +226,10 @@ public void checkWhenCreating() throws DdlException { */ public abstract boolean tableExist(SessionContext ctx, String dbName, String tblName); + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return remoteTableName; + } + /** * init some local objects such as: * hms client, read properties from hive-site.xml, es client @@ -246,15 +257,29 @@ public final synchronized void makeSureInitialized() { if (!initialized) { if (useMetaCache.get()) { if (metaCache == null) { + List> remoteToLocalPairs = getFilteredDatabaseNames(); + metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, - ignored -> getFilteredDatabaseNames(), - dbName -> Optional.ofNullable( - buildDbForInit(dbName, Util.genIdByName(name, dbName), logType, true)), - (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); + ignored -> remoteToLocalPairs.stream().map(Pair::value).collect(Collectors.toList()), + dbName -> { + Optional> pairOpt = remoteToLocalPairs.stream() + .filter(pair -> pair.value().equals(dbName)) + .findFirst(); + if (pairOpt.isPresent()) { + Pair pair = pairOpt.get(); + String remoteDbName = pair.key(); + return Optional.ofNullable( + buildDbForInit(remoteDbName, Util.genIdByName(name, dbName), logType, + true)); + } + return Optional.empty(); + }, + (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit)) + ); } setLastUpdateTime(System.currentTimeMillis()); } else { @@ -350,21 +375,23 @@ private void init() { InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List filteredDatabases = getFilteredDatabaseNames(); - for (String dbName : filteredDatabases) { + List> remoteToLocalPairs = getFilteredDatabaseNames(); + for (Pair pair : remoteToLocalPairs) { + String remoteDbName = pair.key(); + String localDbName = pair.value(); long dbId; - if (dbNameToId != null && dbNameToId.containsKey(dbName)) { - dbId = dbNameToId.get(dbName); - tmpDbNameToId.put(dbName, dbId); + if (dbNameToId != null && dbNameToId.containsKey(localDbName)) { + dbId = dbNameToId.get(localDbName); + tmpDbNameToId.put(localDbName, dbId); ExternalDatabase db = idToDb.get(dbId); tmpIdToDb.put(dbId, db); initCatalogLog.addRefreshDb(dbId); } else { dbId = Env.getCurrentEnv().getNextId(); - tmpDbNameToId.put(dbName, dbId); - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + tmpDbNameToId.put(localDbName, dbId); + ExternalDatabase db = buildDbForInit(remoteDbName, dbId, logType, false); tmpIdToDb.put(dbId, db); - initCatalogLog.addCreateDb(dbId, dbName); + initCatalogLog.addCreateDb(dbId, localDbName, remoteDbName); } } @@ -376,14 +403,18 @@ private void init() { } @NotNull - private List getFilteredDatabaseNames() { + private List> getFilteredDatabaseNames() { List allDatabases = Lists.newArrayList(listDatabaseNames()); allDatabases.remove(InfoSchemaDb.DATABASE_NAME); allDatabases.add(InfoSchemaDb.DATABASE_NAME); allDatabases.remove(MysqlDb.DATABASE_NAME); allDatabases.add(MysqlDb.DATABASE_NAME); + Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); + + List> remoteToLocalPairs = Lists.newArrayList(); + allDatabases = allDatabases.stream().filter(dbName -> { if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) { // Exclude database map take effect with higher priority over include database map @@ -396,7 +427,13 @@ private List getFilteredDatabaseNames() { } return true; }).collect(Collectors.toList()); - return allDatabases; + + for (String remoteDbName : allDatabases) { + String localDbName = fromRemoteDatabaseName(remoteDbName); + remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); + } + + return remoteToLocalPairs; } public void onRefresh(boolean invalidCache) { @@ -468,6 +505,7 @@ public void setComment(String comment) { /** * Different from 'listDatabases()', this method will return dbnames from cache. * while 'listDatabases()' will return dbnames from remote datasource. + * * @return names of database in this catalog. */ @Override @@ -624,8 +662,12 @@ private void removeAccessController() { } public void replayInitCatalog(InitCatalogLog log) { + if (log.getRemoteDbNames() == null || log.getRemoteDbNames().isEmpty()) { + initialized = false; + return; + } Map tmpDbNameToId = Maps.newConcurrentMap(); - Map> tmpIdToDb = Maps.newConcurrentMap(); + Map> tmpIdToDb = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { Optional> db = getDbForReplay(log.getRefreshDbIds().get(i)); // Should not return null. @@ -669,58 +711,59 @@ public Optional> getDbForReplay(long d * Build a database instance. * If checkExists is true, it will check if the database exists in the remote system. * - * @param dbName + * @param remoteDbName * @param dbId * @param logType * @param checkExists * @return */ - protected ExternalDatabase buildDbForInit(String dbName, long dbId, + protected ExternalDatabase buildDbForInit(String remoteDbName, long dbId, InitCatalogLog.Type logType, boolean checkExists) { // When running ut, disable this check to make ut pass. // Because in ut, the database is not created in remote system. if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalCatalog)) { try { - List dbNames = getDbNames(); - if (!dbNames.contains(dbName)) { - dbNames = getFilteredDatabaseNames(); - if (!dbNames.contains(dbName)) { - return null; - } + List> remoteToLocalPairs = getFilteredDatabaseNames(); + List remoteDbNames = remoteToLocalPairs.stream() + .map(Pair::key) + .collect(Collectors.toList()); + if (!remoteDbNames.contains(remoteDbName)) { + return null; } } catch (Throwable t) { // If connection failed, it will throw exception. // ignore it and treat it as not exist. - LOG.warn("Failed to check db {} exist in remote system, ignore it.", dbName, t); + LOG.warn("Failed to check db {} exist in remote system, ignore it.", remoteDbName, t); return null; } } - if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) { + if (remoteDbName.equals(InfoSchemaDb.DATABASE_NAME)) { return new ExternalInfoSchemaDatabase(this, dbId); } - if (dbName.equals(MysqlDb.DATABASE_NAME)) { + if (remoteDbName.equals(MysqlDb.DATABASE_NAME)) { return new ExternalMysqlDatabase(this, dbId); } + String localDbName = fromRemoteDatabaseName(remoteDbName); switch (logType) { case HMS: - return new HMSExternalDatabase(this, dbId, dbName); + return new HMSExternalDatabase(this, dbId, localDbName, remoteDbName); case ES: - return new EsExternalDatabase(this, dbId, dbName); + return new EsExternalDatabase(this, dbId, localDbName, remoteDbName); case JDBC: - return new JdbcExternalDatabase(this, dbId, dbName); + return new JdbcExternalDatabase(this, dbId, localDbName, remoteDbName); case ICEBERG: - return new IcebergExternalDatabase(this, dbId, dbName); + return new IcebergExternalDatabase(this, dbId, localDbName, remoteDbName); case MAX_COMPUTE: - return new MaxComputeExternalDatabase(this, dbId, dbName); + return new MaxComputeExternalDatabase(this, dbId, localDbName, remoteDbName); case LAKESOUL: - return new LakeSoulExternalDatabase(this, dbId, dbName); + return new LakeSoulExternalDatabase(this, dbId, localDbName, remoteDbName); case TEST: - return new TestExternalDatabase(this, dbId, dbName); + return new TestExternalDatabase(this, dbId, localDbName, remoteDbName); case PAIMON: - return new PaimonExternalDatabase(this, dbId, dbName); + return new PaimonExternalDatabase(this, dbId, localDbName, remoteDbName); case TRINO_CONNECTOR: - return new TrinoConnectorExternalDatabase(this, dbId, dbName); + return new TrinoConnectorExternalDatabase(this, dbId, localDbName, remoteDbName); default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index eda98efb9b6a038..a4a9403004fe40a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -26,6 +26,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; @@ -76,6 +77,8 @@ public abstract class ExternalDatabase protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "dbProperties") protected DatabaseProperty dbProperties = new DatabaseProperty(); @SerializedName(value = "initialized") @@ -100,11 +103,14 @@ public abstract class ExternalDatabase * @param extCatalog The catalog this database belongs to. * @param id Database id. * @param name Database name. + * @param remoteName Remote database name. */ - public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, InitDatabaseLog.Type dbLogType) { + public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName, + InitDatabaseLog.Type dbLogType) { this.extCatalog = extCatalog; this.id = id; this.name = name; + this.remoteName = remoteName; this.dbLogType = dbLogType; } @@ -144,16 +150,29 @@ public final synchronized void makeSureInitialized() { if (!initialized) { if (extCatalog.getUseMetaCache().get()) { if (metaCache == null) { + List> remoteToLocalPairs = listTableNames(); metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, - ignored -> listTableNames(), - tableName -> Optional.ofNullable( - buildTableForInit(tableName, - Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)), - (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); + ignored -> remoteToLocalPairs.stream().map(Pair::value).collect(Collectors.toList()), + tableName -> { + Optional> pairOpt = remoteToLocalPairs.stream() + .filter(pair -> pair.value().equals(tableName)) + .findFirst(); + if (pairOpt.isPresent()) { + Pair pair = pairOpt.get(); + String remoteTableName = pair.key(); + return Optional.ofNullable( + buildTableForInit(remoteTableName, + Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog, + this)); + } + return Optional.empty(); + }, + (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated) + ); } setLastUpdateTime(System.currentTimeMillis()); } else { @@ -191,7 +210,8 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } } for (int i = 0; i < log.getCreateCount(); i++) { - T table = buildTableForInit(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog); + T table = + buildTableForInit(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog, this); tmpTableNameToId.put(table.getName(), table.getId()); tmpIdToTbl.put(table.getId(), table); } @@ -206,24 +226,27 @@ private void init() { initDatabaseLog.setType(dbLogType); initDatabaseLog.setCatalogId(extCatalog.getId()); initDatabaseLog.setDbId(id); - List tableNames = listTableNames(); - if (tableNames != null) { + List> tableNamePairs = listTableNames(); + if (tableNamePairs != null) { Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newHashMap(); - for (String tableName : tableNames) { + + for (Pair pair : tableNamePairs) { + String remoteTableName = pair.first; + String localTableName = pair.second; long tblId; - if (tableNameToId != null && tableNameToId.containsKey(tableName)) { - tblId = tableNameToId.get(tableName); - tmpTableNameToId.put(tableName, tblId); + if (tableNameToId != null && tableNameToId.containsKey(localTableName)) { + tblId = tableNameToId.get(localTableName); + tmpTableNameToId.put(localTableName, tblId); T table = idToTbl.get(tblId); tmpIdToTbl.put(tblId, table); initDatabaseLog.addRefreshTable(tblId); } else { tblId = Env.getCurrentEnv().getNextId(); - tmpTableNameToId.put(tableName, tblId); - T table = buildTableForInit(tableName, tblId, extCatalog); + tmpTableNameToId.put(localTableName, tblId); + T table = buildTableForInit(remoteTableName, tblId, extCatalog, this); tmpIdToTbl.put(tblId, table); - initDatabaseLog.addCreateTable(tblId, tableName); + initDatabaseLog.addCreateTable(tblId, localTableName); } } tableNameToId = tmpTableNameToId; @@ -235,26 +258,27 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } - private List listTableNames() { - List tableNames; + private List> listTableNames() { + List> tableNames; if (name.equals(InfoSchemaDb.DATABASE_NAME)) { - tableNames = ExternalInfoSchemaDatabase.listTableNames(); + tableNames = ExternalInfoSchemaDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else if (name.equals(MysqlDb.DATABASE_NAME)) { - tableNames = ExternalMysqlDatabase.listTableNames(); + tableNames = ExternalMysqlDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else { - tableNames = extCatalog.listTableNames(null, name).stream().map(tableName -> { + tableNames = extCatalog.listTableNames(null, remoteName).stream().map(tableName -> { lowerCaseToTableName.put(tableName.toLowerCase(), tableName); - if (Env.isStoredTableNamesLowerCase()) { - return tableName.toLowerCase(); - } else { - return tableName; - } + String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); + return Pair.of(tableName, localTableName); }).collect(Collectors.toList()); } return tableNames; } - protected abstract T buildTableForInit(String tableName, long tblId, ExternalCatalog catalog); + public abstract T buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, ExternalDatabase db); public Optional getTableForReplay(long tableId) { if (extCatalog.getUseMetaCache().get()) { @@ -328,6 +352,10 @@ public String getFullName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public DatabaseProperty getDbProperties() { return dbProperties; @@ -528,7 +556,7 @@ public boolean registerTable(TableIf tableIf) { } else { if (!tableNameToId.containsKey(tableName)) { tableNameToId.put(tableName, tableId); - idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog, this)); lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 1eadb46fe82eedd..150930e0fd3e81e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -65,6 +65,8 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "type") protected TableType type = null; @SerializedName(value = "timestamp") @@ -80,6 +82,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long dbId; protected boolean objectCreated; protected ExternalCatalog catalog; + protected ExternalDatabase db; /** * No args constructor for persist. @@ -93,15 +96,19 @@ public ExternalTable() { * * @param id Table id. * @param name Table name. + * @param remoteName Remote table name. * @param catalog ExternalCatalog this table belongs to. - * @param dbName Name of the db the this table belongs to. + * @param db ExternalDatabase this table belongs to. * @param type Table type. */ - public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) { + public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db, + TableType type) { this.id = id; this.name = name; + this.remoteName = remoteName; this.catalog = catalog; - this.dbName = dbName; + this.db = db; + this.dbName = db.getFullName(); this.type = type; this.objectCreated = false; } @@ -135,6 +142,10 @@ public String getName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public TableType getType() { return type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 7834c0c8826daf0..023eecc2fa4c00e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -64,6 +64,9 @@ public enum Type { @SerializedName(value = "createDbNames") private List createDbNames; + @SerializedName(value = "remoteDbNames") + private List remoteDbNames; + @SerializedName(value = "type") private Type type; @@ -77,6 +80,7 @@ public InitCatalogLog() { refreshDbIds = Lists.newArrayList(); createDbIds = Lists.newArrayList(); createDbNames = Lists.newArrayList(); + remoteDbNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -85,10 +89,11 @@ public void addRefreshDb(long id) { refreshDbIds.add(id); } - public void addCreateDb(long id, String name) { + public void addCreateDb(long id, String name, String remoteName) { createCount += 1; createDbIds.add(id); createDbNames.add(name); + remoteDbNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java index 3c77b112d601602..e1f41a8f020fbe9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java @@ -32,14 +32,17 @@ public class EsExternalDatabase extends ExternalDatabase { * @param extCatalog External data source this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ES); + public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES); } @Override - protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog); + public EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new EsExternalTable(tblId, tableName, tableName, (EsExternalCatalog) extCatalog, + (EsExternalDatabase) db); } public void addTableForTest(EsExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java index 4f05d6d29cd89e6..6e9e5731f415329 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java @@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog EsExternalDataSource. + * @param db Database. */ - public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE); + public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE); } protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 3ae9fbcd6e75a9f..388d868a2b57511 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -32,14 +32,17 @@ public class HMSExternalDatabase extends ExternalDatabase { * @param extCatalog External catalog this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.HMS); + public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS); } @Override - protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog); + public HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new HMSExternalTable(tblId, tableName, tableName, (HMSExternalCatalog) extCatalog, + (HMSExternalDatabase) db); } public void addTableForTest(HMSExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index aacd9268ae35cfc..1c517b84d1254f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -164,11 +164,13 @@ public enum DLAType { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalCatalog. + * @param remoteName Remote table name. + * @param catalog HMSExternalDataSource. + * @param db Database. */ - public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE); + public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog, + HMSExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE); } // Will throw NotSupportedException if not supported hms table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index f56183972e36d22..a181843044c95a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -28,13 +28,15 @@ public class IcebergExternalDatabase extends ExternalDatabase { - public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG); + public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG); } @Override - protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog); + public IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new IcebergExternalTable(tblId, tableName, tableName, (IcebergExternalCatalog) extCatalog, + (IcebergExternalDatabase) db); } public String getLocation() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index feded88ea326f03..a357532f2bf1f81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -36,8 +36,9 @@ public class IcebergExternalTable extends ExternalTable { - public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE); + public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog, + IcebergExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE); } public String getIcebergCatalogType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java index 837f3691962e919..fb7c7204433a195 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java @@ -36,7 +36,7 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,9 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalInfoSchemaTable(tblId, tableName, catalog); + public ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalInfoSchemaTable(tblId, tableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java index 9d1336396128fdf..b3e0ead9a118a6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.infoschema; import org.apache.doris.analysis.SchemaTableType; -import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -31,8 +31,8 @@ public class ExternalInfoSchemaTable extends ExternalTable { - public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java index 5e0653f52781099..eedb2a596631f9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java @@ -36,7 +36,7 @@ public class ExternalMysqlDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalMysqlDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, MysqlDb.DATABASE_NAME, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,9 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalMysqlTable(tblId, tableName, catalog); + public ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalMysqlTable(tblId, tableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java index 6f277a5690619bf..1077abf81d8614e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java @@ -19,8 +19,8 @@ import org.apache.doris.analysis.SchemaTableType; import org.apache.doris.catalog.MysqlDBTable; -import org.apache.doris.catalog.MysqlDb; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -30,8 +30,8 @@ import java.util.Optional; public class ExternalMysqlTable extends ExternalTable { - public ExternalMysqlTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, MysqlDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalMysqlTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index fb26265d19fe93c..82593da76b44780 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -27,11 +27,14 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.jdbc.client.JdbcClientException; +import org.apache.doris.datasource.mapping.JdbcIdentifierMapping; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest; import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult; @@ -53,8 +56,10 @@ import org.apache.thrift.TException; import org.apache.thrift.TSerializer; +import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -77,6 +82,8 @@ public JdbcExternalCatalog(long catalogId, String name, String resource, Map listDatabaseNames() { + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + if (this.identifierMapping == null) { + identifierMapping = new JdbcIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); + } + } + + @Override + public List listDatabaseNames() { return jdbcClient.getDatabaseNameList(); } + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + return identifierMapping.fromRemoteDatabaseName(remoteDatabaseName); + } + @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); return jdbcClient.getTablesNameList(dbName); } + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return identifierMapping.fromRemoteTableName(remoteDatabaseName, remoteTableName); + } + @Override public boolean tableExist(SessionContext ctx, String dbName, String tblName) { makeSureInitialized(); - return jdbcClient.isTableExist(dbName, tblName); + ExternalTable tbl = Objects.requireNonNull(this.getDbNullable(dbName)).getTableNullable(tblName); + String remoteDbName = ((ExternalDatabase) tbl.getDatabase()).getRemoteName(); + String remoteTblName = tbl.getRemoteName(); + return jdbcClient.isTableExist(remoteDbName, remoteTblName); + } + + public List listColumns(String dbName, String tblName) { + makeSureInitialized(); + return jdbcClient.getColumnsFromJdbc(dbName, tblName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java index d078a3e238adbc4..d1c039cf033eb6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java @@ -30,13 +30,17 @@ public class JdbcExternalDatabase extends ExternalDatabase { * @param id database id. * @param name database name. */ - public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.JDBC); + public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.JDBC); } @Override - protected JdbcExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new JdbcExternalTable(tblId, tableName, name, (JdbcExternalCatalog) extCatalog); + public JdbcExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + String remoteDbName = this.getRemoteName(); + String localTblName = extCatalog.fromRemoteTableName(remoteDbName, tableName); + return new JdbcExternalTable(tblId, localTblName, tableName, (JdbcExternalCatalog) extCatalog, + (JdbcExternalDatabase) db); } public void addTableForTest(JdbcExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index 9e188a711b02975..09e31c5ee52dc20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.qe.AutoCloseConnectContext; @@ -32,6 +33,7 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.TTableDescriptor; +import com.google.common.collect.Maps; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * Jdbc external table. @@ -73,11 +76,13 @@ public class JdbcExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog JdbcExternalCatalog. + * @param db JdbcExternalDatabase. */ - public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE); + public JdbcExternalTable(long id, String name, String remoteName, JdbcExternalCatalog catalog, + JdbcExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.JDBC_EXTERNAL_TABLE); } @Override @@ -102,21 +107,51 @@ public TTableDescriptor toThrift() { @Override public Optional initSchema() { - return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient() - .getColumnsFromJdbc(dbName, name))); + String remoteDbName = ((ExternalDatabase) this.getDatabase()).getRemoteName(); + List columns = ((JdbcExternalCatalog) catalog).listColumns(remoteDbName, remoteName); + List remoteColumnNames = columns.stream() + .map(Column::getName) + .collect(Collectors.toList()); + List localColumnNames = remoteColumnNames.stream() + .map(remoteColumnName -> ((JdbcExternalCatalog) catalog).getIdentifierMapping() + .fromRemoteColumnName(remoteDbName, remoteName, remoteColumnName)) + .collect(Collectors.toList()); + for (int i = 0; i < columns.size(); i++) { + columns.get(i).setName(localColumnNames.get(i)); + } + Map remoteColumnNamesMap = Maps.newHashMap(); + for (int i = 0; i < remoteColumnNames.size(); i++) { + remoteColumnNamesMap.put(localColumnNames.get(i), remoteColumnNames.get(i)); + } + return Optional.of(new JdbcSchemaCacheValue(columns, remoteColumnNamesMap)); } private JdbcTable toJdbcTable() { List schema = getFullSchema(); JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog; - String fullDbName = this.dbName + "." + this.name; - JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE); - jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName); + String fullTableName = this.dbName + "." + this.name; + JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE); + jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName); // Set remote properties - jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName)); - jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name)); - jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name)); + jdbcTable.setRemoteDatabaseName(((ExternalDatabase) this.getDatabase()).getRemoteName()); + jdbcTable.setRemoteTableName(this.getRemoteName()); + Map remoteColumnNames = Maps.newHashMap(); + Optional schemaCacheValue = getSchemaCacheValue(); + for (Column column : schema) { + String remoteColumnName = schemaCacheValue.map(value -> ((JdbcSchemaCacheValue) value) + .getremoteColumnName(column.getName())).orElse(column.getName()); + remoteColumnNames.put(column.getName(), remoteColumnName); + } + if (!remoteColumnNames.isEmpty()) { + jdbcTable.setRemoteColumnNames(remoteColumnNames); + } else { + remoteColumnNames = Maps.newHashMap(); + for (Column column : getFullSchema()) { + remoteColumnNames.put(column.getName(), column.getName()); + } + jdbcTable.setRemoteColumnNames(remoteColumnNames); + } return jdbcTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java similarity index 51% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java index 20a74724b3e4965..4e21af4fabd99e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java @@ -17,29 +17,21 @@ package org.apache.doris.datasource.jdbc; -import org.apache.doris.datasource.jdbc.client.JdbcClient; -import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.SchemaCacheValue; -public class JdbcIdentifierMapping extends IdentifierMapping { - private final JdbcClient jdbcClient; +import java.util.List; +import java.util.Map; - public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping, JdbcClient jdbcClient) { - super(isLowerCaseMetaNames, metaNamesMapping); - this.jdbcClient = jdbcClient; - } - - @Override - protected void loadDatabaseNames() { - jdbcClient.getDatabaseNameList(); - } +public class JdbcSchemaCacheValue extends SchemaCacheValue { + private Map remoteColumnNamesMap; - @Override - protected void loadTableNames(String localDbName) { - jdbcClient.getTablesNameList(localDbName); + public JdbcSchemaCacheValue(List schema, Map remoteColumnNamesMap) { + super(schema); + this.remoteColumnNamesMap = remoteColumnNamesMap; } - @Override - protected void loadColumnNames(String localDbName, String localTableName) { - jdbcClient.getColumnsFromJdbc(localDbName, localTableName); + public String getremoteColumnName(String columnName) { + return remoteColumnNamesMap.get(columnName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 458142ff518fe67..36b4c33f4b8af4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -24,7 +24,6 @@ import org.apache.doris.cloud.security.SecurityChecker; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping; import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; @@ -63,11 +62,8 @@ public abstract class JdbcClient { protected ClassLoader classLoader = null; protected HikariDataSource dataSource = null; protected boolean isOnlySpecifiedDatabase; - protected boolean isLowerCaseMetaNames; - protected String metaNamesMapping; protected Map includeDatabaseMap; protected Map excludeDatabaseMap; - protected JdbcIdentifierMapping jdbcLowerCaseMetaMatching; public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -104,8 +100,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.catalogName = jdbcClientConfig.getCatalog(); this.jdbcUser = jdbcClientConfig.getUser(); this.isOnlySpecifiedDatabase = Boolean.parseBoolean(jdbcClientConfig.getOnlySpecifiedDatabase()); - this.isLowerCaseMetaNames = Boolean.parseBoolean(jdbcClientConfig.getIsLowerCaseMetaNames()); - this.metaNamesMapping = jdbcClientConfig.getMetaNamesMapping(); this.includeDatabaseMap = Optional.ofNullable(jdbcClientConfig.getIncludeDatabaseMap()).orElse(Collections.emptyMap()); this.excludeDatabaseMap = @@ -114,7 +108,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.dbType = parseDbType(jdbcUrl); initializeClassLoader(jdbcClientConfig); initializeDataSource(jdbcClientConfig); - this.jdbcLowerCaseMetaMatching = new JdbcIdentifierMapping(isLowerCaseMetaNames, metaNamesMapping, this); } // Initialize DataSource @@ -170,6 +163,7 @@ public static String parseDbType(String jdbcUrl) { public void closeClient() { dataSource.close(); + dataSource = null; } public Connection getConnection() throws JdbcClientException { @@ -308,10 +302,9 @@ public List getDatabaseNameList() { /** * get all tables of one database */ - public List getTablesNameList(String localDbName) { + public List getTablesNameList(String remoteDbName) { List remoteTablesNames = Lists.newArrayList(); String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); processTable(remoteDbName, null, tableTypes, (rs) -> { try { while (rs.next()) { @@ -321,14 +314,12 @@ public List getTablesNameList(String localDbName) { throw new JdbcClientException("failed to get all tables for remote database: `%s`", e, remoteDbName); } }); - return filterTableNames(remoteDbName, remoteTablesNames); + return remoteTablesNames; } - public boolean isTableExist(String localDbName, String localTableName) { + public boolean isTableExist(String remoteDbName, String remoteTableName) { final boolean[] isExist = {false}; String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); processTable(remoteDbName, remoteTableName, tableTypes, (rs) -> { try { if (rs.next()) { @@ -345,12 +336,10 @@ public boolean isTableExist(String localDbName, String localTableName) { /** * get all columns of one table */ - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); @@ -377,21 +366,7 @@ public List getColumnsFromJdbc(String localDbName, String localTableName field.isAllowNull(), field.getRemarks(), true, -1)); } - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - return filterColumnName(remoteDbName, remoteTableName, dorisTableSchema); - } - - public String getRemoteDatabaseName(String localDbname) { - return jdbcLowerCaseMetaMatching.getRemoteDatabaseName(localDbname); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteTableName(localDbName, localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteColumnNames(localDbName, localTableName); + return dorisTableSchema; } // protected methods, for subclass to override @@ -450,7 +425,7 @@ protected List filterDatabaseNames(List remoteDbNames) { } filteredDatabaseNames.add(databaseName); } - return jdbcLowerCaseMetaMatching.setDatabaseNameMapping(filteredDatabaseNames); + return filteredDatabaseNames; } protected Set getFilterInternalDatabases() { @@ -461,14 +436,6 @@ protected Set getFilterInternalDatabases() { .build(); } - protected List filterTableNames(String remoteDbName, List remoteTableNames) { - return jdbcLowerCaseMetaMatching.setTableNameMapping(remoteDbName, remoteTableNames); - } - - protected List filterColumnName(String remoteDbName, String remoteTableName, List remoteColumns) { - return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName, remoteTableName, remoteColumns); - } - protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema); protected Type createDecimalOrStringType(int precision, int scale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java index 7ba393e0d0aae63..086a8a5f393614b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java @@ -87,12 +87,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index a8263f1621a3a88..9a0f6011e166e7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -130,12 +130,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c * get all columns of one table */ @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 9968de79ab3a7de..adffd06c244e54f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -49,12 +49,10 @@ public String getTestQuery() { } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java index 59a7ace0dca3559..d6cf4ae726fb90f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java @@ -23,13 +23,15 @@ public class LakeSoulExternalDatabase extends ExternalDatabase { - public LakeSoulExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.LAKESOUL); + public LakeSoulExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.LAKESOUL); } @Override - protected LakeSoulExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new LakeSoulExternalTable(tblId, tableName, name, (LakeSoulExternalCatalog) catalog); + public LakeSoulExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new LakeSoulExternalTable(tblId, tableName, tableName, (LakeSoulExternalCatalog) extCatalog, + (LakeSoulExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java index 9dd2f4811e98f07..e5e0447f1cb6470 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java @@ -56,8 +56,9 @@ public class LakeSoulExternalTable extends ExternalTable { public final String tableId; - public LakeSoulExternalTable(long id, String name, String dbName, LakeSoulExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.LAKESOUl_EXTERNAL_TABLE); + public LakeSoulExternalTable(long id, String name, String remoteName, LakeSoulExternalCatalog catalog, + LakeSoulExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.LAKESOUl_EXTERNAL_TABLE); TableInfo tableInfo = getLakeSoulTableInfo(); if (tableInfo == null) { throw new RuntimeException(String.format("LakeSoul table %s.%s does not exist", dbName, name)); @@ -88,9 +89,9 @@ private Type arrowFiledToDorisType(Field field) { return Type.BIGINT; default: throw new IllegalArgumentException("Invalid integer bit width: " - + type.getBitWidth() - + " for LakeSoul table: " - + getTableIdentifier()); + + type.getBitWidth() + + " for LakeSoul table: " + + getTableIdentifier()); } } else if (dt instanceof ArrowType.FloatingPoint) { ArrowType.FloatingPoint type = (ArrowType.FloatingPoint) dt; @@ -101,16 +102,16 @@ private Type arrowFiledToDorisType(Field field) { return Type.DOUBLE; default: throw new IllegalArgumentException("Invalid floating point precision: " - + type.getPrecision() - + " for LakeSoul table: " - + getTableIdentifier()); + + type.getPrecision() + + " for LakeSoul table: " + + getTableIdentifier()); } } else if (dt instanceof ArrowType.Utf8) { return Type.STRING; } else if (dt instanceof ArrowType.Decimal) { ArrowType.Decimal decimalType = (ArrowType.Decimal) dt; return ScalarType.createDecimalType(PrimitiveType.DECIMAL64, decimalType.getPrecision(), - decimalType.getScale()); + decimalType.getScale()); } else if (dt instanceof ArrowType.Date) { return ScalarType.createDateV2Type(); } else if (dt instanceof ArrowType.Timestamp) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java index 363ef351152a39d..1c2941ea38bea3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java @@ -17,314 +17,17 @@ package org.apache.doris.datasource.mapping; -import org.apache.doris.catalog.Column; -import org.apache.doris.qe.GlobalVariable; +public interface IdentifierMapping { -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; + String fromRemoteDatabaseName(String remoteDatabaseName); -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; + String fromRemoteTableName(String remoteDatabaseName, String remoteTableName); -public abstract class IdentifierMapping { - private static final Logger LOG = LogManager.getLogger(IdentifierMapping.class); + String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnNames); - private final ObjectMapper mapper = new ObjectMapper(); - private final ConcurrentHashMap localDBToRemoteDB = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> localTableToRemoteTable - = new ConcurrentHashMap<>(); - private final ConcurrentHashMap>> - localColumnToRemoteColumn = new ConcurrentHashMap<>(); + String toRemoteDatabaseName(String localDatabaseName); - private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false); - private final ConcurrentHashMap tableNamesLoadedMap = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> columnNamesLoadedMap - = new ConcurrentHashMap<>(); + String toRemoteTableName(String remoteDatabaseName, String localTableName); - private final boolean isLowerCaseMetaNames; - private final String metaNamesMapping; - - public IdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping) { - this.isLowerCaseMetaNames = isLowerCaseMetaNames; - this.metaNamesMapping = metaNamesMapping; - } - - public List setDatabaseNameMapping(List remoteDatabaseNames) { - JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); - - Map databaseNameMapping = Maps.newTreeMap(); - if (databasesNode.isArray()) { - for (JsonNode node : databasesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String mapping = node.path("mapping").asText(); - databaseNameMapping.put(remoteDatabase, mapping); - } - } - - Map> result = nameListToMapping(remoteDatabaseNames, localDBToRemoteDB, - databaseNameMapping, isLowerCaseMetaNames); - List localDatabaseNames = result.get("localNames"); - List conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict database/schema names found when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the names."); - } - return localDatabaseNames; - } - - public List setTableNameMapping(String remoteDbName, List remoteTableNames) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); - - Map tableNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - if (remoteDbName.equals(remoteDatabase)) { - String remoteTable = node.path("remoteTable").asText(); - String mapping = node.path("mapping").asText(); - tableNameMapping.put(remoteTable, mapping); - } - } - } - - localTableToRemoteTable.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - - List localTableNames; - List conflictNames; - - if (GlobalVariable.lowerCaseTableNames == 1) { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, true); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + " when lower_case_table_names is 1: " + conflictNames - + ". Please use meta_name_mapping to specify the names."); - } - } else { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, isLowerCaseMetaNames); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + "when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the table names."); - } - } - return localTableNames; - } - - public List setColumnNameMapping(String remoteDbName, String remoteTableName, List remoteColumns) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "columns"); - - Map columnNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String remoteTable = node.path("remoteTable").asText(); - if (remoteDbName.equals(remoteDatabase) && remoteTable.equals(remoteTableName)) { - String remoteColumn = node.path("remoteColumn").asText(); - String mapping = node.path("mapping").asText(); - columnNameMapping.put(remoteColumn, mapping); - } - } - } - localColumnToRemoteColumn.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - localColumnToRemoteColumn.get(remoteDbName).putIfAbsent(remoteTableName, new ConcurrentHashMap<>()); - - List localColumnNames; - List conflictNames; - - // Get the name from localColumns and save it to List - List remoteColumnNames = Lists.newArrayList(); - for (Column remoteColumn : remoteColumns) { - remoteColumnNames.add(remoteColumn.getName()); - } - - Map> result = nameListToMapping(remoteColumnNames, - localColumnToRemoteColumn.get(remoteDbName).get(remoteTableName), - columnNameMapping, isLowerCaseMetaNames); - localColumnNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict column names found in remote database/schema: " + remoteDbName - + " in remote table: " + remoteTableName - + " when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the column names."); - } - // Replace the name in remoteColumns with localColumnNames - for (int i = 0; i < remoteColumns.size(); i++) { - remoteColumns.get(i).setName(localColumnNames.get(i)); - } - return remoteColumns; - } - - public String getRemoteDatabaseName(String localDbName) { - return getRequiredMapping(localDBToRemoteDB, localDbName, "database", this::loadDatabaseNamesIfNeeded, - localDbName); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - Map tableMap = localTableToRemoteTable.computeIfAbsent(remoteDbName, - k -> new ConcurrentHashMap<>()); - return getRequiredMapping(tableMap, localTableName, "table", () -> loadTableNamesIfNeeded(localDbName), - localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - ConcurrentHashMap> tableColumnMap - = localColumnToRemoteColumn.computeIfAbsent(remoteDbName, k -> new ConcurrentHashMap<>()); - Map columnMap = tableColumnMap.computeIfAbsent(remoteTableName, k -> new ConcurrentHashMap<>()); - if (columnMap.isEmpty()) { - LOG.info("Column name mapping missing, loading column names for localDbName: {}, localTableName: {}", - localDbName, localTableName); - loadColumnNamesIfNeeded(localDbName, localTableName); - columnMap = tableColumnMap.get(remoteTableName); - } - if (columnMap.isEmpty()) { - LOG.warn("No remote column found for localTableName: {}. Please refresh this catalog.", localTableName); - throw new RuntimeException( - "No remote column found for localTableName: " + localTableName + ". Please refresh this catalog."); - } - return columnMap; - } - - - private void loadDatabaseNamesIfNeeded() { - if (dbNamesLoaded.compareAndSet(false, true)) { - try { - loadDatabaseNames(); - } catch (Exception e) { - dbNamesLoaded.set(false); // Reset on failure - LOG.warn("Error loading database names", e); - } - } - } - - private void loadTableNamesIfNeeded(String localDbName) { - AtomicBoolean isLoaded = tableNamesLoadedMap.computeIfAbsent(localDbName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadTableNames(localDbName); - } catch (Exception e) { - tableNamesLoadedMap.get(localDbName).set(false); // Reset on failure - LOG.warn("Error loading table names for localDbName: {}", localDbName, e); - } - } - } - - private void loadColumnNamesIfNeeded(String localDbName, String localTableName) { - columnNamesLoadedMap.putIfAbsent(localDbName, new ConcurrentHashMap<>()); - AtomicBoolean isLoaded = columnNamesLoadedMap.get(localDbName) - .computeIfAbsent(localTableName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadColumnNames(localDbName, localTableName); - } catch (Exception e) { - columnNamesLoadedMap.get(localDbName).get(localTableName).set(false); // Reset on failure - LOG.warn("Error loading column names for localDbName: {}, localTableName: {}", localDbName, - localTableName, e); - } - } - } - - private V getRequiredMapping(Map map, K key, String typeName, Runnable loadIfNeeded, - String entityName) { - if (map.isEmpty() || !map.containsKey(key) || map.get(key) == null) { - LOG.info("{} mapping missing, loading for {}: {}", typeName, typeName, entityName); - loadIfNeeded.run(); - } - V value = map.get(key); - if (value == null) { - LOG.warn("No remote {} found for {}: {}. Please refresh this catalog.", typeName, typeName, entityName); - throw new RuntimeException("No remote " + typeName + " found for " + typeName + ": " + entityName - + ". Please refresh this catalog."); - } - return value; - } - - // Load the database name from the data source. - // In the corresponding getDatabaseNameList(), setDatabaseNameMapping() must be used to update the mapping. - protected abstract void loadDatabaseNames(); - - // Load the table names for the specified database from the data source. - // In the corresponding getTableNameList(), setTableNameMapping() must be used to update the mapping. - protected abstract void loadTableNames(String localDbName); - - // Load the column names for a specified table in a database from the data source. - // In the corresponding getColumnNameList(), setColumnNameMapping() must be used to update the mapping. - protected abstract void loadColumnNames(String localDbName, String localTableName); - - private JsonNode readAndParseJson(String jsonPath, String nodeName) { - JsonNode rootNode; - try { - rootNode = mapper.readTree(jsonPath); - return rootNode.path(nodeName); - } catch (JsonProcessingException e) { - throw new RuntimeException("parse meta_names_mapping property error", e); - } - } - - private Map> nameListToMapping(List remoteNames, - ConcurrentHashMap localNameToRemoteName, - Map nameMapping, boolean isLowerCaseMetaNames) { - List filteredDatabaseNames = Lists.newArrayList(); - Set lowerCaseNames = Sets.newHashSet(); - Map> nameMap = Maps.newHashMap(); - List conflictNames = Lists.newArrayList(); - - for (String name : remoteNames) { - String mappedName = nameMapping.getOrDefault(name, name); - String localName = isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; - - // Use computeIfAbsent to ensure atomicity - localNameToRemoteName.computeIfAbsent(localName, k -> name); - - if (isLowerCaseMetaNames && !lowerCaseNames.add(localName)) { - if (nameMap.containsKey(localName)) { - nameMap.get(localName).add(mappedName); - } - } else { - nameMap.putIfAbsent(localName, Lists.newArrayList(Collections.singletonList(mappedName))); - } - - filteredDatabaseNames.add(localName); - } - - for (List conflictNameList : nameMap.values()) { - if (conflictNameList.size() > 1) { - conflictNames.addAll(conflictNameList); - } - } - - Map> result = Maps.newConcurrentMap(); - result.put("localNames", filteredDatabaseNames); - result.put("conflictNames", conflictNames); - return result; - } + String toRemoteColumnName(String remoteDatabaseName, String remoteTableName, String localColumnNames); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java new file mode 100644 index 000000000000000..249b9fc163cfcd3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; + +public class JdbcIdentifierMapping implements IdentifierMapping { + private static final Logger LOG = LogManager.getLogger(JdbcIdentifierMapping.class); + + private final ObjectMapper mapper = new ObjectMapper(); + private final boolean isLowerCaseMetaNames; + private final String metaNamesMapping; + + public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping) { + this.isLowerCaseMetaNames = isLowerCaseMetaNames; + this.metaNamesMapping = metaNamesMapping; + } + + private boolean isMappingInvalid() { + return metaNamesMapping == null || metaNamesMapping.isEmpty(); + } + + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteDatabaseName; + } + JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); + + Map databaseNameMapping = Maps.newTreeMap(); + if (databasesNode.isArray()) { + for (JsonNode node : databasesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String mapping = node.path("mapping").asText(); + databaseNameMapping.put(remoteDatabase, mapping); + } + } + + return getMappedName(remoteDatabaseName, databaseNameMapping); + } + + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteTableName; + } + JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); + + Map tableNameMapping = Maps.newTreeMap(); + if (tablesNode.isArray()) { + for (JsonNode node : tablesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + if (remoteDatabaseName.equals(remoteDatabase)) { + String remoteTable = node.path("remoteTable").asText(); + String mapping = node.path("mapping").asText(); + tableNameMapping.put(remoteTable, mapping); + } + } + } + return getMappedName(remoteTableName, tableNameMapping); + } + + @Override + public String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteColumnName; + } + + JsonNode columnsNode = readAndParseJson(metaNamesMapping, "columns"); + + Map columnNameMapping = Maps.newTreeMap(); + if (columnsNode.isArray()) { + for (JsonNode node : columnsNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String remoteTable = node.path("remoteTable").asText(); + if (remoteDatabaseName.equals(remoteDatabase) && remoteTableName.equals(remoteTable)) { + String remoteColumn = node.path("remoteColumn").asText(); + String mapping = node.path("mapping").asText(); + columnNameMapping.put(remoteColumn, mapping); + } + } + } + return getMappedName(remoteColumnName, columnNameMapping); + } + + @Override + public String toRemoteDatabaseName(String localDatabaseName) { + throw new UnsupportedOperationException("toRemoteDatabaseName is not supported"); + } + + @Override + public String toRemoteTableName(String remoteDatabaseName, String localTableName) { + throw new UnsupportedOperationException("toRemoteTableName is not supported"); + } + + @Override + public String toRemoteColumnName(String remoteDatabaseName, String remoteTableName, String localColumnNames) { + throw new UnsupportedOperationException("toRemoteColumnName is not supported"); + } + + private String getMappedName(String name, Map nameMapping) { + String mappedName = nameMapping.getOrDefault(name, name); + return isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; + } + + private JsonNode readAndParseJson(String jsonPath, String nodeName) { + JsonNode rootNode; + try { + rootNode = mapper.readTree(jsonPath); + return rootNode.path(nodeName); + } catch (JsonProcessingException e) { + throw new RuntimeException("parse meta_names_mapping property error", e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java index 0a100e3495ff3d9..c16cf0d9d77fb82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java @@ -32,12 +32,14 @@ public class MaxComputeExternalDatabase extends ExternalDatabase columnNameToOdpsColumn = new HashMap(); + private Map columnNameToOdpsColumn = new HashMap(); @Override protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java index 50265f77463428f..90906a2d65316bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java @@ -23,12 +23,14 @@ public class PaimonExternalDatabase extends ExternalDatabase { - public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.PAIMON); + public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.PAIMON); } @Override - protected PaimonExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new PaimonExternalTable(tblId, tableName, name, (PaimonExternalCatalog) extCatalog); + public PaimonExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new PaimonExternalTable(tblId, tableName, tableName, (PaimonExternalCatalog) extCatalog, + (PaimonExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index c9eaf1b7df32ef1..6fa7ff1786924ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -52,8 +52,9 @@ public class PaimonExternalTable extends ExternalTable { private static final Logger LOG = LogManager.getLogger(PaimonExternalTable.class); - public PaimonExternalTable(long id, String name, String dbName, PaimonExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.PAIMON_EXTERNAL_TABLE); + public PaimonExternalTable(long id, String name, String remoteName, PaimonExternalCatalog catalog, + PaimonExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.PAIMON_EXTERNAL_TABLE); } public String getPaimonCatalogType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java index 2cf1f57d0e4672c..dc14a2490444bd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java @@ -23,12 +23,14 @@ public class TestExternalDatabase extends ExternalDatabase { - public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.TEST); + public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.TEST); } @Override - protected TestExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TestExternalTable(tblId, tableName, name, (TestExternalCatalog) extCatalog); + public TestExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new TestExternalTable(tblId, tableName, tableName, (TestExternalCatalog) extCatalog, + (TestExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java index 6da0981b97ef548..6d08b10403bb76f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java @@ -33,8 +33,8 @@ public class TestExternalTable extends ExternalTable { private static final Logger LOG = LogManager.getLogger(TestExternalTable.class); - public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE); + public TestExternalTable(long id, String name, String remoteName, TestExternalCatalog catalog, TestExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TEST_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java index 33652e7e945f412..9bda856399864c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java @@ -22,12 +22,14 @@ import org.apache.doris.datasource.InitDatabaseLog.Type; public class TrinoConnectorExternalDatabase extends ExternalDatabase { - public TrinoConnectorExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, Type.TRINO_CONNECTOR); + public TrinoConnectorExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, Type.TRINO_CONNECTOR); } @Override - protected TrinoConnectorExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TrinoConnectorExternalTable(tblId, tableName, name, (TrinoConnectorExternalCatalog) extCatalog); + public TrinoConnectorExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog, + ExternalDatabase db) { + return new TrinoConnectorExternalTable(tblId, tableName, tableName, (TrinoConnectorExternalCatalog) extCatalog, + (TrinoConnectorExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java index 27f9b8086a9cefd..f5e63903fff0a73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java @@ -71,8 +71,9 @@ public class TrinoConnectorExternalTable extends ExternalTable { - public TrinoConnectorExternalTable(long id, String name, String dbName, TrinoConnectorExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); + public TrinoConnectorExternalTable(long id, String name, String remoteName, TrinoConnectorExternalCatalog catalog, + TrinoConnectorExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java index 07c7abf7ce0c265..a38b5f49fc070f7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java @@ -239,8 +239,8 @@ void addConstraintLogPersistForExternalTableTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); @@ -302,8 +302,8 @@ void dropConstraintLogPersistForExternalTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es2"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es2"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index aa5fa313be3c91d..9d36d843e9fb7cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -163,15 +163,15 @@ private void createDbAndTableForCatalog(CatalogIf catalog) { schema.add(new Column("k1", PrimitiveType.INT)); if (catalog instanceof HMSExternalCatalog) { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1"); - HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1", "hive_db1"); + HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); hmsCatalog.addDatabaseForTest(db); } else if (catalog instanceof EsExternalCatalog) { EsExternalCatalog esCatalog = (EsExternalCatalog) catalog; - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); esCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index 2a6342a57affa3d..b83b69df26f838b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -217,7 +217,7 @@ public Table getTable(String dbName, String tblName) { @Mock public ExternalDatabase getDbNullable(String dbName) { if (createdDbs.contains(dbName)) { - return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName); + return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName, dbName); } return null; } @@ -228,7 +228,7 @@ public ExternalDatabase getDbNullable(String dbName) { HMSExternalTable getTableNullable(String tableName) { for (Table table : createdTables) { if (table.getTableName().equals(tableName)) { - return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog); + return new HMSExternalTable(0, tableName, tableName, hmsExternalCatalog, (HMSExternalDatabase) hmsExternalCatalog.getDbNullable(mockedDbName)); } } return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index 46d3e1b897d1110..8f268f19426c3ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -68,7 +68,7 @@ public void init() { new MockUp(HMSExternalCatalog.class) { @Mock public ExternalDatabase getDbNullable(String dbName) { - return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb"); + return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb", "mockedDb"); } @Mock diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 1c780d63c9cd8be..d956449ba129710 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -82,7 +82,7 @@ public static void beforeClass() throws Throwable { } else { icebergCatalog.setInitialized(true); } - IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName); + IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); icebergCatalog.addDatabaseForTest(db); // context diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index c875ef6bc2f1414..30a233dd1a999e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -98,7 +98,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index bddb3c8185ae72b..0a981dab8a9dccc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -114,7 +114,7 @@ private void init(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index 026b7fb65b069a9..b8067e7e6bf75b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -117,7 +117,7 @@ public void testSupportAutoAnalyze() { OlapTable table1 = new OlapTable(200, "testTable", schema, null, null, null); Assertions.assertTrue(collector.supportAutoAnalyze(table1)); - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null, null); Assertions.assertFalse(collector.supportAutoAnalyze(externalTable)); new MockUp() { @@ -126,7 +126,7 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, null); Assertions.assertFalse(collector.supportAutoAnalyze(icebergExternalTable)); new MockUp() { @@ -135,7 +135,7 @@ public DLAType getDlaType() { return DLAType.HIVE; } }; - ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, null); Assertions.assertTrue(collector.supportAutoAnalyze(hiveExternalTable)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 32521882939d348..3095ba155e63146 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -30,9 +30,11 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; +import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.AnalysisManager; @@ -180,6 +182,11 @@ void testNeedAnalyzeColumn() throws DdlException { OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); HMSExternalCatalog externalCatalog = new HMSExternalCatalog(); + long dbId = 100; + String dbName = "testDb"; + HMSExternalDatabase externalDatabase = new HMSExternalDatabase(externalCatalog, dbId, dbName, dbName); + + // Test olap table auto analyze disabled. Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, "disable"); @@ -194,7 +201,7 @@ protected synchronized void makeSureInitialized() { }; // Test auto analyze catalog disabled. - HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "dbName", externalCatalog); + HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "name", externalCatalog, externalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsTable, Pair.of("index", column.getName()))); // Test catalog auto analyze enabled. @@ -215,7 +222,7 @@ public TableStatsMeta findTableStatsStatus(long tblId) { } }; externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "false"); - HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "dbName", externalCatalog); + HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "name", externalCatalog, externalDatabase); externalCatalog.setAutoAnalyzePolicy("dbName", "name", "enable"); Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(hmsTable1, Pair.of("index", column.getName()))); @@ -268,8 +275,9 @@ protected synchronized void makeSureInitialized() { } }; // Test not supported external table type. - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", - new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), "")); + JdbcExternalCatalog jdbcExternalCatalog = new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), ""); + JdbcExternalDatabase jdbcExternalDatabase = new JdbcExternalDatabase(jdbcExternalCatalog, 1, "jdbcdb", "jdbcdb"); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbctable", jdbcExternalCatalog, jdbcExternalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(externalTable, Pair.of("index", column.getName()))); // Test hms external table not hive type. @@ -279,7 +287,7 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", externalCatalog); + ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsTable", externalCatalog, externalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsExternalTable, Pair.of("index", column.getName()))); // Test partition first load.