Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

branch-2.1: [fix](meta-cache) fix refreshOnlyCatalogCache when use_meta_cache = false #44363 #44461

Merged
merged 1 commit into from
Nov 22, 2024
Merged

branch-2.1: [fix](meta-cache) fix refreshOnlyCatalogCache when use_meta_cache = false #44363 #44461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ private void refreshOnlyCatalogCache(boolean invalidCache) {
if (useMetaCache.get() && metaCache != null) {
metaCache.invalidateAll();
} else if (!useMetaCache.get()) {
this.initialized = false;
for (ExternalDatabase<? extends ExternalTable> db : idToDb.values()) {
db.setUnInitialized(invalidCache);
}
Expand Down
106 changes: 104 additions & 2 deletions regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,20 @@ test_use_meta_cache_db_hive
test_use_meta_cache_tbl_hive

-- !sql08 --
test_use_meta_cache_tbl_hive

-- !sql09 --
p1=part1
p1=part2

-- !sql10 --
test_use_meta_cache_db_hive

-- !sql11 --

-- !sql12 --
test_use_meta_cache_db_hive

-- !sql13 --

-- !sql01 --

-- !sql02 --
Expand All @@ -61,17 +66,114 @@ test_use_meta_cache_db_hive
-- !sql05 --

-- !sql06 --
test_use_meta_cache_tbl_hive

-- !sql07 --
test_use_meta_cache_tbl_hive

-- !sql08 --

-- !sql09 --
p1=part1
p1=part2

-- !sql10 --
test_use_meta_cache_partitioned_tbl_hive
test_use_meta_cache_tbl_hive

-- !sql11 --

-- !sql12 --
test_use_meta_cache_db_hive

-- !sql13 --

-- !sql01 --

-- !sql02 --
test_use_meta_cache_db

-- !sql03 --
test_use_meta_cache_tbl

-- !sql04 --

-- !sql05 --

-- !sql01 --

-- !sql02 --

-- !sql03 --
test_use_meta_cache_db_hive

-- !sql04 --

-- !sql05 --

-- !sql06 --

-- !sql07 --
test_use_meta_cache_tbl_hive

-- !sql08 --

-- !sql09 --
p1=part1
p1=part2

-- !sql10 --

-- !sql11 --

-- !sql12 --
test_use_meta_cache_db_hive

-- !sql13 --

-- !sql01 --

-- !sql02 --
test_use_meta_cache_db

-- !sql03 --
test_use_meta_cache_tbl

-- !sql04 --

-- !sql05 --

-- !sql01 --

-- !sql02 --

-- !sql03 --
test_use_meta_cache_db_hive

-- !sql04 --

-- !sql05 --

-- !sql06 --
test_use_meta_cache_tbl_hive

-- !sql07 --
test_use_meta_cache_tbl_hive

-- !sql08 --

-- !sql09 --
p1=part1
p1=part2

-- !sql10 --
test_use_meta_cache_partitioned_tbl_hive
test_use_meta_cache_tbl_hive

-- !sql11 --

-- !sql12 --
test_use_meta_cache_db_hive

-- !sql13 --

171 changes: 102 additions & 69 deletions regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,80 +23,113 @@ suite("test_hive_use_meta_cache", "p0,external,hive,external_docker,external_doc
return;
}

for (String hivePrefix : ["hive3", "hive3"]) {
for (String hivePrefix : ["hive2", "hive3"]) {
setHivePrefix(hivePrefix)
try {
String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
String catalog = "test_${hivePrefix}_use_meta_cache"
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
def test_use_meta_cache = { Boolean use_meta_cache ->
String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
String use_meta_cache_string = use_meta_cache ? "true" : "false"
String catalog = "test_${hivePrefix}_use_meta_cache_${use_meta_cache}"
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")

sql """drop catalog if exists ${catalog}"""
sql """create catalog if not exists ${catalog} properties (
'type'='hms',
'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
'use_meta_cache' = 'true'
);"""
sql """drop catalog if exists ${catalog}"""
sql """create catalog if not exists ${catalog} properties (
'type'='hms',
'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
'use_meta_cache' = '${use_meta_cache_string}'
);"""

// create from Doris, the cache will be filled immediately
String database= "test_use_meta_cache_db"
String table = "test_use_meta_cache_tbl"
String database_hive = "test_use_meta_cache_db_hive"
String table_hive = "test_use_meta_cache_tbl_hive"
String partitioned_table_hive = "test_use_meta_cache_partitioned_tbl_hive"

sql "switch ${catalog}"
sql "drop database if exists ${database}"
sql "drop database if exists ${database_hive}"
order_qt_sql01 "show databases like '%${database}%'";
sql "drop database if exists ${database}"
sql "create database ${database}"
order_qt_sql02 "show databases like '%${database}%'";
sql "use ${database}"
sql "create table ${table} (k1 int)"
order_qt_sql03 "show tables"
sql "drop table ${table}"
order_qt_sql04 "show tables"
sql "drop database ${database}"
order_qt_sql05 "show databases like '%${database}%'";

// create from Doris, the cache will be filled immediately
String database= "test_use_meta_cache_db"
String table = "test_use_meta_cache_tbl"
String database_hive = "test_use_meta_cache_db_hive"
String table_hive = "test_use_meta_cache_tbl_hive"
sql "switch ${catalog}"
sql "drop database if exists ${database}"
sql "drop database if exists ${database_hive}"
order_qt_sql01 "show databases like '%${database}%'";
sql "drop database if exists ${database}"
sql "create database ${database}"
order_qt_sql02 "show databases like '%${database}%'";
sql "use ${database}"
sql "create table ${table} (k1 int)"
order_qt_sql03 "show tables"
sql "drop table ${table}"
order_qt_sql04 "show tables"
sql "drop database ${database}"
order_qt_sql05 "show databases like '%${database}%'";

// create from Hive, the cache has different behavior
order_qt_sql01 "show databases like '%${database_hive}%'";
hive_docker "drop database if exists ${database_hive}"
hive_docker "create database ${database_hive}"
// not see
order_qt_sql02 "show databases like '%${database_hive}%'";
// but can use
sql "use ${database_hive}"
sql "refresh catalog ${catalog}"
// can see
order_qt_sql03 "show databases like '%${database_hive}%'";
// show tables first to fill cache
order_qt_sql04 "show tables"
hive_docker "create table ${database_hive}.${table_hive} (k1 int)"
// not see
order_qt_sql05 "show tables"
// but can select
sql "select * from ${table_hive}"
// still not see
order_qt_sql06 "show tables"
sql "refresh database ${database_hive}"
// can see
order_qt_sql07 "show tables"
hive_docker "drop table ${database_hive}.${table_hive}"
// still can see
order_qt_sql08 "show tables"
sql "refresh database ${database_hive}"
// can not see
order_qt_sql09 "show tables"
hive_docker "drop database ${database_hive}"
// still can see
order_qt_sql10 "show databases like '%${database_hive}%'";
sql "refresh catalog ${catalog}"
// can not see
order_qt_sql11 "show databases like '%${database_hive}%'";
// create from Hive, the cache has different behavior
order_qt_sql01 "show databases like '%${database_hive}%'";
hive_docker "drop database if exists ${database_hive}"
hive_docker "create database ${database_hive}"
// not see
order_qt_sql02 "show databases like '%${database_hive}%'";
if (use_meta_cache) {
// if use meta cache, can use
sql "use ${database_hive}"
sql "refresh catalog ${catalog}"
} else {
// if not use meta cache, can not use
sql "refresh catalog ${catalog}"
sql "use ${database_hive}"
}

// can see
order_qt_sql03 "show databases like '%${database_hive}%'";
// show tables first to fill cache
order_qt_sql04 "show tables"
hive_docker "create table ${database_hive}.${table_hive} (k1 int)"
// not see
order_qt_sql05 "show tables"
if (use_meta_cache) {
// but can select
sql "select * from ${table_hive}"
// still not see
order_qt_sql06 "show tables"
sql "refresh database ${database_hive}"
} else {
// if not use meta cache, can not select
sql "refresh database ${database_hive}"
sql "select * from ${table_hive}"
order_qt_sql06 "show tables"
}
// can see
order_qt_sql07 "show tables"

// test Hive Metastore table partition file listing
hive_docker "create table ${database_hive}.${partitioned_table_hive} (k1 int) partitioned by (p1 string)"
sql "refresh catalog ${catalog}"
order_qt_sql08 "show partitions from ${partitioned_table_hive}"
hive_docker "alter table ${database_hive}.${partitioned_table_hive} add partition (p1='part1')"
hive_docker "alter table ${database_hive}.${partitioned_table_hive} add partition (p1='part2')"
// can see because partition file listing is not cached
order_qt_sql09 "show partitions from ${partitioned_table_hive}"

// drop tables
hive_docker "drop table ${database_hive}.${partitioned_table_hive}"
hive_docker "drop table ${database_hive}.${table_hive}"
// still can see
order_qt_sql10 "show tables"
sql "refresh database ${database_hive}"
// can not see
order_qt_sql11 "show tables"

// drop database
hive_docker "drop database ${database_hive}"
// still can see
order_qt_sql12 "show databases like '%${database_hive}%'";
sql "refresh catalog ${catalog}"
// can not see
order_qt_sql13 "show databases like '%${database_hive}%'";
}
test_use_meta_cache(true)
test_use_meta_cache(false)
} finally {
}
}
}


Loading