Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[regression](paimon) test Paimon's connectivity of cloud filesystems #45268

Merged
merged 6 commits into from
Dec 13, 2024
Merged

[regression](paimon) test Paimon's connectivity of cloud filesystems #45268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
78fd688
add obs case for paimon
suxiaogang223 Dec 9, 2024
138473f
fix
suxiaogang223 Dec 10, 2024
e749db1
fix
suxiaogang223 Dec 10, 2024
7174799
finish obs
suxiaogang223 Dec 11, 2024
fa3ec08
fix cosn
suxiaogang223 Dec 12, 2024
0a4bd4b
fix
suxiaogang223 Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !c1 --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364
-- !oss --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !c2 --
7 Hugo
8 Stop
-- !obs --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !c3 --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364
-- !cos --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !c4 --
7 Hugo
8 Stop
-- !cosn --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !oss --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !obs --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !cos --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

65 changes: 51 additions & 14 deletions regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,29 +17,48 @@

suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_docker_doris") {
String enabled = context.config.otherConfigs.get("enablePaimonTest")
// if (enabled == null || !enabled.equalsIgnoreCase("true")) {
if (true) {
// temporary comment out, will add back when env is ready

if (enabled == null || !enabled.equalsIgnoreCase("true")) {
return
}

try {
String catalog_cos = "paimon_base_filesystem_paimon_cos"
String catalog_obs = "paimon_base_filesystem_paimon_obs"
String catalog_oss = "paimon_base_filesystem_paimon_oss"
String catalog_cos = "paimon_base_filesystem_paimon_cos"
String catalog_cosn = "paimon_base_filesystem_paimon_cosn"
String aliYunAk = context.config.otherConfigs.get("aliYunAk")
String aliYunSk = context.config.otherConfigs.get("aliYunSk")
String hwYunAk = context.config.otherConfigs.get("hwYunAk")
String hwYunSk = context.config.otherConfigs.get("hwYunSk")
String txYunAk = context.config.otherConfigs.get("txYunAk")
String txYunSk = context.config.otherConfigs.get("txYunSk")

def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18"""
def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a"""
def obs = """select * from ${catalog_obs}.db1.all_table limit 1;"""
def oss = """select * from ${catalog_oss}.db1.all_table limit 1;"""
def cos = """select * from ${catalog_cos}.db1.all_table limit 1;"""
def cosn = """select * from ${catalog_cosn}.db1.all_table limit 1;"""

sql """drop catalog if exists ${catalog_cos};"""
sql """drop catalog if exists ${catalog_obs};"""
sql """drop catalog if exists ${catalog_oss};"""
sql """drop catalog if exists ${catalog_cos};"""
sql """drop catalog if exists ${catalog_cosn};"""

sql """
create catalog if not exists ${catalog_cos} properties (
"type" = "paimon",
"warehouse" = "cosn://doris-build-1308700295/regression/paimoncos",
"paimon.catalog.type"="filesystem",
"warehouse" = "s3://doris-build-1308700295/regression/paimon1",
"s3.access_key" = "${txYunAk}",
"s3.secret_key" = "${txYunSk}",
"s3.endpoint" = "cos.ap-beijing.myqcloud.com"
);
"""
sql """
create catalog if not exists ${catalog_cosn} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
"warehouse" = "cosn://doris-build-1308700295/regression/paimon1",
"cos.access_key" = "${txYunAk}",
"cos.secret_key" = "${txYunSk}",
"cos.endpoint" = "cos.ap-beijing.myqcloud.com"
Expand All @@ -48,22 +67,40 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock
sql """
create catalog if not exists ${catalog_oss} properties (
"type" = "paimon",
"warehouse" = "oss://doris-regression-bj/regression/paimonoss",
"paimon.catalog.type"="filesystem",
"warehouse" = "oss://doris-regression-bj/regression/paimon1",
"oss.access_key"="${aliYunAk}",
"oss.secret_key"="${aliYunSk}",
"oss.endpoint"="oss-cn-beijing.aliyuncs.com"
);
"""
logger.info("catalog " + catalog_cos + " created")
sql """
create catalog if not exists ${catalog_obs} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
"warehouse" = "obs://doris-build/regression/paimon1",
"obs.access_key"="${hwYunAk}",
"obs.secret_key"="${hwYunSk}",
"obs.endpoint"="obs.cn-north-4.myhuaweicloud.com"
);
"""
logger.info("catalog " + catalog_obs + " created")
logger.info("catalog " + catalog_oss + " created")
logger.info("catalog " + catalog_cos + " created")
logger.info("catalog " + catalog_cosn + " created")

sql """set force_jni_scanner=false"""
qt_c1 cos
qt_c2 oss
qt_oss oss
qt_obs obs
qt_cos cos
qt_cosn cosn

sql """set force_jni_scanner=true"""
qt_c3 cos
qt_c4 oss
qt_oss oss
qt_obs obs
qt_cos cos
// java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.CosFileSystem not found
// qt_cosn cosn

} finally {
sql """set force_jni_scanner=false"""
Expand Down
Loading