From 78fd6880261f268228701948be89de3c3e1682bc Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 9 Dec 2024 21:21:48 +0800 Subject: [PATCH 1/6] add obs case for paimon --- .../paimon/paimon_base_filesystem.groovy | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index 1e4561c406556f..fa06e163ea4fb7 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -25,19 +25,37 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock try { String catalog_cos = "paimon_base_filesystem_paimon_cos" + String catalog_cosn = "paimon_base_filesystem_paimon_cosn" String catalog_oss = "paimon_base_filesystem_paimon_oss" + String catalog_obs = "paimon_base_filesystem_paimon_obs" String aliYunAk = context.config.otherConfigs.get("aliYunAk") String aliYunSk = context.config.otherConfigs.get("aliYunSk") String txYunAk = context.config.otherConfigs.get("txYunAk") String txYunSk = context.config.otherConfigs.get("txYunSk") + String hwYunAk = context.config.otherConfigs.get("hwYunAk") + String hwYunSk = context.config.otherConfigs.get("hwYunSk") - def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18""" + def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cosn}.zd.all_table order by c18""" + def cosn = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cosn}.zd.all_table order by c18""" def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a""" + def obs = """select * from ${catalog_obs}.piamon_obs_db.paimon_table_obs order by k""" sql """drop catalog if exists ${catalog_cos};""" + sql """drop catalog if exists ${catalog_cosn};""" sql """drop catalog if exists ${catalog_oss};""" + sql """drop catalog if exists ${catalog_obs};""" + sql """ create catalog if not exists ${catalog_cos} properties ( + "type" = "paimon", + "warehouse" = "cos://doris-build-1308700295/regression/paimoncos", + "cos.access_key" = "${txYunAk}", + "cos.secret_key" = "${txYunSk}", + "cos.endpoint" = "cos.ap-beijing.myqcloud.com" + ); + """ + sql """ + create catalog if not exists ${catalog_cosn} properties ( "type" = "paimon", "warehouse" = "cosn://doris-build-1308700295/regression/paimoncos", "cos.access_key" = "${txYunAk}", @@ -54,16 +72,31 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock "oss.endpoint"="oss-cn-beijing.aliyuncs.com" ); """ + sql """ + create catalog if not exists ${catalog_obs} properties ( + "type" = "paimon", + "warehouse" = "obs://doris-build/piamon", + "oss.access_key"="${hwYunAk}", + "oss.secret_key"="${hwYunSk}", + "oss.endpoint"="obs.cn-north-4.myhuaweicloud.com" + ); + """ logger.info("catalog " + catalog_cos + " created") + logger.info("catalog " + catalog_cosn + " created") logger.info("catalog " + catalog_oss + " created") + logger.info("catalog " + catalog_obs + " created") sql """set force_jni_scanner=false""" qt_c1 cos - qt_c2 oss + qt_c2 cosn + qt_c3 oss + qt_c4 obs sql """set force_jni_scanner=true""" - qt_c3 cos - qt_c4 oss + qt_c5 cos + qt_c6 cosn + qt_c7 oss + qt_c8 obs } finally { sql """set force_jni_scanner=false""" From 138473f6a365ea802cea450408f8207272f49919 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 10 Dec 2024 23:32:52 +0800 Subject: [PATCH 2/6] fix --- .../paimon/paimon_base_filesystem.groovy | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index fa06e163ea4fb7..bf07006dbe37ca 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -17,9 +17,8 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_docker_doris") { String enabled = context.config.otherConfigs.get("enablePaimonTest") - // if (enabled == null || !enabled.equalsIgnoreCase("true")) { - if (true) { - // temporary comment out, will add back when env is ready + + if (enabled == null || !enabled.equalsIgnoreCase("true")) { return } @@ -76,9 +75,9 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock create catalog if not exists ${catalog_obs} properties ( "type" = "paimon", "warehouse" = "obs://doris-build/piamon", - "oss.access_key"="${hwYunAk}", - "oss.secret_key"="${hwYunSk}", - "oss.endpoint"="obs.cn-north-4.myhuaweicloud.com" + "obs.access_key"="${hwYunAk}", + "obs.secret_key"="${hwYunSk}", + "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com" ); """ logger.info("catalog " + catalog_cos + " created") From e749db11cc100065737237b489249daccdd44e8f Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 11 Dec 2024 00:06:55 +0800 Subject: [PATCH 3/6] fix --- .../paimon/paimon_base_filesystem.groovy | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index bf07006dbe37ca..ae49640f23e609 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -23,31 +23,32 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock } try { + String catalog_obs = "paimon_base_filesystem_paimon_obs" + String catalog_oss = "paimon_base_filesystem_paimon_oss" String catalog_cos = "paimon_base_filesystem_paimon_cos" String catalog_cosn = "paimon_base_filesystem_paimon_cosn" - String catalog_oss = "paimon_base_filesystem_paimon_oss" - String catalog_obs = "paimon_base_filesystem_paimon_obs" String aliYunAk = context.config.otherConfigs.get("aliYunAk") String aliYunSk = context.config.otherConfigs.get("aliYunSk") - String txYunAk = context.config.otherConfigs.get("txYunAk") - String txYunSk = context.config.otherConfigs.get("txYunSk") String hwYunAk = context.config.otherConfigs.get("hwYunAk") String hwYunSk = context.config.otherConfigs.get("hwYunSk") + String txYunAk = context.config.otherConfigs.get("txYunAk") + String txYunSk = context.config.otherConfigs.get("txYunSk") - def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cosn}.zd.all_table order by c18""" - def cosn = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cosn}.zd.all_table order by c18""" - def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a""" - def obs = """select * from ${catalog_obs}.piamon_obs_db.paimon_table_obs order by k""" + def obs = """select * from ${catalog_obs}.paimon_obs_db.paimon_obs_table order by k""" + def oss = """select * from ${catalog_oss}.paimon_oss_db.paimon_oss_table order by k""" + def cos = """select * from ${catalog_cos}.paimon_cos_db.paimon_cos_table order by k""" + def cosn = """select * from ${catalog_cosn}.paimon_cosn_db.paimon_cosn_table order by k""" + + sql """drop catalog if exists ${catalog_obs};""" + sql """drop catalog if exists ${catalog_oss};""" sql """drop catalog if exists ${catalog_cos};""" sql """drop catalog if exists ${catalog_cosn};""" - sql """drop catalog if exists ${catalog_oss};""" - sql """drop catalog if exists ${catalog_obs};""" sql """ create catalog if not exists ${catalog_cos} properties ( "type" = "paimon", - "warehouse" = "cos://doris-build-1308700295/regression/paimoncos", + "warehouse" = "cos://doris-build-1308700295/regression/paimon", "cos.access_key" = "${txYunAk}", "cos.secret_key" = "${txYunSk}", "cos.endpoint" = "cos.ap-beijing.myqcloud.com" @@ -56,16 +57,16 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock sql """ create catalog if not exists ${catalog_cosn} properties ( "type" = "paimon", - "warehouse" = "cosn://doris-build-1308700295/regression/paimoncos", - "cos.access_key" = "${txYunAk}", - "cos.secret_key" = "${txYunSk}", - "cos.endpoint" = "cos.ap-beijing.myqcloud.com" + "warehouse" = "cosn://doris-build-1308700295/regression/paimon", + "cosn.access_key" = "${txYunAk}", + "cosn.secret_key" = "${txYunSk}", + "cosn.endpoint" = "cos.ap-beijing.myqcloud.com" ); """ sql """ create catalog if not exists ${catalog_oss} properties ( "type" = "paimon", - "warehouse" = "oss://doris-regression-bj/regression/paimonoss", + "warehouse" = "oss://doris-regression-bj/regression/paimon", "oss.access_key"="${aliYunAk}", "oss.secret_key"="${aliYunSk}", "oss.endpoint"="oss-cn-beijing.aliyuncs.com" @@ -80,22 +81,22 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com" ); """ + logger.info("catalog " + catalog_obs + " created") + logger.info("catalog " + catalog_oss + " created") logger.info("catalog " + catalog_cos + " created") logger.info("catalog " + catalog_cosn + " created") - logger.info("catalog " + catalog_oss + " created") - logger.info("catalog " + catalog_obs + " created") sql """set force_jni_scanner=false""" - qt_c1 cos - qt_c2 cosn - qt_c3 oss - qt_c4 obs + qt_oss oss + qt_obs obs + qt_cos cos + qt_cosn cosn sql """set force_jni_scanner=true""" - qt_c5 cos - qt_c6 cosn - qt_c7 oss - qt_c8 obs + qt_oss oss + qt_obs obs + qt_cos cos + qt_cosn cosn } finally { sql """set force_jni_scanner=false""" From 7174799dbe01c33f94bd4383e9f6d5d7caa318df Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 12 Dec 2024 00:00:06 +0800 Subject: [PATCH 4/6] finish obs --- .../paimon/paimon_base_filesystem.out | 26 +++++----- .../paimon/paimon_base_filesystem.groovy | 47 ++++++++++--------- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out index d24ebd07ef6b16..71102df6af1266 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out +++ b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out @@ -1,17 +1,19 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !c1 -- -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768 -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364 +-- !oss -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 --- !c2 -- -7 Hugo -8 Stop +-- !obs -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 --- !c3 -- -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768 -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364 +-- !cos -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 --- !c4 -- -7 Hugo -8 Stop +-- !oss -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !obs -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !cos -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index ae49640f23e609..05285c777ee909 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -34,11 +34,10 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock String txYunAk = context.config.otherConfigs.get("txYunAk") String txYunSk = context.config.otherConfigs.get("txYunSk") - def obs = """select * from ${catalog_obs}.paimon_obs_db.paimon_obs_table order by k""" - def oss = """select * from ${catalog_oss}.paimon_oss_db.paimon_oss_table order by k""" - def cos = """select * from ${catalog_cos}.paimon_cos_db.paimon_cos_table order by k""" - def cosn = """select * from ${catalog_cosn}.paimon_cosn_db.paimon_cosn_table order by k""" - + def obs = """select * from ${catalog_obs}.db1.all_table limit 1;""" + def oss = """select * from ${catalog_oss}.db1.all_table limit 1;""" + def cos = """select * from ${catalog_cos}.db1.all_table limit 1;""" + def cosn = """select * from ${catalog_cosn}.db1.all_table limit 1;""" sql """drop catalog if exists ${catalog_obs};""" sql """drop catalog if exists ${catalog_oss};""" @@ -48,25 +47,28 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock sql """ create catalog if not exists ${catalog_cos} properties ( "type" = "paimon", - "warehouse" = "cos://doris-build-1308700295/regression/paimon", - "cos.access_key" = "${txYunAk}", - "cos.secret_key" = "${txYunSk}", - "cos.endpoint" = "cos.ap-beijing.myqcloud.com" - ); - """ - sql """ - create catalog if not exists ${catalog_cosn} properties ( - "type" = "paimon", - "warehouse" = "cosn://doris-build-1308700295/regression/paimon", - "cosn.access_key" = "${txYunAk}", - "cosn.secret_key" = "${txYunSk}", - "cosn.endpoint" = "cos.ap-beijing.myqcloud.com" + "paimon.catalog.type"="filesystem", + "warehouse" = "s3://doris-build-1308700295/regression/paimon1", + "s3.access_key" = "${txYunAk}", + "s3.secret_key" = "${txYunSk}", + "s3.endpoint" = "cos.ap-beijing.myqcloud.com" ); """ + // sql """ + // create catalog if not exists ${catalog_cosn} properties ( + // "type" = "paimon", + // "paimon.catalog.type"="filesystem", + // "warehouse" = "cosn://doris-build-1308700295/regression/paimon1", + // "cosn.access_key" = "${txYunAk}", + // "cosn.secret_key" = "${txYunSk}", + // "cosn.endpoint" = "cos.ap-beijing.myqcloud.com" + // ); + // """ sql """ create catalog if not exists ${catalog_oss} properties ( "type" = "paimon", - "warehouse" = "oss://doris-regression-bj/regression/paimon", + "paimon.catalog.type"="filesystem", + "warehouse" = "oss://doris-regression-bj/regression/paimon1", "oss.access_key"="${aliYunAk}", "oss.secret_key"="${aliYunSk}", "oss.endpoint"="oss-cn-beijing.aliyuncs.com" @@ -75,7 +77,8 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock sql """ create catalog if not exists ${catalog_obs} properties ( "type" = "paimon", - "warehouse" = "obs://doris-build/piamon", + "paimon.catalog.type"="filesystem", + "warehouse" = "obs://doris-build/regression/paimon1", "obs.access_key"="${hwYunAk}", "obs.secret_key"="${hwYunSk}", "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com" @@ -90,13 +93,13 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock qt_oss oss qt_obs obs qt_cos cos - qt_cosn cosn + // qt_cosn cosn sql """set force_jni_scanner=true""" qt_oss oss qt_obs obs qt_cos cos - qt_cosn cosn + // qt_cosn cosn } finally { sql """set force_jni_scanner=false""" From fa3ec084d6e3934f4e98c4cf13caa8fbcc27ddaa Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 12 Dec 2024 10:38:37 +0800 Subject: [PATCH 5/6] fix cosn --- .../paimon/paimon_base_filesystem.out | 3 +++ .../paimon/paimon_base_filesystem.groovy | 23 ++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out index 71102df6af1266..6ce16ab2a38a2d 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out +++ b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out @@ -8,6 +8,9 @@ -- !cos -- 1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 +-- !cosn -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + -- !oss -- 1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index 05285c777ee909..156314c5ccab81 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -54,16 +54,16 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock "s3.endpoint" = "cos.ap-beijing.myqcloud.com" ); """ - // sql """ - // create catalog if not exists ${catalog_cosn} properties ( - // "type" = "paimon", - // "paimon.catalog.type"="filesystem", - // "warehouse" = "cosn://doris-build-1308700295/regression/paimon1", - // "cosn.access_key" = "${txYunAk}", - // "cosn.secret_key" = "${txYunSk}", - // "cosn.endpoint" = "cos.ap-beijing.myqcloud.com" - // ); - // """ + sql """ + create catalog if not exists ${catalog_cosn} properties ( + "type" = "paimon", + "paimon.catalog.type"="filesystem", + "warehouse" = "cosn://doris-build-1308700295/regression/paimon1", + "s3.access_key" = "${txYunAk}", + "s3.secret_key" = "${txYunSk}", + "s3.endpoint" = "cos.ap-beijing.myqcloud.com" + ); + """ sql """ create catalog if not exists ${catalog_oss} properties ( "type" = "paimon", @@ -93,12 +93,13 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock qt_oss oss qt_obs obs qt_cos cos - // qt_cosn cosn + qt_cosn cosn sql """set force_jni_scanner=true""" qt_oss oss qt_obs obs qt_cos cos + // java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.CosFileSystem not found // qt_cosn cosn } finally { From 0a4bd4b2cb1e4ecd43804c0e46688347c35fc2a9 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 12 Dec 2024 15:38:43 +0800 Subject: [PATCH 6/6] fix --- .../external_table_p0/paimon/paimon_base_filesystem.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index 156314c5ccab81..6ac76336986288 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -59,9 +59,9 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock "type" = "paimon", "paimon.catalog.type"="filesystem", "warehouse" = "cosn://doris-build-1308700295/regression/paimon1", - "s3.access_key" = "${txYunAk}", - "s3.secret_key" = "${txYunSk}", - "s3.endpoint" = "cos.ap-beijing.myqcloud.com" + "cos.access_key" = "${txYunAk}", + "cos.secret_key" = "${txYunSk}", + "cos.endpoint" = "cos.ap-beijing.myqcloud.com" ); """ sql """