Reorg db/table sync suites (#219)

Some requirements for adding suites 1. Each suite should be place in a unique dir, to avoid backup/restore job limit per db 2. Some idom 1. Kind: 1. cross_ds/ts: cross-feature related suites 2. db/table_sync: db or table sync related suites 3. table_sync_alias: table sync with alias related suites 4. db/table_ps_sync: db or table partial sync related suites 5. syncer: suites for ccr syncer itself 2. Resource: column,view,table ... 3. Operation: add,alter,drop,rename ... 3. The suite is organized in the directory hierarchy of Kind/Resource/Operation, and Resource is optional
selectdb · Nov 6, 2024 · a134ed7 · a134ed7
1 parent a4d7b01
commit a134ed7
Show file tree

Hide file tree

Showing 66 changed files with 443 additions and 276 deletions.
diff --git a/regression-test/common/helper.groovy b/regression-test/common/helper.groovy
@@ -15,6 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import com.google.common.collect.Maps
+
+import java.util.Map
+
 class Helper {
     def suite
     def context
@@ -31,11 +35,40 @@ class Helper {
     }
 
     String randomSuffix() {
-        return UUID.randomUUID().toString().replace("-", "")
+        def hashCode = UUID.randomUUID().toString().replace("-", "").hashCode()
+        if (hashCode < 0) {
+            hashCode *= -1;
+        }
+        return Integer.toString(hashCode)
+    }
+
+    def get_ccr_body(String table, String db = null) {
+        if (db == null) {
+            db = context.dbName
+        }
+
+        def gson = new com.google.gson.Gson()
+
+        Map<String, String> srcSpec = context.getSrcSpec(db)
+        srcSpec.put("table", table)
+
+        Map<String, String> destSpec = context.getDestSpec(db)
+        destSpec.put("table", table)
+
+        Map<String, Object> body = Maps.newHashMap()
+        String name = context.suiteName
+        if (!table.equals("")) {
+            name = name + "_" + table
+        }
+        body.put("name", name)
+        body.put("src", srcSpec)
+        body.put("dest", destSpec)
+
+        return gson.toJson(body)
     }
 
     void ccrJobDelete(table = "") {
-        def bodyJson = suite.get_ccr_body "${table}"
+        def bodyJson = get_ccr_body "${table}"
         suite.httpTest {
             uri "/delete"
             endpoint syncerAddress
@@ -45,7 +78,7 @@ class Helper {
     }
 
     void ccrJobCreate(table = "") {
-        def bodyJson = suite.get_ccr_body "${table}"
+        def bodyJson = get_ccr_body "${table}"
         suite.httpTest {
             uri "/create_ccr"
             endpoint syncerAddress
@@ -55,7 +88,7 @@ class Helper {
     }
 
     void ccrJobCreateAllowTableExists(table = "") {
-        def bodyJson = suite.get_ccr_body "${table}"
+        def bodyJson = get_ccr_body "${table}"
         def jsonSlurper = new groovy.json.JsonSlurper()
         def object = jsonSlurper.parseText "${bodyJson}"
         object['allow_table_exists'] = true
@@ -71,7 +104,7 @@ class Helper {
     }
 
     void ccrJobPause(table = "") {
-        def bodyJson = suite.get_ccr_body "${table}"
+        def bodyJson = get_ccr_body "${table}"
         suite.httpTest {
             uri "/pause"
             endpoint syncerAddress
@@ -81,7 +114,7 @@ class Helper {
     }
 
     void ccrJobResume(table = "") {
-        def bodyJson = suite.get_ccr_body "${table}"
+        def bodyJson = get_ccr_body "${table}"
         suite.httpTest {
             uri "/resume"
             endpoint syncerAddress
@@ -91,7 +124,7 @@ class Helper {
     }
 
     void ccrJobDesync(table = "") {
-        def bodyJson = suite.get_ccr_body "${table}"
+        def bodyJson = get_ccr_body "${table}"
         suite.httpTest {
             uri "/desync"
             endpoint syncerAddress
@@ -221,7 +254,7 @@ class Helper {
     }
 
     void force_fullsync(tableName = "") {
-        def bodyJson = suite.get_ccr_body "${tableName}"
+        def bodyJson = get_ccr_body "${tableName}"
         suite.httpTest {
             uri "/force_fullsync"
             endpoint syncerAddress
@@ -231,7 +264,7 @@ class Helper {
     }
 
     Object get_job_progress(tableName = "") {
-        def request_body = suite.get_ccr_body(tableName)
+        def request_body = get_ccr_body(tableName)
         def get_job_progress_uri = { check_func ->
             suite.httpTest {
                 uri "/job_progress"

diff --git a/regression-test/data/ccr_user_sync/test_common_sync.out b/regression-test/data/ccr_user_sync/test_common_sync.out
diff --git a/regression-test/data/usercases/cir_8537.out → .../partition/drop_1/test_ds_part_drop_1.out b/regression-test/data/usercases/cir_8537.out → .../partition/drop_1/test_ds_part_drop_1.out
diff --git a/...data/table-sync/test_insert_overwrite.out → ...verwrite/test_ts_dml_insert_overwrite.out b/...data/table-sync/test_insert_overwrite.out → ...verwrite/test_ts_dml_insert_overwrite.out
diff --git a/...t/data/table-sync/test_inverted_index.out → ...rted_idx/test_ts_tbl_res_inverted_idx.out b/...t/data/table-sync/test_inverted_index.out → ...rted_idx/test_ts_tbl_res_inverted_idx.out
diff --git a/...s/test_db_sync_fullsync_with_alias.groovy → ...alias/test_cds_fullsync_with_alias.groovy b/...s/test_db_sync_fullsync_with_alias.groovy → ...alias/test_cds_fullsync_with_alias.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_db_sync_fullsync_with_alias") {
+suite("test_cds_fullsync_with_alias") {
     def helper = new GroovyShell(new Binding(['suite': delegate]))
             .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy"))
 

diff --git a/...test_db_sync_signature_not_matched.groovy → ...hed/test_cds_signature_not_matched.groovy b/...test_db_sync_signature_not_matched.groovy → ...hed/test_cds_signature_not_matched.groovy
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_db_sync_signature_not_matched") {
+suite("test_cds_signature_not_matched") {
     def helper = new GroovyShell(new Binding(['suite': delegate]))
             .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy"))
 
-    def tableName = "tbl_db_sync_sig_not_matched_" + helper.randomSuffix()
+    def tableName = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 20
     def opPartitonName = "less"

diff --git a/.../suites/db-sv/test_sync_view_twice.groovy → ...iew_twice/test_cds_sync_view_twice.groovy b/.../suites/db-sv/test_sync_view_twice.groovy → ...iew_twice/test_cds_sync_view_twice.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_sync_view_twice") {
+suite("test_cds_sync_view_twice") {
     def versions = sql_return_maparray "show variables like 'version_comment'"
     if (versions[0].Value.contains('doris-2.0.') || versions[0].Value.contains('doris-2.1.')) {
         logger.info("2.0/2.1 not support this case, current version is: ${versions[0].Value}")
@@ -96,3 +96,4 @@ suite("test_sync_view_twice") {
     def view_size = target_sql "SHOW VIEW FROM ${tableDuplicate0}"
     assertTrue(view_size.size() == 1);
 }
+
diff --git a/...uites/table-sync/test_keyword_name.groovy → ...s_ts/keyword_name/test_cts_keyword.groovy b/...uites/table-sync/test_keyword_name.groovy → ...s_ts/keyword_name/test_cts_keyword.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_keyword_name") {
+suite("test_cts_keyword_name") {
     def helper = new GroovyShell(new Binding(['suite': delegate]))
             .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy"))
 

diff --git a/..._db_partial_sync_inc_add_partition.groovy → ..._db_partial_sync_inc_add_partition.groovy b/..._db_partial_sync_inc_add_partition.groovy → ..._db_partial_sync_inc_add_partition.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_add_partition") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...ter/test_db_partial_sync_inc_alter.groovy → ...ter/test_db_partial_sync_inc_alter.groovy b/...ter/test_db_partial_sync_inc_alter.groovy → ...ter/test_db_partial_sync_inc_alter.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_alter") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...c-cache/test_db_partial_sync_cache.groovy → ...c/cache/test_db_partial_sync_cache.groovy b/...c-cache/test_db_partial_sync_cache.groovy → ...c/cache/test_db_partial_sync_cache.groovy
@@ -18,7 +18,7 @@ suite("test_db_partial_sync_cache") {
     def helper = new GroovyShell(new Binding(['suite': delegate]))
             .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy"))
 
-    def tableName = "tbl_partial_sync_cache_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 
@@ -38,34 +38,6 @@ suite("test_db_partial_sync_cache") {
         return object.name
     }
 
-    def get_job_progress = { ccr_name ->
-        def request_body = """ {"name":"${ccr_name}"} """
-        def get_job_progress_uri = { check_func ->
-            httpTest {
-                uri "/job_progress"
-                endpoint helper.syncerAddress
-                body request_body
-                op "post"
-                check check_func
-            }
-        }
-
-        def result = null
-        get_job_progress_uri.call() { code, body ->
-            if (!"${code}".toString().equals("200")) {
-                throw "request failed, code: ${code}, body: ${body}"
-            }
-            def jsonSlurper = new groovy.json.JsonSlurper()
-            def object = jsonSlurper.parseText "${body}"
-            if (!object.success) {
-                throw "request failed, error msg: ${object.error_msg}"
-            }
-            logger.info("job progress: ${object.job_progress}")
-            result = jsonSlurper.parseText object.job_progress
-        }
-        return result
-    }
-
     helper.enableDbBinlog()
     sql "DROP TABLE IF EXISTS ${tableName}"
     sql """
@@ -93,15 +65,12 @@ suite("test_db_partial_sync_cache") {
         """
     sql "sync"
 
-    def bodyJson = get_ccr_body ""
-    ccr_name = get_ccr_name(bodyJson)
     helper.ccrJobCreate()
-    logger.info("ccr job name: ${ccr_name}")
 
     assertTrue(helper.checkRestoreFinishTimesOf("${tableName}", 30))
     assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableName}", insert_num, 60))
 
-    first_job_progress = get_job_progress(ccr_name)
+    first_job_progress = helper.get_job_progress()
 
     logger.info("=== Test 1: add first column case ===")
     // binlog type: ALTER_JOB, binlog data:
@@ -140,7 +109,7 @@ suite("test_db_partial_sync_cache") {
     assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableName}", insert_num + 1, 60))
 
     // no full sync triggered.
-    last_job_progress = get_job_progress(ccr_name)
+    last_job_progress = helper.get_job_progress()
     assertTrue(last_job_progress.full_sync_start_at == first_job_progress.full_sync_start_at)
 }
 
diff --git a/...db_partial_sync_inc_drop_partition.groovy → ...db_partial_sync_inc_drop_partition.groovy b/...db_partial_sync_inc_drop_partition.groovy → ...db_partial_sync_inc_drop_partition.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_drop_partition") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...t_db_partial_sync_inc_lightning_sc.groovy → ...t_db_partial_sync_inc_lightning_sc.groovy b/...t_db_partial_sync_inc_lightning_sc.groovy → ...t_db_partial_sync_inc_lightning_sc.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_lightning_sc") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...c/merge/test_db_partial_sync_merge.groovy → ...c/merge/test_db_partial_sync_merge.groovy b/...c/merge/test_db_partial_sync_merge.groovy → ...c/merge/test_db_partial_sync_merge.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_merge") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...partial_sync_inc_replace_partition.groovy → ...partial_sync_inc_replace_partition.groovy b/...partial_sync_inc_replace_partition.groovy → ...partial_sync_inc_replace_partition.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_replace_partition") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...st_db_partial_sync_inc_trunc_table.groovy → ...st_db_partial_sync_inc_trunc_table.groovy b/...st_db_partial_sync_inc_trunc_table.groovy → ...st_db_partial_sync_inc_trunc_table.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_trunc_table") {
         return
     }
 
-    def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5
 

diff --git a/...rt/test_db_partial_sync_inc_upsert.groovy → ...rt/test_db_partial_sync_inc_upsert.groovy b/...rt/test_db_partial_sync_inc_upsert.groovy → ...rt/test_db_partial_sync_inc_upsert.groovy
@@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_upsert") {
         return
     }
 
-    def tableName = "tbl_sync_incremental_" + UUID.randomUUID().toString().replace("-", "")
-    def tableName1 = "tbl_sync_incremental_1_" + UUID.randomUUID().toString().replace("-", "")
+    def tableName = "tbl_" + helper.randomSuffix()
+    def tableName1 = "tbl_" + helper.randomSuffix()
     def test_num = 0
     def insert_num = 5