From 27f82d260d84d3e6a6a422707a7d67d12ad34336 Mon Sep 17 00:00:00 2001 From: lsy3993 Date: Thu, 13 Jun 2024 15:36:54 +0800 Subject: [PATCH 1/4] fix 2.0 insert overwrite --- .../table-sync/test_add_partition.groovy | 16 +- .../table-sync/test_insert_overwrite.groovy | 318 +++++++++--------- 2 files changed, 168 insertions(+), 166 deletions(-) diff --git a/regression-test/suites/table-sync/test_add_partition.groovy b/regression-test/suites/table-sync/test_add_partition.groovy index 9897aa31..7e992e29 100644 --- a/regression-test/suites/table-sync/test_add_partition.groovy +++ b/regression-test/suites/table-sync/test_add_partition.groovy @@ -261,13 +261,13 @@ suite("test_add_partition") { assertTrue(checkRestoreFinishTimesOf("${tableName}", 60)) - sql """ - INSERT OVERWRITE TABLE ${tableName} VALUES (1, 100); - """ + // sql """ + // INSERT OVERWRITE TABLE ${tableName} VALUES (1, 100); + // """ - assertTrue(checkShowTimesOf(""" - SELECT * FROM ${tableName} - WHERE id = 100 - """, - exist, 60, "target")) + // assertTrue(checkShowTimesOf(""" + // SELECT * FROM ${tableName} + // WHERE id = 100 + // """, + // exist, 60, "target")) } diff --git a/regression-test/suites/table-sync/test_insert_overwrite.groovy b/regression-test/suites/table-sync/test_insert_overwrite.groovy index 54d0f2a6..f1a84e6f 100644 --- a/regression-test/suites/table-sync/test_insert_overwrite.groovy +++ b/regression-test/suites/table-sync/test_insert_overwrite.groovy @@ -24,162 +24,164 @@ suite("test_insert_overwrite") { // 1. create temp partitions // 2. insert int temp partitions // 3. replace overlap partitions - def tableName = "tbl_insert_overwrite_" + UUID.randomUUID().toString().replace("-", "") - def uniqueTable = "${tableName}_unique" - def syncerAddress = "127.0.0.1:9190" - def test_num = 0 - def insert_num = 5 - def sync_gap_time = 5000 - String response - - def checkSelectTimesOf = { sqlString, rowSize, times -> Boolean - def tmpRes = target_sql "${sqlString}" - while (tmpRes.size() != rowSize) { - sleep(sync_gap_time) - if (--times > 0) { - tmpRes = target_sql "${sqlString}" - } else { - break - } - } - return tmpRes.size() == rowSize - } - - def checkRestoreFinishTimesOf = { checkTable, times -> Boolean - Boolean ret = false - while (times > 0) { - def sqlInfo = target_sql "SHOW RESTORE FROM TEST_${context.dbName}" - for (List row : sqlInfo) { - if ((row[10] as String).contains(checkTable)) { - ret = row[4] == "FINISHED" - } - } - - if (ret) { - break - } else if (--times > 0) { - sleep(sync_gap_time) - } - } - - return ret - } - - def checkBackupFinishTimesOf = { checkTable, times -> Boolean - Boolean ret = false - while (times > 0) { - def sqlInfo = sql "SHOW BACKUP FROM ${context.dbName}" - for (List row : sqlInfo) { - if ((row[4] as String).contains(checkTable)) { - ret = row[3] == "FINISHED" - } - } - - if (ret) { - break - } else if (--times > 0) { - sleep(sync_gap_time) - } - } - - return ret - } - - def checkData = { data, beginCol, value -> Boolean - if (data.size() < beginCol + value.size()) { - return false - } - - for (int i = 0; i < value.size(); ++i) { - if ((data[beginCol + i] as int) != value[i]) { - return false - } - } - - return true - } - - sql """ - CREATE TABLE if NOT EXISTS ${uniqueTable} - ( - `test` INT, - `id` INT - ) - ENGINE=OLAP - UNIQUE KEY(`test`, `id`) - PARTITION BY RANGE(id) - ( - PARTITION `p1` VALUES LESS THAN ("100"), - PARTITION `p2` VALUES LESS THAN ("200") - ) - DISTRIBUTED BY HASH(id) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "binlog.enable" = "true", - "binlog.ttl_seconds" = "180" - ) - """ - - sql """ - INSERT INTO ${uniqueTable} VALUES - (1, 0), - (1, 1), - (1, 2), - (1, 3), - (1, 4) - """ - sql "sync" - - // test 1: target cluster follow source cluster - logger.info("=== Test 1: backup/restore case ===") - httpTest { - uri "/create_ccr" - endpoint syncerAddress - def bodyJson = get_ccr_body "${uniqueTable}" - body "${bodyJson}" - op "post" - result response - } - assertTrue(checkRestoreFinishTimesOf("${uniqueTable}", 60)) - assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable} WHERE test = 1 ORDER BY id", 5, 60)) - qt_sql "SELECT * FROM ${uniqueTable} WHERE test = 1 ORDER BY id" - qt_target_sql "SELECT * FROM ${uniqueTable} WHERE test = 1 ORDER BY id" - - logger.info("=== Test 2: dest cluster follow source cluster case ===") - - sql """ - INSERT INTO ${uniqueTable} VALUES - (2, 0), - (2, 1), - (2, 2), - (2, 3), - (2, 4) - """ - sql "sync" - assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable} WHERE test=2", 5, 60)) - qt_sql "SELECT * FROM ${uniqueTable} WHERE test=2 ORDER BY id" - qt_target_sql "SELECT * FROM ${uniqueTable} WHERE test=2 ORDER BY id" - - logger.info("=== Test 3: insert overwrite source table ===") - - sql """ - INSERT OVERWRITE TABLE ${uniqueTable} VALUES - (3, 0), - (3, 1), - (3, 2), - (3, 3), - (3, 4) - """ - sql "sync" - - sleep(10000) - assertTrue(checkBackupFinishTimesOf("${uniqueTable}", 60)) - sleep(10000) - assertTrue(checkRestoreFinishTimesOf("${uniqueTable}", 60)) - - assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable} WHERE test=3", 5, 60)) - assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable}", 5, 60)) - - qt_sql "SELECT * FROM ${uniqueTable} ORDER BY test, id" - qt_target_sql "SELECT * FROM ${uniqueTable} ORDER BY test, id" + + + // def tableName = "tbl_insert_overwrite_" + UUID.randomUUID().toString().replace("-", "") + // def uniqueTable = "${tableName}_unique" + // def syncerAddress = "127.0.0.1:9190" + // def test_num = 0 + // def insert_num = 5 + // def sync_gap_time = 5000 + // String response + + // def checkSelectTimesOf = { sqlString, rowSize, times -> Boolean + // def tmpRes = target_sql "${sqlString}" + // while (tmpRes.size() != rowSize) { + // sleep(sync_gap_time) + // if (--times > 0) { + // tmpRes = target_sql "${sqlString}" + // } else { + // break + // } + // } + // return tmpRes.size() == rowSize + // } + + // def checkRestoreFinishTimesOf = { checkTable, times -> Boolean + // Boolean ret = false + // while (times > 0) { + // def sqlInfo = target_sql "SHOW RESTORE FROM TEST_${context.dbName}" + // for (List row : sqlInfo) { + // if ((row[10] as String).contains(checkTable)) { + // ret = row[4] == "FINISHED" + // } + // } + + // if (ret) { + // break + // } else if (--times > 0) { + // sleep(sync_gap_time) + // } + // } + + // return ret + // } + + // def checkBackupFinishTimesOf = { checkTable, times -> Boolean + // Boolean ret = false + // while (times > 0) { + // def sqlInfo = sql "SHOW BACKUP FROM ${context.dbName}" + // for (List row : sqlInfo) { + // if ((row[4] as String).contains(checkTable)) { + // ret = row[3] == "FINISHED" + // } + // } + + // if (ret) { + // break + // } else if (--times > 0) { + // sleep(sync_gap_time) + // } + // } + + // return ret + // } + + // def checkData = { data, beginCol, value -> Boolean + // if (data.size() < beginCol + value.size()) { + // return false + // } + + // for (int i = 0; i < value.size(); ++i) { + // if ((data[beginCol + i] as int) != value[i]) { + // return false + // } + // } + + // return true + // } + + // sql """ + // CREATE TABLE if NOT EXISTS ${uniqueTable} + // ( + // `test` INT, + // `id` INT + // ) + // ENGINE=OLAP + // UNIQUE KEY(`test`, `id`) + // PARTITION BY RANGE(id) + // ( + // PARTITION `p1` VALUES LESS THAN ("100"), + // PARTITION `p2` VALUES LESS THAN ("200") + // ) + // DISTRIBUTED BY HASH(id) BUCKETS 1 + // PROPERTIES ( + // "replication_allocation" = "tag.location.default: 1", + // "binlog.enable" = "true", + // "binlog.ttl_seconds" = "180" + // ) + // """ + + // sql """ + // INSERT INTO ${uniqueTable} VALUES + // (1, 0), + // (1, 1), + // (1, 2), + // (1, 3), + // (1, 4) + // """ + // sql "sync" + + // // test 1: target cluster follow source cluster + // logger.info("=== Test 1: backup/restore case ===") + // httpTest { + // uri "/create_ccr" + // endpoint syncerAddress + // def bodyJson = get_ccr_body "${uniqueTable}" + // body "${bodyJson}" + // op "post" + // result response + // } + // assertTrue(checkRestoreFinishTimesOf("${uniqueTable}", 60)) + // assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable} WHERE test = 1 ORDER BY id", 5, 60)) + // qt_sql "SELECT * FROM ${uniqueTable} WHERE test = 1 ORDER BY id" + // qt_target_sql "SELECT * FROM ${uniqueTable} WHERE test = 1 ORDER BY id" + + // logger.info("=== Test 2: dest cluster follow source cluster case ===") + + // sql """ + // INSERT INTO ${uniqueTable} VALUES + // (2, 0), + // (2, 1), + // (2, 2), + // (2, 3), + // (2, 4) + // """ + // sql "sync" + // assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable} WHERE test=2", 5, 60)) + // qt_sql "SELECT * FROM ${uniqueTable} WHERE test=2 ORDER BY id" + // qt_target_sql "SELECT * FROM ${uniqueTable} WHERE test=2 ORDER BY id" + + // logger.info("=== Test 3: insert overwrite source table ===") + + // sql """ + // INSERT OVERWRITE TABLE ${uniqueTable} VALUES + // (3, 0), + // (3, 1), + // (3, 2), + // (3, 3), + // (3, 4) + // """ + // sql "sync" + + // sleep(10000) + // assertTrue(checkBackupFinishTimesOf("${uniqueTable}", 60)) + // sleep(10000) + // assertTrue(checkRestoreFinishTimesOf("${uniqueTable}", 60)) + + // assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable} WHERE test=3", 5, 60)) + // assertTrue(checkSelectTimesOf("SELECT * FROM ${uniqueTable}", 5, 60)) + + // qt_sql "SELECT * FROM ${uniqueTable} ORDER BY test, id" + // qt_target_sql "SELECT * FROM ${uniqueTable} ORDER BY test, id" } From 1c33216bd1b6e6d8157f446fe370da546d60ac1e Mon Sep 17 00:00:00 2001 From: lsy3993 Date: Thu, 13 Jun 2024 15:39:25 +0800 Subject: [PATCH 2/4] add not null --- regression-test/suites/table-sync/test_add_partition.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/table-sync/test_add_partition.groovy b/regression-test/suites/table-sync/test_add_partition.groovy index 7e992e29..b81b2692 100644 --- a/regression-test/suites/table-sync/test_add_partition.groovy +++ b/regression-test/suites/table-sync/test_add_partition.groovy @@ -139,8 +139,8 @@ suite("test_add_partition") { sql """ CREATE TABLE if NOT EXISTS ${tableName} ( - `test` INT, - `id` INT + `test` INT NOT NULL, + `id` INT NOT NULL ) ENGINE=OLAP UNIQUE KEY(`test`, `id`) From aa6e823992490fdff3fa8ba23574ce4da6a3071e Mon Sep 17 00:00:00 2001 From: walter Date: Thu, 13 Jun 2024 15:46:58 +0800 Subject: [PATCH 3/4] Update regression-test/suites/table-sync/test_insert_overwrite.groovy --- regression-test/suites/table-sync/test_insert_overwrite.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/table-sync/test_insert_overwrite.groovy b/regression-test/suites/table-sync/test_insert_overwrite.groovy index f1a84e6f..afb83bc4 100644 --- a/regression-test/suites/table-sync/test_insert_overwrite.groovy +++ b/regression-test/suites/table-sync/test_insert_overwrite.groovy @@ -26,6 +26,7 @@ suite("test_insert_overwrite") { // 3. replace overlap partitions + // INSERT OVERWRITE is not supported by branch-2.0 table sync. // def tableName = "tbl_insert_overwrite_" + UUID.randomUUID().toString().replace("-", "") // def uniqueTable = "${tableName}_unique" // def syncerAddress = "127.0.0.1:9190" From e9c1174bec5e65bf4f79f8d645fd0c7c8bad30df Mon Sep 17 00:00:00 2001 From: walter Date: Thu, 13 Jun 2024 15:47:06 +0800 Subject: [PATCH 4/4] Update regression-test/suites/table-sync/test_add_partition.groovy --- regression-test/suites/table-sync/test_add_partition.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/table-sync/test_add_partition.groovy b/regression-test/suites/table-sync/test_add_partition.groovy index b81b2692..25fb986f 100644 --- a/regression-test/suites/table-sync/test_add_partition.groovy +++ b/regression-test/suites/table-sync/test_add_partition.groovy @@ -261,6 +261,7 @@ suite("test_add_partition") { assertTrue(checkRestoreFinishTimesOf("${tableName}", 60)) + // INSERT OVERWRITE is not supported by branch-2.0 table sync. // sql """ // INSERT OVERWRITE TABLE ${tableName} VALUES (1, 100); // """