From 3a0ca8b4bbf8a491eeb995e45dd1d9ae7338af7b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 15:12:20 +0800 Subject: [PATCH 1/9] remove some useless code. --- tests/integration_tests/cdc/cdc.go | 11 ------- tests/integration_tests/util/db.go | 52 ------------------------------ 2 files changed, 63 deletions(-) diff --git a/tests/integration_tests/cdc/cdc.go b/tests/integration_tests/cdc/cdc.go index 1336d7f119f..7031101aa48 100644 --- a/tests/integration_tests/cdc/cdc.go +++ b/tests/integration_tests/cdc/cdc.go @@ -56,16 +56,5 @@ func main() { } }() - // TODO(dongmen): remove the useless code - sourceDBs, err := util.CreateSourceDBs() - if err != nil { - log.S().Fatal(err) - } - defer func() { - if err := util.CloseDBs(sourceDBs); err != nil { - log.S().Errorf("Failed to close source databases: %s\n", err) - } - }() - dailytest.Run(sourceDB, targetDB, cfg.SourceDBCfg[0].Name, cfg.WorkerCount, cfg.JobCount, cfg.Batch) } diff --git a/tests/integration_tests/util/db.go b/tests/integration_tests/util/db.go index f8db3de7415..2a65459c745 100644 --- a/tests/integration_tests/util/db.go +++ b/tests/integration_tests/util/db.go @@ -156,55 +156,3 @@ func MustExecWithConn(ctx context.Context, conn *sql.Conn, sql string, args ...i log.S().Fatal(err) } } - -// CreateSourceDBs return source sql.DB for test -// we create two TiDB instance now in tests/integration_tests/run.sh, change it if needed -func CreateSourceDBs() (dbs []*sql.DB, err error) { - cfg := DBConfig{ - Host: "127.0.0.1", - User: "root", - Password: "", - Name: "test", - Port: 4000, - } - - src1, err := CreateDB(cfg) - if err != nil { - return nil, errors.Trace(err) - } - - cfg.Port = 4001 - src2, err := CreateDB(cfg) - if err != nil { - return nil, errors.Trace(err) - } - - dbs = append(dbs, src1, src2) - return -} - -// CreateSourceDB return source sql.DB for test -func CreateSourceDB() (db *sql.DB, err error) { - cfg := DBConfig{ - Host: "127.0.0.1", - User: "root", - Password: "", - Name: "test", - Port: 4000, - } - - return CreateDB(cfg) -} - -// CreateSinkDB return sink sql.DB for test -func CreateSinkDB() (db *sql.DB, err error) { - cfg := DBConfig{ - Host: "127.0.0.1", - User: "root", - Password: "", - Name: "test", - Port: 3306, - } - - return CreateDB(cfg) -} From 20171c189c3ae56dad14dec5a74892393857ce67 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 15:18:41 +0800 Subject: [PATCH 2/9] only run the simple cases --- .../cdc/dailytest/dailytest.go | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/integration_tests/cdc/dailytest/dailytest.go b/tests/integration_tests/cdc/dailytest/dailytest.go index b6493962583..d7326039ac3 100644 --- a/tests/integration_tests/cdc/dailytest/dailytest.go +++ b/tests/integration_tests/cdc/dailytest/dailytest.go @@ -21,49 +21,49 @@ import ( // Run runs the daily test func Run(sourceDB *sql.DB, targetDB *sql.DB, schema string, workerCount int, jobCount int, batch int) { - TableSQLs := []string{ - ` - create table ptest( - a int primary key, - b double NOT NULL DEFAULT 2.0, - c varchar(10) NOT NULL, - d time unique - ); - `, - `create table itest( - a int, - b double NOT NULL DEFAULT 2.0, - c varchar(10) NOT NULL, - d time unique, - PRIMARY KEY(a, b) - ); - `, - `create table ntest( - a int, - b double NOT NULL DEFAULT 2.0, - c varchar(10) NOT NULL, - d time unique not null - ); - `, - } + //TableSQLs := []string{ + // ` + // create table ptest( + // a int primary key, + // b double NOT NULL DEFAULT 2.0, + // c varchar(10) NOT NULL, + // d time unique + // ); + // `, + // `create table itest( + // a int, + // b double NOT NULL DEFAULT 2.0, + // c varchar(10) NOT NULL, + // d time unique, + // PRIMARY KEY(a, b) + // ); + // `, + // `create table ntest( + // a int, + // b double NOT NULL DEFAULT 2.0, + // c varchar(10) NOT NULL, + // d time unique not null + // ); + // `, + //} // run the simple test case RunCase(sourceDB, targetDB, schema) - RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { - // generate insert/update/delete sqls and execute - RunDailyTest(sourceDB, TableSQLs, workerCount, jobCount, batch) - }) - - RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { - // truncate test data - TruncateTestTable(sourceDB, TableSQLs) - }) - - RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { - // drop test table - DropTestTable(sourceDB, TableSQLs) - }) + //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { + // // generate insert/update/delete sqls and execute + // RunDailyTest(sourceDB, TableSQLs, workerCount, jobCount, batch) + //}) + // + //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { + // // truncate test data + // TruncateTestTable(sourceDB, TableSQLs) + //}) + // + //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { + // // drop test table + // DropTestTable(sourceDB, TableSQLs) + //}) log.S().Info("test pass!!!") } From 2dbf114efa9300b693d94d325a2b2aa21fc5b56a Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 15:51:24 +0800 Subject: [PATCH 3/9] also add the finish mark --- tests/integration_tests/cdc/dailytest/case.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/cdc/dailytest/case.go b/tests/integration_tests/cdc/dailytest/case.go index 3dd44770ad7..350c6026045 100644 --- a/tests/integration_tests/cdc/dailytest/case.go +++ b/tests/integration_tests/cdc/dailytest/case.go @@ -191,6 +191,8 @@ func ineligibleTable(tr *testRunner, src *sql.DB, dst *sql.DB) { "insert into eligible_table (uk, ncol) values (2,2);", "ALTER TABLE eligible_table ADD COLUMN c1 INT NOT NULL;", "insert into eligible_table (uk, ncol, c1) values (3,4,5);", + + "CREATE TABLE finish_mark(id int primary key);", } // execute SQL but don't check for _, sql := range sqls { @@ -216,7 +218,7 @@ TestLoop: if synced { break TestLoop } - if tableName == "eligible_table" { + if tableName == "finish_mark" { synced = true } } @@ -227,6 +229,7 @@ TestLoop: "DROP TABLE ineligible_table1;", "DROP TABLE ineligible_table2;", "DROP TABLE eligible_table;", + "DROP TABLE finish_mark;", } tr.execSQLs(sqls) } From 8b85cfc040c36f049cdf932981f82670cc36c35c Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 17:04:39 +0800 Subject: [PATCH 4/9] add tests --- tests/integration_tests/default_value/main.go | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/integration_tests/default_value/main.go b/tests/integration_tests/default_value/main.go index b6b6dd6f4a2..bba57419cbe 100644 --- a/tests/integration_tests/default_value/main.go +++ b/tests/integration_tests/default_value/main.go @@ -74,15 +74,21 @@ func main() { log.S().Infof("DefaultValue integration tests take %v", time.Since(start)) }() wg.Add(2) - go testMultiDDLs([]*sql.DB{sourceDB0, sourceDB1}, &wg) - go testGetDefaultValue([]*sql.DB{sourceDB0, sourceDB1}, &wg) + + go func() { + defer wg.Done() + testMultiDDLs([]*sql.DB{sourceDB0, sourceDB1}) + }() + go func() { + defer wg.Done() + testGetDefaultValue([]*sql.DB{sourceDB0, sourceDB1}) + }() wg.Wait() util.MustExec(sourceDB0, "create table mark.finish_mark(a int primary key);") } // for every DDL, run the DDL continuously, and one goroutine for one TiDB instance to do some DML op -func testGetDefaultValue(srcs []*sql.DB, wg *sync.WaitGroup) { - defer wg.Done() +func testGetDefaultValue(srcs []*sql.DB) { start := time.Now() defer func() { log.S().Infof("testGetDefaultValue take %v", time.Since(start)) @@ -342,9 +348,7 @@ func ddlDefaultValueFunc(ctx context.Context, db *sql.DB, format string, table s } } -func testMultiDDLs(srcs []*sql.DB, wg *sync.WaitGroup) { - defer wg.Done() - +func testMultiDDLs(srcs []*sql.DB) { type Unit struct { Fmt string DefaultValue interface{} @@ -886,9 +890,3 @@ func mustCreateTable(db *sql.DB, tableName string) { sql := fmt.Sprintf(createTableSQL, tableName) util.MustExec(db, sql) } - -func mustCreateTableWithConn(ctx context.Context, conn *sql.Conn, tableName string) { - util.MustExecWithConn(ctx, conn, createDatabaseSQL) - sql := fmt.Sprintf(createTableSQL, tableName) - util.MustExecWithConn(ctx, conn, sql) -} From 754ec901661a78332b27e319dd32a4e88fda8878 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 17:34:13 +0800 Subject: [PATCH 5/9] hard code the tikv commit-hash to 8b006a5bdfae691c7286933ed8ab3b7063a2b76d --- scripts/download-integration-test-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download-integration-test-binaries.sh b/scripts/download-integration-test-binaries.sh index ec12db47193..4ab47b5c7bf 100755 --- a/scripts/download-integration-test-binaries.sh +++ b/scripts/download-integration-test-binaries.sh @@ -118,7 +118,7 @@ function download_binaries() { # Get sha1 based on branch name. tidb_sha1=$(curl "${file_server_url}/download/refs/pingcap/tidb/${branch}/sha1") - tikv_sha1=$(curl "${file_server_url}/download/refs/pingcap/tikv/${branch}/sha1") + tikv_sha1="8b006a5bdfae691c7286933ed8ab3b7063a2b76d" pd_sha1=$(curl "${file_server_url}/download/refs/pingcap/pd/${branch}/sha1") tiflash_sha1=$(curl "${file_server_url}/download/refs/pingcap/tiflash/${branch}/sha1") From e5b6f7403d1ddd1c6b342447a9ee8addb0f8fb9e Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 18:29:07 +0800 Subject: [PATCH 6/9] use 7.5.4 tikv --- scripts/download-integration-test-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download-integration-test-binaries.sh b/scripts/download-integration-test-binaries.sh index 4ab47b5c7bf..6703d34f64e 100755 --- a/scripts/download-integration-test-binaries.sh +++ b/scripts/download-integration-test-binaries.sh @@ -118,7 +118,7 @@ function download_binaries() { # Get sha1 based on branch name. tidb_sha1=$(curl "${file_server_url}/download/refs/pingcap/tidb/${branch}/sha1") - tikv_sha1="8b006a5bdfae691c7286933ed8ab3b7063a2b76d" + tikv_sha1="b4bddeeb995e7bedc1973ce9e856eeb2d856ce9b" pd_sha1=$(curl "${file_server_url}/download/refs/pingcap/pd/${branch}/sha1") tiflash_sha1=$(curl "${file_server_url}/download/refs/pingcap/tiflash/${branch}/sha1") From a5b07540e5e9ac77b5e3198ad1badd313afc822f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 17 Dec 2024 19:41:10 +0800 Subject: [PATCH 7/9] uncomment one more case --- .../cdc/dailytest/dailytest.go | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/integration_tests/cdc/dailytest/dailytest.go b/tests/integration_tests/cdc/dailytest/dailytest.go index d7326039ac3..0a4991f7820 100644 --- a/tests/integration_tests/cdc/dailytest/dailytest.go +++ b/tests/integration_tests/cdc/dailytest/dailytest.go @@ -21,39 +21,39 @@ import ( // Run runs the daily test func Run(sourceDB *sql.DB, targetDB *sql.DB, schema string, workerCount int, jobCount int, batch int) { - //TableSQLs := []string{ - // ` - // create table ptest( - // a int primary key, - // b double NOT NULL DEFAULT 2.0, - // c varchar(10) NOT NULL, - // d time unique - // ); - // `, - // `create table itest( - // a int, - // b double NOT NULL DEFAULT 2.0, - // c varchar(10) NOT NULL, - // d time unique, - // PRIMARY KEY(a, b) - // ); - // `, - // `create table ntest( - // a int, - // b double NOT NULL DEFAULT 2.0, - // c varchar(10) NOT NULL, - // d time unique not null - // ); - // `, - //} + TableSQLs := []string{ + ` + create table ptest( + a int primary key, + b double NOT NULL DEFAULT 2.0, + c varchar(10) NOT NULL, + d time unique + ); + `, + `create table itest( + a int, + b double NOT NULL DEFAULT 2.0, + c varchar(10) NOT NULL, + d time unique, + PRIMARY KEY(a, b) + ); + `, + `create table ntest( + a int, + b double NOT NULL DEFAULT 2.0, + c varchar(10) NOT NULL, + d time unique not null + ); + `, + } // run the simple test case RunCase(sourceDB, targetDB, schema) - //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { - // // generate insert/update/delete sqls and execute - // RunDailyTest(sourceDB, TableSQLs, workerCount, jobCount, batch) - //}) + RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { + // generate insert/update/delete sqls and execute + RunDailyTest(sourceDB, TableSQLs, workerCount, jobCount, batch) + }) // //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { // // truncate test data From ed31471e8f8a73366f8842eff16f3d16a322cdde Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 18 Dec 2024 17:56:17 +0800 Subject: [PATCH 8/9] revert the test --- scripts/download-integration-test-binaries.sh | 2 +- .../cdc/dailytest/dailytest.go | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/download-integration-test-binaries.sh b/scripts/download-integration-test-binaries.sh index 6703d34f64e..1dca0c48b6d 100755 --- a/scripts/download-integration-test-binaries.sh +++ b/scripts/download-integration-test-binaries.sh @@ -118,7 +118,7 @@ function download_binaries() { # Get sha1 based on branch name. tidb_sha1=$(curl "${file_server_url}/download/refs/pingcap/tidb/${branch}/sha1") - tikv_sha1="b4bddeeb995e7bedc1973ce9e856eeb2d856ce9b" + tikv_sha1=$(curl "${file_server_url}/download/refs/pingcap/tikv/${branch}/sha1") pd_sha1=$(curl "${file_server_url}/download/refs/pingcap/pd/${branch}/sha1") tiflash_sha1=$(curl "${file_server_url}/download/refs/pingcap/tiflash/${branch}/sha1") diff --git a/tests/integration_tests/cdc/dailytest/dailytest.go b/tests/integration_tests/cdc/dailytest/dailytest.go index 0a4991f7820..b6493962583 100644 --- a/tests/integration_tests/cdc/dailytest/dailytest.go +++ b/tests/integration_tests/cdc/dailytest/dailytest.go @@ -54,16 +54,16 @@ func Run(sourceDB *sql.DB, targetDB *sql.DB, schema string, workerCount int, job // generate insert/update/delete sqls and execute RunDailyTest(sourceDB, TableSQLs, workerCount, jobCount, batch) }) - // - //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { - // // truncate test data - // TruncateTestTable(sourceDB, TableSQLs) - //}) - // - //RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { - // // drop test table - // DropTestTable(sourceDB, TableSQLs) - //}) + + RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { + // truncate test data + TruncateTestTable(sourceDB, TableSQLs) + }) + + RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { + // drop test table + DropTestTable(sourceDB, TableSQLs) + }) log.S().Info("test pass!!!") } From b924432a306854f4f814e7ca7af5a7b44e162478 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 19 Dec 2024 13:49:27 +0800 Subject: [PATCH 9/9] fix the fmt --- scripts/download-integration-test-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download-integration-test-binaries.sh b/scripts/download-integration-test-binaries.sh index 1dca0c48b6d..ec12db47193 100755 --- a/scripts/download-integration-test-binaries.sh +++ b/scripts/download-integration-test-binaries.sh @@ -118,7 +118,7 @@ function download_binaries() { # Get sha1 based on branch name. tidb_sha1=$(curl "${file_server_url}/download/refs/pingcap/tidb/${branch}/sha1") - tikv_sha1=$(curl "${file_server_url}/download/refs/pingcap/tikv/${branch}/sha1") + tikv_sha1=$(curl "${file_server_url}/download/refs/pingcap/tikv/${branch}/sha1") pd_sha1=$(curl "${file_server_url}/download/refs/pingcap/pd/${branch}/sha1") tiflash_sha1=$(curl "${file_server_url}/download/refs/pingcap/tiflash/${branch}/sha1")