From 94ac40a5415b7f6ee6fec325fe2e7e93d9d6e1df Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Fri, 11 Oct 2024 20:30:59 +0200 Subject: [PATCH 01/10] Added global index to partition_table test --- .../partition_table/data/prepare.sql | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/tests/integration_tests/partition_table/data/prepare.sql b/tests/integration_tests/partition_table/data/prepare.sql index bde99a305b7..ef6dcc2d43d 100644 --- a/tests/integration_tests/partition_table/data/prepare.sql +++ b/tests/integration_tests/partition_table/data/prepare.sql @@ -2,56 +2,64 @@ drop database if exists `partition_table`; drop database if exists `partition_table2`; create database `partition_table`; use `partition_table`; +set tidb_enable_global_index = ON; -create table t (a int, primary key (a)) partition by hash(a) partitions 5; -insert into t values (1),(2),(3),(4),(5),(6); -insert into t values (7),(8),(9); +create table t (a int, b varchar(255), primary key (a), unique key (b) global) partition by hash(a) partitions 5; +insert into t values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); +insert into t values (7,7),(8,8),(9,9); alter table t truncate partition p3; -update t set a=a+10 where a=2; +update t set a=a+10, b = "12" where a=2; -create table t1 (a int primary key) PARTITION BY RANGE ( a ) ( PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11),PARTITION p2 VALUES LESS THAN (21)); -insert into t1 values (1),(2),(3),(4),(5),(6); -insert into t1 values (7),(8),(9); -insert into t1 values (11),(12),(20); +create table t1 (a int primary key, b bigint, unique key (b) global) PARTITION BY RANGE ( a ) ( PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11),PARTITION p2 VALUES LESS THAN (21)); +insert into t1 (a) values (1),(2),(3),(4),(5),(6); +insert into t1 (a) values (7),(8),(9); +insert into t1 (a) values (11),(12),(20); +update t1 set b = a; alter table t1 add partition (partition p3 values less than (30), partition p4 values less than (40)); -insert into t1 values (25),(29),(35); /*these values in p3,p4*/ +insert into t1 values (25,25),(29,29),(35,35); /*these values in p3,p4*/ alter table t1 truncate partition p0; alter table t1 drop partition p1; -insert into t1 values (7),(8),(9); -update t1 set a=a+10 where a=9; +insert into t1 values (7,7),(8,8),(9,9); +update t1 set a=a+10, b = 19 where a=9; /* Remove partitioning + add partitioning back again */ alter table t remove partitioning; -insert into t values (20),(21),(22),(23),(24),(25); -alter table t partition by hash (a) partitions 5; -insert into t values (30),(31),(32),(33),(34),(35); +insert into t values (20,20),(21,21),(22,22),(23,23),(24,24),(25,25); +alter table t partition by hash (a) partitions 5 update indexes (b global); +insert into t values (30,30),(31,31),(32,32),(33,33),(34,34),(35,35); /* exchange partition case 1: source table and target table in same database */ -create table t2 (a int primary key); -ALTER TABLE t1 EXCHANGE PARTITION p3 WITH TABLE t2; -insert into t2 values (100),(101),(102),(103),(104),(105); /*these values will be replicated to in downstream t2*/ -insert into t1 values (25),(29); /*these values will be replicated to in downstream t1.p3*/ +/* exchange partition does not support global index */ +create table t1b like t1; +alter table t1b drop index b; +insert into t1b select * from t1; +create table t2 (a int primary key, b bigint); +ALTER TABLE t1b EXCHANGE PARTITION p3 WITH TABLE t2; +insert into t2 values (100,100),(101,101),(102,102),(103,103),(104,104),(105,105); /*these values will be replicated to in downstream t2*/ +insert into t1b values (25,25),(29,29); /*these values will be replicated to in downstream t1b.p3*/ /* exchange partition ccase 2: source table and target table in different database */ create database `partition_table2`; -create table partition_table2.t2 (a int primary key); -ALTER TABLE t1 EXCHANGE PARTITION p3 WITH TABLE partition_table2.t2; -insert into partition_table2.t2 values (1002),(1012),(1022),(1032),(1042),(1052); /*these values will be replicated to in downstream t2*/ -insert into t1 values (21),(28); /*these values will be replicated to in downstream t1.p3*/ +create table partition_table2.t2 (a int primary key, b bigint); +ALTER TABLE t1b EXCHANGE PARTITION p3 WITH TABLE partition_table2.t2; +insert into partition_table2.t2 values (1002,1002),(1012,1012),(1022,1022),(1032,1032),(1042,1042),(1052,1052); /*these values will be replicated to in downstream t2*/ +insert into t1b values (21,21),(28,28); /*these values will be replicated to in downstream t1.p3*/ +truncate table t1; +insert into t1 select * from t1b; ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21)); -insert into t1 values (-1),(6),(13); -update t1 set a=a-22 where a=20; +insert into t1 values (-1,-1),(6,6),(13,13); +update t1 set a=a-22, b = -2 where a=20; delete from t1 where a = 5; ALTER TABLE t1 REORGANIZE PARTITION p2,p3,p4 INTO (PARTITION p2 VALUES LESS THAN (20), PARTITION p3 VALUES LESS THAN (26), PARTITION p4 VALUES LESS THAN (35), PARTITION pMax VALUES LESS THAN (MAXVALUE)); -insert into t1 values (-3),(5),(14),(22),(30),(100); -update t1 set a=a-16 where a=12; +insert into t1 values (-3,-3),(5,5),(14,14),(22,22),(30,30),(100,100); +update t1 set a=a-16, b = -4 where a=12; delete from t1 where a = 29; /* Change partitioning to key based and then back to range */ alter table t1 partition by key(a) partitions 7; -insert into t1 values (-2001),(2001),(2002),(-2002),(-2003),(2003),(-2004),(2004),(-2005),(2005),(2006),(-2006),(2007),(-2007); +insert into t1 values (-2001,-2001),(2001,2001),(2002,2002),(-2002,-2002),(-2003,-2003),(2003,2003),(-2004,-2004),(2004,2004),(-2005,-2005),(2005,2005),(2006,2006),(-2006,-2006),(2007,2007),(-2007,-2007); ALTER TABLE t1 partition by range(a) (partition p0 values less than (5), PARTITION p2 VALUES LESS THAN (20), PARTITION p3 VALUES LESS THAN (26), PARTITION p4 VALUES LESS THAN (35), PARTITION pMax VALUES LESS THAN (MAXVALUE)); create table finish_mark (a int primary key); From 08a8f49eebcaba6ff894cb1eb0cc3e51e5f33dd8 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Fri, 22 Nov 2024 11:13:22 +0100 Subject: [PATCH 02/10] Update tests/integration_tests/partition_table/data/prepare.sql Co-authored-by: Hangjie Mo --- tests/integration_tests/partition_table/data/prepare.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_tests/partition_table/data/prepare.sql b/tests/integration_tests/partition_table/data/prepare.sql index ef6dcc2d43d..39f0c419a06 100644 --- a/tests/integration_tests/partition_table/data/prepare.sql +++ b/tests/integration_tests/partition_table/data/prepare.sql @@ -2,7 +2,6 @@ drop database if exists `partition_table`; drop database if exists `partition_table2`; create database `partition_table`; use `partition_table`; -set tidb_enable_global_index = ON; create table t (a int, b varchar(255), primary key (a), unique key (b) global) partition by hash(a) partitions 5; insert into t values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6); From 23b538cbda1110dab098d173fe3530a01980b85d Mon Sep 17 00:00:00 2001 From: wlwilliamx <53336371+wlwilliamx@users.noreply.github.com> Date: Fri, 15 Nov 2024 20:08:08 +0800 Subject: [PATCH 03/10] ddl_puller.go(ticdc): fix DDLs are ignored when schema versions are out of order (#11733) close pingcap/tiflow#11714 --- cdc/puller/ddl_puller.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cdc/puller/ddl_puller.go b/cdc/puller/ddl_puller.go index f4b9e456a62..f9fc521f3d8 100644 --- a/cdc/puller/ddl_puller.go +++ b/cdc/puller/ddl_puller.go @@ -73,7 +73,6 @@ type ddlJobPullerImpl struct { kvStorage tidbkv.Storage schemaStorage entry.SchemaStorage resolvedTs uint64 - schemaVersion int64 filter filter.Filter // ddlTableInfo is initialized when receive the first concurrent DDL job. ddlTableInfo *entry.DDLTableInfo @@ -317,8 +316,7 @@ func (p *ddlJobPullerImpl) handleJob(job *timodel.Job) (skip bool, err error) { return false, nil } - if job.BinlogInfo.FinishedTS <= p.getResolvedTs() || - job.BinlogInfo.SchemaVersion <= p.schemaVersion { + if job.BinlogInfo.FinishedTS <= p.getResolvedTs() { log.Info("ddl job finishedTs less than puller resolvedTs,"+ "discard the ddl job", zap.String("namespace", p.changefeedID.Namespace), @@ -480,7 +478,6 @@ func (p *ddlJobPullerImpl) handleJob(job *timodel.Job) (skip bool, err error) { errors.Trace(err), job.Query, job.StartTS, job.StartTS) } p.setResolvedTs(job.BinlogInfo.FinishedTS) - p.schemaVersion = job.BinlogInfo.SchemaVersion return p.checkIneligibleTableDDL(snap, job) } From 8155a4c091dd1d8a899ad4ea684a4320bb11839f Mon Sep 17 00:00:00 2001 From: CharlesCheung <61726649+CharlesCheung96@users.noreply.github.com> Date: Sat, 16 Nov 2024 13:07:11 +0800 Subject: [PATCH 04/10] ticdc(redo, sink): return correct error in redo writer & fix default retryer (#11747) close pingcap/tiflow#11744 --- cdc/redo/writer/memory/encoding_worker.go | 2 +- cdc/redo/writer/memory/mem_log_writer_test.go | 26 ++++++++++++++++--- pkg/redo/config.go | 2 +- pkg/sink/kafka/claimcheck/claim_check.go | 6 +---- pkg/util/external_storage.go | 16 +++++++----- 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/cdc/redo/writer/memory/encoding_worker.go b/cdc/redo/writer/memory/encoding_worker.go index 9b0cc6a8652..f62f236f888 100644 --- a/cdc/redo/writer/memory/encoding_worker.go +++ b/cdc/redo/writer/memory/encoding_worker.go @@ -131,7 +131,7 @@ func (e *encodingWorkerGroup) Run(ctx context.Context) (err error) { zap.String("namespace", e.changefeed.Namespace), zap.String("changefeed", e.changefeed.ID), zap.Error(err)) - if err != nil && errors.Cause(err) != context.Canceled { + if err != nil { e.closed <- err } close(e.closed) diff --git a/cdc/redo/writer/memory/mem_log_writer_test.go b/cdc/redo/writer/memory/mem_log_writer_test.go index 33e1f426489..12f34ce11e8 100644 --- a/cdc/redo/writer/memory/mem_log_writer_test.go +++ b/cdc/redo/writer/memory/mem_log_writer_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/tiflow/cdc/model" "github.com/pingcap/tiflow/cdc/redo/writer" + "github.com/pingcap/tiflow/pkg/errors" "github.com/pingcap/tiflow/pkg/redo" "github.com/pingcap/tiflow/pkg/util" "github.com/stretchr/testify/require" @@ -99,10 +100,27 @@ func testWriteEvents(t *testing.T, events []writer.RedoEvent) { }) require.NoError(t, err) + require.ErrorIs(t, lw.Close(), context.Canceled) + // duplicate close should return the same error require.ErrorIs(t, lw.Close(), context.Canceled) - err = lw.WriteEvents(ctx, events...) - require.NoError(t, err) - err = lw.FlushLog(ctx) - require.NoError(t, err) + functions := map[string]func(error){ + "WriteEvents": func(expected error) { + err := lw.WriteEvents(ctx, events...) + require.ErrorIs(t, errors.Cause(err), expected) + }, + "FlushLog": func(expected error) { + err := lw.FlushLog(ctx) + require.ErrorIs(t, errors.Cause(err), expected) + }, + } + firstCall := true + for _, f := range functions { + if firstCall { + firstCall = false + f(context.Canceled) + } else { + f(nil) + } + } } diff --git a/pkg/redo/config.go b/pkg/redo/config.go index 8865e9f4f61..4b7821b1048 100644 --- a/pkg/redo/config.go +++ b/pkg/redo/config.go @@ -190,7 +190,7 @@ func IsBlackholeStorage(scheme string) bool { // InitExternalStorage init an external storage. var InitExternalStorage = func(ctx context.Context, uri url.URL) (storage.ExternalStorage, error) { - s, err := util.GetExternalStorageWithTimeout(ctx, uri.String(), DefaultTimeout) + s, err := util.GetExternalStorageWithDefaultTimeout(ctx, uri.String()) if err != nil { return nil, errors.WrapError(errors.ErrStorageInitialize, err, fmt.Sprintf("can't init external storage for %s", uri.String())) diff --git a/pkg/sink/kafka/claimcheck/claim_check.go b/pkg/sink/kafka/claimcheck/claim_check.go index 488186f2d81..f4bca23dfe7 100644 --- a/pkg/sink/kafka/claimcheck/claim_check.go +++ b/pkg/sink/kafka/claimcheck/claim_check.go @@ -31,10 +31,6 @@ import ( "go.uber.org/zap" ) -const ( - defaultTimeout = 5 * time.Minute -) - // ClaimCheck manage send message to the claim-check external storage. type ClaimCheck struct { storage storage.ExternalStorage @@ -59,7 +55,7 @@ func New(ctx context.Context, config *config.LargeMessageHandleConfig, changefee zap.String("storageURI", util.MaskSensitiveDataInURI(config.ClaimCheckStorageURI))) start := time.Now() - externalStorage, err := util.GetExternalStorageWithTimeout(ctx, config.ClaimCheckStorageURI, defaultTimeout) + externalStorage, err := util.GetExternalStorageWithDefaultTimeout(ctx, config.ClaimCheckStorageURI) if err != nil { log.Error("create external storage failed", zap.String("namespace", changefeedID.Namespace), diff --git a/pkg/util/external_storage.go b/pkg/util/external_storage.go index 1d7a23c661b..c557cde3172 100644 --- a/pkg/util/external_storage.go +++ b/pkg/util/external_storage.go @@ -35,6 +35,8 @@ import ( "golang.org/x/sync/errgroup" ) +const defaultTimeout = 5 * time.Minute + // GetExternalStorageFromURI creates a new storage.ExternalStorage from a uri. func GetExternalStorageFromURI( ctx context.Context, uri string, @@ -42,18 +44,18 @@ func GetExternalStorageFromURI( return GetExternalStorage(ctx, uri, nil, DefaultS3Retryer()) } -// GetExternalStorageWithTimeout creates a new storage.ExternalStorage from a uri +// GetExternalStorageWithDefaultTimeout creates a new storage.ExternalStorage from a uri // without retry. It is the caller's responsibility to set timeout to the context. -func GetExternalStorageWithTimeout( - ctx context.Context, uri string, timeout time.Duration, -) (storage.ExternalStorage, error) { - ctx, cancel := context.WithTimeout(ctx, timeout) +func GetExternalStorageWithDefaultTimeout(ctx context.Context, uri string) (storage.ExternalStorage, error) { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() - s, err := GetExternalStorage(ctx, uri, nil, nil) + // total retry time is [1<<7, 1<<8] = [128, 256] + 30*6 = [308, 436] seconds + r := NewS3Retryer(7, 1*time.Second, 2*time.Second) + s, err := GetExternalStorage(ctx, uri, nil, r) return &extStorageWithTimeout{ ExternalStorage: s, - timeout: timeout, + timeout: defaultTimeout, }, err } From e06bd9c977b79fec91a8581febf1f24dfe957b90 Mon Sep 17 00:00:00 2001 From: Jianyuan Jiang Date: Tue, 19 Nov 2024 16:26:53 +0800 Subject: [PATCH 05/10] fix resign owner api not processed by owner node bug (#11770) close pingcap/tiflow#11769 --- cdc/api/v2/api.go | 2 +- tests/integration_tests/availability/owner.sh | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cdc/api/v2/api.go b/cdc/api/v2/api.go index 9d70f3ca37f..033fe177bd5 100644 --- a/cdc/api/v2/api.go +++ b/cdc/api/v2/api.go @@ -86,7 +86,7 @@ func RegisterOpenAPIV2Routes(router *gin.Engine, api OpenAPIV2) { // owner apis ownerGroup := v2.Group("/owner") - unsafeGroup.Use(ownerMiddleware) + ownerGroup.Use(ownerMiddleware) ownerGroup.POST("/resign", api.resignOwner) // common APIs diff --git a/tests/integration_tests/availability/owner.sh b/tests/integration_tests/availability/owner.sh index b78c0ecd78e..82eb2ad69c2 100755 --- a/tests/integration_tests/availability/owner.sh +++ b/tests/integration_tests/availability/owner.sh @@ -13,6 +13,7 @@ function test_owner_ha() { test_owner_retryable_error test_gap_between_watch_capture test_delete_owner_key + test_resign_owner } # test_kill_owner starts two captures and kill the owner # we expect the live capture will be elected as the new @@ -219,7 +220,7 @@ function test_delete_owner_key() { run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --addr "127.0.0.1:8301" --logsuffix test_gap_between_watch_capture.server2 ensure $MAX_RETRIES "$CDC_BINARY cli capture list 2>&1 | grep -v \"$owner_id\" | grep id" capture_pid=$(ps -C $CDC_BINARY -o pid= | awk '{print $1}' | grep -v "$owner_pid") - capture_id=$($CDC_BINARY cli capture list 2>&1 | awk -F '"' '/id/{print $4}' | grep -v "$owner_id") + capture_id=$($CDC_BINARY cli capture list 2>&1 | awk -F '"' '/\"id/{print $4}' | grep -v "$owner_id") echo "capture_id:" $capture_id etcdctl del $owner_key @@ -242,3 +243,30 @@ function test_delete_owner_key() { echo "delete_owner_key pass" cleanup_process $CDC_BINARY } + +# test_resign_owner resign the owner by sending +# the resign owner v2 API +# We expect when the owner is resigned, the new owner will be elected +function test_resign_owner() { + echo "run test case test_resign_owner" + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --addr "127.0.0.1:8300" --logsuffix test_resign_owner.server1 + # ensure the server become the owner + ensure $MAX_RETRIES "$CDC_BINARY cli capture list 2>&1 | grep '\"is-owner\": true'" + owner_pid=$(ps -C $CDC_BINARY -o pid= | awk '{print $1}') + owner_id=$($CDC_BINARY cli capture list 2>&1 | awk -F '"' '/\"id/{print $4}') + echo "owner pid:" $owner_pid + echo "owner id" $owner_id + + # run another server + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --addr "127.0.0.1:8301" --logsuffix test_resign_owner.server2 + ensure $MAX_RETRIES "$CDC_BINARY cli capture list 2>&1 | grep -v \"$owner_id\" | grep id" + capture_id=$($CDC_BINARY cli capture list 2>&1 | awk -F '"' '/\"id/{print $4}' | grep -v "$owner_id") + echo "capture_id:" $capture_id + + # resign the owner + curl -X POST http://127.0.0.1:8301/api/v2/owner/resign + # check that the new owner is elected + ensure $MAX_RETRIES "$CDC_BINARY cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep $capture_id -A1 | grep '\"is-owner\": true'" + echo "test_resign_owner: pass" + cleanup_process $CDC_BINARY +} From 0603517978435f1dc7aa95a541c36be383907fa3 Mon Sep 17 00:00:00 2001 From: D3Hunter Date: Wed, 20 Nov 2024 12:09:35 +0800 Subject: [PATCH 06/10] *: upgrade tidb to include fix (#11776) close pingcap/tiflow#11768 --- dm/pkg/parser/common_test.go | 6 ++-- go.mod | 32 +++++++++--------- go.sum | 64 ++++++++++++++++++------------------ 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/dm/pkg/parser/common_test.go b/dm/pkg/parser/common_test.go index 4f0a3b66925..f81e7506593 100644 --- a/dm/pkg/parser/common_test.go +++ b/dm/pkg/parser/common_test.go @@ -278,11 +278,11 @@ var testCases = []testCase{ []string{"ALTER TABLE `xtest`.`xt1` DROP INDEX IF EXISTS `i1`"}, }, { - "alter table `t1` drop foreign key if exists fk_t2_id", - []string{"ALTER TABLE `test`.`t1` DROP FOREIGN KEY IF EXISTS `fk_t2_id`"}, + "alter table `t1` drop foreign key fk_t2_id", + []string{"ALTER TABLE `test`.`t1` DROP FOREIGN KEY `fk_t2_id`"}, [][]*filter.Table{{genTableName("test", "t1")}}, [][]*filter.Table{{genTableName("xtest", "xt1")}}, - []string{"ALTER TABLE `xtest`.`xt1` DROP FOREIGN KEY IF EXISTS `fk_t2_id`"}, + []string{"ALTER TABLE `xtest`.`xt1` DROP FOREIGN KEY `fk_t2_id`"}, }, { "alter table `t1` drop partition if exists p2", diff --git a/go.mod b/go.mod index 84f0c3f609e..0a6f5dbfa92 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/docker/go-units v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 - github.com/fatih/color v1.17.0 + github.com/fatih/color v1.18.0 github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 github.com/getkin/kin-openapi v0.80.0 github.com/gin-gonic/gin v1.9.1 @@ -69,9 +69,9 @@ require ( github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 github.com/pingcap/kvproto v0.0.0-20240924080114-4a3e17f5e62d github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d - github.com/pingcap/tidb v1.1.0-beta.0.20241107131230-e2505e95a03c + github.com/pingcap/tidb v1.1.0-beta.0.20241119124618-50b5cd27d413 github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 - github.com/pingcap/tidb/pkg/parser v0.0.0-20241014034929-94b2ac04a0c4 + github.com/pingcap/tidb/pkg/parser v0.0.0-20241119124618-50b5cd27d413 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/r3labs/diff v1.1.0 @@ -89,9 +89,9 @@ require ( github.com/swaggo/swag v1.16.3 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 github.com/thanhpk/randstr v1.0.6 - github.com/tikv/client-go/v2 v2.0.8-0.20241023023120-691e80ae0ea9 + github.com/tikv/client-go/v2 v2.0.8-0.20241111090227-70049ae310bf github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b - github.com/tikv/pd/client v0.0.0-20241016064947-b70107ec31e6 + github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 github.com/tinylib/msgp v1.1.6 github.com/uber-go/atomic v1.4.0 github.com/vmihailenco/msgpack/v5 v5.3.5 @@ -111,11 +111,11 @@ require ( go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - golang.org/x/net v0.30.0 + golang.org/x/net v0.31.0 golang.org/x/oauth2 v0.23.0 - golang.org/x/sync v0.8.0 - golang.org/x/sys v0.26.0 - golang.org/x/text v0.19.0 + golang.org/x/sync v0.9.0 + golang.org/x/sys v0.27.0 + golang.org/x/text v0.20.0 golang.org/x/time v0.7.0 google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 @@ -173,7 +173,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/ks3sdklib/aws-sdk-go v1.2.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lance6716/pebble v0.0.0-20241104073946-6f55c09bd183 // indirect + github.com/lance6716/pebble v0.0.0-20241108073934-da961314c63f // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -330,7 +330,7 @@ require ( github.com/pingcap/fn v1.0.0 // indirect github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 - github.com/pingcap/tipb v0.0.0-20241008083645-0bcddae67837 // indirect + github.com/pingcap/tipb v0.0.0-20241022082558-0607513e7fa4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -339,7 +339,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/rs/cors v1.7.0 // indirect github.com/sasha-s/go-deadlock v0.3.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -382,10 +382,10 @@ require ( go.opentelemetry.io/otel/sdk v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/term v0.25.0 - golang.org/x/tools v0.26.0 // indirect + golang.org/x/crypto v0.29.0 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/term v0.26.0 + golang.org/x/tools v0.27.0 // indirect google.golang.org/api v0.170.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index 4c5951e860b..4d4837cb879 100644 --- a/go.sum +++ b/go.sum @@ -308,8 +308,8 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= @@ -719,8 +719,8 @@ github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjF github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= -github.com/lance6716/pebble v0.0.0-20241104073946-6f55c09bd183 h1:CrFmpCAT5PGMgmQadTa2lXZrjuvpknONB1/pyxiyDsM= -github.com/lance6716/pebble v0.0.0-20241104073946-6f55c09bd183/go.mod h1:ZxnWA3Ab0ufDIyppyzL16j6HFNpdXeiU/1cE4Wlv/lQ= +github.com/lance6716/pebble v0.0.0-20241108073934-da961314c63f h1:KyBFXtlZDJTrwuD3wBHGk+TtdOpCB17AEm0X4eqhRdg= +github.com/lance6716/pebble v0.0.0-20241108073934-da961314c63f/go.mod h1:ZxnWA3Ab0ufDIyppyzL16j6HFNpdXeiU/1cE4Wlv/lQ= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= @@ -906,14 +906,14 @@ github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfU github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= -github.com/pingcap/tidb v1.1.0-beta.0.20241107131230-e2505e95a03c h1:PLop/4ZOM5Uxd9co41gIKztalJfe+jSxYpuu9DdndBE= -github.com/pingcap/tidb v1.1.0-beta.0.20241107131230-e2505e95a03c/go.mod h1:tAyfBFGzdPPLJrSU76pAtddlNtWMLynRplC/ESWvvuM= +github.com/pingcap/tidb v1.1.0-beta.0.20241119124618-50b5cd27d413 h1:EpfEfHIdolNMUHy/qM6+eC7kGx8M6PJteTTc0YeKUx0= +github.com/pingcap/tidb v1.1.0-beta.0.20241119124618-50b5cd27d413/go.mod h1:6iRUCOS1jqgfXVtLO8RaRC91Sv/+dPuAt551k1nAyRw= github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 h1:eFu98FbfJB7PKWOtkaV6YNXXJWqDhczQX56j/iucgU4= github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= -github.com/pingcap/tidb/pkg/parser v0.0.0-20241014034929-94b2ac04a0c4 h1:+HMT6yCrQ0GGkhOPtP22u0/PnPmfXir9WC3NezF3OHo= -github.com/pingcap/tidb/pkg/parser v0.0.0-20241014034929-94b2ac04a0c4/go.mod h1:Hju1TEWZvrctQKbztTRwXH7rd41Yq0Pgmq4PrEKcq7o= -github.com/pingcap/tipb v0.0.0-20241008083645-0bcddae67837 h1:tyIymn821fB8gUmqafdvLlcFkVOpgyJXImoYJ8n9oJE= -github.com/pingcap/tipb v0.0.0-20241008083645-0bcddae67837/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tidb/pkg/parser v0.0.0-20241119124618-50b5cd27d413 h1:Yr9Hc6y70JvAcRMCqUu5fdpzHc+tnHxxnQLmeVLnxxo= +github.com/pingcap/tidb/pkg/parser v0.0.0-20241119124618-50b5cd27d413/go.mod h1:Hju1TEWZvrctQKbztTRwXH7rd41Yq0Pgmq4PrEKcq7o= +github.com/pingcap/tipb v0.0.0-20241022082558-0607513e7fa4 h1:wvaUybJT0fUReCDcFtV3CEvMuI9iu+G7IW72tbSlil4= +github.com/pingcap/tipb v0.0.0-20241022082558-0607513e7fa4/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -969,8 +969,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -1108,12 +1108,12 @@ github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0I github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/tikv/client-go/v2 v2.0.8-0.20241023023120-691e80ae0ea9 h1:1Fgp6FqjgXEj/CKcegdXu3wLo77sx7JM9NPC7sF0io0= -github.com/tikv/client-go/v2 v2.0.8-0.20241023023120-691e80ae0ea9/go.mod h1:WAp0oZxDL+3GX+QhJdG0quubJUzEH8LrFofmIxleJhs= +github.com/tikv/client-go/v2 v2.0.8-0.20241111090227-70049ae310bf h1:qCi6BiBUPk3Ky4f2CCgBxgUmi3ZpuQLYDLgxw1ilXPA= +github.com/tikv/client-go/v2 v2.0.8-0.20241111090227-70049ae310bf/go.mod h1:p9zPFlKBrxhp3b/cBmKBWL9M0X4HtJjgi1ThUtQYF7o= github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b h1:t2XoZp4UHrkPpYPsxbRTRVExJnriWlh+ZsDIfpYyd98= github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b/go.mod h1:7HJMdb0O5umNpZIFt8e/wKAcEmH99n2HsYgXX+vZj3k= -github.com/tikv/pd/client v0.0.0-20241016064947-b70107ec31e6 h1:u0z6yR68sg0pextuabJv/bD4mvwBe8iFeQOdymBUy0E= -github.com/tikv/pd/client v0.0.0-20241016064947-b70107ec31e6/go.mod h1:W5a0sDadwUpI9k8p7M77d3jo253ZHdmua+u4Ho4Xw8U= +github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 h1:oAYc4m5Eu1OY9ogJ103VO47AYPHvhtzbUPD8L8B67Qk= +github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31/go.mod h1:W5a0sDadwUpI9k8p7M77d3jo253ZHdmua+u4Ho4Xw8U= github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= @@ -1301,8 +1301,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1358,8 +1358,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1403,8 +1403,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1423,8 +1423,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1498,8 +1498,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1508,8 +1508,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1524,8 +1524,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1589,8 +1589,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 3d99b81a471ddd74fd10a27ae810f192bd80895b Mon Sep 17 00:00:00 2001 From: will <87208113+db-will@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:43:44 -0500 Subject: [PATCH 07/10] Redact query arguments in dm log (#11783) close pingcap/tiflow#11489 --- dm/syncer/dbconn/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/syncer/dbconn/db.go b/dm/syncer/dbconn/db.go index 64090843d78..ef8f5276d76 100644 --- a/dm/syncer/dbconn/db.go +++ b/dm/syncer/dbconn/db.go @@ -246,7 +246,7 @@ func (conn *DBConn) retryableFn(tctx *tcontext.Context, queries, args any) func( if err != nil { tctx.L().Error("reset connection failed", zap.Int("retry", retryTime), zap.String("queries", utils.TruncateInterface(queries, -1)), - zap.String("arguments", utils.TruncateInterface(args, -1)), + log.ZapRedactString("arguments", utils.TruncateInterface(args, -1)), log.ShortError(err)) return false } @@ -257,7 +257,7 @@ func (conn *DBConn) retryableFn(tctx *tcontext.Context, queries, args any) func( if dbutil.IsRetryableError(err) { tctx.L().Warn("execute statements", zap.Int("retry", retryTime), zap.String("queries", utils.TruncateInterface(queries, -1)), - zap.String("arguments", utils.TruncateInterface(args, -1)), + log.ZapRedactString("arguments", utils.TruncateInterface(args, -1)), log.ShortError(err)) return true } From 94082c3c5d1eb580075999053fa475904bf596f1 Mon Sep 17 00:00:00 2001 From: Zack Zhao <57036248+joccau@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:47:53 +0800 Subject: [PATCH 08/10] dm: handle the query in metadata from binlog (#11743) close pingcap/tiflow#11735 --- dm/pkg/dumpling/utils.go | 3 ++- dm/pkg/dumpling/utils_test.go | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/dm/pkg/dumpling/utils.go b/dm/pkg/dumpling/utils.go index 0b6cf273917..01efb4b4f8d 100644 --- a/dm/pkg/dumpling/utils.go +++ b/dm/pkg/dumpling/utils.go @@ -123,7 +123,8 @@ func ParseMetaDataByReader(filename string, rd io.Reader) (*binlog.Location, *bi } switch line { - case "SHOW MASTER STATUS:": + case "SHOW BINARY LOG STATUS:", + "SHOW MASTER STATUS:": if err3 := parsePosAndGTID(&pos, >idStr); err3 != nil { return nil, nil, err3 } diff --git a/dm/pkg/dumpling/utils_test.go b/dm/pkg/dumpling/utils_test.go index 55bf0610a1c..7aa373e6b5a 100644 --- a/dm/pkg/dumpling/utils_test.go +++ b/dm/pkg/dumpling/utils_test.go @@ -84,6 +84,29 @@ Finished dump at: 2018-12-27 19:51:22`, mysql.Position{}, "", }, + { + `Started dump at: 2018-12-27 19:51:22 +SHOW BINARY LOG STATUS: + Log: mysql-bin.000004 + Pos: 3295818 + GTID: + +SHOW SLAVE STATUS: + Host: 10.128.27.98 + Log: mysql-bin.000003 + Pos: 329635 + GTID: + +Finished dump at: 2018-12-27 19:51:22`, + mysql.Position{ + Name: "mysql-bin.000004", + Pos: 3295818, + }, + "", + false, + mysql.Position{}, + "", + }, { // with empty line after multiple GTID sets `Started dump at: 2020-05-21 18:14:49 SHOW MASTER STATUS: From 205b58257c9a0878bda0172c1bb3893ba5431d6b Mon Sep 17 00:00:00 2001 From: nhsmw Date: Mon, 25 Nov 2024 15:25:02 +0800 Subject: [PATCH 09/10] sink(ticdc): limit encoder-concurrency to avoid crash (#11775) close pingcap/tiflow#11773 --- pkg/sink/codec/encoder_group.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/sink/codec/encoder_group.go b/pkg/sink/codec/encoder_group.go index ad828f030a5..160ffc4253c 100644 --- a/pkg/sink/codec/encoder_group.go +++ b/pkg/sink/codec/encoder_group.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" + "github.com/pingcap/tidb/pkg/util/cpu" "github.com/pingcap/tiflow/cdc/model" "github.com/pingcap/tiflow/cdc/sink/dmlsink" "github.com/pingcap/tiflow/pkg/config" @@ -69,6 +70,11 @@ func NewEncoderGroup( if concurrency <= 0 { concurrency = config.DefaultEncoderGroupConcurrency } + limitConcurrency := cpu.GetCPUCount() * 10 + if concurrency > limitConcurrency { + concurrency = limitConcurrency + log.Warn("limit concurrency to avoid crash", zap.Int("concurrency", concurrency), zap.Any("limitConcurrency", limitConcurrency)) + } inputCh := make([]chan *future, concurrency) for i := 0; i < concurrency; i++ { inputCh[i] = make(chan *future, defaultInputChanSize) From 1dcc9f501b3594c27b76018c23e2efc6ab0068cd Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 25 Nov 2024 13:37:36 +0100 Subject: [PATCH 10/10] Updated sync_diff dependency --- scripts/download-integration-test-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download-integration-test-binaries.sh b/scripts/download-integration-test-binaries.sh index 765d848aede..5b9f4f9c4ad 100755 --- a/scripts/download-integration-test-binaries.sh +++ b/scripts/download-integration-test-binaries.sh @@ -147,7 +147,7 @@ download_binaries() { local minio_download_url="${FILE_SERVER_URL}/download/minio.tar.gz" local go_ycsb_download_url="${FILE_SERVER_URL}/download/builds/pingcap/go-ycsb/test-br/go-ycsb" local etcd_download_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz" - local sync_diff_inspector_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/sync_diff_inspector_hash-a129f096_linux-amd64.tar.gz" + local sync_diff_inspector_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/sync_diff_inspector_hash-d28498c1_linux-amd64.tar.gz" local jq_download_url="${FILE_SERVER_URL}/download/builds/pingcap/test/jq-1.6/jq-linux64" local schema_registry_url="${FILE_SERVER_URL}/download/builds/pingcap/cdc/schema-registry.tar.gz"