From 2c0d131a89852ea57d8a5bfc2f37d6f77b32136d Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 26 Jan 2024 16:58:51 +0800 Subject: [PATCH 1/3] This is an automated cherry-pick of #7765 close tikv/pd#4399 Signed-off-by: ti-chi-bot --- pkg/schedule/config/config.go | 5 +++- server/cluster/cluster_test.go | 14 ++++++++++ .../mcs/scheduling/server_test.go | 27 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index c8fa62b8aff..8c6829f769e 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -525,9 +525,12 @@ type SchedulerConfig struct { var DefaultSchedulers = SchedulerConfigs{ {Type: "balance-region"}, {Type: "balance-leader"}, - {Type: "balance-witness"}, {Type: "hot-region"}, +<<<<<<< HEAD {Type: "transfer-witness-leader"}, +======= + {Type: "evict-slow-store"}, +>>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) } // IsDefaultScheduler checks whether the scheduler is enable by default. diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index d1c8c8fe9cb..2fa47cb1085 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -3041,8 +3041,12 @@ func TestAddScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) +<<<<<<< HEAD re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) +======= + re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) +>>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) re.Empty(controller.GetSchedulerNames()) stream := mockhbstream.NewHeartbeatStream() @@ -3137,9 +3141,15 @@ func TestPersistScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) +<<<<<<< HEAD re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) re.Len(controller.GetSchedulerNames(), defaultCount-3) +======= + re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + // only remains 2 items with independent config. + re.Len(controller.GetSchedulerNames(), 2) +>>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) re.NoError(co.GetCluster().GetSchedulerConfig().Persist(storage)) co.Stop() co.GetSchedulersController().Wait() @@ -3249,8 +3259,12 @@ func TestRemoveScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) +<<<<<<< HEAD re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) +======= + re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) +>>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) // all removed sches, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 324c8e5cad5..4654ff1c8af 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -265,6 +265,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { api.MustDeleteScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName) checkEvictLeaderSchedulerExist(re, schedulersController, false) +<<<<<<< HEAD // TODO: test more schedulers. // Fixme: the following code will fail because the scheduler is not removed but not synced. // checkDelete := func(schedulerName string) { @@ -277,6 +278,32 @@ func (suite *serverTestSuite) TestSchedulerSync() { // checkDelete(schedulers.BalanceLeaderName) // checkDelete(schedulers.BalanceRegionName) // checkDelete(schedulers.HotRegionName) +======= + // The default scheduler could not be deleted, it could only be disabled. + defaultSchedulerNames := []string{ + schedulers.BalanceLeaderName, + schedulers.BalanceRegionName, + schedulers.HotRegionName, + } + checkDisabled := func(name string, shouldDisabled bool) { + re.NotNil(schedulersController.GetScheduler(name), name) + testutil.Eventually(re, func() bool { + disabled, err := schedulersController.IsSchedulerDisabled(name) + re.NoError(err, name) + return disabled == shouldDisabled + }) + } + for _, name := range defaultSchedulerNames { + checkDisabled(name, false) + api.MustDeleteScheduler(re, suite.backendEndpoints, name) + checkDisabled(name, true) + } + for _, name := range defaultSchedulerNames { + checkDisabled(name, true) + api.MustAddScheduler(re, suite.backendEndpoints, name, nil) + checkDisabled(name, false) + } +>>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) } func checkEvictLeaderSchedulerExist(re *require.Assertions, sc *schedulers.Controller, exist bool) { From 7a84c4bcb77cc408894428fe7d7a31d7a36cba95 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 11 Dec 2024 13:33:17 +0800 Subject: [PATCH 2/3] fix Signed-off-by: Ryan Leung --- pkg/schedule/config/config.go | 5 -- server/cluster/cluster_test.go | 27 +----- .../mcs/scheduling/server_test.go | 31 +------ tests/pdctl/scheduler/scheduler_test.go | 84 +++++++------------ tests/server/cluster/cluster_test.go | 6 +- 5 files changed, 40 insertions(+), 113 deletions(-) diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 8c6829f769e..3b1557d6a48 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -526,11 +526,6 @@ var DefaultSchedulers = SchedulerConfigs{ {Type: "balance-region"}, {Type: "balance-leader"}, {Type: "hot-region"}, -<<<<<<< HEAD - {Type: "transfer-witness-leader"}, -======= - {Type: "evict-slow-store"}, ->>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) } // IsDefaultScheduler checks whether the scheduler is enable by default. diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 2fa47cb1085..e84f880a50a 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -3041,12 +3041,6 @@ func TestAddScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) -<<<<<<< HEAD - re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) - re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) -======= - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) ->>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) re.Empty(controller.GetSchedulerNames()) stream := mockhbstream.NewHeartbeatStream() @@ -3137,19 +3131,12 @@ func TestPersistScheduler(t *testing.T) { re.NoError(err) re.Len(sches, defaultCount+2) - // remove 5 schedulers + // remove 3 schedulers re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) -<<<<<<< HEAD - re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) - re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) - re.Len(controller.GetSchedulerNames(), defaultCount-3) -======= - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) // only remains 2 items with independent config. re.Len(controller.GetSchedulerNames(), 2) ->>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) re.NoError(co.GetCluster().GetSchedulerConfig().Persist(storage)) co.Stop() co.GetSchedulersController().Wait() @@ -3200,7 +3187,7 @@ func TestPersistScheduler(t *testing.T) { brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) re.NoError(err) re.NoError(controller.AddScheduler(brs)) - re.Len(controller.GetSchedulerNames(), defaultCount) + re.Len(controller.GetSchedulerNames(), 5) // the scheduler option should contain 6 items // the `hot scheduler` are disabled @@ -3221,9 +3208,9 @@ func TestPersistScheduler(t *testing.T) { co.Run() controller = co.GetSchedulersController() - re.Len(controller.GetSchedulerNames(), defaultCount-1) + re.Len(controller.GetSchedulerNames(), 4) re.NoError(controller.RemoveScheduler(schedulers.EvictLeaderName)) - re.Len(controller.GetSchedulerNames(), defaultCount-2) + re.Len(controller.GetSchedulerNames(), 3) } func TestRemoveScheduler(t *testing.T) { @@ -3259,12 +3246,6 @@ func TestRemoveScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) -<<<<<<< HEAD - re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) - re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) -======= - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) ->>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) // all removed sches, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 4654ff1c8af..98ffb1ac8ee 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -126,7 +126,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { tc.WaitForPrimaryServing(re) primary := tc.GetPrimaryServer() oldPrimaryAddr := primary.GetAddr() - re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 5) + re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 4) testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) return ok && oldPrimaryAddr == watchedAddr @@ -137,7 +137,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { primary = tc.GetPrimaryServer() newPrimaryAddr := primary.GetAddr() re.NotEqual(oldPrimaryAddr, newPrimaryAddr) - re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 5) + re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 4) testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) return ok && newPrimaryAddr == watchedAddr @@ -265,7 +265,6 @@ func (suite *serverTestSuite) TestSchedulerSync() { api.MustDeleteScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName) checkEvictLeaderSchedulerExist(re, schedulersController, false) -<<<<<<< HEAD // TODO: test more schedulers. // Fixme: the following code will fail because the scheduler is not removed but not synced. // checkDelete := func(schedulerName string) { @@ -278,32 +277,6 @@ func (suite *serverTestSuite) TestSchedulerSync() { // checkDelete(schedulers.BalanceLeaderName) // checkDelete(schedulers.BalanceRegionName) // checkDelete(schedulers.HotRegionName) -======= - // The default scheduler could not be deleted, it could only be disabled. - defaultSchedulerNames := []string{ - schedulers.BalanceLeaderName, - schedulers.BalanceRegionName, - schedulers.HotRegionName, - } - checkDisabled := func(name string, shouldDisabled bool) { - re.NotNil(schedulersController.GetScheduler(name), name) - testutil.Eventually(re, func() bool { - disabled, err := schedulersController.IsSchedulerDisabled(name) - re.NoError(err, name) - return disabled == shouldDisabled - }) - } - for _, name := range defaultSchedulerNames { - checkDisabled(name, false) - api.MustDeleteScheduler(re, suite.backendEndpoints, name) - checkDisabled(name, true) - } - for _, name := range defaultSchedulerNames { - checkDisabled(name, true) - api.MustAddScheduler(re, suite.backendEndpoints, name, nil) - checkDisabled(name, false) - } ->>>>>>> 5b939c6fc (config: disable witness related schedulers by default (#7765)) } func checkEvictLeaderSchedulerExist(re *require.Assertions, sc *schedulers.Controller, exist bool) { diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index fb344ecebee..eb62029a608 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -111,11 +111,9 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // scheduler show command expected := map[string]bool{ - "balance-region-scheduler": true, - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, } checkSchedulerCommand(nil, expected) @@ -123,10 +121,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { args := []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"} time.Sleep(10 * time.Second) expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, } checkSchedulerCommand(args, expected) @@ -136,11 +132,9 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // scheduler add command args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "2"} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - schedulers[idx]: true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + schedulers[idx]: true, } checkSchedulerCommand(args, expected) @@ -152,11 +146,9 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // scheduler config update command args = []string{"-u", pdAddr, "scheduler", "config", schedulers[idx], "add-store", "3"} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - schedulers[idx]: true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + schedulers[idx]: true, } checkSchedulerCommand(args, expected) @@ -167,32 +159,26 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // scheduler delete command args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx]} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, } checkSchedulerCommand(args, expected) // scheduler add command args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "2"} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - schedulers[idx]: true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + schedulers[idx]: true, } checkSchedulerCommand(args, expected) // scheduler add command twice args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "4"} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - schedulers[idx]: true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + schedulers[idx]: true, } checkSchedulerCommand(args, expected) @@ -203,11 +189,9 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // scheduler remove command [old] args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx] + "-4"} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - schedulers[idx]: true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + schedulers[idx]: true, } checkSchedulerCommand(args, expected) @@ -218,21 +202,17 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // scheduler remove command, when remove the last store, it should remove whole scheduler args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx] + "-2"} expected = map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, } checkSchedulerCommand(args, expected) } // test shuffle region config checkSchedulerCommand([]string{"-u", pdAddr, "scheduler", "add", "shuffle-region-scheduler"}, map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - "shuffle-region-scheduler": true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + "shuffle-region-scheduler": true, }) var roles []string mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) @@ -245,12 +225,10 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // test grant hot region scheduler config checkSchedulerCommand([]string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{ - "balance-leader-scheduler": true, - "balance-hot-region-scheduler": true, - "shuffle-region-scheduler": true, - "grant-hot-region-scheduler": true, - "transfer-witness-leader-scheduler": true, - "balance-witness-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + "shuffle-region-scheduler": true, + "grant-hot-region-scheduler": true, }) var conf3 map[string]interface{} expected3 := map[string]interface{}{ diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 3b70e837ac5..2c371190cd9 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -1416,7 +1416,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { }) // Check scheduler updated. schedulersController := rc.GetCoordinator().GetSchedulersController() - re.Len(schedulersController.GetSchedulerNames(), 6) + re.Len(schedulersController.GetSchedulerNames(), 4) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) @@ -1436,7 +1436,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.True(leaderServer.GetRaftCluster().IsPrepared()) // Check scheduler updated. schedulersController = rc1.GetCoordinator().GetSchedulersController() - re.Len(schedulersController.GetSchedulerNames(), 6) + re.Len(schedulersController.GetSchedulerNames(), 4) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) @@ -1455,7 +1455,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.True(leaderServer.GetRaftCluster().IsPrepared()) // Check scheduler updated schedulersController = rc.GetCoordinator().GetSchedulersController() - re.Len(schedulersController.GetSchedulerNames(), 6) + re.Len(schedulersController.GetSchedulerNames(), 4) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) From f57ab8d821a4e092efd6001bc1f147d997a1a657 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 12 Dec 2024 11:15:52 +0800 Subject: [PATCH 3/3] fix Signed-off-by: Ryan Leung --- tests/integrations/mcs/scheduling/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 98ffb1ac8ee..f1312c71ebf 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -126,7 +126,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { tc.WaitForPrimaryServing(re) primary := tc.GetPrimaryServer() oldPrimaryAddr := primary.GetAddr() - re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 4) + re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 3) testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) return ok && oldPrimaryAddr == watchedAddr @@ -137,7 +137,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { primary = tc.GetPrimaryServer() newPrimaryAddr := primary.GetAddr() re.NotEqual(oldPrimaryAddr, newPrimaryAddr) - re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 4) + re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 3) testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) return ok && newPrimaryAddr == watchedAddr