From 844aa846697ad2f0833eb470e66ddd80230e661b Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Mon, 17 Jun 2024 17:43:15 +0800
Subject: [PATCH 01/28] skip evict-leader-scheduler

Signed-off-by: okJiang <819421878@qq.com>
---
 client/http/interface.go                        | 4 ++++
 pkg/schedule/schedulers/scheduler_controller.go | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/client/http/interface.go b/client/http/interface.go
index 3684e19b1f5..4e3fe68c386 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -81,6 +81,10 @@ type Client interface {
 	DeletePlacementRuleGroupByID(context.Context, string) error
 	GetAllRegionLabelRules(context.Context) ([]*LabelRule, error)
 	GetRegionLabelRulesByIDs(context.Context, []string) ([]*LabelRule, error)
+	// `SetRegionLabelRule` sets the label rule for a region.
+	// When you set a label rule(deny scheduler),
+	//  1. All of the scheduler will be disabled expect the evict-leader-scheduler.
+	// 	2. Merge-checker will be disabled, which means those regions will not be merged.
 	SetRegionLabelRule(context.Context, *LabelRule) error
 	PatchRegionLabelRules(context.Context, *LabelRulePatch) error
 	/* Scheduling-related interfaces */
diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go
index ea480a06845..aa7bfc6b139 100644
--- a/pkg/schedule/schedulers/scheduler_controller.go
+++ b/pkg/schedule/schedulers/scheduler_controller.go
@@ -486,7 +486,8 @@ retry:
 			if labelMgr == nil {
 				continue
 			}
-			if labelMgr.ScheduleDisabled(region) {
+			_, ok := s.Scheduler.(*evictLeaderScheduler)
+			if labelMgr.ScheduleDisabled(region) && !ok {
 				denySchedulersByLabelerCounter.Inc()
 				continue retry
 			}

From 0f95d6e3d43e462e0ea7c8bf370f3f4f6d897469 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Mon, 17 Jun 2024 17:45:28 +0800
Subject: [PATCH 02/28] refine comment

Signed-off-by: okJiang <819421878@qq.com>
---
 client/http/interface.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/http/interface.go b/client/http/interface.go
index 4e3fe68c386..6495d904ca2 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -82,9 +82,9 @@ type Client interface {
 	GetAllRegionLabelRules(context.Context) ([]*LabelRule, error)
 	GetRegionLabelRulesByIDs(context.Context, []string) ([]*LabelRule, error)
 	// `SetRegionLabelRule` sets the label rule for a region.
-	// When you set a label rule(deny scheduler),
-	//  1. All of the scheduler will be disabled expect the evict-leader-scheduler.
-	// 	2. Merge-checker will be disabled, which means those regions will not be merged.
+	// When a label rule (deny scheduler) is set,
+	//  1. All schedulers will be disabled except for the evict-leader-scheduler.
+	//  2. The merge-checker will be disabled, preventing these regions from being merged.
 	SetRegionLabelRule(context.Context, *LabelRule) error
 	PatchRegionLabelRules(context.Context, *LabelRulePatch) error
 	/* Scheduling-related interfaces */

From 78200a44fac7f73a093af1974175fe9bb0071f05 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 10:42:40 +0800
Subject: [PATCH 03/28] fix comment

Signed-off-by: okJiang <819421878@qq.com>
---
 pkg/schedule/schedulers/scheduler_controller.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go
index aa7bfc6b139..04c74af7964 100644
--- a/pkg/schedule/schedulers/scheduler_controller.go
+++ b/pkg/schedule/schedulers/scheduler_controller.go
@@ -456,6 +456,7 @@ func (s *ScheduleController) Stop() {
 
 // Schedule tries to create some operators.
 func (s *ScheduleController) Schedule(diagnosable bool) []*operator.Operator {
+	_, isEvictLeaderScheduler := s.Scheduler.(*evictLeaderScheduler)
 retry:
 	for i := 0; i < maxScheduleRetries; i++ {
 		// no need to retry if schedule should stop to speed exit
@@ -486,8 +487,10 @@ retry:
 			if labelMgr == nil {
 				continue
 			}
-			_, ok := s.Scheduler.(*evictLeaderScheduler)
-			if labelMgr.ScheduleDisabled(region) && !ok {
+
+			// If the evict-leader-scheduler is disabled, it will obstruct the restart operation of tikv by the operator.
+			// Refer: https://docs.pingcap.com/tidb-in-kubernetes/stable/restart-a-tidb-cluster#perform-a-graceful-restart-to-a-single-tikv-pod
+			if labelMgr.ScheduleDisabled(region) && !isEvictLeaderScheduler {
 				denySchedulersByLabelerCounter.Inc()
 				continue retry
 			}

From de0c4409ad612fc82f7ef99f15d9576b3a5ce675 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 16:30:34 +0800
Subject: [PATCH 04/28] add real cluster test

Signed-off-by: okJiang <819421878@qq.com>
---
 client/http/interface.go                      |   8 +
 client/http/request_info.go                   |   1 +
 .../realcluster/scheduler_test.go             | 188 ++++++++++++++++++
 .../realcluster/transfer_leader_test.go       |  73 -------
 4 files changed, 197 insertions(+), 73 deletions(-)
 create mode 100644 tests/integrations/realcluster/scheduler_test.go
 delete mode 100644 tests/integrations/realcluster/transfer_leader_test.go

diff --git a/client/http/interface.go b/client/http/interface.go
index 6495d904ca2..89174623169 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -65,6 +65,7 @@ type Client interface {
 	/* Scheduler-related interfaces */
 	GetSchedulers(context.Context) ([]string, error)
 	CreateScheduler(ctx context.Context, name string, storeID uint64) error
+	DeleteScheduler(ctx context.Context, name string) error
 	SetSchedulerDelay(context.Context, string, int64) error
 	/* Rule-related interfaces */
 	GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error)
@@ -766,6 +767,13 @@ func (c *client) CreateScheduler(ctx context.Context, name string, storeID uint6
 		WithBody(inputJSON))
 }
 
+func (c *client) DeleteScheduler(ctx context.Context, name string) error {
+	return c.request(ctx, newRequestInfo().
+		WithName(deleteSchedulerName).
+		WithURI(SchedulerByName(name)).
+		WithMethod(http.MethodDelete))
+}
+
 // AccelerateSchedule accelerates the scheduling of the regions within the given key range.
 // The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes).
 func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) error {
diff --git a/client/http/request_info.go b/client/http/request_info.go
index 40bd0368250..23e8aaea07c 100644
--- a/client/http/request_info.go
+++ b/client/http/request_info.go
@@ -53,6 +53,7 @@ const (
 	getReplicateConfigName                  = "GetReplicateConfig"
 	getSchedulersName                       = "GetSchedulers"
 	createSchedulerName                     = "CreateScheduler"
+	deleteSchedulerName                     = "DeleteScheduler"
 	setSchedulerDelayName                   = "SetSchedulerDelay"
 	getAllPlacementRuleBundlesName          = "GetAllPlacementRuleBundles"
 	getPlacementRuleBundleByGroupName       = "GetPlacementRuleBundleByGroup"
diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
new file mode 100644
index 00000000000..5929da5f48e
--- /dev/null
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -0,0 +1,188 @@
+// Copyright 2023 TiKV Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package realcluster
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	pd "github.com/tikv/pd/client/http"
+	"github.com/tikv/pd/client/testutil"
+	"github.com/tikv/pd/pkg/schedule/labeler"
+	"github.com/tikv/pd/pkg/schedule/schedulers"
+)
+
+// https://github.com/tikv/pd/issues/6988#issuecomment-1694924611
+// https://github.com/tikv/pd/issues/6897
+func TestTransferLeader(t *testing.T) {
+	re := require.New(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resp, err := pdHTTPCli.GetLeader(ctx)
+	re.NoError(err)
+	oldLeader := resp.Name
+
+	var newLeader string
+	for i := 0; i < 2; i++ {
+		if resp.Name != fmt.Sprintf("pd-%d", i) {
+			newLeader = fmt.Sprintf("pd-%d", i)
+		}
+	}
+
+	// record scheduler
+	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, 1))
+	defer func() {
+		re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName))
+	}()
+	res, err := pdHTTPCli.GetSchedulers(ctx)
+	re.NoError(err)
+	oldSchedulersLen := len(res)
+
+	re.NoError(pdHTTPCli.TransferLeader(ctx, newLeader))
+	// wait for transfer leader to new leader
+	time.Sleep(1 * time.Second)
+	resp, err = pdHTTPCli.GetLeader(ctx)
+	re.NoError(err)
+	re.Equal(newLeader, resp.Name)
+
+	res, err = pdHTTPCli.GetSchedulers(ctx)
+	re.NoError(err)
+	re.Len(res, oldSchedulersLen)
+
+	// transfer leader to old leader
+	re.NoError(pdHTTPCli.TransferLeader(ctx, oldLeader))
+	// wait for transfer leader
+	time.Sleep(1 * time.Second)
+	resp, err = pdHTTPCli.GetLeader(ctx)
+	re.NoError(err)
+	re.Equal(oldLeader, resp.Name)
+
+	res, err = pdHTTPCli.GetSchedulers(ctx)
+	re.NoError(err)
+	re.Len(res, oldSchedulersLen)
+}
+
+func TestRegionLabel_DenyScheduler(t *testing.T) {
+	re := require.New(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	regions, err := pdHTTPCli.GetRegions(ctx)
+	re.NoError(err)
+	re.True(len(regions.Regions) >= 2)
+	region1, region2 := regions.Regions[0], regions.Regions[1]
+
+	pdHTTPCli.DeleteScheduler(ctx, schedulers.ShuffleLeaderName)
+	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.ShuffleLeaderName, 0))
+	defer func() {
+		re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.ShuffleLeaderName))
+	}()
+
+	// wait leader transfer
+	testutil.Eventually(re, func() bool {
+		regions, err := pdHTTPCli.GetRegions(ctx)
+		re.NoError(err)
+		re.True(len(regions.Regions) > 0)
+		for _, region := range regions.Regions {
+			if region.ID == region1.ID && region.Leader.StoreID != region1.ID {
+				return true
+			}
+			if region.ID == region2.ID && region.Leader.StoreID != region2.ID {
+				return true
+			}
+		}
+		return false
+	})
+
+	// disable schedule for region1
+	labelRule := &pd.LabelRule{
+		ID:       "rule1",
+		Labels:   []pd.RegionLabel{{Key: "schedule", Value: "deny"}},
+		RuleType: "key-range",
+		Data:     labeler.MakeKeyRanges(region1.StartKey, region1.EndKey),
+	}
+	err = pdHTTPCli.SetRegionLabelRule(ctx, labelRule)
+	re.NoError(err)
+	time.Sleep(time.Second)
+	labelRules, err := pdHTTPCli.GetAllRegionLabelRules(ctx)
+	re.NoError(err)
+	re.Len(labelRules, 2)
+	sort.Slice(labelRules, func(i, j int) bool {
+		return labelRules[i].ID < labelRules[j].ID
+	})
+	re.Equal(labelRule.ID, labelRules[1].ID)
+	re.Equal(labelRule.Labels, labelRules[1].Labels)
+	re.Equal(labelRule.RuleType, labelRules[1].RuleType)
+
+	regions, err = pdHTTPCli.GetRegions(ctx)
+	re.NoError(err)
+	for _, region := range regions.Regions {
+		if region.ID == region1.ID {
+			region1 = region
+		}
+		if region.ID == region2.ID {
+			region2 = region
+		}
+	}
+	// check shuffle leader scheduler of region1 has been disabled
+	for i := 0; i < 10; i++ {
+		regions, err := pdHTTPCli.GetRegions(ctx)
+		re.NoError(err)
+		for _, region := range regions.Regions {
+			if region.ID == region1.ID {
+				re.True(region.Leader.StoreID == region1.Leader.StoreID)
+			}
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	// check shuffle leader scheduler of region2 has not been disabled
+	testutil.Eventually(re, func() bool {
+		regions, err := pdHTTPCli.GetRegions(ctx)
+		re.NoError(err)
+		for _, region := range regions.Regions {
+			if region.ID == region2.ID && region.Leader.StoreID != region2.Leader.StoreID {
+				return true
+			}
+		}
+		return false
+	})
+
+	oldRegions, err := pdHTTPCli.GetRegionsByStoreID(ctx, uint64(region1.Leader.StoreID))
+	re.NoError(err)
+	oldRegionMap := make(map[int64]bool, len(oldRegions.Regions))
+	for _, region := range oldRegions.Regions {
+		oldRegionMap[region.ID] = true
+	}
+	// enable evict leader scheduler, and check it works
+	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, uint64(region1.Leader.StoreID)))
+	defer func() {
+		re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName))
+	}()
+	testutil.Eventually(re, func() bool {
+		regions, err := pdHTTPCli.GetRegions(ctx)
+		re.NoError(err)
+		for _, region := range regions.Regions {
+			if oldRegionMap[region.ID] && region.Leader.StoreID == region1.Leader.StoreID {
+				return false
+			}
+		}
+		return true
+	})
+}
diff --git a/tests/integrations/realcluster/transfer_leader_test.go b/tests/integrations/realcluster/transfer_leader_test.go
deleted file mode 100644
index 0000f7e14a5..00000000000
--- a/tests/integrations/realcluster/transfer_leader_test.go
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright 2023 TiKV Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package realcluster
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-)
-
-// https://github.com/tikv/pd/issues/6988#issuecomment-1694924611
-// https://github.com/tikv/pd/issues/6897
-func TestTransferLeader(t *testing.T) {
-	re := require.New(t)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	resp, err := pdHTTPCli.GetLeader(ctx)
-	re.NoError(err)
-	oldLeader := resp.Name
-
-	var newLeader string
-	for i := 0; i < 2; i++ {
-		if resp.Name != fmt.Sprintf("pd-%d", i) {
-			newLeader = fmt.Sprintf("pd-%d", i)
-		}
-	}
-
-	// record scheduler
-	err = pdHTTPCli.CreateScheduler(ctx, "evict-leader-scheduler", 1)
-	re.NoError(err)
-	res, err := pdHTTPCli.GetSchedulers(ctx)
-	re.NoError(err)
-	oldSchedulersLen := len(res)
-
-	re.NoError(pdHTTPCli.TransferLeader(ctx, newLeader))
-	// wait for transfer leader to new leader
-	time.Sleep(1 * time.Second)
-	resp, err = pdHTTPCli.GetLeader(ctx)
-	re.NoError(err)
-	re.Equal(newLeader, resp.Name)
-
-	res, err = pdHTTPCli.GetSchedulers(ctx)
-	re.NoError(err)
-	re.Len(res, oldSchedulersLen)
-
-	// transfer leader to old leader
-	re.NoError(pdHTTPCli.TransferLeader(ctx, oldLeader))
-	// wait for transfer leader
-	time.Sleep(1 * time.Second)
-	resp, err = pdHTTPCli.GetLeader(ctx)
-	re.NoError(err)
-	re.Equal(oldLeader, resp.Name)
-
-	res, err = pdHTTPCli.GetSchedulers(ctx)
-	re.NoError(err)
-	re.Len(res, oldSchedulersLen)
-}

From c3e122d13aefa144e440bbd71782048f8b3b18d4 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 16:41:42 +0800
Subject: [PATCH 05/28] fix lint

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/scheduler_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index 5929da5f48e..40812277153 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -89,7 +89,6 @@ func TestRegionLabel_DenyScheduler(t *testing.T) {
 	re.True(len(regions.Regions) >= 2)
 	region1, region2 := regions.Regions[0], regions.Regions[1]
 
-	pdHTTPCli.DeleteScheduler(ctx, schedulers.ShuffleLeaderName)
 	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.ShuffleLeaderName, 0))
 	defer func() {
 		re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.ShuffleLeaderName))

From d37ce0261b51acbc9970c7d7f4f227a9730ff38f Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 17:19:24 +0800
Subject: [PATCH 06/28] add log

Signed-off-by: okJiang <819421878@qq.com>
---
 .gitignore                                      |  1 +
 client/http/interface.go                        |  1 +
 tests/integrations/realcluster/deploy.sh        |  4 ++--
 .../integrations/realcluster/scheduler_test.go  | 17 ++++++++++++-----
 4 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index b9be6099e24..b743f854405 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,4 @@ coverage
 *.txt
 go.work*
 embedded_assets_handler.go
+*.log
\ No newline at end of file
diff --git a/client/http/interface.go b/client/http/interface.go
index 89174623169..639a810fc00 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -767,6 +767,7 @@ func (c *client) CreateScheduler(ctx context.Context, name string, storeID uint6
 		WithBody(inputJSON))
 }
 
+// DeleteScheduler deletes a scheduler from PD cluster.
 func (c *client) DeleteScheduler(ctx context.Context, name string) error {
 	return c.request(ctx, newRequestInfo().
 		WithName(deleteSchedulerName).
diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index 8cce60e8ee6..64124615b2a 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -19,13 +19,13 @@ if [ ! -d "bin" ] || [ ! -e "bin/tikv-server" ] && [ ! -e "bin/tidb-server" ] &&
 	color-green "downloading binaries..."
 	color-green "this may take a few minutes, you can also download them manually and put them in the bin directory."
 	make pd-server WITH_RACE=1
-	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_test \
+	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_real_cluster_test \
 		--pd.binpath ./bin/pd-server \
 		> $CUR_PATH/playground.log 2>&1 &
 else
 	color-green "using existing binaries..."
 	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
-		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_test \
+		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test \
 		> $CUR_PATH/playground.log 2>&1 &
 fi
 
diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index 40812277153..fc2c1fc48ea 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -1,4 +1,4 @@
-// Copyright 2023 TiKV Authors
+// Copyright 2024 TiKV Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pingcap/log"
 	"github.com/stretchr/testify/require"
 	pd "github.com/tikv/pd/client/http"
 	"github.com/tikv/pd/client/testutil"
@@ -79,7 +80,7 @@ func TestTransferLeader(t *testing.T) {
 	re.Len(res, oldSchedulersLen)
 }
 
-func TestRegionLabel_DenyScheduler(t *testing.T) {
+func TestRegionLabelDenyScheduler(t *testing.T) {
 	re := require.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -141,11 +142,12 @@ func TestRegionLabel_DenyScheduler(t *testing.T) {
 		}
 	}
 	// check shuffle leader scheduler of region1 has been disabled
-	for i := 0; i < 10; i++ {
+	for i := 0; i < 20; i++ {
 		regions, err := pdHTTPCli.GetRegions(ctx)
 		re.NoError(err)
 		for _, region := range regions.Regions {
 			if region.ID == region1.ID {
+				log.Info("hit region 1")
 				re.True(region.Leader.StoreID == region1.Leader.StoreID)
 			}
 		}
@@ -156,6 +158,9 @@ func TestRegionLabel_DenyScheduler(t *testing.T) {
 		regions, err := pdHTTPCli.GetRegions(ctx)
 		re.NoError(err)
 		for _, region := range regions.Regions {
+			if region.ID == region2.ID {
+				log.Info("hit region 2")
+			}
 			if region.ID == region2.ID && region.Leader.StoreID != region2.Leader.StoreID {
 				return true
 			}
@@ -163,11 +168,13 @@ func TestRegionLabel_DenyScheduler(t *testing.T) {
 		return false
 	})
 
-	oldRegions, err := pdHTTPCli.GetRegionsByStoreID(ctx, uint64(region1.Leader.StoreID))
+	oldRegions, err := pdHTTPCli.GetRegions(ctx)
 	re.NoError(err)
 	oldRegionMap := make(map[int64]bool, len(oldRegions.Regions))
 	for _, region := range oldRegions.Regions {
-		oldRegionMap[region.ID] = true
+		if region.Leader.ID == region1.Leader.ID {
+			oldRegionMap[region.ID] = true
+		}
 	}
 	// enable evict leader scheduler, and check it works
 	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, uint64(region1.Leader.StoreID)))

From 15c9eee5edfa7558057bd3488b2271ea3118b6cb Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 17:20:26 +0800
Subject: [PATCH 07/28] fix

Signed-off-by: okJiang <819421878@qq.com>
---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index b743f854405..fb9f0424418 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,4 +26,4 @@ coverage
 *.txt
 go.work*
 embedded_assets_handler.go
-*.log
\ No newline at end of file
+*.log

From fbca57799485b4cb3b387669bf29800b20c79759 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 17:25:56 +0800
Subject: [PATCH 08/28] fix comment

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/wait_tiup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integrations/realcluster/wait_tiup.sh b/tests/integrations/realcluster/wait_tiup.sh
index 497774f9e96..3a8c02a969e 100755
--- a/tests/integrations/realcluster/wait_tiup.sh
+++ b/tests/integrations/realcluster/wait_tiup.sh
@@ -12,7 +12,7 @@ fi
 
 for ((i=0; i<${MAX_TIMES}; i++)); do
     sleep ${INTERVAL}
-    $TIUP_BIN_DIR playground display --tag pd_test
+    $TIUP_BIN_DIR playground display --tag pd_real_cluster_test
     if [ $? -eq 0 ]; then
         exit 0
     fi

From 61865f3f98d94e70ecaeb2db44dc602e6b6ef3f0 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 18:11:15 +0800
Subject: [PATCH 09/28] add log

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/Makefile          | 2 +-
 tests/integrations/realcluster/deploy.sh         | 4 ++--
 tests/integrations/realcluster/scheduler_test.go | 5 +++--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile
index e161d52a86e..d209198b99d 100644
--- a/tests/integrations/realcluster/Makefile
+++ b/tests/integrations/realcluster/Makefile
@@ -50,7 +50,7 @@ kill_cluster:
 	fi
 
 test:
-	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || { exit 1; }
+	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (cat ~/.tiup/data/pd_real_cluster_test/pd-0/pd.log ; exit 1)
 
 install-tools:
 	cd $(ROOT_PATH) && $(MAKE) install-tools
diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index 64124615b2a..a733060c842 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -19,12 +19,12 @@ if [ ! -d "bin" ] || [ ! -e "bin/tikv-server" ] && [ ! -e "bin/tidb-server" ] &&
 	color-green "downloading binaries..."
 	color-green "this may take a few minutes, you can also download them manually and put them in the bin directory."
 	make pd-server WITH_RACE=1
-	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_real_cluster_test \
+	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 1 --without-monitor --tag pd_real_cluster_test \
 		--pd.binpath ./bin/pd-server \
 		> $CUR_PATH/playground.log 2>&1 &
 else
 	color-green "using existing binaries..."
-	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
+	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 1 --without-monitor \
 		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test \
 		> $CUR_PATH/playground.log 2>&1 &
 fi
diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index fc2c1fc48ea..1b0dda6fece 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/tikv/pd/client/testutil"
 	"github.com/tikv/pd/pkg/schedule/labeler"
 	"github.com/tikv/pd/pkg/schedule/schedulers"
+	"go.uber.org/zap"
 )
 
 // https://github.com/tikv/pd/issues/6988#issuecomment-1694924611
@@ -147,7 +148,7 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 		re.NoError(err)
 		for _, region := range regions.Regions {
 			if region.ID == region1.ID {
-				log.Info("hit region 1")
+				log.Info("hit region 1", zap.Int64("region id", region.ID))
 				re.True(region.Leader.StoreID == region1.Leader.StoreID)
 			}
 		}
@@ -159,7 +160,7 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 		re.NoError(err)
 		for _, region := range regions.Regions {
 			if region.ID == region2.ID {
-				log.Info("hit region 2")
+				log.Info("hit region 2", zap.Int64("region id", region.ID))
 			}
 			if region.ID == region2.ID && region.Leader.StoreID != region2.Leader.StoreID {
 				return true

From 3430f19716a0ddb76b9ccbe0e6da4e375b695c1d Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Tue, 18 Jun 2024 18:21:15 +0800
Subject: [PATCH 10/28] revert pd count

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/Makefile  | 6 +++++-
 tests/integrations/realcluster/deploy.sh | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile
index d209198b99d..18fade38805 100644
--- a/tests/integrations/realcluster/Makefile
+++ b/tests/integrations/realcluster/Makefile
@@ -50,7 +50,11 @@ kill_cluster:
 	fi
 
 test:
-	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (cat ~/.tiup/data/pd_real_cluster_test/pd-0/pd.log ; exit 1)
+	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (\
+	cat ~/.tiup/data/pd_real_cluster_test/pd-0/pd.log ; \
+	cat ~/.tiup/data/pd_real_cluster_test/pd-1/pd.log ; \
+	cat ~/.tiup/data/pd_real_cluster_test/pd-2/pd.log ; \
+	exit 1)
 
 install-tools:
 	cd $(ROOT_PATH) && $(MAKE) install-tools
diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index a733060c842..64124615b2a 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -19,12 +19,12 @@ if [ ! -d "bin" ] || [ ! -e "bin/tikv-server" ] && [ ! -e "bin/tidb-server" ] &&
 	color-green "downloading binaries..."
 	color-green "this may take a few minutes, you can also download them manually and put them in the bin directory."
 	make pd-server WITH_RACE=1
-	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 1 --without-monitor --tag pd_real_cluster_test \
+	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_real_cluster_test \
 		--pd.binpath ./bin/pd-server \
 		> $CUR_PATH/playground.log 2>&1 &
 else
 	color-green "using existing binaries..."
-	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 1 --without-monitor \
+	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
 		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test \
 		> $CUR_PATH/playground.log 2>&1 &
 fi

From 2efe6ecbae3d367f91636e73dc01815dca952787 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 11:03:17 +0800
Subject: [PATCH 11/28] fix ci

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/Makefile  | 3 +++
 tests/integrations/realcluster/deploy.sh | 1 +
 2 files changed, 4 insertions(+)

diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile
index 18fade38805..28c918ec2bf 100644
--- a/tests/integrations/realcluster/Makefile
+++ b/tests/integrations/realcluster/Makefile
@@ -51,8 +51,11 @@ kill_cluster:
 
 test:
 	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (\
+	echo "follow is pd-0 log\n" ; \
 	cat ~/.tiup/data/pd_real_cluster_test/pd-0/pd.log ; \
+	echo "follow is pd-1 log\n" ; \
 	cat ~/.tiup/data/pd_real_cluster_test/pd-1/pd.log ; \
+	echo "follow is pd-2 log\n" ; \
 	cat ~/.tiup/data/pd_real_cluster_test/pd-2/pd.log ; \
 	exit 1)
 
diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index 64124615b2a..eb90f1c386c 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -24,6 +24,7 @@ if [ ! -d "bin" ] || [ ! -e "bin/tikv-server" ] && [ ! -e "bin/tidb-server" ] &&
 		> $CUR_PATH/playground.log 2>&1 &
 else
 	color-green "using existing binaries..."
+	make pd-server WITH_RACE=1
 	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
 		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test \
 		> $CUR_PATH/playground.log 2>&1 &

From 52e4c96e48767e7a92976e2583b0931031a44b5e Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 11:21:14 +0800
Subject: [PATCH 12/28] clean data after test

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile
index 28c918ec2bf..c1b01b269e0 100644
--- a/tests/integrations/realcluster/Makefile
+++ b/tests/integrations/realcluster/Makefile
@@ -48,6 +48,7 @@ kill_cluster:
 		echo "waiting for cluster to exit..."; \
 		sleep 30; \
 	fi
+	@ rm -rf ~/.tiup/data/pd_real_cluster_test
 
 test:
 	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (\

From 616363b6f3cfe211ca207a5f808c99a580e9cd51 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 12:51:12 +0800
Subject: [PATCH 13/28] update log level to debug

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/deploy.sh | 2 +-
 tests/integrations/realcluster/pd.toml   | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 tests/integrations/realcluster/pd.toml

diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index eb90f1c386c..c2f6e8dfc74 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -20,7 +20,7 @@ if [ ! -d "bin" ] || [ ! -e "bin/tikv-server" ] && [ ! -e "bin/tidb-server" ] &&
 	color-green "this may take a few minutes, you can also download them manually and put them in the bin directory."
 	make pd-server WITH_RACE=1
 	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_real_cluster_test \
-		--pd.binpath ./bin/pd-server \
+		--pd.binpath ./bin/pd-server --pd.config ./tests/integrations/realcluster/pd.toml \
 		> $CUR_PATH/playground.log 2>&1 &
 else
 	color-green "using existing binaries..."
diff --git a/tests/integrations/realcluster/pd.toml b/tests/integrations/realcluster/pd.toml
new file mode 100644
index 00000000000..876c7f13af2
--- /dev/null
+++ b/tests/integrations/realcluster/pd.toml
@@ -0,0 +1,5 @@
+[schedule]
+patrol-region-interval = "100ms"
+
+[log]
+level = "debug"

From 182188eb47ab1bda0a48a7fb437639e1c78ae505 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 13:11:58 +0800
Subject: [PATCH 14/28] fix ci

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/deploy.sh         |  5 ++++-
 tests/integrations/realcluster/scheduler_test.go | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index c2f6e8dfc74..ba41fed8277 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 # deploy `tiup playground`
 
+set -x
+
 TIUP_BIN_DIR=$HOME/.tiup/bin/tiup
 CUR_PATH=$(pwd)
 
@@ -26,7 +28,8 @@ else
 	color-green "using existing binaries..."
 	make pd-server WITH_RACE=1
 	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
-		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test \
+		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server \
+		--tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test --pd.config ./tests/integrations/realcluster/pd.toml \
 		> $CUR_PATH/playground.log 2>&1 &
 fi
 
diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index 1b0dda6fece..a877bffdc14 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -16,6 +16,7 @@ package realcluster
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"testing"
@@ -81,6 +82,19 @@ func TestTransferLeader(t *testing.T) {
 	re.Len(res, oldSchedulersLen)
 }
 
+func TestX(t *testing.T) {
+	type test struct {
+		A []string `json:"-"`
+		B []string `json:"b"`
+	}
+
+	tt := test{A: []string{"a", "b"}, B: []string{"c", "d"}}
+	b, _ := json.Marshal(tt)
+	var tt2 test
+	json.Unmarshal(b, &tt2)
+	require.Equal(t, tt, tt2)
+}
+
 func TestRegionLabelDenyScheduler(t *testing.T) {
 	re := require.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -192,4 +206,5 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 		}
 		return true
 	})
+	re.True(false)
 }

From 00ea6d4eb8aba6388f7872ce2d862dd1473b88e8 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 14:19:10 +0800
Subject: [PATCH 15/28] fix ci

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/reboot_pd_test.go |  9 ++++++---
 tests/integrations/realcluster/scheduler_test.go | 15 ---------------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index 8e99b0822f0..91877611c8b 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -47,10 +47,13 @@ func TestReloadLabel(t *testing.T) {
 	for _, label := range setStore.Store.Labels {
 		storeLabel[label.Key] = label.Value
 	}
-	err := pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel)
-	re.NoError(err)
 
-	resp, err = pdHTTPCli.GetStores(ctx)
+	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, nil))
+	defer func() {
+		re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, nil))
+	}()
+
+	resp, err := pdHTTPCli.GetStores(ctx)
 	re.NoError(err)
 	for _, store := range resp.Stores {
 		if store.Store.ID == setStore.Store.ID {
diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index a877bffdc14..1b0dda6fece 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -16,7 +16,6 @@ package realcluster
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"sort"
 	"testing"
@@ -82,19 +81,6 @@ func TestTransferLeader(t *testing.T) {
 	re.Len(res, oldSchedulersLen)
 }
 
-func TestX(t *testing.T) {
-	type test struct {
-		A []string `json:"-"`
-		B []string `json:"b"`
-	}
-
-	tt := test{A: []string{"a", "b"}, B: []string{"c", "d"}}
-	b, _ := json.Marshal(tt)
-	var tt2 test
-	json.Unmarshal(b, &tt2)
-	require.Equal(t, tt, tt2)
-}
-
 func TestRegionLabelDenyScheduler(t *testing.T) {
 	re := require.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -206,5 +192,4 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 		}
 		return true
 	})
-	re.True(false)
 }

From 4cbf3621dbbf2f59ca380152bd8c65e0c09ec723 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 14:22:00 +0800
Subject: [PATCH 16/28] fix typo

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/reboot_pd_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index 91877611c8b..5a7b3cff127 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -48,7 +48,7 @@ func TestReloadLabel(t *testing.T) {
 		storeLabel[label.Key] = label.Value
 	}
 
-	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, nil))
+	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
 	defer func() {
 		re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, nil))
 	}()

From 1ae13f2863ff91121dc677a443913ff298265686 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 14:36:12 +0800
Subject: [PATCH 17/28] add force param

Signed-off-by: okJiang <819421878@qq.com>
---
 client/http/api.go                               |  5 ++++-
 client/http/interface.go                         | 16 +++++++++++++++-
 tests/integrations/realcluster/reboot_pd_test.go |  2 +-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/client/http/api.go b/client/http/api.go
index 3376a48770d..472f2b4813f 100644
--- a/client/http/api.go
+++ b/client/http/api.go
@@ -137,7 +137,10 @@ func StoreLabelByID(id uint64) string {
 }
 
 // LabelByStoreID returns the path of PD HTTP API to set store label.
-func LabelByStoreID(storeID int64) string {
+func LabelByStoreID(storeID int64, force bool) string {
+	if force {
+		return fmt.Sprintf("%s/%d/label?force=true", store, storeID)
+	}
 	return fmt.Sprintf("%s/%d/label", store, storeID)
 }
 
diff --git a/client/http/interface.go b/client/http/interface.go
index 639a810fc00..b4c0085f90a 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -51,6 +51,7 @@ type Client interface {
 	GetStore(context.Context, uint64) (*StoreInfo, error)
 	DeleteStore(context.Context, uint64) error
 	SetStoreLabels(context.Context, int64, map[string]string) error
+	SetStoreLabelsV2(ctx context.Context, storeID int64, force bool, storeLabels map[string]string) error
 	GetHealthStatus(context.Context) ([]Health, error)
 	/* Config-related interfaces */
 	GetConfig(context.Context) (map[string]any, error)
@@ -339,7 +340,20 @@ func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels
 	}
 	return c.request(ctx, newRequestInfo().
 		WithName(setStoreLabelsName).
-		WithURI(LabelByStoreID(storeID)).
+		WithURI(LabelByStoreID(storeID, false)).
+		WithMethod(http.MethodPost).
+		WithBody(jsonInput))
+}
+
+// SetStoreLabelsV2 sets the labels of a store.
+func (c *client) SetStoreLabelsV2(ctx context.Context, storeID int64, force bool, storeLabels map[string]string) error {
+	jsonInput, err := json.Marshal(storeLabels)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	return c.request(ctx, newRequestInfo().
+		WithName(setStoreLabelsName).
+		WithURI(LabelByStoreID(storeID, force)).
 		WithMethod(http.MethodPost).
 		WithBody(jsonInput))
 }
diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index 5a7b3cff127..1a1f895e9b1 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -50,7 +50,7 @@ func TestReloadLabel(t *testing.T) {
 
 	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
 	defer func() {
-		re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, nil))
+		re.NoError(pdHTTPCli.SetStoreLabelsV2(ctx, setStore.Store.ID, true, nil))
 	}()
 
 	resp, err := pdHTTPCli.GetStores(ctx)

From 96ae0a6d8c1d71b0d4ad9eb8d2287a14c44afe99 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 17:39:27 +0800
Subject: [PATCH 18/28] update test

Signed-off-by: okJiang <819421878@qq.com>
---
 .../realcluster/scheduler_test.go             | 97 +++++++++----------
 1 file changed, 45 insertions(+), 52 deletions(-)

diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index 1b0dda6fece..caf97db8bb8 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -21,13 +21,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/stretchr/testify/require"
 	pd "github.com/tikv/pd/client/http"
 	"github.com/tikv/pd/client/testutil"
 	"github.com/tikv/pd/pkg/schedule/labeler"
 	"github.com/tikv/pd/pkg/schedule/schedulers"
-	"go.uber.org/zap"
 )
 
 // https://github.com/tikv/pd/issues/6988#issuecomment-1694924611
@@ -88,28 +86,31 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 
 	regions, err := pdHTTPCli.GetRegions(ctx)
 	re.NoError(err)
-	re.True(len(regions.Regions) >= 2)
-	region1, region2 := regions.Regions[0], regions.Regions[1]
+	re.True(len(regions.Regions) >= 1)
+	region1 := regions.Regions[0]
+
+	err = pdHTTPCli.DeleteScheduler(ctx, schedulers.BalanceLeaderName)
+	if err == nil {
+		defer func() {
+			pdHTTPCli.CreateScheduler(ctx, schedulers.BalanceLeaderName, 0)
+		}()
+	}
 
-	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.ShuffleLeaderName, 0))
+	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID)))
 	defer func() {
-		re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.ShuffleLeaderName))
+		pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName)
 	}()
 
 	// wait leader transfer
 	testutil.Eventually(re, func() bool {
 		regions, err := pdHTTPCli.GetRegions(ctx)
 		re.NoError(err)
-		re.True(len(regions.Regions) > 0)
 		for _, region := range regions.Regions {
-			if region.ID == region1.ID && region.Leader.StoreID != region1.ID {
-				return true
-			}
-			if region.ID == region2.ID && region.Leader.StoreID != region2.ID {
-				return true
+			if region.Leader.StoreID != region1.Leader.StoreID {
+				return false
 			}
 		}
-		return false
+		return true
 	})
 
 	// disable schedule for region1
@@ -119,9 +120,10 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 		RuleType: "key-range",
 		Data:     labeler.MakeKeyRanges(region1.StartKey, region1.EndKey),
 	}
-	err = pdHTTPCli.SetRegionLabelRule(ctx, labelRule)
-	re.NoError(err)
-	time.Sleep(time.Second)
+	re.NoError(pdHTTPCli.SetRegionLabelRule(ctx, labelRule))
+	defer func() {
+		pdHTTPCli.PatchRegionLabelRules(ctx, &pd.LabelRulePatch{DeleteRules: []string{labelRule.ID}})
+	}()
 	labelRules, err := pdHTTPCli.GetAllRegionLabelRules(ctx)
 	re.NoError(err)
 	re.Len(labelRules, 2)
@@ -132,61 +134,52 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 	re.Equal(labelRule.Labels, labelRules[1].Labels)
 	re.Equal(labelRule.RuleType, labelRules[1].RuleType)
 
-	regions, err = pdHTTPCli.GetRegions(ctx)
-	re.NoError(err)
-	for _, region := range regions.Regions {
-		if region.ID == region1.ID {
-			region1 = region
-		}
-		if region.ID == region2.ID {
-			region2 = region
-		}
-	}
-	// check shuffle leader scheduler of region1 has been disabled
-	for i := 0; i < 20; i++ {
+	// enable evict leader scheduler, and check it works
+	re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName))
+	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, uint64(region1.Leader.StoreID)))
+	defer func() {
+		pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName)
+	}()
+	testutil.Eventually(re, func() bool {
 		regions, err := pdHTTPCli.GetRegions(ctx)
 		re.NoError(err)
 		for _, region := range regions.Regions {
-			if region.ID == region1.ID {
-				log.Info("hit region 1", zap.Int64("region id", region.ID))
-				re.True(region.Leader.StoreID == region1.Leader.StoreID)
+			if region.Leader.StoreID == region1.Leader.StoreID {
+				return false
 			}
 		}
-		time.Sleep(50 * time.Millisecond)
-	}
-	// check shuffle leader scheduler of region2 has not been disabled
+		return true
+	})
+
+	re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName))
+	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID)))
+	defer func() {
+		pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName)
+	}()
 	testutil.Eventually(re, func() bool {
 		regions, err := pdHTTPCli.GetRegions(ctx)
 		re.NoError(err)
 		for _, region := range regions.Regions {
-			if region.ID == region2.ID {
-				log.Info("hit region 2", zap.Int64("region id", region.ID))
+			if region.ID == region1.ID {
+				continue
 			}
-			if region.ID == region2.ID && region.Leader.StoreID != region2.Leader.StoreID {
-				return true
+			if region.Leader.StoreID != region1.Leader.StoreID {
+				return false
 			}
 		}
-		return false
+		return true
 	})
 
-	oldRegions, err := pdHTTPCli.GetRegions(ctx)
+	pdHTTPCli.PatchRegionLabelRules(ctx, &pd.LabelRulePatch{DeleteRules: []string{labelRule.ID}})
+	labelRules, err = pdHTTPCli.GetAllRegionLabelRules(ctx)
 	re.NoError(err)
-	oldRegionMap := make(map[int64]bool, len(oldRegions.Regions))
-	for _, region := range oldRegions.Regions {
-		if region.Leader.ID == region1.Leader.ID {
-			oldRegionMap[region.ID] = true
-		}
-	}
-	// enable evict leader scheduler, and check it works
-	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, uint64(region1.Leader.StoreID)))
-	defer func() {
-		re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName))
-	}()
+	re.Len(labelRules, 1)
+
 	testutil.Eventually(re, func() bool {
 		regions, err := pdHTTPCli.GetRegions(ctx)
 		re.NoError(err)
 		for _, region := range regions.Regions {
-			if oldRegionMap[region.ID] && region.Leader.StoreID == region1.Leader.StoreID {
+			if region.Leader.StoreID != region1.Leader.StoreID {
 				return false
 			}
 		}

From 4355569a83794501a6cb5ec75365dc29e929f42c Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Wed, 19 Jun 2024 21:15:17 +0800
Subject: [PATCH 19/28] update api

Signed-off-by: okJiang <819421878@qq.com>
---
 client/http/api.go                               |  5 +----
 client/http/interface.go                         | 16 ++++++++--------
 client/http/request_info.go                      |  1 +
 tests/integrations/realcluster/reboot_pd_test.go |  2 +-
 4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/client/http/api.go b/client/http/api.go
index 472f2b4813f..3376a48770d 100644
--- a/client/http/api.go
+++ b/client/http/api.go
@@ -137,10 +137,7 @@ func StoreLabelByID(id uint64) string {
 }
 
 // LabelByStoreID returns the path of PD HTTP API to set store label.
-func LabelByStoreID(storeID int64, force bool) string {
-	if force {
-		return fmt.Sprintf("%s/%d/label?force=true", store, storeID)
-	}
+func LabelByStoreID(storeID int64) string {
 	return fmt.Sprintf("%s/%d/label", store, storeID)
 }
 
diff --git a/client/http/interface.go b/client/http/interface.go
index b4c0085f90a..d87bb003cee 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -51,7 +51,7 @@ type Client interface {
 	GetStore(context.Context, uint64) (*StoreInfo, error)
 	DeleteStore(context.Context, uint64) error
 	SetStoreLabels(context.Context, int64, map[string]string) error
-	SetStoreLabelsV2(ctx context.Context, storeID int64, force bool, storeLabels map[string]string) error
+	DeleteStoreLabel(ctx context.Context, storeID int64, labelKey string) error
 	GetHealthStatus(context.Context) ([]Health, error)
 	/* Config-related interfaces */
 	GetConfig(context.Context) (map[string]any, error)
@@ -340,21 +340,21 @@ func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels
 	}
 	return c.request(ctx, newRequestInfo().
 		WithName(setStoreLabelsName).
-		WithURI(LabelByStoreID(storeID, false)).
+		WithURI(LabelByStoreID(storeID)).
 		WithMethod(http.MethodPost).
 		WithBody(jsonInput))
 }
 
-// SetStoreLabelsV2 sets the labels of a store.
-func (c *client) SetStoreLabelsV2(ctx context.Context, storeID int64, force bool, storeLabels map[string]string) error {
-	jsonInput, err := json.Marshal(storeLabels)
+// DeleteStoreLabel deletes the labels of a store.
+func (c *client) DeleteStoreLabel(ctx context.Context, storeID int64, labelKey string) error {
+	jsonInput, err := json.Marshal(labelKey)
 	if err != nil {
 		return errors.Trace(err)
 	}
 	return c.request(ctx, newRequestInfo().
-		WithName(setStoreLabelsName).
-		WithURI(LabelByStoreID(storeID, force)).
-		WithMethod(http.MethodPost).
+		WithName(deleteStoreLableName).
+		WithURI(LabelByStoreID(storeID)).
+		WithMethod(http.MethodDelete).
 		WithBody(jsonInput))
 }
 
diff --git a/client/http/request_info.go b/client/http/request_info.go
index 23e8aaea07c..2345895cd54 100644
--- a/client/http/request_info.go
+++ b/client/http/request_info.go
@@ -41,6 +41,7 @@ const (
 	getStoreName                            = "GetStore"
 	deleteStoreName                         = "DeleteStore"
 	setStoreLabelsName                      = "SetStoreLabels"
+	deleteStoreLableName                    = "DeleteStoreLabel"
 	getHealthStatusName                     = "GetHealthStatus"
 	getConfigName                           = "GetConfig"
 	setConfigName                           = "SetConfig"
diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index 1a1f895e9b1..f5dc3b55b52 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -50,7 +50,7 @@ func TestReloadLabel(t *testing.T) {
 
 	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
 	defer func() {
-		re.NoError(pdHTTPCli.SetStoreLabelsV2(ctx, setStore.Store.ID, true, nil))
+		re.NoError(pdHTTPCli.DeleteStoreLabel(ctx, setStore.Store.ID, "zone"))
 	}()
 
 	resp, err := pdHTTPCli.GetStores(ctx)

From 231bd8a557377ee019eb05156c848ae46dc1df5e Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Thu, 20 Jun 2024 11:41:10 +0800
Subject: [PATCH 20/28] disable restart test

Signed-off-by: okJiang <819421878@qq.com>
---
 .../realcluster/reboot_pd_test.go             | 101 ++++++++----------
 1 file changed, 46 insertions(+), 55 deletions(-)

diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index f5dc3b55b52..313179e262d 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -14,64 +14,55 @@
 
 package realcluster
 
-import (
-	"context"
-	"os/exec"
-	"testing"
+// func restartTiUP() {
+// 	log.Info("start to restart TiUP")
+// 	cmd := exec.Command("make", "deploy")
+// 	err := cmd.Run()
+// 	if err != nil {
+// 		panic(err)
+// 	}
+// 	log.Info("TiUP restart success")
+// }
 
-	"github.com/pingcap/log"
-	"github.com/stretchr/testify/require"
-)
+// // https://github.com/tikv/pd/issues/6467
+// func TestReloadLabel(t *testing.T) {
+// 	re := require.New(t)
+// 	ctx := context.Background()
 
-func restartTiUP() {
-	log.Info("start to restart TiUP")
-	cmd := exec.Command("make", "deploy")
-	err := cmd.Run()
-	if err != nil {
-		panic(err)
-	}
-	log.Info("TiUP restart success")
-}
+// 	resp, _ := pdHTTPCli.GetStores(ctx)
+// 	setStore := resp.Stores[0]
+// 	// TiFlash labels will be ["engine": "tiflash"]
+// 	storeLabel := map[string]string{
+// 		"zone": "zone1",
+// 	}
+// 	for _, label := range setStore.Store.Labels {
+// 		storeLabel[label.Key] = label.Value
+// 	}
 
-// https://github.com/tikv/pd/issues/6467
-func TestReloadLabel(t *testing.T) {
-	re := require.New(t)
-	ctx := context.Background()
+// 	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
+// 	defer func() {
+// 		pdHTTPCli.DeleteStoreLabel(ctx, setStore.Store.ID, "zone")
+// 	}()
 
-	resp, _ := pdHTTPCli.GetStores(ctx)
-	setStore := resp.Stores[0]
-	// TiFlash labels will be ["engine": "tiflash"]
-	storeLabel := map[string]string{
-		"zone": "zone1",
-	}
-	for _, label := range setStore.Store.Labels {
-		storeLabel[label.Key] = label.Value
-	}
+// 	resp, err := pdHTTPCli.GetStores(ctx)
+// 	re.NoError(err)
+// 	for _, store := range resp.Stores {
+// 		if store.Store.ID == setStore.Store.ID {
+// 			for _, label := range store.Store.Labels {
+// 				re.Equal(label.Value, storeLabel[label.Key])
+// 			}
+// 		}
+// 	}
 
-	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
-	defer func() {
-		re.NoError(pdHTTPCli.DeleteStoreLabel(ctx, setStore.Store.ID, "zone"))
-	}()
+// 	restartTiUP()
 
-	resp, err := pdHTTPCli.GetStores(ctx)
-	re.NoError(err)
-	for _, store := range resp.Stores {
-		if store.Store.ID == setStore.Store.ID {
-			for _, label := range store.Store.Labels {
-				re.Equal(label.Value, storeLabel[label.Key])
-			}
-		}
-	}
-
-	restartTiUP()
-
-	resp, err = pdHTTPCli.GetStores(ctx)
-	re.NoError(err)
-	for _, store := range resp.Stores {
-		if store.Store.ID == setStore.Store.ID {
-			for _, label := range store.Store.Labels {
-				re.Equal(label.Value, storeLabel[label.Key])
-			}
-		}
-	}
-}
+// 	resp, err = pdHTTPCli.GetStores(ctx)
+// 	re.NoError(err)
+// 	for _, store := range resp.Stores {
+// 		if store.Store.ID == setStore.Store.ID {
+// 			for _, label := range store.Store.Labels {
+// 				re.Equal(label.Value, storeLabel[label.Key])
+// 			}
+// 		}
+// 	}
+// }

From 6b72f31908f37b8b3af79533ab2ed4dbf40f6eae Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Thu, 20 Jun 2024 11:54:06 +0800
Subject: [PATCH 21/28] add wait for time

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/scheduler_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index caf97db8bb8..6fd56d01ba9 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -111,7 +111,7 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 			}
 		}
 		return true
-	})
+	}, testutil.WithWaitFor(time.Minute))
 
 	// disable schedule for region1
 	labelRule := &pd.LabelRule{
@@ -149,7 +149,7 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 			}
 		}
 		return true
-	})
+	}, testutil.WithWaitFor(time.Minute))
 
 	re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName))
 	re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID)))
@@ -168,7 +168,7 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 			}
 		}
 		return true
-	})
+	}, testutil.WithWaitFor(time.Minute))
 
 	pdHTTPCli.PatchRegionLabelRules(ctx, &pd.LabelRulePatch{DeleteRules: []string{labelRule.ID}})
 	labelRules, err = pdHTTPCli.GetAllRegionLabelRules(ctx)
@@ -184,5 +184,5 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 			}
 		}
 		return true
-	})
+	}, testutil.WithWaitFor(time.Minute))
 }

From 2a51bd1d2850363cf7da1b744e0dda6d2026a38e Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Thu, 20 Jun 2024 14:01:07 +0800
Subject: [PATCH 22/28] fix ci

Signed-off-by: okJiang <819421878@qq.com>
---
 server/cluster/cluster.go                        | 3 +++
 tests/integrations/realcluster/scheduler_test.go | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go
index 534d8361b2a..18efdf1763c 100644
--- a/server/cluster/cluster.go
+++ b/server/cluster/cluster.go
@@ -1207,6 +1207,9 @@ func (c *RaftCluster) DeleteStoreLabel(storeID uint64, labelKey string) error {
 		return errs.ErrInvalidStoreID.FastGenByArgs(storeID)
 	}
 	newStore := typeutil.DeepClone(store.GetMeta(), core.StoreFactory)
+	if len(newStore.GetLabels()) == 0 {
+		return errors.Errorf("the label key %s does not exist", labelKey)
+	}
 	labels := make([]*metapb.StoreLabel, 0, len(newStore.GetLabels())-1)
 	for _, label := range newStore.GetLabels() {
 		if label.Key == labelKey {
diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go
index 6fd56d01ba9..0ed6f6c6b76 100644
--- a/tests/integrations/realcluster/scheduler_test.go
+++ b/tests/integrations/realcluster/scheduler_test.go
@@ -86,7 +86,7 @@ func TestRegionLabelDenyScheduler(t *testing.T) {
 
 	regions, err := pdHTTPCli.GetRegions(ctx)
 	re.NoError(err)
-	re.True(len(regions.Regions) >= 1)
+	re.GreaterOrEqual(len(regions.Regions), 1)
 	region1 := regions.Regions[0]
 
 	err = pdHTTPCli.DeleteScheduler(ctx, schedulers.BalanceLeaderName)

From ab2761a0710d6d618eb7ee3f5e5ffb32d6af024d Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Thu, 20 Jun 2024 15:56:16 +0800
Subject: [PATCH 23/28] fix comment and add some tests

Signed-off-by: okJiang <819421878@qq.com>
---
 client/http/interface.go                      |  2 +-
 client/http/request_info.go                   |  2 +-
 tests/integrations/client/http_client_test.go | 12 +++++++++++-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/client/http/interface.go b/client/http/interface.go
index d87bb003cee..f90ab19624f 100644
--- a/client/http/interface.go
+++ b/client/http/interface.go
@@ -352,7 +352,7 @@ func (c *client) DeleteStoreLabel(ctx context.Context, storeID int64, labelKey s
 		return errors.Trace(err)
 	}
 	return c.request(ctx, newRequestInfo().
-		WithName(deleteStoreLableName).
+		WithName(deleteStoreLabelName).
 		WithURI(LabelByStoreID(storeID)).
 		WithMethod(http.MethodDelete).
 		WithBody(jsonInput))
diff --git a/client/http/request_info.go b/client/http/request_info.go
index 2345895cd54..783220bcc60 100644
--- a/client/http/request_info.go
+++ b/client/http/request_info.go
@@ -41,7 +41,7 @@ const (
 	getStoreName                            = "GetStore"
 	deleteStoreName                         = "DeleteStore"
 	setStoreLabelsName                      = "SetStoreLabels"
-	deleteStoreLableName                    = "DeleteStoreLabel"
+	deleteStoreLabelName                    = "DeleteStoreLabel"
 	getHealthStatusName                     = "GetHealthStatus"
 	getConfigName                           = "GetConfig"
 	setConfigName                           = "SetConfig"
diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go
index f4a48dcd63e..86839f59770 100644
--- a/tests/integrations/client/http_client_test.go
+++ b/tests/integrations/client/http_client_test.go
@@ -560,9 +560,14 @@ func (suite *httpClientTestSuite) TestSchedulers() {
 	re.NoError(err)
 	err = client.SetSchedulerDelay(ctx, "not-exist", 100)
 	re.ErrorContains(err, "500 Internal Server Error") // TODO: should return friendly error message
+
+	re.NoError(client.DeleteScheduler(ctx, schedulerName))
+	schedulers, err = client.GetSchedulers(ctx)
+	re.NoError(err)
+	re.NotContains(schedulers, schedulerName)
 }
 
-func (suite *httpClientTestSuite) TestSetStoreLabels() {
+func (suite *httpClientTestSuite) TestStoreLabels() {
 	re := suite.Require()
 	client := suite.client
 	ctx, cancel := context.WithCancel(suite.ctx)
@@ -586,6 +591,11 @@ func (suite *httpClientTestSuite) TestSetStoreLabels() {
 			}
 		}
 	}
+
+	re.NoError(client.DeleteStoreLabel(ctx, 1, "zone"))
+	store, err := client.GetStore(ctx, 1)
+	re.NoError(err)
+	re.Empty(store.Store.Labels)
 }
 
 func (suite *httpClientTestSuite) TestTransferLeader() {

From ad19424c860167302b3b0146900eecc57391d0d7 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Thu, 20 Jun 2024 16:10:24 +0800
Subject: [PATCH 24/28] revert comment

Signed-off-by: okJiang <819421878@qq.com>
---
 .../realcluster/reboot_pd_test.go             | 101 +++++++++---------
 1 file changed, 53 insertions(+), 48 deletions(-)

diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index 313179e262d..03adb854f79 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -4,65 +4,70 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 package realcluster
 
-// func restartTiUP() {
-// 	log.Info("start to restart TiUP")
-// 	cmd := exec.Command("make", "deploy")
-// 	err := cmd.Run()
-// 	if err != nil {
-// 		panic(err)
-// 	}
-// 	log.Info("TiUP restart success")
-// }
+import (
+	"context"
+	"os/exec"
+	"testing"
+
+	"github.com/pingcap/log"
+	"github.com/stretchr/testify/require"
+)
 
-// // https://github.com/tikv/pd/issues/6467
-// func TestReloadLabel(t *testing.T) {
-// 	re := require.New(t)
-// 	ctx := context.Background()
+func restartTiUP() {
+	log.Info("start to restart TiUP")
+	cmd := exec.Command("make", "deploy")
+	err := cmd.Run()
+	if err != nil {
+		panic(err)
+	}
+	log.Info("TiUP restart success")
+}
 
-// 	resp, _ := pdHTTPCli.GetStores(ctx)
-// 	setStore := resp.Stores[0]
-// 	// TiFlash labels will be ["engine": "tiflash"]
-// 	storeLabel := map[string]string{
-// 		"zone": "zone1",
-// 	}
-// 	for _, label := range setStore.Store.Labels {
-// 		storeLabel[label.Key] = label.Value
-// 	}
+// https://github.com/tikv/pd/issues/6467
+func TestReloadLabel(t *testing.T) {
+	re := require.New(t)
+	ctx := context.Background()
 
-// 	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
-// 	defer func() {
-// 		pdHTTPCli.DeleteStoreLabel(ctx, setStore.Store.ID, "zone")
-// 	}()
+	resp, _ := pdHTTPCli.GetStores(ctx)
+	setStore := resp.Stores[0]
+	// TiFlash labels will be ["engine": "tiflash"]
+	storeLabel := map[string]string{
+		"zone": "zone1",
+	}
+	for _, label := range setStore.Store.Labels {
+		storeLabel[label.Key] = label.Value
+	}
+	err := pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel)
+	re.NoError(err)
 
-// 	resp, err := pdHTTPCli.GetStores(ctx)
-// 	re.NoError(err)
-// 	for _, store := range resp.Stores {
-// 		if store.Store.ID == setStore.Store.ID {
-// 			for _, label := range store.Store.Labels {
-// 				re.Equal(label.Value, storeLabel[label.Key])
-// 			}
-// 		}
-// 	}
+	resp, err = pdHTTPCli.GetStores(ctx)
+	re.NoError(err)
+	for _, store := range resp.Stores {
+		if store.Store.ID == setStore.Store.ID {
+			for _, label := range store.Store.Labels {
+				re.Equal(label.Value, storeLabel[label.Key])
+			}
+		}
+	}
 
-// 	restartTiUP()
+	restartTiUP()
 
-// 	resp, err = pdHTTPCli.GetStores(ctx)
-// 	re.NoError(err)
-// 	for _, store := range resp.Stores {
-// 		if store.Store.ID == setStore.Store.ID {
-// 			for _, label := range store.Store.Labels {
-// 				re.Equal(label.Value, storeLabel[label.Key])
-// 			}
-// 		}
-// 	}
-// }
+	resp, err = pdHTTPCli.GetStores(ctx)
+	re.NoError(err)
+	for _, store := range resp.Stores {
+		if store.Store.ID == setStore.Store.ID {
+			for _, label := range store.Store.Labels {
+				re.Equal(label.Value, storeLabel[label.Key])
+			}
+		}
+	}
+}

From fa47700c0aa37c4d5302c9a91217aed4e489b7f4 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Thu, 20 Jun 2024 16:54:54 +0800
Subject: [PATCH 25/28] fix delete not found

Signed-off-by: okJiang <819421878@qq.com>
---
 Makefile                                         | 1 +
 tests/integrations/realcluster/Makefile          | 1 -
 tests/integrations/realcluster/reboot_pd_test.go | 8 +++++---
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index dca00012114..5f5ac871f18 100644
--- a/Makefile
+++ b/Makefile
@@ -280,6 +280,7 @@ test-tso-consistency: install-tools
 REAL_CLUSTER_TEST_PATH := $(ROOT_PATH)/tests/integrations/realcluster
 
 test-real-cluster:
+	@ rm -rf ~/.tiup/data/pd_real_cluster_test
 	# testing with the real cluster...
 	cd $(REAL_CLUSTER_TEST_PATH) && $(MAKE) check
 
diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile
index c1b01b269e0..28c918ec2bf 100644
--- a/tests/integrations/realcluster/Makefile
+++ b/tests/integrations/realcluster/Makefile
@@ -48,7 +48,6 @@ kill_cluster:
 		echo "waiting for cluster to exit..."; \
 		sleep 30; \
 	fi
-	@ rm -rf ~/.tiup/data/pd_real_cluster_test
 
 test:
 	CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (\
diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index 03adb854f79..46ddcaa8b2a 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -46,10 +46,12 @@ func TestReloadLabel(t *testing.T) {
 	for _, label := range setStore.Store.Labels {
 		storeLabel[label.Key] = label.Value
 	}
-	err := pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel)
-	re.NoError(err)
+	re.NoError(pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel))
+	defer func() {
+		re.NoError(pdHTTPCli.DeleteStoreLabel(ctx, setStore.Store.ID, "zone"))
+	}()
 
-	resp, err = pdHTTPCli.GetStores(ctx)
+	resp, err := pdHTTPCli.GetStores(ctx)
 	re.NoError(err)
 	for _, store := range resp.Stores {
 		if store.Store.ID == setStore.Store.ID {

From 18d1a97a95cb5c62a98f150be76f17465298d281 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Mon, 24 Jun 2024 15:15:22 +0800
Subject: [PATCH 26/28] fix ci

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/client/http_client_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go
index d3563135a33..1d7d4488692 100644
--- a/tests/integrations/client/http_client_test.go
+++ b/tests/integrations/client/http_client_test.go
@@ -596,8 +596,8 @@ func (suite *httpClientTestSuite) TestStoreLabels() {
 		re.Equal(value, labelsMap[key])
 	}
 
-	re.NoError(client.DeleteStoreLabel(ctx, 1, "zone"))
-	store, err := client.GetStore(ctx, 1)
+	re.NoError(client.DeleteStoreLabel(ctx, firstStore.Store.ID, "zone"))
+	store, err := client.GetStore(ctx, uint64(firstStore.Store.ID))
 	re.NoError(err)
 	re.Empty(store.Store.Labels)
 }

From 1a31857829b25a63e3d474471165f997b6d43376 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Mon, 24 Jun 2024 16:25:24 +0800
Subject: [PATCH 27/28] fix comment

Signed-off-by: okJiang <819421878@qq.com>
---
 tests/integrations/realcluster/deploy.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh
index edbb6037e40..f6f567314f0 100755
--- a/tests/integrations/realcluster/deploy.sh
+++ b/tests/integrations/realcluster/deploy.sh
@@ -28,7 +28,6 @@ else
   # CI will download the binaries in the prepare phase.
   # ref https://github.com/PingCAP-QE/ci/blob/387e9e533b365174962ccb1959442a7070f9cd66/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy#L55-L68
 	color-green "using existing binaries..."
-	make pd-server WITH_RACE=1
 	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
 		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server \
 		--tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test --pd.config ./tests/integrations/realcluster/pd.toml \

From f981b697246685271cb4d27d98e9c1a026a70948 Mon Sep 17 00:00:00 2001
From: okJiang <819421878@qq.com>
Date: Mon, 24 Jun 2024 16:42:27 +0800
Subject: [PATCH 28/28] fix comment

Signed-off-by: okJiang <819421878@qq.com>
---
 server/cluster/cluster.go                        | 4 ++--
 tests/integrations/realcluster/reboot_pd_test.go | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go
index 18efdf1763c..5c15856cec6 100644
--- a/server/cluster/cluster.go
+++ b/server/cluster/cluster.go
@@ -1206,10 +1206,10 @@ func (c *RaftCluster) DeleteStoreLabel(storeID uint64, labelKey string) error {
 	if store == nil {
 		return errs.ErrInvalidStoreID.FastGenByArgs(storeID)
 	}
-	newStore := typeutil.DeepClone(store.GetMeta(), core.StoreFactory)
-	if len(newStore.GetLabels()) == 0 {
+	if len(store.GetLabels()) == 0 {
 		return errors.Errorf("the label key %s does not exist", labelKey)
 	}
+	newStore := typeutil.DeepClone(store.GetMeta(), core.StoreFactory)
 	labels := make([]*metapb.StoreLabel, 0, len(newStore.GetLabels())-1)
 	for _, label := range newStore.GetLabels() {
 		if label.Key == labelKey {
diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go
index ff70b92ff89..14c86f2dedb 100644
--- a/tests/integrations/realcluster/reboot_pd_test.go
+++ b/tests/integrations/realcluster/reboot_pd_test.go
@@ -4,13 +4,14 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//	http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package realcluster
 
 import (