syncer: exit watch leader immediately (#8824) (#8830)

close #8823

Signed-off-by: ti-chi-bot <[email protected]>
Signed-off-by: Ryan Leung <[email protected]>

Co-authored-by: Ryan Leung <[email protected]>
Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

pkg/mock/mockserver/mockserver.go

@@ -0,0 +1,88 @@
+// Copyright 2024 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mockserver
+
+import (
+	"context"
+
+	"github.com/pingcap/kvproto/pkg/pdpb"
+	"github.com/tikv/pd/pkg/core"
+	"github.com/tikv/pd/pkg/storage"
+	"github.com/tikv/pd/pkg/utils/grpcutil"
+)
+
+// MockServer is used to mock Server for test use.
+type MockServer struct {
+	ctx            context.Context
+	member, leader *pdpb.Member
+	storage        storage.Storage
+	bc             *core.BasicCluster
+}
+
+// NewMockServer creates a new MockServer.
+func NewMockServer(ctx context.Context, member, leader *pdpb.Member, storage storage.Storage, bc *core.BasicCluster) *MockServer {
+	return &MockServer{
+		ctx:     ctx,
+		member:  member,
+		leader:  leader,
+		storage: storage,
+		bc:      bc,
+	}
+}
+
+// LoopContext returns the context of the server.
+func (s *MockServer) LoopContext() context.Context {
+	return s.ctx
+}
+
+// ClusterID returns the cluster ID of the server.
+func (*MockServer) ClusterID() uint64 {
+	return 1
+}
+
+// GetMemberInfo returns the member info of the server.
+func (s *MockServer) GetMemberInfo() *pdpb.Member {
+	return s.member
+}
+
+// GetLeader returns the leader of the server.
+func (s *MockServer) GetLeader() *pdpb.Member {
+	return s.leader
+}
+
+// GetStorage returns the storage of the server.
+func (s *MockServer) GetStorage() storage.Storage {
+	return s.storage
+}
+
+// Name returns the name of the server.
+func (*MockServer) Name() string {
+	return "mock-server"
+}
+
+// GetRegions returns the regions of the server.
+func (s *MockServer) GetRegions() []*core.RegionInfo {
+	return s.bc.GetRegions()
+}
+
+// GetTLSConfig returns the TLS config of the server.
+func (*MockServer) GetTLSConfig() *grpcutil.TLSConfig {
+	return &grpcutil.TLSConfig{}
+}
+
+// GetBasicCluster returns the basic cluster of the server.
+func (s *MockServer) GetBasicCluster() *core.BasicCluster {
+	return s.bc
+}

pkg/syncer/client.go

@@ -41,6 +41,7 @@ const (
 	keepaliveTime    = 10 * time.Second
 	keepaliveTimeout = 3 * time.Second
 	msgSize          = 8 * units.MiB
+	retryInterval    = time.Second
 )
 
 // StopSyncWithLeader stop to sync the region with leader.
@@ -150,7 +151,12 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) {
 					}
 				}
 				log.Error("server failed to establish sync stream with leader", zap.String("server", s.server.Name()), zap.String("leader", s.server.GetLeader().GetName()), errs.ZapError(err))
-				time.Sleep(time.Second)
+				select {
+				case <-ctx.Done():
+					log.Info("stop synchronizing with leader due to context canceled")
+					return
+				case <-time.After(retryInterval):
+				}
 				continue
 			}
 			log.Info("server starts to synchronize with leader", zap.String("server", s.server.Name()), zap.String("leader", s.server.GetLeader().GetName()), zap.Uint64("request-index", s.history.GetNextIndex()))
@@ -162,7 +168,12 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) {
 					if err = stream.CloseSend(); err != nil {
 						log.Error("failed to terminate client stream", errs.ZapError(errs.ErrGRPCCloseSend, err))
 					}
-					time.Sleep(time.Second)
+					select {
+					case <-ctx.Done():
+						log.Info("stop synchronizing with leader due to context canceled")
+						return
+					case <-time.After(retryInterval):
+					}
 					break
 				}
 				if s.history.GetNextIndex() != resp.GetStartIndex() {

pkg/syncer/client_test.go

@@ -21,9 +21,9 @@ import (
 
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/metapb"
-	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/stretchr/testify/require"
 	"github.com/tikv/pd/pkg/core"
+	"github.com/tikv/pd/pkg/mock/mockserver"
 	"github.com/tikv/pd/pkg/storage"
 	"github.com/tikv/pd/pkg/utils/grpcutil"
 	"google.golang.org/grpc/codes"
@@ -37,11 +37,13 @@ func TestLoadRegion(t *testing.T) {
 	rs, err := storage.NewRegionStorageWithLevelDBBackend(context.Background(), tempDir, nil)
 	re.NoError(err)
 
-	server := &mockServer{
-		ctx:     context.Background(),
-		storage: storage.NewCoreStorage(storage.NewStorageWithMemoryBackend(), rs),
-		bc:      core.NewBasicCluster(),
-	}
+	server := mockserver.NewMockServer(
+		context.Background(),
+		nil,
+		nil,
+		storage.NewCoreStorage(storage.NewStorageWithMemoryBackend(), rs),
+		core.NewBasicCluster(),
+	)
 	for i := 0; i < 30; i++ {
 		rs.SaveRegion(&metapb.Region{Id: uint64(i) + 1})
 	}
@@ -64,11 +66,13 @@ func TestErrorCode(t *testing.T) {
 	tempDir := t.TempDir()
 	rs, err := storage.NewRegionStorageWithLevelDBBackend(context.Background(), tempDir, nil)
 	re.NoError(err)
-	server := &mockServer{
-		ctx:     context.Background(),
-		storage: storage.NewCoreStorage(storage.NewStorageWithMemoryBackend(), rs),
-		bc:      core.NewBasicCluster(),
-	}
+	server := mockserver.NewMockServer(
+		context.Background(),
+		nil,
+		nil,
+		storage.NewCoreStorage(storage.NewStorageWithMemoryBackend(), rs),
+		core.NewBasicCluster(),
+	)
 	ctx, cancel := context.WithCancel(context.TODO())
 	rc := NewRegionSyncer(server)
 	conn, err := grpcutil.GetClientConn(ctx, "http://127.0.0.1", nil)
@@ -79,46 +83,3 @@ func TestErrorCode(t *testing.T) {
 	re.True(ok)
 	re.Equal(codes.Canceled, ev.Code())
 }
-
-type mockServer struct {
-	ctx            context.Context
-	member, leader *pdpb.Member
-	storage        storage.Storage
-	bc             *core.BasicCluster
-}
-
-func (s *mockServer) LoopContext() context.Context {
-	return s.ctx
-}
-
-func (s *mockServer) ClusterID() uint64 {
-	return 1
-}
-
-func (s *mockServer) GetMemberInfo() *pdpb.Member {
-	return s.member
-}
-
-func (s *mockServer) GetLeader() *pdpb.Member {
-	return s.leader
-}
-
-func (s *mockServer) GetStorage() storage.Storage {
-	return s.storage
-}
-
-func (s *mockServer) Name() string {
-	return "mock-server"
-}
-
-func (s *mockServer) GetRegions() []*core.RegionInfo {
-	return s.bc.GetRegions()
-}
-
-func (s *mockServer) GetTLSConfig() *grpcutil.TLSConfig {
-	return &grpcutil.TLSConfig{}
-}
-
-func (s *mockServer) GetBasicCluster() *core.BasicCluster {
-	return s.bc
-}

tests/integrations/client/client_test.go

@@ -228,11 +228,11 @@ func TestLeaderTransferAndMoveCluster(t *testing.T) {
 	oldServers := cluster.GetServers()
 	oldLeaderName := cluster.WaitLeader()
 	for i := 0; i < 3; i++ {
+		time.Sleep(5 * time.Second)
 		newPD, err := cluster.Join(ctx)
 		re.NoError(err)
 		re.NoError(newPD.Run())
 		oldLeaderName = cluster.WaitLeader()
-		time.Sleep(5 * time.Second)
 	}
 
 	// ABCDEF->DEF