From 99a232e663d293cd4aa8fc8598cf0eaf2cc79de1 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 28 Feb 2024 15:14:14 +0800 Subject: [PATCH 1/3] change the backoff Signed-off-by: Ryan Leung --- client/grpcutil/grpcutil.go | 12 +++++++++++- pkg/utils/grpcutil/grpcutil.go | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index b6be2594b4d..d1abbcf4b9a 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -27,6 +27,7 @@ import ( "github.com/tikv/pd/client/errs" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/backoff" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" ) @@ -62,7 +63,16 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g if err != nil { return nil, errs.ErrURLParse.Wrap(err).GenWithStackByCause() } - cc, err := grpc.DialContext(ctx, u.Host, append(do, opt)...) + backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 100 * time.Millisecond, + Multiplier: 1.6, + Jitter: 0.2, + MaxDelay: 3 * time.Second, + }, + }) + do = append(do, opt, backoffOpts) + cc, err := grpc.DialContext(ctx, u.Host, do...) if err != nil { return nil, errs.ErrGRPCDial.Wrap(err).GenWithStackByCause() } diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index e54ddfe1711..1d3ab988a6f 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -30,6 +30,7 @@ import ( "go.etcd.io/etcd/pkg/transport" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/backoff" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" @@ -147,7 +148,16 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g if err != nil { return nil, errs.ErrURLParse.Wrap(err).GenWithStackByCause() } - cc, err := grpc.DialContext(ctx, u.Host, append(do, opt)...) + backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 100 * time.Millisecond, + Multiplier: 1.6, + Jitter: 0.2, + MaxDelay: 3 * time.Second, + }, + }) + do = append(do, opt, backoffOpts) + cc, err := grpc.DialContext(ctx, u.Host, do...) if err != nil { return nil, errs.ErrGRPCDial.Wrap(err).GenWithStackByCause() } From c2996e7c22200218cc4940fb8462370684f17877 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 28 Feb 2024 15:34:54 +0800 Subject: [PATCH 2/3] change the backoff Signed-off-by: Ryan Leung --- client/grpcutil/grpcutil.go | 2 +- pkg/utils/grpcutil/grpcutil.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index d1abbcf4b9a..268319b11b1 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -65,7 +65,7 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g } backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{ - BaseDelay: 100 * time.Millisecond, + BaseDelay: time.Second, Multiplier: 1.6, Jitter: 0.2, MaxDelay: 3 * time.Second, diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 1d3ab988a6f..6294740da86 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -150,7 +150,7 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g } backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{ - BaseDelay: 100 * time.Millisecond, + BaseDelay: time.Second, Multiplier: 1.6, Jitter: 0.2, MaxDelay: 3 * time.Second, From 22638b5c17caf8c9ac4e9142e6a0a6ffeca23a6e Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 29 Feb 2024 13:18:45 +0800 Subject: [PATCH 3/3] add comment Signed-off-by: Ryan Leung --- client/grpcutil/grpcutil.go | 2 ++ pkg/utils/grpcutil/grpcutil.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index 268319b11b1..742ee872b07 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -63,6 +63,8 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g if err != nil { return nil, errs.ErrURLParse.Wrap(err).GenWithStackByCause() } + // Here we use a shorter MaxDelay to make the connection recover faster. + // The default MaxDelay is 120s, which is too long for us. backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{ BaseDelay: time.Second, diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 6294740da86..d108a6f5d44 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -148,6 +148,8 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g if err != nil { return nil, errs.ErrURLParse.Wrap(err).GenWithStackByCause() } + // Here we use a shorter MaxDelay to make the connection recover faster. + // The default MaxDelay is 120s, which is too long for us. backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{ BaseDelay: time.Second,