Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kv-client(cdc): handle ServerIsBusy errors #10252

Merged
merged 7 commits into from
Dec 14, 2023
Merged

kv-client(cdc): handle ServerIsBusy errors #10252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
676e48c
init
hicqu Dec 5, 2023
bffda31
fix
hicqu Dec 5, 2023
1873ac9
Merge branch 'master' into handle-server-is-busy
hicqu Dec 6, 2023
cbbcee0
fix
hicqu Dec 6, 2023
57af0c1
fix
hicqu Dec 6, 2023
bd1562a
fix
hicqu Dec 11, 2023
aa9ad1a
fix
hicqu Dec 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cdc/kv/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ var (
metricFeedRPCCtxUnavailable = eventFeedErrorCounter.WithLabelValues("RPCCtxUnavailable")
metricStoreSendRequestErr = eventFeedErrorCounter.WithLabelValues("SendRequestToStore")
metricConnectToStoreErr = eventFeedErrorCounter.WithLabelValues("ConnectToStore")
metricKvIsBusyCounter = eventFeedErrorCounter.WithLabelValues("KvIsBusy")
)

var (
Expand Down Expand Up @@ -912,6 +913,10 @@ func (s *eventFeedSession) handleError(ctx context.Context, errInfo regionErrorI
metricFeedRegionNotFoundCounter.Inc()
s.scheduleDivideRegionAndRequest(ctx, errInfo.span)
return nil
} else if busy := innerErr.GetServerIsBusy(); busy != nil {
metricKvIsBusyCounter.Inc()
s.scheduleRegionRequest(ctx, errInfo.singleRegionInfo)
return nil
} else if duplicatedRequest := innerErr.GetDuplicateRequest(); duplicatedRequest != nil {
metricFeedDuplicateRequestCounter.Inc()
logPanic("tikv reported duplicated request to the same region, which is not expected",
Expand Down
5 changes: 5 additions & 0 deletions cdc/kv/shared_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -600,6 +600,11 @@ func (s *SharedClient) handleError(ctx context.Context, errInfo regionErrorInfo)
s.scheduleRangeRequest(ctx, errInfo.span, errInfo.requestedTable)
return nil
}
if innerErr.GetServerIsBusy() != nil {
metricKvIsBusyCounter.Inc()
s.scheduleRegionRequest(ctx, errInfo.singleRegionInfo)
return nil
}
if duplicated := innerErr.GetDuplicateRequest(); duplicated != nil {
metricFeedDuplicateRequestCounter.Inc()
// TODO(qupeng): It's better to add a new machanism to deregister one region.
Expand Down
37 changes: 21 additions & 16 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ require (
github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0
github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071
github.com/pingcap/kvproto v0.0.0-20231204093812-96c40585233f
github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22
github.com/pingcap/tidb v1.1.0-beta.0.20231010115255-ec2731b8f539
github.com/pingcap/tidb-tools v0.0.0-20231031074420-a020d2cdfc5a
Expand Down Expand Up @@ -109,15 +109,16 @@ require (
go.uber.org/ratelimit v0.2.0
go.uber.org/zap v1.26.0
golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4
golang.org/x/net v0.15.0
golang.org/x/net v0.17.0
golang.org/x/oauth2 v0.8.0
golang.org/x/sync v0.3.0
golang.org/x/sys v0.12.0
golang.org/x/sys v0.13.0
golang.org/x/text v0.13.0
golang.org/x/time v0.3.0
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
google.golang.org/grpc v1.54.0
google.golang.org/protobuf v1.30.0
google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.31.0
gopkg.in/yaml.v2 v2.4.0
gorm.io/driver/mysql v1.3.3
gorm.io/gorm v1.23.8
Expand All @@ -143,13 +144,15 @@ require (
github.com/go-ldap/ldap/v3 v3.4.4 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/influxdata/tdigest v0.0.1 // indirect
github.com/jellydator/ttlcache/v3 v3.0.1 // indirect
github.com/jfcg/sixb v1.3.8 // indirect
github.com/jfcg/sorty/v2 v2.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/segmentio/asm v1.2.0 // indirect
google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/api v0.27.2 // indirect
k8s.io/apimachinery v0.27.2 // indirect
Expand All @@ -160,10 +163,10 @@ require (
)

require (
cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go/compute v1.19.0 // indirect
cloud.google.com/go v0.110.8 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.13.0 // indirect
cloud.google.com/go/iam v1.1.2 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.1 // indirect
github.com/AthenZ/athenz v1.10.39 // indirect
Expand Down Expand Up @@ -228,12 +231,12 @@ require (
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/glog v1.1.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
Expand Down Expand Up @@ -351,13 +354,13 @@ require (
go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/term v0.12.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/tools v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.114.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/api v0.128.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand All @@ -383,3 +386,5 @@ replace go.opencensus.io => go.opencensus.io v0.23.1-0.20220331163232-052120675f
replace github.com/go-ldap/ldap/v3 v3.4.4 => github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117

replace github.com/tildeleb/hashland => leb.io/hashland v0.1.5

replace google.golang.org/grpc v1.59.0 => google.golang.org/grpc v1.54.0
Loading
Loading