tests: add integration test and ci workflow for real tiup

.github/workflows/real_tiup.yaml

@@ -0,0 +1,35 @@
+name: PD Real TiUP Test
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request:
+    branches:
+      - master
+      - release-*
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+jobs:
+  real-cluster:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.21'
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Restore cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            **/.tools
+            **/.dashboard_download_cache
+          key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }}
+
+      - name: Test
+        run: make check
+        working-directory: tests/integrations/realtiup

Makefile

@@ -46,7 +46,12 @@ ifeq ($(PLUGIN), 1)
 	BUILD_TAGS += with_plugin
 endif
 
-LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDReleaseVersion=$(shell git describe --tags --dirty --always)"
+RELEASE_VERSION ?= $(shell git describe --tags --dirty --always)
+ifeq ($(RUN_CI), 1)
+	RELEASE_VERSION := None
+endif
+
+LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDReleaseVersion=$(RELEASE_VERSION)"
 LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
 LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitHash=$(shell git rev-parse HEAD)"
 LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitBranch=$(shell git rev-parse --abbrev-ref HEAD)"

client/http/api.go

@@ -29,7 +29,13 @@
 	regionsByKey        = "/pd/api/v1/regions/key"
 	regionsByStoreID    = "/pd/api/v1/regions/store"
 	Stores              = "/pd/api/v1/stores"
+	Store               = "/pd/api/v1/store"
 	MinResolvedTSPrefix = "/pd/api/v1/min-resolved-ts"
+
+	Leader         = "/pd/api/v1/leader"
+	TransferLeader = "/pd/api/v1/leader/transfer"
+
+	Schedulers = "/pd/api/v1/schedulers"
 )
 
 // RegionByID returns the path of PD HTTP API to get region by ID.
@@ -52,3 +58,13 @@
 func RegionsByStoreID(storeID uint64) string {
 	return fmt.Sprintf("%s/%d", regionsByStoreID, storeID)
 }
+
+// LabelByStore returns the path of PD HTTP API to set store label.
+func LabelByStore(storeID int64) string {
+	return fmt.Sprintf("%s/%d/label", Store, storeID)
+}
+
+// TransferLeaderID returns the path of PD HTTP API to transfer leader by ID.
+func TransferLeaderID(leaderID string) string {
+	return fmt.Sprintf("%s/%s", TransferLeader, leaderID)
+}

client/http/client.go

@@ -15,6 +15,7 @@
 package http
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
@@ -25,6 +26,7 @@
 	"time"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
@@ -49,6 +51,14 @@
 	GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error)
 	GetStores(context.Context) (*StoresInfo, error)
 	GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error)
+	SetStoreLabel(context.Context, int64, map[string]string) error
+
+	GetLeader(context.Context) (*pdpb.Member, error)
+	TransferLeader(context.Context, string) error
+
+	GetSchedulers(context.Context) ([]string, error)
+	AddScheduler(context.Context, string, map[string]interface{}) error
+
 	Close()
 }
 
@@ -154,7 +164,8 @@
 // it consistent with the current implementation of some clients (e.g. TiDB).
 func (c *client) requestWithRetry(
 	ctx context.Context,
-	name, uri string,
+	name, uri, method string,
+	body io.Reader,
 	res interface{},
 ) error {
 	var (
@@ -163,7 +174,7 @@
 	)
 	for idx := 0; idx < len(c.pdAddrs); idx++ {
 		addr = c.pdAddrs[idx]
-		err = c.request(ctx, name, addr, uri, res)
+		err = c.request(ctx, name, addr, uri, method, body, res)
 		if err == nil {
 			break
 		}
@@ -175,16 +186,18 @@
 
 func (c *client) request(
 	ctx context.Context,
-	name, addr, uri string,
+	name, addr, uri, method string,
+	body io.Reader,
 	res interface{},
 ) error {
 	reqURL := fmt.Sprintf("%s%s", addr, uri)
 	logFields := []zap.Field{
 		zap.String("name", name),
 		zap.String("url", reqURL),
+		zap.String("method", method),
 	}
 	log.Debug("[pd] request the http url", logFields...)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
+	req, err := http.NewRequestWithContext(ctx, method, reqURL, body)
 	if err != nil {
 		log.Error("[pd] create http request failed", append(logFields, zap.Error(err))...)
 		return errors.Trace(err)
@@ -219,17 +232,19 @@
 		return errors.Errorf("request pd http api failed with status: '%s'", resp.Status)
 	}
 
-	err = json.NewDecoder(resp.Body).Decode(res)
-	if err != nil {
-		return errors.Trace(err)
+	if res != nil {
+		err = json.NewDecoder(resp.Body).Decode(&res)
+		if err != nil {
+			return errors.Trace(err)
+		}
 	}
 	return nil
 }
 
 // GetRegionByID gets the region info by ID.
 func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) {
 	var region RegionInfo
-	err := c.requestWithRetry(ctx, "GetRegionByID", RegionByID(regionID), &region)
+	err := c.requestWithRetry(ctx, "GetRegionByID", RegionByID(regionID), http.MethodGet, nil, &region)
 	if err != nil {
 		return nil, err
 	}
@@ -239,7 +254,7 @@
 // GetRegionByKey gets the region info by key.
 func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, error) {
 	var region RegionInfo
-	err := c.requestWithRetry(ctx, "GetRegionByKey", RegionByKey(key), &region)
+	err := c.requestWithRetry(ctx, "GetRegionByKey", RegionByKey(key), http.MethodGet, nil, &region)
 	if err != nil {
 		return nil, err
 	}
@@ -249,7 +264,7 @@
 // GetRegions gets the regions info.
 func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) {
 	var regions RegionsInfo
-	err := c.requestWithRetry(ctx, "GetRegions", Regions, &regions)
+	err := c.requestWithRetry(ctx, "GetRegions", Regions, http.MethodGet, nil, &regions)
 	if err != nil {
 		return nil, err
 	}
@@ -259,7 +274,7 @@
 // GetRegionsByKey gets the regions info by key range. If the limit is -1, it will return all regions within the range.
 func (c *client) GetRegionsByKey(ctx context.Context, startKey, endKey []byte, limit int) (*RegionsInfo, error) {
 	var regions RegionsInfo
-	err := c.requestWithRetry(ctx, "GetRegionsByKey", RegionsByKey(startKey, endKey, limit), &regions)
+	err := c.requestWithRetry(ctx, "GetRegionsByKey", RegionsByKey(startKey, endKey, limit), http.MethodGet, nil, &regions)
 	if err != nil {
 		return nil, err
 	}
@@ -269,7 +284,7 @@
 // GetRegionsByStoreID gets the regions info by store ID.
 func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*RegionsInfo, error) {
 	var regions RegionsInfo
-	err := c.requestWithRetry(ctx, "GetRegionsByStoreID", RegionsByStoreID(storeID), &regions)
+	err := c.requestWithRetry(ctx, "GetRegionsByStoreID", RegionsByStoreID(storeID), http.MethodGet, nil, &regions)
 	if err != nil {
 		return nil, err
 	}
@@ -279,7 +294,7 @@
 // GetHotReadRegions gets the hot read region statistics info.
 func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, error) {
 	var hotReadRegions StoreHotPeersInfos
-	err := c.requestWithRetry(ctx, "GetHotReadRegions", HotRead, &hotReadRegions)
+	err := c.requestWithRetry(ctx, "GetHotReadRegions", HotRead, http.MethodGet, nil, &hotReadRegions)
 	if err != nil {
 		return nil, err
 	}
@@ -289,7 +304,7 @@
 // GetHotWriteRegions gets the hot write region statistics info.
 func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, error) {
 	var hotWriteRegions StoreHotPeersInfos
-	err := c.requestWithRetry(ctx, "GetHotWriteRegions", HotWrite, &hotWriteRegions)
+	err := c.requestWithRetry(ctx, "GetHotWriteRegions", HotWrite, http.MethodGet, nil, &hotWriteRegions)
 	if err != nil {
 		return nil, err
 	}
@@ -299,7 +314,7 @@
 // GetStores gets the stores info.
 func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) {
 	var stores StoresInfo
-	err := c.requestWithRetry(ctx, "GetStores", Stores, &stores)
+	err := c.requestWithRetry(ctx, "GetStores", Stores, http.MethodGet, nil, &stores)
 	if err != nil {
 		return nil, err
 	}
@@ -326,7 +341,7 @@
 		IsRealTime          bool              `json:"is_real_time,omitempty"`
 		StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"`
 	}{}
-	err := c.requestWithRetry(ctx, "GetMinResolvedTSByStoresIDs", uri, &resp)
+	err := c.requestWithRetry(ctx, "GetMinResolvedTSByStoresIDs", uri, http.MethodGet, nil, &resp)
 	if err != nil {
 		return 0, nil, err
 	}
@@ -335,3 +350,54 @@
 	}
 	return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil
 }
+
+// SetStoreLabel sets the label of a store.
+func (c *client) SetStoreLabel(ctx context.Context, storeID int64, storeLabel map[string]string) error {
+	jsonBody, err := json.Marshal(storeLabel)
+	if err != nil {
+		return err
+	}
+
+	return c.requestWithRetry(ctx, "SetStoreLabel", LabelByStore(storeID),
+		http.MethodPost, bytes.NewBuffer(jsonBody), nil)
+}
+
+// GetLeader gets the leader of PD cluster.
+func (c *client) GetLeader(context.Context) (*pdpb.Member, error) {
+	var leader pdpb.Member
+	err := c.requestWithRetry(context.Background(), "GetLeader", Leader, http.MethodGet, nil, &leader)
+	if err != nil {
+		return nil, err
+	}
+	return &leader, nil
+}
+
+// TransferLeader transfers the PD leader.
+func (c *client) TransferLeader(ctx context.Context, newLeader string) error {
+	return c.requestWithRetry(ctx, "TransferLeader", TransferLeaderID(newLeader), http.MethodPost, nil, nil)
+}
+
+// GetSchedulers gets the schedulers from PD cluster.
+func (c *client) GetSchedulers(ctx context.Context) ([]string, error) {
+	var schedulers []string
+	err := c.requestWithRetry(ctx, "GetSchedulers", Schedulers, http.MethodGet, nil, &schedulers)
+	if err != nil {
+		return nil, err
+	}
+	return schedulers, nil
+}
+
+// AddScheduler adds a scheduler to PD cluster.
+func (c *client) AddScheduler(ctx context.Context, name string, args map[string]interface{}) error {
+	request := map[string]interface{}{
+		"name": name,
+	}
+	for arg, val := range args {
+		request[arg] = val
+	}
+	data, err := json.Marshal(request)
+	if err != nil {
+		return err
+	}
+	return c.requestWithRetry(ctx, "AddScheduler", Schedulers, http.MethodPost, bytes.NewBuffer(data), nil)
+}

go.mod

@@ -55,7 +55,7 @@ require (
 	go.uber.org/atomic v1.10.0
 	go.uber.org/goleak v1.1.12
 	go.uber.org/zap v1.24.0
-	golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a
+	golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4
 	golang.org/x/text v0.13.0
 	golang.org/x/time v0.1.0
 	golang.org/x/tools v0.6.0
@@ -179,7 +179,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/image v0.5.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

go.sum

@@ -655,8 +655,8 @@ golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU=
-golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM=
+golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI=
 golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
@@ -672,8 +672,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/member/member.go

@@ -54,7 +54,7 @@ type EmbeddedEtcdMember struct {
 	id       uint64       // etcd server id.
 	member   *pdpb.Member // current PD's info.
 	rootPath string
-	// memberValue is the serialized string of `member`. It will be save in
+	// memberValue is the serialized string of `member`. It will be saved in
 	// etcd leader key when the PD node is successfully elected as the PD leader
 	// of the cluster. Every write will use it to check PD leadership.
 	memberValue string
@@ -186,7 +186,7 @@ func (m *EmbeddedEtcdMember) KeepLeader(ctx context.Context) {
 	m.leadership.Keep(ctx)
 }
 
-// PreCheckLeader does some pre-check before checking whether or not it's the leader.
+// PreCheckLeader does some pre-check before checking whether it's the leader.
 func (m *EmbeddedEtcdMember) PreCheckLeader() error {
 	if m.GetEtcdLeader() == 0 {
 		return errs.ErrEtcdLeaderNotFound

tests/integrations/client/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
 	github.com/stretchr/testify v1.8.2
 	github.com/tikv/pd v0.0.0-00010101000000-000000000000
-	github.com/tikv/pd/client v0.0.0-00010101000000-000000000000
+	github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793
 	go.uber.org/goleak v1.1.12
 	go.uber.org/zap v1.24.0

tests/integrations/mcs/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
 	github.com/stretchr/testify v1.8.2
 	github.com/tikv/pd v0.0.0-00010101000000-000000000000
-	github.com/tikv/pd/client v0.0.0-00010101000000-000000000000
+	github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793
 	go.uber.org/goleak v1.1.12
 	go.uber.org/zap v1.24.0