test: refactor real cluster test and implement cluster id case (#8685)

close #8683, close #8684

Signed-off-by: okJiang <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

.gitignore

@@ -29,3 +29,4 @@ go.work*
 embedded_assets_handler.go
 *.log
 *.bin
+third_bin

tests/integrations/realcluster/Makefile

@@ -28,9 +28,17 @@ tidy:
 	git diff go.mod go.sum | cat
 	git diff --quiet go.mod go.sum
 
-check: deploy test kill_cluster
+check: tiup test
 
-deploy: kill_cluster
+tiup:
+	# if tiup binary not exist, download it
+	if ! which tiup > /dev/null 2>&1; then \
+        curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh; \
+    fi
+
+deploy: kill_cluster deploy_only
+
+deploy_only:
 	@ echo "deploying..."
 	./deploy.sh
 	@ echo "wait cluster ready..."

tests/integrations/realcluster/cluster_id_test.go

@@ -0,0 +1,85 @@
+// Copyright 2024 TiKV Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package realcluster
+
+import (
+	"context"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	pd "github.com/tikv/pd/client"
+)
+
+type clusterIDSuite struct {
+	realClusterSuite
+}
+
+func TestClusterID(t *testing.T) {
+	suite.Run(t, &clusterIDSuite{
+		realClusterSuite: realClusterSuite{
+			suiteName: "cluster_id",
+		},
+	})
+}
+
+func (s *clusterIDSuite) TestClientClusterID() {
+	re := require.New(s.T())
+	ctx := context.Background()
+	// deploy second cluster
+	s.startRealCluster(s.T())
+	defer s.stopRealCluster(s.T())
+
+	pdEndpoints := getPDEndpoints(s.T())
+	// Try to create a client with the mixed endpoints.
+	_, err := pd.NewClientWithContext(
+		ctx, pdEndpoints,
+		pd.SecurityOption{}, pd.WithMaxErrorRetry(1),
+	)
+	re.Error(err)
+	re.Contains(err.Error(), "unmatched cluster id")
+}
+
+func getPDEndpoints(t *testing.T) []string {
+	cmd := exec.Command("sh", "-c", "ps -ef | grep tikv-server | awk -F '--pd-endpoints=' '{print $2}' | awk '{print $1}'")
+	bytes, err := cmd.Output()
+	require.NoError(t, err)
+	pdAddrsForEachTikv := strings.Split(string(bytes), "\n")
+	var pdAddrs []string
+	for _, addr := range pdAddrsForEachTikv {
+		// length of addr is less than 5 means it must not be a valid address
+		if len(addr) < 5 {
+			continue
+		}
+		pdAddrs = append(pdAddrs, strings.Split(addr, ",")...)
+	}
+	return removeDuplicates(pdAddrs)
+}
+
+func removeDuplicates(arr []string) []string {
+	uniqueMap := make(map[string]bool)
+	var result []string
+
+	for _, item := range arr {
+		if _, exists := uniqueMap[item]; !exists {
+			uniqueMap[item] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

tests/integrations/realcluster/deploy.sh

@@ -28,9 +28,9 @@ else
   # CI will download the binaries in the prepare phase.
   # ref https://github.com/PingCAP-QE/ci/blob/387e9e533b365174962ccb1959442a7070f9cd66/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy#L55-L68
 	color-green "using existing binaries..."
-	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \
+	$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_real_cluster_test \
 		--pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server \
-		--tiflash.binpath ./bin/tiflash --tag pd_real_cluster_test --pd.config ./tests/integrations/realcluster/pd.toml \
+		--tiflash.binpath ./bin/tiflash --pd.config ./tests/integrations/realcluster/pd.toml \
 		> $CUR_PATH/playground.log 2>&1 &
 fi
 

tests/integrations/realcluster/download_integration_test_binaries.sh

@@ -0,0 +1,80 @@
+#! /usr/bin/env bash
+
+# help
+# download some third party tools for integration test
+# example: ./download_integration_test_binaries.sh master
+
+
+set -o errexit
+set -o pipefail
+
+
+# Specify which branch to be utilized for executing the test, which is
+# exclusively accessible when obtaining binaries from
+# http://fileserver.pingcap.net.
+branch=${1:-master}
+file_server_url=${2:-http://fileserver.pingcap.net}
+
+tidb_sha1_url="${file_server_url}/download/refs/pingcap/tidb/${branch}/sha1"
+tikv_sha1_url="${file_server_url}/download/refs/pingcap/tikv/${branch}/sha1"
+tiflash_sha1_url="${file_server_url}/download/refs/pingcap/tiflash/${branch}/sha1"
+
+tidb_sha1=$(curl "$tidb_sha1_url")
+tikv_sha1=$(curl "$tikv_sha1_url")
+tiflash_sha1=$(curl "$tiflash_sha1_url")
+
+# download tidb / tikv / tiflash binary build from tibuid multibranch pipeline
+tidb_download_url="${file_server_url}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz"
+tikv_download_url="${file_server_url}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz"
+tiflash_download_url="${file_server_url}/download/builds/pingcap/tiflash/${branch}/${tiflash_sha1}/centos7/tiflash.tar.gz"
+
+set -o nounset
+
+# See https://misc.flogisoft.com/bash/tip_colors_and_formatting.
+color_green() { # Green
+	echo -e "\x1B[1;32m${*}\x1B[0m"
+}
+
+function download() {
+    local url=$1
+    local file_name=$2
+    local file_path=$3
+    if [[ -f "${file_path}" ]]; then
+        echo "file ${file_name} already exists, skip download"
+        return
+    fi
+    echo "download ${file_name} from ${url}"
+    wget --no-verbose --retry-connrefused --waitretry=1 -t 3 -O "${file_path}" "${url}"
+}
+
+function main() { 
+    rm -rf third_bin
+    rm -rf tmp
+    mkdir third_bin
+    mkdir tmp
+
+    # tidb server
+    download "$tidb_download_url" "tidb-server.tar.gz" "tmp/tidb-server.tar.gz"
+    tar -xzf tmp/tidb-server.tar.gz -C third_bin --wildcards 'bin/*'
+    mv third_bin/bin/* third_bin/ 
+
+    # TiKV server
+    download "$tikv_download_url" "tikv-server.tar.gz" "tmp/tikv-server.tar.gz"
+    tar -xzf tmp/tikv-server.tar.gz -C third_bin --wildcards 'bin/*'
+    mv third_bin/bin/* third_bin/
+
+    # TiFlash
+    download "$tiflash_download_url" "tiflash.tar.gz" "tmp/tiflash.tar.gz"
+    tar -xzf tmp/tiflash.tar.gz -C third_bin
+    mv third_bin/tiflash third_bin/_tiflash
+    mv third_bin/_tiflash/* third_bin && rm -rf third_bin/_tiflash
+
+    chmod +x third_bin/*
+    rm -rf tmp
+    rm -rf third_bin/bin
+    ls -alh third_bin/
+}
+
+main "$@"
+
+color_green "Download SUCCESS"

tests/integrations/realcluster/real_cluster.go

@@ -0,0 +1,169 @@
+// Copyright 2024 TiKV Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package realcluster
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/pingcap/log"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"go.uber.org/zap"
+)
+
+type realClusterSuite struct {
+	suite.Suite
+
+	clusterCnt int
+	suiteName  string
+}
+
+var tiupBin = os.Getenv("HOME") + "/.tiup/bin/tiup"
+
+// SetupSuite will run before the tests in the suite are run.
+func (s *realClusterSuite) SetupSuite() {
+	t := s.T()
+
+	// Clean the data dir. It is the default data dir of TiUP.
+	dataDir := filepath.Join(os.Getenv("HOME"), ".tiup", "data", "pd_real_cluster_test_"+s.suiteName+"_*")
+	matches, err := filepath.Glob(dataDir)
+	require.NoError(t, err)
+
+	for _, match := range matches {
+		require.NoError(t, runCommand("rm", "-rf", match))
+	}
+	s.startRealCluster(t)
+	t.Cleanup(func() {
+		s.stopRealCluster(t)
+	})
+}
+
+// TearDownSuite will run after all the tests in the suite have been run.
+func (s *realClusterSuite) TearDownSuite() {
+	// Even if the cluster deployment fails, we still need to destroy the cluster.
+	// If the cluster does not fail to deploy, the cluster will be destroyed in
+	// the cleanup function. And these code will not work.
+	s.clusterCnt++
+	s.stopRealCluster(s.T())
+}
+
+func (s *realClusterSuite) startRealCluster(t *testing.T) {
+	log.Info("start to deploy a real cluster")
+
+	s.deploy(t)
+	s.clusterCnt++
+}
+
+func (s *realClusterSuite) stopRealCluster(t *testing.T) {
+	s.clusterCnt--
+
+	log.Info("start to destroy a real cluster", zap.String("tag", s.tag()))
+	destroy(t, s.tag())
+	time.Sleep(5 * time.Second)
+}
+
+func (s *realClusterSuite) tag() string {
+	return fmt.Sprintf("pd_real_cluster_test_%s_%d", s.suiteName, s.clusterCnt)
+}
+
+// func restartTiUP() {
+// 	log.Info("start to restart TiUP")
+// 	cmd := exec.Command("make", "deploy")
+// 	cmd.Stdout = os.Stdout
+// 	cmd.Stderr = os.Stderr
+// 	err := cmd.Run()
+// 	if err != nil {
+// 		panic(err)
+// 	}
+// 	log.Info("TiUP restart success")
+// }
+
+func (s *realClusterSuite) deploy(t *testing.T) {
+	tag := s.tag()
+	deployTiupPlayground(t, tag)
+	waitTiupReady(t, tag)
+}
+
+func destroy(t *testing.T, tag string) {
+	cmdStr := fmt.Sprintf("ps -ef | grep 'tiup playground' | grep %s | awk '{print $2}' | head -n 1", tag)
+	cmd := exec.Command("sh", "-c", cmdStr)
+	bytes, err := cmd.Output()
+	require.NoError(t, err)
+	pid := string(bytes)
+	// nolint:errcheck
+	runCommand("sh", "-c", "kill -9 "+pid)
+	log.Info("destroy success", zap.String("pid", pid))
+}
+
+func deployTiupPlayground(t *testing.T, tag string) {
+	curPath, err := os.Getwd()
+	require.NoError(t, err)
+
+	log.Info(curPath)
+	require.NoError(t, os.Chdir("../../.."))
+
+	if !fileExists("third_bin") || !fileExists("third_bin/tikv-server") || !fileExists("third_bin/tidb-server") || !fileExists("third_bin/tiflash") {
+		log.Info("downloading binaries...")
+		log.Info("this may take a few minutes, you can also download them manually and put them in the bin directory.")
+		require.NoError(t, runCommand("sh",
+			"./tests/integrations/realcluster/download_integration_test_binaries.sh"))
+	}
+	if !fileExists("bin") || !fileExists("bin/pd-server") {
+		log.Info("complie pd binaries...")
+		require.NoError(t, runCommand("make", "pd-server"))
+	}
+	if !fileExists(filepath.Join(curPath, "playground")) {
+		require.NoError(t, os.Mkdir(filepath.Join(curPath, "playground"), 0755))
+	}
+	// nolint:errcheck
+	go runCommand("sh", "-c",
+		tiupBin+` playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 \
+		--without-monitor --tag `+tag+` --pd.binpath ./bin/pd-server \
+		// --kv.binpath ./third_bin/tikv-server \
+		// --db.binpath ./third_bin/tidb-server --tiflash.binpath ./third_bin/tiflash \
+		--kv.binpath ./bin/tikv-server \
+		--db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash \
+		--pd.config ./tests/integrations/realcluster/pd.toml \
+		> `+filepath.Join(curPath, "playground", tag+".log")+` 2>&1 & `)
+
+	// Avoid to change the dir before execute `tiup playground`.
+	time.Sleep(10 * time.Second)
+	require.NoError(t, os.Chdir(curPath))
+}
+
+func waitTiupReady(t *testing.T, tag string) {
+	const (
+		interval = 5
+		maxTimes = 20
+	)
+	log.Info("start to wait TiUP ready", zap.String("tag", tag))
+	for i := 0; i < maxTimes; i++ {
+		err := runCommand(tiupBin, "playground", "display", "--tag", tag)
+		if err == nil {
+			log.Info("TiUP is ready", zap.String("tag", tag))
+			return
+		}
+
+		log.Info("TiUP is not ready, will retry", zap.Int("retry times", i),
+			zap.String("tag", tag), zap.Error(err))
+		time.Sleep(time.Duration(interval) * time.Second)
+	}
+	require.Failf(t, "TiUP is not ready", "tag: %s", tag)
+}