diff --git a/tests/integrations/realcluster/real_cluster.go b/tests/integrations/realcluster/real_cluster.go index 21284f285b0..1843b78a528 100644 --- a/tests/integrations/realcluster/real_cluster.go +++ b/tests/integrations/realcluster/real_cluster.go @@ -19,6 +19,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "time" @@ -37,17 +38,9 @@ type realClusterSuite struct { var ( playgroundLogDir = filepath.Join("tmp", "real_cluster", "playground") - tiupBin string + tiupBin = os.Getenv("HOME") + "/.tiup/bin/tiup" ) -func init() { - var err error - tiupBin, err = exec.LookPath("tiup") - if err != nil { - panic(err) - } -} - // SetupSuite will run before the tests in the suite are run. func (s *realClusterSuite) SetupSuite() { t := s.T() @@ -78,7 +71,9 @@ func (s *realClusterSuite) TearDownSuite() { func (s *realClusterSuite) startRealCluster(t *testing.T) { log.Info("start to deploy a real cluster") - s.deploy(t) + tag := s.tag() + deployTiupPlayground(t, tag) + waitTiupReady(t, tag) s.clusterCnt++ } @@ -94,33 +89,26 @@ func (s *realClusterSuite) tag() string { return fmt.Sprintf("pd_real_cluster_test_%s_%d", s.suiteName, s.clusterCnt) } -// func restartTiUP() { -// log.Info("start to restart TiUP") -// cmd := exec.Command("make", "deploy") -// cmd.Stdout = os.Stdout -// cmd.Stderr = os.Stderr -// err := cmd.Run() -// if err != nil { -// panic(err) -// } -// log.Info("TiUP restart success") -// } - -func (s *realClusterSuite) deploy(t *testing.T) { +func (s *realClusterSuite) restart() { tag := s.tag() - deployTiupPlayground(t, tag) - waitTiupReady(t, tag) + log.Info("start to restart", zap.String("tag", tag)) + s.stopRealCluster(s.T()) + s.startRealCluster(s.T()) + log.Info("TiUP restart success") } func destroy(t *testing.T, tag string) { - cmdStr := fmt.Sprintf("ps -ef | grep 'tiup playground' | grep %s | awk '{print $2}' | head -n 1", tag) + cmdStr := fmt.Sprintf("ps -ef | grep %s | awk '{print $2}'", tag) cmd := exec.Command("sh", "-c", cmdStr) bytes, err := cmd.Output() require.NoError(t, err) - pid := string(bytes) - // nolint:errcheck - runCommand("sh", "-c", "kill -9 "+pid) - log.Info("destroy success", zap.String("pid", pid)) + pids := string(bytes) + pidArr := strings.Split(pids, "\n") + for _, pid := range pidArr { + // nolint:errcheck + runCommand("sh", "-c", "kill -9 "+pid) + } + log.Info("destroy success", zap.String("tag", tag)) } func deployTiupPlayground(t *testing.T, tag string) { @@ -146,11 +134,11 @@ func deployTiupPlayground(t *testing.T, tag string) { go func() { runCommand("sh", "-c", tiupBin+` playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 \ - --without-monitor --tag `+tag+` --pd.binpath ./bin/pd-server \ - --kv.binpath ./third_bin/tikv-server \ - --db.binpath ./third_bin/tidb-server --tiflash.binpath ./third_bin/tiflash \ - --pd.config ./tests/integrations/realcluster/pd.toml \ - > `+filepath.Join(playgroundLogDir, tag+".log")+` 2>&1 & `) + --without-monitor --tag `+tag+` --pd.binpath ./bin/pd-server \ + --kv.binpath ./third_bin/tikv-server \ + --db.binpath ./third_bin/tidb-server --tiflash.binpath ./third_bin/tiflash \ + --pd.config ./tests/integrations/realcluster/pd.toml \ + > `+filepath.Join(playgroundLogDir, tag+".log")+` 2>&1 & `) }() // Avoid to change the dir before execute `tiup playground`. diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go index 50b4bee2055..e3f37ac0605 100644 --- a/tests/integrations/realcluster/reboot_pd_test.go +++ b/tests/integrations/realcluster/reboot_pd_test.go @@ -14,45 +14,68 @@ package realcluster +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/client/http" +) + +type rebootPDSuite struct { + realClusterSuite +} + +func TestRebootPD(t *testing.T) { + suite.Run(t, &rebootPDSuite{ + realClusterSuite: realClusterSuite{ + suiteName: "reboot_pd", + }, + }) +} + // https://github.com/tikv/pd/issues/6467 -// func TestReloadLabel(t *testing.T) { -// re := require.New(t) -// ctx := context.Background() - -// resp, err := pdHTTPCli.GetStores(ctx) -// re.NoError(err) -// re.NotEmpty(resp.Stores) -// firstStore := resp.Stores[0] -// // TiFlash labels will be ["engine": "tiflash"] -// // So we need to merge the labels -// storeLabels := map[string]string{ -// "zone": "zone1", -// } -// for _, label := range firstStore.Store.Labels { -// storeLabels[label.Key] = label.Value -// } -// re.NoError(pdHTTPCli.SetStoreLabels(ctx, firstStore.Store.ID, storeLabels)) -// defer func() { -// re.NoError(pdHTTPCli.DeleteStoreLabel(ctx, firstStore.Store.ID, "zone")) -// }() - -// checkLabelsAreEqual := func() { -// resp, err := pdHTTPCli.GetStore(ctx, uint64(firstStore.Store.ID)) -// re.NoError(err) - -// labelsMap := make(map[string]string) -// for _, label := range resp.Store.Labels { -// re.NotNil(label) -// labelsMap[label.Key] = label.Value -// } - -// for key, value := range storeLabels { -// re.Equal(value, labelsMap[key]) -// } -// } -// // Check the label is set -// checkLabelsAreEqual() -// // Restart TiUP to reload the label -// restartTiUP() -// checkLabelsAreEqual() -// } +func (s *rebootPDSuite) TestReloadLabel() { + re := require.New(s.T()) + ctx := context.Background() + + pdHTTPCli := http.NewClient("pd-real-cluster-test", getPDEndpoints(s.T())) + resp, err := pdHTTPCli.GetStores(ctx) + re.NoError(err) + re.NotEmpty(resp.Stores) + firstStore := resp.Stores[0] + // TiFlash labels will be ["engine": "tiflash"] + // So we need to merge the labels + storeLabels := map[string]string{ + "zone": "zone1", + } + for _, label := range firstStore.Store.Labels { + storeLabels[label.Key] = label.Value + } + re.NoError(pdHTTPCli.SetStoreLabels(ctx, firstStore.Store.ID, storeLabels)) + defer func() { + re.NoError(pdHTTPCli.DeleteStoreLabel(ctx, firstStore.Store.ID, "zone")) + }() + + checkLabelsAreEqual := func() { + resp, err := pdHTTPCli.GetStore(ctx, uint64(firstStore.Store.ID)) + re.NoError(err) + + labelsMap := make(map[string]string) + for _, label := range resp.Store.Labels { + re.NotNil(label) + labelsMap[label.Key] = label.Value + } + + for key, value := range storeLabels { + re.Equal(value, labelsMap[key]) + } + } + // Check the label is set + checkLabelsAreEqual() + // Restart to reload the label + s.restart() + pdHTTPCli = http.NewClient("pd-real-cluster-test", getPDEndpoints(s.T())) + checkLabelsAreEqual() +} diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go index 7e5087627fb..69da846b491 100644 --- a/tests/integrations/realcluster/scheduler_test.go +++ b/tests/integrations/realcluster/scheduler_test.go @@ -14,161 +14,190 @@ package realcluster +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/client/http" + "github.com/tikv/pd/client/testutil" + "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/types" +) + +type schedulerSuite struct { + realClusterSuite +} + +func TestScheduler(t *testing.T) { + suite.Run(t, &schedulerSuite{ + realClusterSuite: realClusterSuite{ + suiteName: "scheduler", + }, + }) +} + // https://github.com/tikv/pd/issues/6988#issuecomment-1694924611 // https://github.com/tikv/pd/issues/6897 -// func TestTransferLeader(t *testing.T) { -// re := require.New(t) -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// resp, err := pdHTTPCli.GetLeader(ctx) -// re.NoError(err) -// oldLeader := resp.Name - -// var newLeader string -// for i := 0; i < 2; i++ { -// if resp.Name != fmt.Sprintf("pd-%d", i) { -// newLeader = fmt.Sprintf("pd-%d", i) -// } -// } - -// // record scheduler -// re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), 1)) -// defer func() { -// re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) -// }() -// res, err := pdHTTPCli.GetSchedulers(ctx) -// re.NoError(err) -// oldSchedulersLen := len(res) - -// re.NoError(pdHTTPCli.TransferLeader(ctx, newLeader)) -// // wait for transfer leader to new leader -// time.Sleep(1 * time.Second) -// resp, err = pdHTTPCli.GetLeader(ctx) -// re.NoError(err) -// re.Equal(newLeader, resp.Name) - -// res, err = pdHTTPCli.GetSchedulers(ctx) -// re.NoError(err) -// re.Len(res, oldSchedulersLen) - -// // transfer leader to old leader -// re.NoError(pdHTTPCli.TransferLeader(ctx, oldLeader)) -// // wait for transfer leader -// time.Sleep(1 * time.Second) -// resp, err = pdHTTPCli.GetLeader(ctx) -// re.NoError(err) -// re.Equal(oldLeader, resp.Name) - -// res, err = pdHTTPCli.GetSchedulers(ctx) -// re.NoError(err) -// re.Len(res, oldSchedulersLen) -// } - -// func TestRegionLabelDenyScheduler(t *testing.T) { -// re := require.New(t) -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// regions, err := pdHTTPCli.GetRegions(ctx) -// re.NoError(err) -// re.NotEmpty(regions.Regions) -// region1 := regions.Regions[0] - -// err = pdHTTPCli.DeleteScheduler(ctx, types.BalanceLeaderScheduler.String()) -// if err == nil { -// defer func() { -// pdHTTPCli.CreateScheduler(ctx, types.BalanceLeaderScheduler.String(), 0) -// }() -// } - -// re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) -// defer func() { -// pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) -// }() - -// // wait leader transfer -// testutil.Eventually(re, func() bool { -// regions, err := pdHTTPCli.GetRegions(ctx) -// re.NoError(err) -// for _, region := range regions.Regions { -// if region.Leader.StoreID != region1.Leader.StoreID { -// return false -// } -// } -// return true -// }, testutil.WithWaitFor(time.Minute)) - -// // disable schedule for region1 -// labelRule := &pd.LabelRule{ -// ID: "rule1", -// Labels: []pd.RegionLabel{{Key: "schedule", Value: "deny"}}, -// RuleType: "key-range", -// Data: labeler.MakeKeyRanges(region1.StartKey, region1.EndKey), -// } -// re.NoError(pdHTTPCli.SetRegionLabelRule(ctx, labelRule)) -// defer func() { -// pdHTTPCli.PatchRegionLabelRules(ctx, &pd.LabelRulePatch{DeleteRules: []string{labelRule.ID}}) -// }() -// labelRules, err := pdHTTPCli.GetAllRegionLabelRules(ctx) -// re.NoError(err) -// re.Len(labelRules, 2) -// sort.Slice(labelRules, func(i, j int) bool { -// return labelRules[i].ID < labelRules[j].ID -// }) -// re.Equal(labelRule.ID, labelRules[1].ID) -// re.Equal(labelRule.Labels, labelRules[1].Labels) -// re.Equal(labelRule.RuleType, labelRules[1].RuleType) - -// // enable evict leader scheduler, and check it works -// re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String())) -// re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), uint64(region1.Leader.StoreID))) -// defer func() { -// pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String()) -// }() -// testutil.Eventually(re, func() bool { -// regions, err := pdHTTPCli.GetRegions(ctx) -// re.NoError(err) -// for _, region := range regions.Regions { -// if region.Leader.StoreID == region1.Leader.StoreID { -// return false -// } -// } -// return true -// }, testutil.WithWaitFor(time.Minute)) - -// re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) -// re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) -// defer func() { -// pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) -// }() -// testutil.Eventually(re, func() bool { -// regions, err := pdHTTPCli.GetRegions(ctx) -// re.NoError(err) -// for _, region := range regions.Regions { -// if region.ID == region1.ID { -// continue -// } -// if region.Leader.StoreID != region1.Leader.StoreID { -// return false -// } -// } -// return true -// }, testutil.WithWaitFor(time.Minute)) - -// pdHTTPCli.PatchRegionLabelRules(ctx, &pd.LabelRulePatch{DeleteRules: []string{labelRule.ID}}) -// labelRules, err = pdHTTPCli.GetAllRegionLabelRules(ctx) -// re.NoError(err) -// re.Len(labelRules, 1) - -// testutil.Eventually(re, func() bool { -// regions, err := pdHTTPCli.GetRegions(ctx) -// re.NoError(err) -// for _, region := range regions.Regions { -// if region.Leader.StoreID != region1.Leader.StoreID { -// return false -// } -// } -// return true -// }, testutil.WithWaitFor(time.Minute)) -// } +func (s *schedulerSuite) TestTransferLeader() { + re := require.New(s.T()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pdHTTPCli := http.NewClient("pd-real-cluster-test", getPDEndpoints(s.T())) + resp, err := pdHTTPCli.GetLeader(ctx) + re.NoError(err) + oldLeader := resp.Name + + var newLeader string + for i := 0; i < 2; i++ { + if resp.Name != fmt.Sprintf("pd-%d", i) { + newLeader = fmt.Sprintf("pd-%d", i) + } + } + + // record scheduler + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), 1)) + defer func() { + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) + }() + res, err := pdHTTPCli.GetSchedulers(ctx) + re.NoError(err) + oldSchedulersLen := len(res) + + re.NoError(pdHTTPCli.TransferLeader(ctx, newLeader)) + // wait for transfer leader to new leader + time.Sleep(1 * time.Second) + resp, err = pdHTTPCli.GetLeader(ctx) + re.NoError(err) + re.Equal(newLeader, resp.Name) + + res, err = pdHTTPCli.GetSchedulers(ctx) + re.NoError(err) + re.Len(res, oldSchedulersLen) + + // transfer leader to old leader + re.NoError(pdHTTPCli.TransferLeader(ctx, oldLeader)) + // wait for transfer leader + time.Sleep(1 * time.Second) + resp, err = pdHTTPCli.GetLeader(ctx) + re.NoError(err) + re.Equal(oldLeader, resp.Name) + + res, err = pdHTTPCli.GetSchedulers(ctx) + re.NoError(err) + re.Len(res, oldSchedulersLen) +} + +func (s *schedulerSuite) TestRegionLabelDenyScheduler() { + re := require.New(s.T()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pdHTTPCli := http.NewClient("pd-real-cluster-test", getPDEndpoints(s.T())) + regions, err := pdHTTPCli.GetRegions(ctx) + re.NoError(err) + re.NotEmpty(regions.Regions) + region1 := regions.Regions[0] + + err = pdHTTPCli.DeleteScheduler(ctx, types.BalanceLeaderScheduler.String()) + if err == nil { + defer func() { + pdHTTPCli.CreateScheduler(ctx, types.BalanceLeaderScheduler.String(), 0) + }() + } + + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) + defer func() { + pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) + }() + + // wait leader transfer + testutil.Eventually(re, func() bool { + regions, err := pdHTTPCli.GetRegions(ctx) + re.NoError(err) + for _, region := range regions.Regions { + if region.Leader.StoreID != region1.Leader.StoreID { + return false + } + } + return true + }, testutil.WithWaitFor(time.Minute)) + + // disable schedule for region1 + labelRule := &http.LabelRule{ + ID: "rule1", + Labels: []http.RegionLabel{{Key: "schedule", Value: "deny"}}, + RuleType: "key-range", + Data: labeler.MakeKeyRanges(region1.StartKey, region1.EndKey), + } + re.NoError(pdHTTPCli.SetRegionLabelRule(ctx, labelRule)) + defer func() { + pdHTTPCli.PatchRegionLabelRules(ctx, &http.LabelRulePatch{DeleteRules: []string{labelRule.ID}}) + }() + labelRules, err := pdHTTPCli.GetAllRegionLabelRules(ctx) + re.NoError(err) + re.Len(labelRules, 2) + sort.Slice(labelRules, func(i, j int) bool { + return labelRules[i].ID < labelRules[j].ID + }) + re.Equal(labelRule.ID, labelRules[1].ID) + re.Equal(labelRule.Labels, labelRules[1].Labels) + re.Equal(labelRule.RuleType, labelRules[1].RuleType) + + // enable evict leader scheduler, and check it works + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String())) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), uint64(region1.Leader.StoreID))) + defer func() { + pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String()) + }() + testutil.Eventually(re, func() bool { + regions, err := pdHTTPCli.GetRegions(ctx) + re.NoError(err) + for _, region := range regions.Regions { + if region.Leader.StoreID == region1.Leader.StoreID { + return false + } + } + return true + }, testutil.WithWaitFor(time.Minute)) + + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) + defer func() { + pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) + }() + testutil.Eventually(re, func() bool { + regions, err := pdHTTPCli.GetRegions(ctx) + re.NoError(err) + for _, region := range regions.Regions { + if region.ID == region1.ID { + continue + } + if region.Leader.StoreID != region1.Leader.StoreID { + return false + } + } + return true + }, testutil.WithWaitFor(time.Minute)) + + pdHTTPCli.PatchRegionLabelRules(ctx, &http.LabelRulePatch{DeleteRules: []string{labelRule.ID}}) + labelRules, err = pdHTTPCli.GetAllRegionLabelRules(ctx) + re.NoError(err) + re.Len(labelRules, 1) + + testutil.Eventually(re, func() bool { + regions, err := pdHTTPCli.GetRegions(ctx) + re.NoError(err) + for _, region := range regions.Regions { + if region.Leader.StoreID != region1.Leader.StoreID { + return false + } + } + return true + }, testutil.WithWaitFor(time.Minute)) +} diff --git a/tests/integrations/realcluster/ts_test.go b/tests/integrations/realcluster/ts_test.go index 156e3d63e71..f19124d04a4 100644 --- a/tests/integrations/realcluster/ts_test.go +++ b/tests/integrations/realcluster/ts_test.go @@ -14,26 +14,45 @@ package realcluster -// func TestTS(t *testing.T) { -// re := require.New(t) - -// db := OpenTestDB(t) -// db.MustExec("use test") -// db.MustExec("drop table if exists t") -// db.MustExec("create table t(a int, index i(a))") -// db.MustExec("insert t values (1), (2), (3)") -// var rows int -// err := db.inner.Raw("select count(*) from t").Row().Scan(&rows) -// re.NoError(err) -// re.Equal(3, rows) - -// re.NoError(err) -// re.Equal(3, rows) - -// var ts uint64 -// err = db.inner.Begin().Raw("select @@tidb_current_ts").Scan(&ts).Rollback().Error -// re.NoError(err) -// re.NotEqual(0, GetTimeFromTS(ts)) - -// db.MustClose() -// } +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type tsSuite struct { + realClusterSuite +} + +func TestTS(t *testing.T) { + suite.Run(t, &tsSuite{ + realClusterSuite: realClusterSuite{ + suiteName: "ts", + }, + }) +} + +func (s *tsSuite) TestTS() { + re := require.New(s.T()) + + db := OpenTestDB(s.T()) + db.MustExec("use test") + db.MustExec("drop table if exists t") + db.MustExec("create table t(a int, index i(a))") + db.MustExec("insert t values (1), (2), (3)") + var rows int + err := db.inner.Raw("select count(*) from t").Row().Scan(&rows) + re.NoError(err) + re.Equal(3, rows) + + re.NoError(err) + re.Equal(3, rows) + + var ts uint64 + err = db.inner.Begin().Raw("select @@tidb_current_ts").Scan(&ts).Rollback().Error + re.NoError(err) + re.NotEqual(0, GetTimeFromTS(ts)) + + db.MustClose() +} diff --git a/tests/integrations/realcluster/util.go b/tests/integrations/realcluster/util.go index 013c41da7f3..789ceaa29c2 100644 --- a/tests/integrations/realcluster/util.go +++ b/tests/integrations/realcluster/util.go @@ -22,11 +22,6 @@ import ( const physicalShiftBits = 18 -var ( -// pdAddrs = []string{"http://127.0.0.1:2379"} -// pdHTTPCli = http.NewClient("pd-real-cluster-test", pdAddrs) -) - // GetTimeFromTS extracts time.Time from a timestamp. func GetTimeFromTS(ts uint64) time.Time { ms := ExtractPhysical(ts)