Skip to content

Commit

Permalink
simulator: make store,region,replica configurable in cases (#8215)
Browse files Browse the repository at this point in the history 
ref #8135

Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
  • Loading branch information
@lhy1024 @ti-chi-bot
lhy1024 and ti-chi-bot[bot] authored May 30, 2024
1 parent c498063 commit 52389b0
Show file tree
Hide file tree
Showing 24 changed files with 303 additions and 350 deletions.
7 changes: 7 additions & 0 deletions pkg/utils/configutil/configutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,10 @@ func AdjustPath(p *string) {
*p = absPath
}
}

// AdjustBool adjusts the value of a bool variable.
func AdjustBool(v *bool, defValue bool) {
if !*v {
*v = defValue
}
}
34 changes: 15 additions & 19 deletions tools/pd-simulator/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,19 @@ import (
"github.com/tikv/pd/tools/pd-analysis/analysis"
"github.com/tikv/pd/tools/pd-simulator/simulator"
"github.com/tikv/pd/tools/pd-simulator/simulator/cases"
sc "github.com/tikv/pd/tools/pd-simulator/simulator/config"
"github.com/tikv/pd/tools/pd-simulator/simulator/simutil"
"go.uber.org/zap"
)

var (
pdAddr = flag.String("pd-endpoints", "", "pd address")
configFile = flag.String("config", "conf/simconfig.toml", "config file")
caseName = flag.String("case", "", "case name")
serverLogLevel = flag.String("serverLog", "info", "pd server log level")
simLogLevel = flag.String("simLog", "info", "simulator log level")
simLogFile = flag.String("log-file", "", "simulator log file")
regionNum = flag.Int("regionNum", 0, "regionNum of one store")
storeNum = flag.Int("storeNum", 0, "storeNum")
enableTransferRegionCounter = flag.Bool("enableTransferRegionCounter", false, "enableTransferRegionCounter")
statusAddress = flag.String("status-addr", "0.0.0.0:20180", "status address")
pdAddr = flag.String("pd-endpoints", "", "pd address")
configFile = flag.String("config", "conf/simconfig.toml", "config file")
caseName = flag.String("case", "", "case name")
serverLogLevel = flag.String("serverLog", "info", "pd server log level")
simLogLevel = flag.String("simLog", "info", "simulator log level")
simLogFile = flag.String("log-file", "", "simulator log file")
statusAddress = flag.String("status-addr", "0.0.0.0:20180", "status address")
)

func main() {
Expand All @@ -63,14 +61,12 @@ func main() {
flag.Parse()

simutil.InitLogger(*simLogLevel, *simLogFile)
simutil.InitCaseConfig(*storeNum, *regionNum, *enableTransferRegionCounter)
statistics.Denoising = false
if simutil.CaseConfigure.EnableTransferRegionCounter {
analysis.GetTransferCounter().Init(simutil.CaseConfigure.StoreNum, simutil.CaseConfigure.RegionNum)
}

schedulers.Register() // register schedulers, which is needed by simConfig.Adjust
simConfig := simulator.NewSimConfig(*serverLogLevel)
simConfig := sc.NewSimConfig(*serverLogLevel)
if simConfig.EnableTransferRegionCounter {
analysis.GetTransferCounter().Init(simConfig.TotalStore, simConfig.TotalRegion)
}
var meta toml.MetaData
var err error
if *configFile != "" {
Expand All @@ -97,7 +93,7 @@ func main() {
}
}

func run(simCase string, simConfig *simulator.SimConfig) {
func run(simCase string, simConfig *sc.SimConfig) {
if *pdAddr != "" {
go runHTTPServer()
simStart(*pdAddr, simCase, simConfig)
Expand Down Expand Up @@ -136,7 +132,7 @@ func runHTTPServer() {
}

// NewSingleServer creates a pd server for simulator.
func NewSingleServer(ctx context.Context, simConfig *simulator.SimConfig) (*server.Server, testutil.CleanupFunc) {
func NewSingleServer(ctx context.Context, simConfig *sc.SimConfig) (*server.Server, testutil.CleanupFunc) {
err := logutil.SetupLogger(simConfig.ServerConfig.Log, &simConfig.ServerConfig.Logger, &simConfig.ServerConfig.LogProps)
if err == nil {
log.ReplaceGlobals(simConfig.ServerConfig.Logger, simConfig.ServerConfig.LogProps)
Expand All @@ -161,7 +157,7 @@ func cleanServer(cfg *config.Config) {
os.RemoveAll(cfg.DataDir)
}

func simStart(pdAddr string, simCase string, simConfig *simulator.SimConfig, clean ...testutil.CleanupFunc) {
func simStart(pdAddr string, simCase string, simConfig *sc.SimConfig, clean ...testutil.CleanupFunc) {
start := time.Now()
driver, err := simulator.NewDriver(pdAddr, simCase, simConfig)
if err != nil {
Expand Down
49 changes: 23 additions & 26 deletions tools/pd-simulator/simulator/cases/add_nodes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,35 @@
package cases

import (
"math/rand"

"github.com/docker/go-units"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/tikv/pd/pkg/core"
sc "github.com/tikv/pd/tools/pd-simulator/simulator/config"
"github.com/tikv/pd/tools/pd-simulator/simulator/info"
"github.com/tikv/pd/tools/pd-simulator/simulator/simutil"
"go.uber.org/zap"
)

func newAddNodes() *Case {
func newAddNodes(config *sc.SimConfig) *Case {
var simCase Case

storeNum, regionNum := getStoreNum(), getRegionNum()
noEmptyRatio := rand.Float64() // the ratio of noEmpty store to total store
noEmptyStoreNum := getNoEmptyStoreNum(storeNum, noEmptyRatio)
totalStore := config.TotalStore
totalRegion := config.TotalRegion
replica := int(config.ServerConfig.Replication.MaxReplicas)
noEmptyStoreNum := getNoEmptyStoreNum(totalStore, replica)

for i := 1; i <= storeNum; i++ {
for i := 0; i < totalStore; i++ {
simCase.Stores = append(simCase.Stores, &Store{
ID: IDAllocator.nextID(),
Status: metapb.StoreState_Up,
})
}

for i := 0; i < regionNum*storeNum/3; i++ {
peers := []*metapb.Peer{
{Id: IDAllocator.nextID(), StoreId: uint64(i)%noEmptyStoreNum + 1},
{Id: IDAllocator.nextID(), StoreId: uint64(i+1)%noEmptyStoreNum + 1},
{Id: IDAllocator.nextID(), StoreId: uint64(i+2)%noEmptyStoreNum + 1},
for i := 0; i < totalRegion; i++ {
peers := make([]*metapb.Peer, 0, replica)
for j := 0; j < replica; j++ {
peers = append(peers, &metapb.Peer{
Id: IDAllocator.nextID(),
StoreId: uint64((i+j)%noEmptyStoreNum + 1),
})
}
simCase.Regions = append(simCase.Regions, Region{
ID: IDAllocator.nextID(),
Expand All @@ -54,21 +54,18 @@ func newAddNodes() *Case {
})
}

threshold := 0.05
simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool {
res := true
leaderCounts := make([]int, 0, storeNum)
regionCounts := make([]int, 0, storeNum)
for i := 1; i <= storeNum; i++ {
for i := 1; i <= totalStore; i++ {
leaderCount := regions.GetStoreLeaderCount(uint64(i))
regionCount := regions.GetStoreRegionCount(uint64(i))
leaderCounts = append(leaderCounts, leaderCount)
regionCounts = append(regionCounts, regionCount)
res = res && leaderAndRegionIsUniform(leaderCount, regionCount, regionNum, threshold)
peerCount := regions.GetStoreRegionCount(uint64(i))
if !isUniform(leaderCount, totalRegion/totalStore) {
return false
}
if !isUniform(peerCount, totalRegion*replica/totalStore) {
return false
}
}

simutil.Logger.Info("current counts", zap.Ints("leader", leaderCounts), zap.Ints("region", regionCounts))
return res
return true
}
return &simCase
}
60 changes: 30 additions & 30 deletions tools/pd-simulator/simulator/cases/add_nodes_dynamic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,40 +15,40 @@
package cases

import (
"math/rand"

"github.com/docker/go-units"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/tikv/pd/pkg/core"
sc "github.com/tikv/pd/tools/pd-simulator/simulator/config"
"github.com/tikv/pd/tools/pd-simulator/simulator/info"
"github.com/tikv/pd/tools/pd-simulator/simulator/simutil"
"go.uber.org/zap"
)

func newAddNodesDynamic() *Case {
func newAddNodesDynamic(config *sc.SimConfig) *Case {
var simCase Case

storeNum, regionNum := getStoreNum(), getRegionNum()
noEmptyRatio := rand.Float64() // the ratio of noEmpty store to total store
noEmptyStoreNum := getNoEmptyStoreNum(storeNum, noEmptyRatio)
totalStore := config.TotalStore
totalRegion := config.TotalRegion
replica := int(config.ServerConfig.Replication.MaxReplicas)
noEmptyStoreNum := getNoEmptyStoreNum(totalStore, replica)

for i := 1; i <= int(noEmptyStoreNum); i++ {
for i := 0; i < noEmptyStoreNum; i++ {
simCase.Stores = append(simCase.Stores, &Store{
ID: IDAllocator.nextID(),
Status: metapb.StoreState_Up,
})
}

var ids []uint64
for i := 1; i <= storeNum-int(noEmptyStoreNum); i++ {
for i := 0; i < totalStore-noEmptyStoreNum; i++ {
ids = append(ids, IDAllocator.nextID())
}

for i := 0; i < regionNum*storeNum/3; i++ {
peers := []*metapb.Peer{
{Id: IDAllocator.nextID(), StoreId: uint64(i)%noEmptyStoreNum + 1},
{Id: IDAllocator.nextID(), StoreId: uint64(i+1)%noEmptyStoreNum + 1},
{Id: IDAllocator.nextID(), StoreId: uint64(i+2)%noEmptyStoreNum + 1},
for i := 0; i < totalRegion; i++ {
peers := make([]*metapb.Peer, 0, replica)
for j := 0; j < replica; j++ {
peers = append(peers, &metapb.Peer{
Id: IDAllocator.nextID(),
StoreId: uint64((i+j)%noEmptyStoreNum + 1),
})
}
simCase.Regions = append(simCase.Regions, Region{
ID: IDAllocator.nextID(),
Expand All @@ -59,11 +59,11 @@ func newAddNodesDynamic() *Case {
})
}

numNodes := int(noEmptyStoreNum)
currentStoreCount := noEmptyStoreNum
e := &AddNodesDescriptor{}
e.Step = func(tick int64) uint64 {
if tick%100 == 0 && numNodes < storeNum {
numNodes++
if tick%100 == 0 && currentStoreCount < totalStore {
currentStoreCount++
nodeID := ids[0]
ids = append(ids[:0], ids[1:]...)
return nodeID
Expand All @@ -72,21 +72,21 @@ func newAddNodesDynamic() *Case {
}
simCase.Events = []EventDescriptor{e}

threshold := 0.05
simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool {
res := numNodes == storeNum
leaderCounts := make([]int, 0, numNodes)
regionCounts := make([]int, 0, numNodes)
for i := 1; i <= numNodes; i++ {
if currentStoreCount != totalStore {
return false
}
for i := 1; i <= currentStoreCount; i++ {
leaderCount := regions.GetStoreLeaderCount(uint64(i))
regionCount := regions.GetStoreRegionCount(uint64(i))
leaderCounts = append(leaderCounts, leaderCount)
regionCounts = append(regionCounts, regionCount)
res = res && leaderAndRegionIsUniform(leaderCount, regionCount, regionNum, threshold)
peerCount := regions.GetStoreRegionCount(uint64(i))
if !isUniform(leaderCount, totalRegion/totalStore) {
return false
}
if !isUniform(peerCount, totalRegion*replica/totalStore) {
return false
}
}

simutil.Logger.Info("current counts", zap.Ints("leader", leaderCounts), zap.Ints("region", regionCounts))
return res
return true
}
return &simCase
}
42 changes: 23 additions & 19 deletions tools/pd-simulator/simulator/cases/balance_leader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,35 @@ import (
"github.com/docker/go-units"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/tikv/pd/pkg/core"
sc "github.com/tikv/pd/tools/pd-simulator/simulator/config"
"github.com/tikv/pd/tools/pd-simulator/simulator/info"
"github.com/tikv/pd/tools/pd-simulator/simulator/simutil"
"go.uber.org/zap"
)

func newBalanceLeader() *Case {
func newBalanceLeader(config *sc.SimConfig) *Case {
var simCase Case

storeNum, regionNum := getStoreNum(), getRegionNum()

for i := 1; i <= storeNum; i++ {
totalStore := config.TotalStore
totalRegion := config.TotalRegion
replica := int(config.ServerConfig.Replication.MaxReplicas)
for i := 0; i < totalStore; i++ {
simCase.Stores = append(simCase.Stores, &Store{
ID: IDAllocator.nextID(),
Status: metapb.StoreState_Up,
})
}

for i := 0; i < storeNum*regionNum/3; i++ {
peers := []*metapb.Peer{
{Id: IDAllocator.nextID(), StoreId: uint64(storeNum)},
{Id: IDAllocator.nextID(), StoreId: uint64((i+1)%(storeNum-1)) + 1},
{Id: IDAllocator.nextID(), StoreId: uint64((i+2)%(storeNum-1)) + 1},
leaderStoreID := simCase.Stores[totalStore-1].ID
for i := 0; i < totalRegion; i++ {
peers := make([]*metapb.Peer, 0, replica)
peers = append(peers, &metapb.Peer{
Id: IDAllocator.nextID(),
StoreId: leaderStoreID,
})
for j := 1; j < replica; j++ {
peers = append(peers, &metapb.Peer{
Id: IDAllocator.nextID(),
StoreId: uint64((i+j)%(totalStore-1) + 1),
})
}
simCase.Regions = append(simCase.Regions, Region{
ID: IDAllocator.nextID(),
Expand All @@ -50,17 +57,14 @@ func newBalanceLeader() *Case {
})
}

threshold := 0.05
simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool {
res := true
leaderCounts := make([]int, 0, storeNum)
for i := 1; i <= storeNum; i++ {
for i := 1; i <= totalStore; i++ {
leaderCount := regions.GetStoreLeaderCount(uint64(i))
leaderCounts = append(leaderCounts, leaderCount)
res = res && isUniform(leaderCount, regionNum/3, threshold)
if !isUniform(leaderCount, totalRegion/totalStore) {
return false
}
}
simutil.Logger.Info("current counts", zap.Ints("leader", leaderCounts))
return res
return true
}
return &simCase
}
Loading

0 comments on commit 52389b0

Please sign in to comment.