Skip to content

Commit

Permalink
core/region: use a separate tree to check for overlaps in the subtree (…
Browse files Browse the repository at this point in the history 
…#8185)

ref #7897, close #8136

Previously, removing overlapped regions in subtrees required finding overlaps for each subtree individually.
This PR introduces a separate tree specifically for checking overlaps in the subtrees, thereby reducing lookup times and improving performance.

Signed-off-by: JmPotato <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
Co-authored-by: Hu# <[email protected]>
  • Loading branch information
@JmPotato @ti-chi-bot @HuSharp
3 people authored May 17, 2024
1 parent 4477285 commit 36a1fa2
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 138 deletions.
170 changes: 56 additions & 114 deletions pkg/core/region.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,8 @@ type RegionsInfo struct {
learners map[uint64]*regionTree // storeID -> sub regionTree
witnesses map[uint64]*regionTree // storeID -> sub regionTree
pendingPeers map[uint64]*regionTree // storeID -> sub regionTree
// This tree is used to check the overlaps among all the subtrees.
overlapTree *regionTree
}

// NewRegionsInfo creates RegionsInfo with tree, regions, leaders and followers
Expand All @@ -927,6 +929,7 @@ func NewRegionsInfo() *RegionsInfo {
learners: make(map[uint64]*regionTree),
witnesses: make(map[uint64]*regionTree),
pendingPeers: make(map[uint64]*regionTree),
overlapTree: newRegionTree(),
}
}

Expand Down Expand Up @@ -1066,48 +1069,68 @@ func (r *RegionsInfo) UpdateSubTreeOrderInsensitive(region *RegionInfo) {
origin = originItem.RegionInfo
}
rangeChanged := true

if origin != nil {
rangeChanged = !origin.rangeEqualsTo(region)
if r.preUpdateSubTreeLocked(rangeChanged, !origin.peersEqualTo(region), true, origin, region) {
return
}
}
r.updateSubTreeLocked(rangeChanged, nil, region)
}

func (r *RegionsInfo) preUpdateSubTreeLocked(
rangeChanged, peerChanged, orderInsensitive bool,
origin, region *RegionInfo,
) (done bool) {
if orderInsensitive {
re := region.GetRegionEpoch()
oe := origin.GetRegionEpoch()
isTermBehind := region.GetTerm() > 0 && region.GetTerm() < origin.GetTerm()
if (isTermBehind || re.GetVersion() < oe.GetVersion() || re.GetConfVer() < oe.GetConfVer()) && !region.isRegionRecreated() {
// Region meta is stale, skip.
return
return true
}
rangeChanged = !origin.rangeEqualsTo(region)

if rangeChanged || !origin.peersEqualTo(region) {
// If the range or peers have changed, the sub regionTree needs to be cleaned up.
// TODO: Improve performance by deleting only the different peers.
r.removeRegionFromSubTreeLocked(origin)
} else {
// The region tree and the subtree update is not atomic and the region tree is updated first.
// If there are two thread needs to update region tree,
// t1: thread-A update region tree
// t2: thread-B: update region tree again
// t3: thread-B: update subtree
// t4: thread-A: update region subtree
// to keep region tree consistent with subtree, we need to drop this update.
if tree, ok := r.subRegions[region.GetID()]; ok {
r.updateSubTreeStat(origin, region)
tree.RegionInfo = region
}
return
}
if rangeChanged || peerChanged {
// If the range or peers have changed, clean up the subtrees before updating them.
// TODO: improve performance by deleting only the different peers.
r.removeRegionFromSubTreeLocked(origin)
} else {
// The region tree and the subtree update is not atomic and the region tree is updated first.
// If there are two thread needs to update region tree,
// t1: thread-A update region tree
// t2: thread-B: update region tree again
// t3: thread-B: update subtree
// t4: thread-A: update region subtree
// to keep region tree consistent with subtree, we need to drop this update.
if tree, ok := r.subRegions[region.GetID()]; ok {
r.updateSubTreeStat(origin, region)
tree.RegionInfo = region
}
return true
}
return false
}

func (r *RegionsInfo) updateSubTreeLocked(rangeChanged bool, overlaps []*RegionInfo, region *RegionInfo) {
if rangeChanged {
overlaps := r.getOverlapRegionFromSubTreeLocked(region)
for _, re := range overlaps {
r.removeRegionFromSubTreeLocked(re)
// TODO: only perform the remove operation on the overlapped peer.
if len(overlaps) == 0 {
// If the range has changed but the overlapped regions are not provided, collect them by `[]*regionItem`.
for _, item := range r.getOverlapRegionFromOverlapTreeLocked(region) {
r.removeRegionFromSubTreeLocked(item.RegionInfo)
}
} else {
// Remove all provided overlapped regions from the subtrees.
for _, overlap := range overlaps {
r.removeRegionFromSubTreeLocked(overlap)
}
}
}

// Reinsert the region into all subtrees.
item := &regionItem{region}
r.subRegions[region.GetID()] = item
// It has been removed and all information needs to be updated again.
// Set peers then.
r.overlapTree.update(item, false)
setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem, countRef bool) {
store, ok := peersMap[storeID]
if !ok {
Expand Down Expand Up @@ -1147,29 +1170,8 @@ func (r *RegionsInfo) UpdateSubTreeOrderInsensitive(region *RegionInfo) {
setPeers(r.pendingPeers, region.GetPendingPeers())
}

func (r *RegionsInfo) getOverlapRegionFromSubTreeLocked(region *RegionInfo) []*RegionInfo {
it := &regionItem{RegionInfo: region}
overlaps := make([]*RegionInfo, 0)
overlapsMap := make(map[uint64]struct{})
collectFromItemSlice := func(peersMap map[uint64]*regionTree, storeID uint64) {
if tree, ok := peersMap[storeID]; ok {
items := tree.overlaps(it)
for _, item := range items {
if _, ok := overlapsMap[item.GetID()]; !ok {
overlapsMap[item.GetID()] = struct{}{}
overlaps = append(overlaps, item.RegionInfo)
}
}
}
}
for _, peer := range region.GetMeta().GetPeers() {
storeID := peer.GetStoreId()
collectFromItemSlice(r.leaders, storeID)
collectFromItemSlice(r.followers, storeID)
collectFromItemSlice(r.learners, storeID)
collectFromItemSlice(r.witnesses, storeID)
}
return overlaps
func (r *RegionsInfo) getOverlapRegionFromOverlapTreeLocked(region *RegionInfo) []*regionItem {
return r.overlapTree.overlaps(&regionItem{RegionInfo: region})
}

// GetRelevantRegions returns the relevant regions for a given region.
Expand Down Expand Up @@ -1275,72 +1277,11 @@ func (r *RegionsInfo) UpdateSubTree(region, origin *RegionInfo, overlaps []*Regi
r.st.Lock()
defer r.st.Unlock()
if origin != nil {
if rangeChanged || !origin.peersEqualTo(region) {
// If the range or peers have changed, the sub regionTree needs to be cleaned up.
// TODO: Improve performance by deleting only the different peers.
r.removeRegionFromSubTreeLocked(origin)
} else {
// The region tree and the subtree update is not atomic and the region tree is updated first.
// If there are two thread needs to update region tree,
// t1: thread-A update region tree
// t2: thread-B: update region tree again
// t3: thread-B: update subtree
// t4: thread-A: update region subtree
// to keep region tree consistent with subtree, we need to drop this update.
if tree, ok := r.subRegions[region.GetID()]; ok {
r.updateSubTreeStat(origin, region)
tree.RegionInfo = region
}
if r.preUpdateSubTreeLocked(rangeChanged, !origin.peersEqualTo(region), false, origin, region) {
return
}
}
if rangeChanged {
for _, re := range overlaps {
r.removeRegionFromSubTreeLocked(re)
}
}

item := &regionItem{region}
r.subRegions[region.GetID()] = item
// It has been removed and all information needs to be updated again.
// Set peers then.
setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem, countRef bool) {
store, ok := peersMap[storeID]
if !ok {
if !countRef {
store = newRegionTree()
} else {
store = newRegionTreeWithCountRef()
}
peersMap[storeID] = store
}
store.update(item, false)
}

// Add to leaders and followers.
for _, peer := range region.GetVoters() {
storeID := peer.GetStoreId()
if peer.GetId() == region.leader.GetId() {
// Add leader peer to leaders.
setPeer(r.leaders, storeID, item, true)
} else {
// Add follower peer to followers.
setPeer(r.followers, storeID, item, false)
}
}

setPeers := func(peersMap map[uint64]*regionTree, peers []*metapb.Peer) {
for _, peer := range peers {
storeID := peer.GetStoreId()
setPeer(peersMap, storeID, item, false)
}
}
// Add to learners.
setPeers(r.learners, region.GetLearners())
// Add to witnesses.
setPeers(r.witnesses, region.GetWitnesses())
// Add to PendingPeers
setPeers(r.pendingPeers, region.GetPendingPeers())
r.updateSubTreeLocked(rangeChanged, overlaps, region)
}

func (r *RegionsInfo) updateSubTreeStat(origin *RegionInfo, region *RegionInfo) {
Expand Down Expand Up @@ -1404,6 +1345,7 @@ func (r *RegionsInfo) ResetRegionCache() {
r.learners = make(map[uint64]*regionTree)
r.witnesses = make(map[uint64]*regionTree)
r.pendingPeers = make(map[uint64]*regionTree)
r.overlapTree = newRegionTree()
}

// RemoveRegionFromSubTree removes RegionInfo from regionSubTrees
Expand All @@ -1416,7 +1358,6 @@ func (r *RegionsInfo) RemoveRegionFromSubTree(region *RegionInfo) {

// removeRegionFromSubTreeLocked removes RegionInfo from regionSubTrees
func (r *RegionsInfo) removeRegionFromSubTreeLocked(region *RegionInfo) {
// Remove from leaders and followers.
for _, peer := range region.GetMeta().GetPeers() {
storeID := peer.GetStoreId()
r.leaders[storeID].remove(region)
Expand All @@ -1425,6 +1366,7 @@ func (r *RegionsInfo) removeRegionFromSubTreeLocked(region *RegionInfo) {
r.witnesses[storeID].remove(region)
r.pendingPeers[storeID].remove(region)
}
r.overlapTree.remove(region)
delete(r.subRegions, region.GetMeta().GetId())
}

Expand Down
87 changes: 69 additions & 18 deletions pkg/core/region_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -778,27 +778,24 @@ func BenchmarkRandomSetRegionWithGetRegionSizeByRangeParallel(b *testing.B) {
)
}

const keyLength = 100

func randomBytes(n int) []byte {
bytes := make([]byte, n)
_, err := rand.Read(bytes)
if err != nil {
panic(err)
}
return bytes
}
const (
peerNum = 3
storeNum = 10
keyLength = 100
)

func newRegionInfoIDRandom(idAllocator id.Allocator) *RegionInfo {
var (
peers []*metapb.Peer
leader *metapb.Peer
)
storeNum := 10
for i := 0; i < 3; i++ {
// Randomly select a peer as the leader.
leaderIdx := mrand.Intn(peerNum)
for i := 0; i < peerNum; i++ {
id, _ := idAllocator.Alloc()
p := &metapb.Peer{Id: id, StoreId: uint64(i%storeNum + 1)}
if i == 0 {
// Randomly distribute the peers to different stores.
p := &metapb.Peer{Id: id, StoreId: uint64(mrand.Intn(storeNum) + 1)}
if i == leaderIdx {
leader = p
}
peers = append(peers, p)
Expand All @@ -817,20 +814,74 @@ func newRegionInfoIDRandom(idAllocator id.Allocator) *RegionInfo {
)
}

func randomBytes(n int) []byte {
bytes := make([]byte, n)
_, err := rand.Read(bytes)
if err != nil {
panic(err)
}
return bytes
}

func BenchmarkAddRegion(b *testing.B) {
regions := NewRegionsInfo()
idAllocator := mockid.NewIDAllocator()
var items []*RegionInfo
for i := 0; i < 10000000; i++ {
items = append(items, newRegionInfoIDRandom(idAllocator))
}
items := generateRegionItems(idAllocator, 10000000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
origin, overlaps, rangeChanged := regions.SetRegion(items[i])
regions.UpdateSubTree(items[i], origin, overlaps, rangeChanged)
}
}

func BenchmarkUpdateSubTreeOrderInsensitive(b *testing.B) {
idAllocator := mockid.NewIDAllocator()
for _, size := range []int{10, 100, 1000, 10000, 100000, 1000000, 10000000} {
regions := NewRegionsInfo()
items := generateRegionItems(idAllocator, size)
// Update the subtrees from an empty `*RegionsInfo`.
b.Run(fmt.Sprintf("from empty with size %d", size), func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
for idx := range items {
regions.UpdateSubTreeOrderInsensitive(items[idx])
}
}
})

// Update the subtrees from a non-empty `*RegionsInfo` with the same regions,
// which means the regions are completely non-overlapped.
b.Run(fmt.Sprintf("from non-overlapped regions with size %d", size), func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
for idx := range items {
regions.UpdateSubTreeOrderInsensitive(items[idx])
}
}
})

// Update the subtrees from a non-empty `*RegionsInfo` with different regions,
// which means the regions are most likely overlapped.
b.Run(fmt.Sprintf("from overlapped regions with size %d", size), func(b *testing.B) {
items = generateRegionItems(idAllocator, size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for idx := range items {
regions.UpdateSubTreeOrderInsensitive(items[idx])
}
}
})
}
}

func generateRegionItems(idAllocator *mockid.IDAllocator, size int) []*RegionInfo {
items := make([]*RegionInfo, size)
for i := 0; i < size; i++ {
items[i] = newRegionInfoIDRandom(idAllocator)
}
return items
}

func BenchmarkRegionFromHeartbeat(b *testing.B) {
peers := make([]*metapb.Peer, 0, 3)
for i := uint64(1); i <= 3; i++ {
Expand Down
Loading

0 comments on commit 36a1fa2

Please sign in to comment.