From ed1e5c63f6b306cdc32d3904348e4c960991e43c Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 11 Oct 2023 20:12:40 +0800 Subject: [PATCH] refine delete resource group logic to avoid corner case of tiflash Signed-off-by: guo-shaoge --- pkg/mcs/resourcemanager/server/manager.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go index 1731faf8af1..0b0b193d5f6 100644 --- a/pkg/mcs/resourcemanager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -268,12 +268,25 @@ func (m *Manager) DeleteResourceGroup(name string) error { if name == reservedDefaultGroupName { return errs.ErrDeleteReservedGroup } - if err := m.storage.DeleteResourceGroupSetting(name); err != nil { - return err - } + + // First delete meta info from memory, then storage. + // This is to avoid an corner case of tiflash. + // 1. tiflash received etcd watch to delete resource group. + // 2. queries of that deleted resource group come, and tiflash ask PD to check, + // PD may not have deleted the meta info of the group in memory yet. So tiflash will setup the meta info of the group. + // And the meta info of the deleted group will stay in tiflash forever. m.Lock() + group, ok := m.groups[name] + if !ok { + return errors.Errorf("resource group %s not found", name) + } delete(m.groups, name) m.Unlock() + + if err := m.storage.DeleteResourceGroupSetting(name); err != nil { + m.groups[name] = group + return err + } return nil }