[DNM]GC: Support keyspace level GC

pkg/domain/infosync/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//pkg/ddl/placement",
         "//pkg/ddl/util",
         "//pkg/errno",
+        "//pkg/keyspace",
         "//pkg/kv",
         "//pkg/meta/model",
         "//pkg/metrics",

pkg/domain/infosync/info.go

@@ -38,6 +38,7 @@ import (
 	"github.com/pingcap/tidb/pkg/ddl/placement"
 	"github.com/pingcap/tidb/pkg/ddl/util"
 	"github.com/pingcap/tidb/pkg/errno"
+	"github.com/pingcap/tidb/pkg/keyspace"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	"github.com/pingcap/tidb/pkg/metrics"
@@ -126,6 +127,7 @@ type InfoSyncer struct {
 	scheduleManager       ScheduleManager
 	tiflashReplicaManager TiFlashReplicaManager
 	resourceManagerClient pd.ResourceManagerClient
+	tikvCodec             tikv.Codec
 }
 
 // ServerInfo is server static information.
@@ -224,6 +226,7 @@ func GlobalInfoSyncerInit(
 		info:              getServerInfo(id, serverIDGetter),
 		serverInfoPath:    fmt.Sprintf("%s/%s", ServerInformationPath, id),
 		minStartTSPath:    fmt.Sprintf("%s/%s", ServerMinStartTSPath, id),
+		tikvCodec:         codec,
 	}
 	err := is.init(ctx, skipRegisterToDashBoard)
 	if err != nil {
@@ -734,12 +737,23 @@ func (is *InfoSyncer) GetMinStartTS() uint64 {
 	return is.minStartTS
 }
 
+func (is *InfoSyncer) getMinStartTsEtcdCli() *clientv3.Client {
+	if keyspace.IsKeyspaceUseKeyspaceLevelGC(is.tikvCodec.GetKeyspaceMeta()) {
+		// If the current keyspace uses keyspace level GC,
+		// we should use the etcd namespace with the keyspace prefix
+		// to access the etcd key for min start timestamp of the current keyspace.
+		return is.etcdCli
+	}
+	return is.unprefixedEtcdCli
+}
+
 // storeMinStartTS stores self server min start timestamp to etcd.
 func (is *InfoSyncer) storeMinStartTS(ctx context.Context) error {
-	if is.unprefixedEtcdCli == nil {
+	minStartTsEtcdCli := is.getMinStartTsEtcdCli()
+	if minStartTsEtcdCli == nil {
 		return nil
 	}
-	return util.PutKVToEtcd(ctx, is.unprefixedEtcdCli, keyOpDefaultRetryCnt, is.minStartTSPath,
+	return util.PutKVToEtcd(ctx, minStartTsEtcdCli, keyOpDefaultRetryCnt, is.minStartTSPath,
 		strconv.FormatUint(is.minStartTS, 10),
 		clientv3.WithLease(is.session.Lease()))
 }

pkg/keyspace/BUILD.bazel

@@ -7,6 +7,8 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/config",
+        "//pkg/util/codec",
+        "@com_github_pingcap_kvproto//pkg/keyspacepb",
         "@com_github_pingcap_kvproto//pkg/kvrpcpb",
         "@com_github_tikv_client_go_v2//tikv",
         "@org_uber_go_zap//:zap",
@@ -20,8 +22,11 @@ go_test(
     srcs = ["keyspace_test.go"],
     embed = [":keyspace"],
     flaky = True,
+    shard_count = 4,
     deps = [
         "//pkg/config",
+        "//pkg/util/codec",
+        "@com_github_pingcap_kvproto//pkg/keyspacepb",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/keyspace/keyspace.go

@@ -15,10 +15,13 @@
 package keyspace
 
 import (
+	"encoding/binary"
 	"fmt"
 
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
 	"github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/tikv/client-go/v2/tikv"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
@@ -27,6 +30,30 @@ import (
 const (
 	// tidbKeyspaceEtcdPathPrefix is the keyspace prefix for etcd namespace
 	tidbKeyspaceEtcdPathPrefix = "/keyspaces/tidb/"
+
+	// KeyspaceMetaConfigGCManagementType is the name of the key for GC management type in keyspace meta config.
+	KeyspaceMetaConfigGCManagementType = "gc_management_type"
+	// KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC is a type of GC management in keyspace meta config,
+	// it means this keyspace will calculate GC safe point by its own.
+	KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC = "keyspace_level_gc"
+	// KeyspaceMetaConfigGCManagementTypeGlobalGC is a type of GC management in keyspace meta config, it means this keyspace will use GC safe point by global GC(default).
+	KeyspaceMetaConfigGCManagementTypeGlobalGC = "global_gc"
+
+	// maxKeyspaceID is the maximum keyspace id that can be created, no keyspace can be created greater than this value.
+	maxKeyspaceID = 0xffffff
+
+	// API V2 key bounds:
+	// See https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md for more keyspace and API V2 details.
+
+	// KeyspaceTxnModePrefix is txn data prefix of keyspace.
+	KeyspaceTxnModePrefix byte = 'x'
+	// MaxKeyspaceRightBoundaryPrefix is the max keyspace txn right boundary.
+	// Because the keyspace ranges are all under the 'x' prefix, so MaxKeyspaceRightBoundaryPrefix = 'y'
+	MaxKeyspaceRightBoundaryPrefix byte = 'y'
+	// RawKVLeftBoundaryPrefix is RawKV left boundary.
+	RawKVLeftBoundaryPrefix byte = 'r'
+	// RawKVRightBoundaryPrefix is RawKV right boundary.
+	RawKVRightBoundaryPrefix byte = 's'
 )
 
 // CodecV1 represents api v1 codec.
@@ -40,7 +67,7 @@ func MakeKeyspaceEtcdNamespace(c tikv.Codec) string {
 	return fmt.Sprintf(tidbKeyspaceEtcdPathPrefix+"%d", c.GetKeyspaceID())
 }
 
-// MakeKeyspaceEtcdNamespaceSlash return the keyspace prefix path for etcd namespace, and end with a slash.
+// MakeKeyspaceEtcdNamespaceSlash returns the keyspace prefix path for etcd namespace, and end with a slash.
 func MakeKeyspaceEtcdNamespaceSlash(c tikv.Codec) string {
 	if c.GetAPIVersion() == kvrpcpb.APIVersion_V1 {
 		return ""
@@ -69,3 +96,53 @@ func WrapZapcoreWithKeyspace() zap.Option {
 		return core
 	})
 }
+
+// IsKeyspaceUseKeyspaceLevelGC returns true if keyspace meta config set "gc_management_type" = "keyspace_level_gc" explicitly.
+func IsKeyspaceUseKeyspaceLevelGC(keyspaceMeta *keyspacepb.KeyspaceMeta) bool {
+	if keyspaceMeta == nil {
+		return false
+	}
+	if val, ok := keyspaceMeta.Config[KeyspaceMetaConfigGCManagementType]; ok {
+		return val == KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC
+	}
+	return false
+}
+
+// IsKeyspaceMetaNotNilAndUseGlobalGC returns whether the specified keyspace meta use global GC.
+func IsKeyspaceMetaNotNilAndUseGlobalGC(keyspaceMeta *keyspacepb.KeyspaceMeta) bool {
+	if keyspaceMeta == nil {
+		return false
+	}
+	if val, ok := keyspaceMeta.Config[KeyspaceMetaConfigGCManagementType]; ok {
+		return val == KeyspaceMetaConfigGCManagementTypeGlobalGC
+	}
+	return true
+}
+
+// GetKeyspaceTxnLeftBound returns the specified keyspace txn left boundary.
+func GetKeyspaceTxnLeftBound(keyspaceID uint32) []byte {
+	keyspaceIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(keyspaceIDBytes, keyspaceID)
+
+	// The first byte is keyspaceTxnModePrefix, and the next three bytes are converted from uint32
+	txnLeftBound := codec.EncodeBytes(nil, append([]byte{KeyspaceTxnModePrefix}, keyspaceIDBytes[1:]...))
+	return txnLeftBound
+}
+
+// GetKeyspaceTxnRange returns the specified keyspace txn left boundary and txn right boundary.
+func GetKeyspaceTxnRange(keyspaceID uint32) ([]byte, []byte) {
+	// Get keyspace txn left boundary
+	txnLeftBound := GetKeyspaceTxnLeftBound(keyspaceID)
+
+	var txnRightBound []byte
+	if keyspaceID == maxKeyspaceID {
+		// Directly set the right boundary of maxKeyspaceID to be {MaxKeyspaceRightBoundaryPrefix}
+		maxKeyspaceIDTxnRightBound := []byte{MaxKeyspaceRightBoundaryPrefix}
+		txnRightBound = codec.EncodeBytes(nil, maxKeyspaceIDTxnRightBound[:])
+	} else {
+		// The right boundary of the specified keyspace is the left boundary of keyspaceID + 1.
+		txnRightBound = GetKeyspaceTxnLeftBound(keyspaceID + 1)
+	}
+
+	return txnLeftBound, txnRightBound
+}

pkg/keyspace/keyspace_test.go

@@ -17,7 +17,9 @@ package keyspace
 import (
 	"testing"
 
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
 	"github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/stretchr/testify/require"
 )
 
@@ -50,3 +52,58 @@ func TestNoKeyspaceNameSet(t *testing.T) {
 	require.Equal(t, "", getKeyspaceName)
 	require.Equal(t, true, IsKeyspaceNameEmpty(getKeyspaceName))
 }
+
+func TestGetKeyspaceRange(t *testing.T) {
+	keyspaceID := uint32(1)
+	leftBound, rightBound := GetKeyspaceTxnRange(keyspaceID)
+	expectLeftBound := codec.EncodeBytes(nil, []byte{'x', 0, 0, 1})
+	expectRightBound := codec.EncodeBytes(nil, []byte{'x', 0, 0, 2})
+	require.Equal(t, expectLeftBound, leftBound)
+	require.Equal(t, expectRightBound, rightBound)
+
+	// Check the max keyspace ID txn left boundary and txn right boundary.
+	maxKeyspaceIDLeftBound, maxKeyspaceIDRightBound := GetKeyspaceTxnRange(maxKeyspaceID)
+	expectMaxKeyspaceIDLeftBound := codec.EncodeBytes(nil, []byte{'x', 0xff, 0xff, 0xff})
+	maxKeyspaceIDexpectRightBound := codec.EncodeBytes(nil, []byte{MaxKeyspaceRightBoundaryPrefix})
+	require.Equal(t, expectMaxKeyspaceIDLeftBound, maxKeyspaceIDLeftBound)
+	require.Equal(t, maxKeyspaceIDexpectRightBound, maxKeyspaceIDRightBound)
+}
+
+func TestGetKeyspaceGCManagementType(t *testing.T) {
+	// Case 1: The keyspace use global GC by default.
+	keyspaceMeta := keyspacepb.KeyspaceMeta{
+		Id: 1,
+	}
+
+	require.Equal(t, false, IsKeyspaceUseKeyspaceLevelGC(&keyspaceMeta))
+	require.Equal(t, true, IsKeyspaceMetaNotNilAndUseGlobalGC(&keyspaceMeta))
+
+	// Case 2: The keyspace is set to use global GC.
+	keyspaceConfig := map[string]string{
+		KeyspaceMetaConfigGCManagementType: KeyspaceMetaConfigGCManagementTypeGlobalGC,
+	}
+	keyspaceMeta = keyspacepb.KeyspaceMeta{
+		Id:     1,
+		Config: keyspaceConfig,
+	}
+
+	require.Equal(t, false, IsKeyspaceUseKeyspaceLevelGC(&keyspaceMeta))
+	require.Equal(t, true, IsKeyspaceMetaNotNilAndUseGlobalGC(&keyspaceMeta))
+
+	// Case 3: The keyspace is set to use keyspace level GC.
+	keyspaceConfig = map[string]string{
+		KeyspaceMetaConfigGCManagementType: KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC,
+	}
+	keyspaceMeta = keyspacepb.KeyspaceMeta{
+		Id:     1,
+		Config: keyspaceConfig,
+	}
+	require.Equal(t, true, IsKeyspaceUseKeyspaceLevelGC(&keyspaceMeta))
+	require.Equal(t, false, IsKeyspaceMetaNotNilAndUseGlobalGC(&keyspaceMeta))
+
+	// Case 4: The keyspace meta is nil, it doesn't use keyspace level GC.
+	require.Equal(t, false, IsKeyspaceUseKeyspaceLevelGC(nil))
+
+	// Case 5: The keyspace meta is nil, IsKeyspaceMetaNotNilAndUseGlobalGC will return false.
+	require.Equal(t, false, IsKeyspaceMetaNotNilAndUseGlobalGC(nil))
+}

pkg/kv/interface_mock_test.go

@@ -197,6 +197,10 @@ func newMockTxn() Transaction {
 // mockStorage is used to start a must commit-failed txn.
 type mockStorage struct{}
 
+func (s *mockStorage) CurrentAllTSOKeyspaceGroupMinTs() (uint64, error) {
+	return 0, nil
+}
+
 func (s *mockStorage) GetOption(k any) (any, bool) {
 	return nil, false
 }

pkg/kv/kv.go

@@ -703,6 +703,8 @@ type Storage interface {
 	UUID() string
 	// CurrentVersion returns current max committed version with the given txnScope (local or global).
 	CurrentVersion(txnScope string) (Version, error)
+	// CurrentAllTSOKeyspaceGroupMinTs returns current minimum ts from all TSO keyspace groups.
+	CurrentAllTSOKeyspaceGroupMinTs() (uint64, error)
 	// GetOracle gets a timestamp oracle client.
 	GetOracle() oracle.Oracle
 	// SupportDeleteRange gets the storage support delete range or not.

pkg/store/driver/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/pingcap/tidb/pkg/store/driver",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/keyspace",
         "//pkg/kv",
         "//pkg/metrics",
         "//pkg/sessionctx/variable",

pkg/store/driver/tikv_driver.go

@@ -27,6 +27,7 @@ import (
 	"github.com/pingcap/errors"
 	deadlockpb "github.com/pingcap/kvproto/pkg/deadlock"
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
+	"github.com/pingcap/tidb/pkg/keyspace"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/metrics"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
@@ -122,6 +123,16 @@ func (d *TiKVDriver) setDefaultAndOptions(options ...Option) {
 	}
 }
 
+// NewEtcdSafePointKVWithKeyspacePrefixIfNeeded is used to add etcd namespace with keyspace prefix,
+// if the current keyspace use keyspace level GC.
+func NewEtcdSafePointKVWithKeyspacePrefixIfNeeded(etcdAddrs []string, codec tikv.Codec, tlsConfig *tls.Config) (*tikv.EtcdSafePointKV, error) {
+	var etcdNameSpace string
+	if keyspace.IsKeyspaceUseKeyspaceLevelGC(codec.GetKeyspaceMeta()) {
+		etcdNameSpace = keyspace.MakeKeyspaceEtcdNamespace(codec)
+	}
+	return tikv.NewEtcdSafePointKV(etcdAddrs, tlsConfig, tikv.WithPrefix(etcdNameSpace))
+}
+
 // OpenWithOptions is used by other program that use tidb as a library, to avoid modifying GlobalConfig
 // unspecified options will be set to global config
 func (d TiKVDriver) OpenWithOptions(path string, options ...Option) (resStore kv.Storage, err error) {
@@ -187,11 +198,6 @@ func (d TiKVDriver) OpenWithOptions(path string, options ...Option) (resStore kv
 		return nil, errors.Trace(err)
 	}
 
-	spkv, err = tikv.NewEtcdSafePointKV(etcdAddrs, tlsConfig)
-	if err != nil {
-		return nil, errors.Trace(err)
-	}
-
 	// ---------------- keyspace logic  ----------------
 	var (
 		pdClient *tikv.CodecPDClient
@@ -215,6 +221,11 @@ func (d TiKVDriver) OpenWithOptions(path string, options ...Option) (resStore kv
 		tikv.WithCodec(codec),
 	)
 
+	spkv, err = NewEtcdSafePointKVWithKeyspacePrefixIfNeeded(etcdAddrs, codec, tlsConfig)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
 	s, err = tikv.NewKVStore(uuid, pdClient, spkv, &injectTraceClient{Client: rpcClient},
 		tikv.WithPDHTTPClient("tikv-driver", etcdAddrs, pdhttp.WithTLSConfig(tlsConfig), pdhttp.WithMetrics(metrics.PDAPIRequestCounter, metrics.PDAPIExecutionHistogram)))
 	if err != nil {
@@ -395,6 +406,12 @@ func (s *tikvStore) CurrentVersion(txnScope string) (kv.Version, error) {
 	return kv.NewVersion(ver), derr.ToTiDBErr(err)
 }
 
+// CurrentAllTSOKeyspaceGroupMinTs returns a minimum timestamp from all TSO keyspace groups.
+func (s *tikvStore) CurrentAllTSOKeyspaceGroupMinTs() (uint64, error) {
+	ts, err := s.KVStore.CurrentAllTSOKeyspaceGroupMinTs()
+	return ts, derr.ToTiDBErr(err)
+}
+
 // ShowStatus returns the specified status of the storage
 func (s *tikvStore) ShowStatus(ctx context.Context, key string) (any, error) {
 	return nil, kv.ErrNotImplemented

pkg/store/gcworker/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "//pkg/ddl/placement",
         "//pkg/ddl/util",
         "//pkg/domain/infosync",
+        "//pkg/keyspace",
         "//pkg/kv",
         "//pkg/meta/model",
         "//pkg/metrics",
@@ -27,6 +28,7 @@ go_library(
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_pingcap_kvproto//pkg/errorpb",
+        "@com_github_pingcap_kvproto//pkg/keyspacepb",
         "@com_github_pingcap_kvproto//pkg/kvrpcpb",
         "@com_github_pingcap_kvproto//pkg/metapb",
         "@com_github_tikv_client_go_v2//error",