[DNM]GC: Support keyspace level GC

cmd/tidb-server/main.go

@@ -279,7 +279,9 @@ func main() {
 		os.Exit(0)
 	}
 	registerStores()
-	err := metricsutil.RegisterMetrics()
+	err := keyspace.InitGlobalKeyspaceMeta()
+	terror.MustNil(err)
+	err = metricsutil.RegisterMetrics()
 	terror.MustNil(err)
 
 	if variable.EnableTmpStorageOnOOM.Load() {

pkg/domain/infosync/BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
         "//pkg/ddl/util",
         "//pkg/domain/resourcegroup",
         "//pkg/errno",
+        "//pkg/keyspace",
         "//pkg/kv",
         "//pkg/metrics",
         "//pkg/parser/model",

pkg/domain/infosync/info.go

@@ -39,6 +39,7 @@ import (
 	"github.com/pingcap/tidb/pkg/ddl/util"
 	"github.com/pingcap/tidb/pkg/domain/resourcegroup"
 	"github.com/pingcap/tidb/pkg/errno"
+	"github.com/pingcap/tidb/pkg/keyspace"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/metrics"
 	"github.com/pingcap/tidb/pkg/parser/model"
@@ -724,12 +725,21 @@ func (is *InfoSyncer) GetMinStartTS() uint64 {
 	return is.minStartTS
 }
 
+func (is *InfoSyncer) getMinStartTsEtcdCli() *clientv3.Client {
+	if keyspace.IsCurrentKeyspaceUseKeyspaceLevelGC() {
+		// If keyspace use keyspace leve gc, we should use etcd client with keyspace prefix to access min start timestamp of the current keyspace.
+		return is.etcdCli
+	}
+	return is.unprefixedEtcdCli
+}
+
 // storeMinStartTS stores self server min start timestamp to etcd.
 func (is *InfoSyncer) storeMinStartTS(ctx context.Context) error {
-	if is.unprefixedEtcdCli == nil {
+	minStartTsEtcdCli := is.getMinStartTsEtcdCli()
+	if minStartTsEtcdCli == nil {
 		return nil
 	}
-	return util.PutKVToEtcd(ctx, is.unprefixedEtcdCli, keyOpDefaultRetryCnt, is.minStartTSPath,
+	return util.PutKVToEtcd(ctx, minStartTsEtcdCli, keyOpDefaultRetryCnt, is.minStartTSPath,
 		strconv.FormatUint(is.minStartTS, 10),
 		clientv3.WithLease(is.session.Lease()))
 }

pkg/keyspace/BUILD.bazel

@@ -7,8 +7,14 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/config",
+        "//pkg/util",
+        "//pkg/util/codec",
+        "@com_github_pingcap_kvproto//pkg/keyspacepb",
         "@com_github_pingcap_kvproto//pkg/kvrpcpb",
+        "@com_github_pingcap_kvproto//pkg/pdpb",
+        "@com_github_tikv_client_go_v2//config",
         "@com_github_tikv_client_go_v2//tikv",
+        "@com_github_tikv_pd_client//:client",
         "@org_uber_go_zap//:zap",
         "@org_uber_go_zap//zapcore",
     ],
@@ -20,8 +26,10 @@ go_test(
     srcs = ["keyspace_test.go"],
     embed = [":keyspace"],
     flaky = True,
+    shard_count = 3,
     deps = [
         "//pkg/config",
+        "//pkg/util/codec",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/keyspace/keyspace.go

@@ -15,23 +15,50 @@
 package keyspace
 
 import (
+	"context"
+	"encoding/binary"
 	"fmt"
+	"strings"
+	"sync/atomic"
+	"time"
 
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
+	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/util"
+	"github.com/pingcap/tidb/pkg/util/codec"
+	tikvconfig "github.com/tikv/client-go/v2/config"
 	"github.com/tikv/client-go/v2/tikv"
+	pd "github.com/tikv/pd/client"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
 
 const (
 	// tidbKeyspaceEtcdPathPrefix is the keyspace prefix for etcd namespace
 	tidbKeyspaceEtcdPathPrefix = "/keyspaces/tidb/"
+
+	// KeyspaceMetaConfigGCManagementType is gc management type in keyspace meta config.
+	KeyspaceMetaConfigGCManagementType = "gc_management_type"
+	// KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC is a type of GC management in keyspace meta config, it means this keyspace will calculate GC safe point by its own.
+	KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC = "keyspace_level_gc"
+	// KeyspaceMetaConfigGCManagementTypeGlobalGC is a type of GC management in keyspace meta config, it means this keyspace will use GC safe point by global GC.
+	KeyspaceMetaConfigGCManagementTypeGlobalGC = "global_gc"
+
+	// maxKeyspaceID is the maximum keyspace id that can be created, no keyspace can be created greater than this value.
+	maxKeyspaceID = 0xffffff
+
+	// keyspaceTxnModePrefix is txn data prefix of keyspace.
+	keyspaceTxnModePrefix byte = 'x'
 )
 
 // CodecV1 represents api v1 codec.
 var CodecV1 = tikv.NewCodecV1(tikv.ModeTxn)
 
+// globalKeyspaceMeta is the keyspace meta of the current TiDB, if TiDB without set "keyspace-name" then globalKeyspaceMeta.Load() == nil.
+var globalKeyspaceMeta atomic.Pointer[keyspacepb.KeyspaceMeta]
+
 // MakeKeyspaceEtcdNamespace return the keyspace prefix path for etcd namespace
 func MakeKeyspaceEtcdNamespace(c tikv.Codec) string {
 	if c.GetAPIVersion() == kvrpcpb.APIVersion_V1 {
@@ -69,3 +96,135 @@ func WrapZapcoreWithKeyspace() zap.Option {
 		return core
 	})
 }
+
+// IsCurrentKeyspaceUseKeyspaceLevelGC return true if globalKeyspaceMeta not nil and globalKeyspaceMeta config has "gc_management_type" = "keyspace_level_gc".
+func IsCurrentKeyspaceUseKeyspaceLevelGC() bool {
+	return IsKeyspaceUseKeyspaceLevelGC(GetCurrentKeyspaceMeta())
+}
+
+// IsKeyspaceUseKeyspaceLevelGC return true if keyspace meta config has "gc_management_type" = "keyspace_level_gc".
+func IsKeyspaceUseKeyspaceLevelGC(keyspaceMeta *keyspacepb.KeyspaceMeta) bool {
+	if keyspaceMeta == nil {
+		return false
+	}
+	if val, ok := keyspaceMeta.Config[KeyspaceMetaConfigGCManagementType]; ok {
+		return val == KeyspaceMetaConfigGCManagementTypeKeyspaceLevelGC
+	}
+	return false
+}
+
+// IsCurrentKeyspaceUseGlobalGC return true if TiDB set 'keyspace-name' and use global gc.
+func IsCurrentKeyspaceUseGlobalGC() bool {
+	if GetCurrentKeyspaceMeta() == nil {
+		return true
+	}
+	if val, ok := GetCurrentKeyspaceMeta().Config[KeyspaceMetaConfigGCManagementType]; ok {
+		return val == KeyspaceMetaConfigGCManagementTypeGlobalGC
+	}
+	return true
+}
+
+// GetKeyspaceTxnLeftBound return the keyspace txn left boundary.
+func GetKeyspaceTxnLeftBound(keyspaceID uint32) []byte {
+	keyspaceIDBytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(keyspaceIDBytes, keyspaceID)
+
+	// The first byte is keyspaceTxnModePrefix, and the next three bytes are converted from uint32
+	txnLeftBound := codec.EncodeBytes(nil, append([]byte{keyspaceTxnModePrefix}, keyspaceIDBytes[1:]...))
+	return txnLeftBound
+}
+
+// GetKeyspaceTxnRange return the keyspace txn left boundary and txn right boundary.
+func GetKeyspaceTxnRange(keyspaceID uint32) ([]byte, []byte) {
+	// Get keyspace txn left boundary
+	txnLeftBound := GetKeyspaceTxnLeftBound(keyspaceID)
+
+	var txnRightBound []byte
+	if keyspaceID == maxKeyspaceID {
+		// Directly set the right boundary of maxKeyspaceID to be {keyspaceTxnModePrefix + 1, 0, 0, 0}
+		maxKeyspaceIDTxnRightBound := [4]byte{keyspaceTxnModePrefix + 1, 0, 0, 0}
+		txnRightBound = codec.EncodeBytes(nil, maxKeyspaceIDTxnRightBound[:])
+	} else {
+		// The right boundary of the specified keyspace is the left boundary of keyspaceID + 1.
+		txnRightBound = GetKeyspaceTxnLeftBound(keyspaceID + 1)
+	}
+
+	return txnLeftBound, txnRightBound
+}
+
+// InitGlobalKeyspaceMeta is used to get the keyspace meta from PD during TiDB startup and set global keyspace meta.
+func InitGlobalKeyspaceMeta() error {
+	cfg := config.GetGlobalConfig()
+	if IsKeyspaceNameEmpty(GetKeyspaceNameBySettings()) {
+		return nil
+	}
+	pdAddrs, _, _, err := tikvconfig.ParsePath("tikv://" + cfg.Path)
+	if err != nil {
+		return err
+	}
+	timeoutSec := time.Duration(cfg.PDClient.PDServerTimeout) * time.Second
+	pdCli, err := pd.NewClient(pdAddrs, pd.SecurityOption{
+		CAPath:   cfg.Security.ClusterSSLCA,
+		CertPath: cfg.Security.ClusterSSLCert,
+		KeyPath:  cfg.Security.ClusterSSLKey,
+	}, pd.WithCustomTimeoutOption(timeoutSec))
+	if err != nil {
+		return err
+	}
+	defer pdCli.Close()
+
+	keyspaceMeta, err := GetKeyspaceMeta(pdCli, cfg.KeyspaceName)
+
+	setCurrentKeyspaceMeta(keyspaceMeta)
+	return err
+}
+
+// GetKeyspaceMeta return keyspace meta of the given keyspace name.
+func GetKeyspaceMeta(pdCli pd.Client, keyspaceName string) (*keyspacepb.KeyspaceMeta, error) {
+	// Load Keyspace meta with retry.
+	var keyspaceMeta *keyspacepb.KeyspaceMeta
+	err := util.RunWithRetry(util.DefaultMaxRetries, util.RetryInterval, func() (bool, error) {
+		var errInner error
+		keyspaceMeta, errInner = pdCli.LoadKeyspace(context.TODO(), keyspaceName)
+		// Retry when pd not bootstrapped or if keyspace not exists.
+		if IsNotBootstrappedError(errInner) || IsKeyspaceNotExistError(errInner) {
+			return true, errInner
+		}
+		// Do not retry when success or encountered unexpected error.
+		return false, errInner
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return keyspaceMeta, nil
+}
+
+// IsNotBootstrappedError returns true if the error is pd not bootstrapped error.
+func IsNotBootstrappedError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), pdpb.ErrorType_NOT_BOOTSTRAPPED.String())
+}
+
+// IsKeyspaceNotExistError returns true the error is caused by keyspace not exists.
+func IsKeyspaceNotExistError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), pdpb.ErrorType_ENTRY_NOT_FOUND.String())
+}
+
+// GetCurrentKeyspaceMeta return global keyspace meta if TiDB set "keyspace-name"
+func GetCurrentKeyspaceMeta() *keyspacepb.KeyspaceMeta {
+	v := globalKeyspaceMeta.Load()
+	if v == nil {
+		return nil
+	}
+	return v
+}
+
+func setCurrentKeyspaceMeta(ks *keyspacepb.KeyspaceMeta) {
+	globalKeyspaceMeta.Store(ks)
+}

pkg/keyspace/keyspace_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/stretchr/testify/require"
 )
 
@@ -50,3 +51,17 @@ func TestNoKeyspaceNameSet(t *testing.T) {
 	require.Equal(t, "", getKeyspaceName)
 	require.Equal(t, true, IsKeyspaceNameEmpty(getKeyspaceName))
 }
+
+func TestGetKeyspaceRange(t *testing.T) {
+	leftBound, rightBound := GetKeyspaceTxnRange(1)
+	expectLeftBound := codec.EncodeBytes(nil, []byte{'x', 0, 0, 1})
+	expectRightBound := codec.EncodeBytes(nil, []byte{'x', 0, 0, 2})
+	require.Equal(t, expectLeftBound, leftBound)
+	require.Equal(t, expectRightBound, rightBound)
+
+	maxKeyspaceIDLeftBound, maxKeyspaceIDRightBound := GetKeyspaceTxnRange(maxKeyspaceID)
+	expectMaxKeyspaceIDLeftBound := codec.EncodeBytes(nil, []byte{'x', 0xff, 0xff, 0xff})
+	maxKeyspaceIDexpectRightBound := codec.EncodeBytes(nil, []byte{'y', 0, 0, 0})
+	require.Equal(t, expectMaxKeyspaceIDLeftBound, maxKeyspaceIDLeftBound)
+	require.Equal(t, maxKeyspaceIDexpectRightBound, maxKeyspaceIDRightBound)
+}

pkg/kv/interface_mock_test.go

@@ -195,6 +195,10 @@ func newMockTxn() Transaction {
 // mockStorage is used to start a must commit-failed txn.
 type mockStorage struct{}
 
+func (s *mockStorage) CurrentAllTSOKeyspaceGroupMinTs() (uint64, error) {
+	return 0, nil
+}
+
 func (s *mockStorage) GetCodec() tikv.Codec {
 	return nil
 }

pkg/kv/kv.go

@@ -694,6 +694,8 @@ type Storage interface {
 	UUID() string
 	// CurrentVersion returns current max committed version with the given txnScope (local or global).
 	CurrentVersion(txnScope string) (Version, error)
+	// CurrentAllTSOKeyspaceGroupMinTs returns current minimum ts from all TSO keyspace groups.
+	CurrentAllTSOKeyspaceGroupMinTs() (uint64, error)
 	// GetOracle gets a timestamp oracle client.
 	GetOracle() oracle.Oracle
 	// SupportDeleteRange gets the storage support delete range or not.

pkg/store/driver/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/pingcap/tidb/pkg/store/driver",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/keyspace",
         "//pkg/kv",
         "//pkg/metrics",
         "//pkg/sessionctx/variable",