support e2e test

Signed-off-by: husharp <[email protected]>

examples/basic/pd-micro-service-cluster.yaml

@@ -17,8 +17,8 @@ spec:
     image: alpine:3.16.0
   pd:
     # TODO: use a stable version
-    baseImage: pingcap/pd
-    version: nightly
+    baseImage: hub-new.pingcap.net/orchestration/pd
+    version: v8.0.0
     maxFailoverCount: 0
     replicas: 1
     # if storageClassName is not set, the default Storage Class of the Kubernetes cluster will be used
@@ -29,16 +29,18 @@ spec:
     mode: "ms"
   pdms:
     - name: "tso"
-      baseImage: pingcap/pd
-      version: nightly
+      # TODO: use a stable version
+      baseImage: hub-new.pingcap.net/orchestration/pd
+      version: v8.0.0
       replicas: 2
     - name: "scheduling"
-      baseImage: pingcap/pd
-      version: nightly
+      # TODO: use a stable version
+      baseImage: hub-new.pingcap.net/orchestration/pd
+      version: v8.0.0
       replicas: 1
   tikv:
     baseImage: pingcap/tikv
-    version: v7.4.0
+    version: v7.5.0
     maxFailoverCount: 0
     # If only 1 TiKV is deployed, the TiKV region leader
     # cannot be transferred during upgrade, so we have
@@ -61,7 +63,7 @@ spec:
         max-open-files: 256
   tidb:
     baseImage: pingcap/tidb
-    version: v7.4.0
+    version: v7.5.0
     maxFailoverCount: 0
     replicas: 1
     service:

pkg/controller/tidbcluster/tidb_cluster_control.go

@@ -14,21 +14,20 @@
 package tidbcluster
 
 import (
-	"github.com/pingcap/tidb-operator/pkg/features"
-	v1 "k8s.io/api/core/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	errorutils "k8s.io/apimachinery/pkg/util/errors"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/klog/v2"
-
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1/defaulting"
 	v1alpha1validation "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1/validation"
 	"github.com/pingcap/tidb-operator/pkg/controller"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/manager"
 	"github.com/pingcap/tidb-operator/pkg/manager/member"
 	"github.com/pingcap/tidb-operator/pkg/manager/volumes"
 	"github.com/pingcap/tidb-operator/pkg/metrics"
+	v1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
 )
 
 // ControlInterface implements the control logic for updating TidbClusters and their children StatefulSets.
@@ -188,17 +187,6 @@ func (c *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster)
 		}
 	}
 
-	// works that should be done to make the pd microservice current state match the desired state:
-	//   - create or update the pdms service
-	//   - create or update the pdms headless service
-	//   - create the pdms statefulset
-	//   - sync pdms cluster status from pdms to TidbCluster object
-	//   - upgrade the pdms cluster
-	//   - scale out/in the pdms cluster
-	if err := c.pdMSMemberManager.Sync(tc); err != nil {
-		return err
-	}
-
 	// works that should be done to make the pd cluster current state match the desired state:
 	//   - create or update the pd service
 	//   - create or update the pd headless service
@@ -215,6 +203,17 @@ func (c *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster)
 		return err
 	}
 
+	// works that should be done to make the pd microservice current state match the desired state:
+	//   - create or update the pdms service
+	//   - create or update the pdms headless service
+	//   - create the pdms statefulset
+	//   - sync pdms cluster status from pdms to TidbCluster object
+	//   - upgrade the pdms cluster
+	//   - scale out/in the pdms cluster
+	if err := c.pdMSMemberManager.Sync(tc); err != nil {
+		return err
+	}
+
 	// works that should be done to make the tiproxy cluster current state match the desired state:
 	//   - create or update the tiproxy service
 	//   - create or update the tiproxy headless service

pkg/manager/member/pd_member_manager.go

@@ -80,8 +80,9 @@ func (m *pdMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
 		return nil
 	}
 
-	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS == nil {
-		return fmt.Errorf("tidbcluster: [%s/%s]'s enable micro service but pdMS spec is nil, please check `PDMS`", tc.GetNamespace(), tc.GetName())
+	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS == nil ||
+		tc.Spec.PDMS != nil && tc.Spec.PD.Mode != "ms" {
+		klog.Infof("tidbcluster: [%s/%s]'s enable micro service failed, please check `PD.Mode` and `PDMS`", tc.GetNamespace(), tc.GetName())
 	}
 
 	// skip sync if pd is suspended

pkg/manager/member/pd_ms_member_manager.go

@@ -59,37 +59,40 @@ func NewPDMSMemberManager(dependencies *controller.Dependencies, pdMSScaler Scal
 
 // Sync for all PD Micro Service components.
 func (m *pdMSMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
-	if tc.Spec.PDMS == nil {
-		return nil
-	}
-
 	// Need to start PD API
-	if tc.Spec.PD == nil {
+	if tc.Spec.PDMS != nil && tc.Spec.PD == nil {
 		klog.Infof("PD Micro Service is enabled, but PD is not enabled, skip syncing PD Micro Service")
 		return nil
 	}
-	if tc.Spec.PD.Mode != "ms" {
-		// remove all micro service components
-		for _, comp := range tc.Spec.PDMS {
+	// remove all micro service components if PDMS is not enabled
+	// PDMS need to be enabled when PD.Mode is ms && PDMS is not nil
+	if tc.Spec.PDMS == nil || tc.Spec.PD != nil && tc.Spec.PD.Mode != "ms" {
+		for _, comp := range tc.Status.PDMS {
 			ns := tc.GetNamespace()
 			tcName := tc.GetName()
 			curService := comp.Name
 
 			oldPDMSSetTmp, err := m.deps.StatefulSetLister.StatefulSets(ns).Get(controller.PDMSMemberName(tcName, curService))
-			if err != nil && !errors.IsNotFound(err) {
+			if err != nil {
+				if errors.IsNotFound(err) {
+					continue
+				}
 				return fmt.Errorf("syncPDMSStatefulSet: fail to get sts %s PDMS component %s for cluster [%s/%s], error: %s",
 					controller.PDMSMemberName(tcName, curService), curService, ns, tcName, err)
 			}
 
 			oldPDMSSet := oldPDMSSetTmp.DeepCopy()
 			newPDMSSet := oldPDMSSetTmp.DeepCopy()
+			if oldPDMSSet.Status.Replicas == 0 {
+				continue
+			}
+			tc.Status.PDMS[curService].Synced = true
 			*newPDMSSet.Spec.Replicas = 0
 			if err := m.scaler.Scale(tc, oldPDMSSet, newPDMSSet); err != nil {
 				return err
 			}
 			mngerutils.UpdateStatefulSetWithPrecheck(m.deps, tc, "FailedUpdatePDMSSTS", newPDMSSet, oldPDMSSet)
 		}
-		klog.Infof("PD Micro Service is enabled, but PD is not in `ms` mode, skip syncing PD Micro Service")
 		return nil
 	}
 

pkg/manager/member/startscript/v1/render_script.go

@@ -74,7 +74,7 @@ func RenderPDStartScript(tc *v1alpha1.TidbCluster) (string, error) {
 
 	pdStartSubScript := ``
 	mode := ""
-	if tc.Spec.PD.Mode == "ms" {
+	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS != nil {
 		mode = "api"
 	}
 	pdStartScriptTpl := template.Must(

pkg/manager/member/startscript/v2/pd_start_script.go

@@ -94,7 +94,7 @@ func RenderPDStartScript(tc *v1alpha1.TidbCluster) (string, error) {
 		tc.Spec.StartScriptV2FeatureFlags, v1alpha1.StartScriptV2FeatureFlagWaitForDnsNameIpMatch)
 
 	mode := ""
-	if tc.Spec.PD.Mode == "ms" {
+	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS != nil {
 		mode = "api"
 	}
 	pdStartScriptTpl := template.Must(

pkg/manager/member/tikv_member_manager.go

@@ -111,17 +111,14 @@ func (m *tikvMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
 	if tc.Spec.PD != nil && !tc.PDIsAvailable() {
 		return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName)
 	}
-	// Need to wait for PDMS
-	if tc.Spec.PD != nil && tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS == nil {
-		return controller.RequeueErrorf("TidbCluster: [%s/%s], please make sure pdms is not nil, "+
-			"then now waiting for PD's micro service running", ns, tcName)
-	}
 	// Check if all PD Micro Services are available
-	for _, pdms := range tc.Spec.PDMS {
-		_, err = controller.GetPDMSClient(m.deps.PDControl, tc, pdms.Name)
-		if err != nil {
-			return controller.RequeueErrorf("PDMS component %s for TidbCluster: [%s/%s], "+
-				"waiting for PD micro service cluster running, error: %v", pdms.Name, ns, tcName, err)
+	if tc.Spec.PDMS != nil && tc.Spec.PD != nil && tc.Spec.PD.Mode == "ms" {
+		for _, pdms := range tc.Spec.PDMS {
+			_, err = controller.GetPDMSClient(m.deps.PDControl, tc, pdms.Name)
+			if err != nil {
+				return controller.RequeueErrorf("PDMS component %s for TidbCluster: [%s/%s], "+
+					"waiting for PD micro service cluster running, error: %v", pdms.Name, ns, tcName, err)
+			}
 		}
 	}
 

tests/actions.go

@@ -684,18 +684,19 @@ func (oa *OperatorActions) memberCheckContextForTC(tc *v1alpha1.TidbCluster, com
 		services = []string{controller.PDMemberName(name), controller.PDPeerMemberName(name)}
 		checkComponent = oa.isPDMembersReady
 	case v1alpha1.PDMSTSOMemberType, v1alpha1.PDMSSchedulingMemberType:
-		curService := component.String()
 		skip = true
-		for _, service := range tc.Spec.PDMS {
-			log.Logf("check pdms service ready, curService: %s, service.Name: %s", curService, service.Name)
-			if curService == service.Name {
-				skip = false
-				break
+		if tc.Spec.PD != nil && tc.Spec.PD.Mode == "ms" {
+			curService := component.String()
+			for _, service := range tc.Spec.PDMS {
+				if curService == service.Name {
+					skip = false
+					break
+				}
 			}
+			expectedImage = tc.PDImage()
+			services = []string{controller.PDMSMemberName(name, curService), controller.PDMSPeerMemberName(name, curService)}
+			checkComponent = oa.isPDMSMembersReady
 		}
-		expectedImage = tc.PDImage()
-		services = []string{controller.PDMSMemberName(name, curService), controller.PDMSPeerMemberName(name, curService)}
-		checkComponent = oa.isPDMSMembersReady
 	case v1alpha1.TiDBMemberType:
 		skip = tc.Spec.TiDB == nil
 		expectedImage = tc.TiDBImage()
@@ -1350,47 +1351,6 @@ func (oa *OperatorActions) eventWorker() {
 // FIXME: this duplicates with WaitForTidbClusterReady in crd_test_utils.go, and all functions in it
 // TODO: sync with e2e doc
 func (oa *OperatorActions) WaitForTidbClusterReady(tc *v1alpha1.TidbCluster, timeout, pollInterval time.Duration) error {
-	if tc == nil {
-		return fmt.Errorf("tidbcluster is nil, cannot call WaitForTidbClusterReady")
-	}
-	var checkErr, err error
-	var local *v1alpha1.TidbCluster
-	tcID := fmt.Sprintf("%s/%s", tc.Namespace, tc.Name)
-	err = wait.PollImmediate(pollInterval, timeout, func() (bool, error) {
-		if local, err = oa.cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(context.TODO(), tc.Name, metav1.GetOptions{}); err != nil {
-			checkErr = fmt.Errorf("failed to get TidbCluster: %q, %v", tcID, err)
-			return false, nil
-		}
-
-		components := []v1alpha1.MemberType{
-			v1alpha1.PDMemberType,
-			v1alpha1.TiKVMemberType,
-			v1alpha1.TiDBMemberType,
-			v1alpha1.TiDBMemberType,
-			v1alpha1.TiFlashMemberType,
-			v1alpha1.PumpMemberType,
-			v1alpha1.TiCDCMemberType,
-		}
-
-		for _, component := range components {
-			if err := oa.IsMembersReady(local, component); err != nil {
-				checkErr = fmt.Errorf("%s members for tc %q are not ready: %v", component, tcID, err)
-				return false, nil
-			}
-		}
-
-		log.Logf("TidbCluster %q is ready", tcID)
-		return true, nil
-	})
-
-	if err == wait.ErrWaitTimeout {
-		err = checkErr
-	}
-
-	return err
-}
-
-func (oa *OperatorActions) WaitForPDMSClusterReady(tc *v1alpha1.TidbCluster, timeout, pollInterval time.Duration) error {
 	if tc == nil {
 		return fmt.Errorf("tidbcluster is nil, cannot call WaitForTidbClusterReady")
 	}
@@ -1417,7 +1377,6 @@ func (oa *OperatorActions) WaitForPDMSClusterReady(tc *v1alpha1.TidbCluster, tim
 
 		for _, component := range components {
 			if err := oa.IsMembersReady(local, component); err != nil {
-				log.Logf("%s members for tc %q are not ready: %v", component, tcID, err)
 				checkErr = fmt.Errorf("%s members for tc %q are not ready: %v", component, tcID, err)
 				return false, nil
 			}