From 011da18ca178030cd0013b098b76dda7e11377dc Mon Sep 17 00:00:00 2001 From: husharp Date: Wed, 31 Jan 2024 17:45:29 +0800 Subject: [PATCH] add pv Signed-off-by: husharp --- docs/api-references/docs.md | 46 ++++++++++++++- examples/basic/pd-micro-service-cluster.yaml | 2 - manifests/crd.yaml | 57 +++++++++++++++++++ .../crd/v1/pingcap.com_tidbclusters.yaml | 57 +++++++++++++++++++ pkg/apis/pingcap/v1alpha1/component_status.go | 6 +- .../pingcap/v1alpha1/openapi_generated.go | 27 ++++++++- pkg/apis/pingcap/v1alpha1/types.go | 15 ++++- .../pingcap/v1alpha1/validation/validation.go | 14 +++++ .../v1alpha1/validation/validation_test.go | 1 + .../pingcap/v1alpha1/zz_generated.deepcopy.go | 27 +++++++++ .../tidbcluster/tidb_cluster_controller.go | 2 +- pkg/manager/member/constants/constants.go | 3 - pkg/manager/member/pd_ms_member_manager.go | 57 +++++++++++++------ .../member/pd_ms_member_manager_test.go | 41 +++++++++++-- pkg/manager/member/pvc_resizer.go | 14 +++++ .../member/startscript/v1/render_script.go | 3 +- pkg/manager/member/startscript/v1/template.go | 1 - .../member/startscript/v2/pd_start_script.go | 3 - pkg/manager/volumes/vol_compare_utils.go | 10 +++- pkg/util/utils_test.go | 36 ++++++++++++ 20 files changed, 378 insertions(+), 44 deletions(-) diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index e92c42cf4a..b1545f2cd9 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -10827,7 +10827,7 @@ ServiceSpec (Optional) -

Service defines a Kubernetes service of PD cluster. +

Service defines a Kubernetes service of PD Micro Service cluster. Optional: Defaults to .spec.services in favor of backward compatibility

@@ -10855,7 +10855,7 @@ PDConfigWraper (Optional) -

Config is the Configuration of pd-servers

+

Config is the Configuration of pd Micro Service servers

@@ -10897,6 +10897,33 @@ string +storageClassName
+ +string + + + +(Optional) +

The storageClassName of the persistent volume for PD Micro Service log storage. +Defaults to Kubernetes default storage class.

+ + + + +storageVolumes
+ + +[]StorageVolume + + + + +(Optional) +

StorageVolumes configure additional storage for PD Micro Service pods.

+ + + + startTimeout
int @@ -10971,6 +10998,19 @@ Kubernetes apps/v1.StatefulSetStatus +volumes
+ + +map[github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.StorageVolumeName]*github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.StorageVolumeStatus + + + + +

Volumes contains the status of all volumes.

+ + + + members
[]string @@ -15604,6 +15644,7 @@ LocalStorageProvider

(Appears on: NGMonitoringSpec, +PDMSSpec, PDSpec, TiCDCSpec, TiDBSpec, @@ -15680,6 +15721,7 @@ string

(Appears on: MasterStatus, +PDMSStatus, PDStatus, PumpStatus, TiCDCStatus, diff --git a/examples/basic/pd-micro-service-cluster.yaml b/examples/basic/pd-micro-service-cluster.yaml index b48d724c04..44ea28a4ae 100644 --- a/examples/basic/pd-micro-service-cluster.yaml +++ b/examples/basic/pd-micro-service-cluster.yaml @@ -32,12 +32,10 @@ spec: baseImage: pingcap/pd version: nightly replicas: 2 - config: {} - name: "scheduling" baseImage: pingcap/pd version: nightly replicas: 1 - config: {} tikv: baseImage: pingcap/tikv version: v7.4.0 diff --git a/manifests/crd.yaml b/manifests/crd.yaml index b280a0bb3f..4923bdce0a 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -27379,6 +27379,24 @@ spec: type: string statefulSetUpdateStrategy: type: string + storageClassName: + type: string + storageVolumes: + items: + properties: + mountPath: + type: string + name: + type: string + storageClassName: + type: string + storageSize: + type: string + required: + - name + - storageSize + type: object + type: array suspendAction: properties: suspendStatefulSet: @@ -44079,6 +44097,45 @@ spec: type: object synced: type: boolean + volumes: + additionalProperties: + properties: + boundCount: + type: integer + currentCapacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + currentCount: + type: integer + currentStorageClass: + type: string + modifiedCapacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + modifiedCount: + type: integer + modifiedStorageClass: + type: string + name: + type: string + resizedCapacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resizedCount: + type: integer + required: + - name + type: object + type: object type: object type: object pump: diff --git a/manifests/crd/v1/pingcap.com_tidbclusters.yaml b/manifests/crd/v1/pingcap.com_tidbclusters.yaml index 03581c6933..48bdf43291 100644 --- a/manifests/crd/v1/pingcap.com_tidbclusters.yaml +++ b/manifests/crd/v1/pingcap.com_tidbclusters.yaml @@ -8380,6 +8380,24 @@ spec: type: string statefulSetUpdateStrategy: type: string + storageClassName: + type: string + storageVolumes: + items: + properties: + mountPath: + type: string + name: + type: string + storageClassName: + type: string + storageSize: + type: string + required: + - name + - storageSize + type: object + type: array suspendAction: properties: suspendStatefulSet: @@ -25080,6 +25098,45 @@ spec: type: object synced: type: boolean + volumes: + additionalProperties: + properties: + boundCount: + type: integer + currentCapacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + currentCount: + type: integer + currentStorageClass: + type: string + modifiedCapacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + modifiedCount: + type: integer + modifiedStorageClass: + type: string + name: + type: string + resizedCapacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resizedCount: + type: integer + required: + - name + type: object + type: object type: object type: object pump: diff --git a/pkg/apis/pingcap/v1alpha1/component_status.go b/pkg/apis/pingcap/v1alpha1/component_status.go index e7f45bc6fb..1d1cbc9492 100644 --- a/pkg/apis/pingcap/v1alpha1/component_status.go +++ b/pkg/apis/pingcap/v1alpha1/component_status.go @@ -195,7 +195,8 @@ func (s *PDMSStatus) GetSynced() bool { func (s *PDMSStatus) GetPhase() MemberPhase { return s.Phase } -func (s *PDMSStatus) GetVolumes() map[StorageVolumeName]*StorageVolumeStatus { return nil } +func (s *PDMSStatus) GetVolumes() map[StorageVolumeName]*StorageVolumeStatus { return s.Volumes } +func (s *PDMSStatus) SetVolumes(vols map[StorageVolumeName]*StorageVolumeStatus) { s.Volumes = vols } func (s *PDMSStatus) GetConditions() []metav1.Condition { return s.Conditions } @@ -228,8 +229,7 @@ func (s *PDMSStatus) SetPhase(phase MemberPhase) { func (s *PDMSStatus) SetStatefulSet(sts *appsv1.StatefulSetStatus) { s.StatefulSet = sts } -func (s *PDMSStatus) SetVolumes(vols map[StorageVolumeName]*StorageVolumeStatus) {} -func (s *PDMSStatus) SetVolReplaceInProgress(status bool) {} +func (s *PDMSStatus) SetVolReplaceInProgress(status bool) {} func (s *TiKVStatus) MemberType() MemberType { return TiKVMemberType diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 75f6625c6f..2a2c9146c5 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -5930,7 +5930,7 @@ func schema_pkg_apis_pingcap_v1alpha1_PDMSSpec(ref common.ReferenceCallback) com }, "service": { SchemaProps: spec.SchemaProps{ - Description: "Service defines a Kubernetes service of PD cluster. Optional: Defaults to `.spec.services` in favor of backward compatibility", + Description: "Service defines a Kubernetes service of PD Micro Service cluster. Optional: Defaults to `.spec.services` in favor of backward compatibility", Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.ServiceSpec"), }, }, @@ -5943,7 +5943,7 @@ func schema_pkg_apis_pingcap_v1alpha1_PDMSSpec(ref common.ReferenceCallback) com }, "config": { SchemaProps: spec.SchemaProps{ - Description: "Config is the Configuration of pd-servers", + Description: "Config is the Configuration of pd Micro Service servers", Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDConfigWraper"), }, }, @@ -5968,6 +5968,27 @@ func schema_pkg_apis_pingcap_v1alpha1_PDMSSpec(ref common.ReferenceCallback) com Format: "", }, }, + "storageClassName": { + SchemaProps: spec.SchemaProps{ + Description: "The storageClassName of the persistent volume for PD Micro Service log storage. Defaults to Kubernetes default storage class.", + Type: []string{"string"}, + Format: "", + }, + }, + "storageVolumes": { + SchemaProps: spec.SchemaProps{ + Description: "StorageVolumes configure additional storage for PD Micro Service pods.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.StorageVolume"), + }, + }, + }, + }, + }, "startTimeout": { SchemaProps: spec.SchemaProps{ Description: "Timeout threshold when pd get started", @@ -5980,7 +6001,7 @@ func schema_pkg_apis_pingcap_v1alpha1_PDMSSpec(ref common.ReferenceCallback) com }, }, Dependencies: []string{ - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDConfigWraper", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.Probe", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.ServiceSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.SuspendAction", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TopologySpreadConstraint", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Container", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PodDNSConfig", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.ResourceClaim", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume", "k8s.io/api/core/v1.VolumeMount", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDConfigWraper", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.Probe", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.ServiceSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.StorageVolume", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.SuspendAction", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TopologySpreadConstraint", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Container", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PodDNSConfig", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.ResourceClaim", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume", "k8s.io/api/core/v1.VolumeMount", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index da9fd8f0cc..28c900684b 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -587,7 +587,7 @@ type PDMSSpec struct { // +optional BaseImage *string `json:"baseImage"` - // Service defines a Kubernetes service of PD cluster. + // Service defines a Kubernetes service of PD Micro Service cluster. // Optional: Defaults to `.spec.services` in favor of backward compatibility // +optional Service *ServiceSpec `json:"service,omitempty"` @@ -598,7 +598,7 @@ type PDMSSpec struct { // +optional MaxFailoverCount *int32 `json:"maxFailoverCount,omitempty"` - // Config is the Configuration of pd-servers + // Config is the Configuration of pd Micro Service servers // +optional // +kubebuilder:validation:Schemaless // +kubebuilder:validation:XPreserveUnknownFields @@ -618,6 +618,15 @@ type PDMSSpec struct { // +kubebuilder:validation:Enum:="";"v1" StartUpScriptVersion string `json:"startUpScriptVersion,omitempty"` + // The storageClassName of the persistent volume for PD Micro Service log storage. + // Defaults to Kubernetes default storage class. + // +optional + StorageClassName *string `json:"storageClassName,omitempty"` + + // StorageVolumes configure additional storage for PD Micro Service pods. + // +optional + StorageVolumes []StorageVolume `json:"storageVolumes,omitempty"` + // Timeout threshold when pd get started // +kubebuilder:default=30 StartTimeout int `json:"startTimeout,omitempty"` @@ -1414,6 +1423,8 @@ type PDMSStatus struct { Synced bool `json:"synced"` Phase MemberPhase `json:"phase,omitempty"` StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + // Volumes contains the status of all volumes. + Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Members contains other service in current TidbCluster Members []string `json:"members,omitempty"` Image string `json:"image,omitempty"` diff --git a/pkg/apis/pingcap/v1alpha1/validation/validation.go b/pkg/apis/pingcap/v1alpha1/validation/validation.go index 5fb0496ce9..5428a70833 100644 --- a/pkg/apis/pingcap/v1alpha1/validation/validation.go +++ b/pkg/apis/pingcap/v1alpha1/validation/validation.go @@ -130,6 +130,11 @@ func validateTiDBClusterSpec(spec *v1alpha1.TidbClusterSpec, fldPath *field.Path if spec.PD != nil { allErrs = append(allErrs, validatePDSpec(spec.PD, fldPath.Child("pd"))...) } + if spec.PDMS != nil { + for _, comp := range spec.PDMS { + allErrs = append(allErrs, validatePDMSSpec(comp, fldPath.Child(comp.Name))...) + } + } if spec.TiKV != nil { allErrs = append(allErrs, validateTiKVSpec(spec.TiKV, fldPath.Child("tikv"))...) } @@ -175,6 +180,15 @@ func validatePDSpec(spec *v1alpha1.PDSpec, fldPath *field.Path) field.ErrorList return allErrs } +func validatePDMSSpec(spec *v1alpha1.PDMSSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...) + if len(spec.StorageVolumes) > 0 { + allErrs = append(allErrs, validateStorageVolumes(spec.StorageVolumes, fldPath.Child("storageVolumes"))...) + } + return allErrs +} + func validatePDAddresses(arrayOfAddresses []string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} for i, address := range arrayOfAddresses { diff --git a/pkg/apis/pingcap/v1alpha1/validation/validation_test.go b/pkg/apis/pingcap/v1alpha1/validation/validation_test.go index 5e676e081e..3f5ea81b32 100644 --- a/pkg/apis/pingcap/v1alpha1/validation/validation_test.go +++ b/pkg/apis/pingcap/v1alpha1/validation/validation_test.go @@ -541,6 +541,7 @@ func newTidbCluster() *v1alpha1.TidbCluster { PD: &v1alpha1.PDSpec{}, TiKV: &v1alpha1.TiKVSpec{}, TiDB: &v1alpha1.TiDBSpec{}, + PDMS: []*v1alpha1.PDMSSpec{}, }, } tc.Name = "test-validate-requests-storage" diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 745c2bdfa7..43ea190109 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -3400,6 +3400,18 @@ func (in *PDMSSpec) DeepCopyInto(out *PDMSSpec) { *out = new(bool) **out = **in } + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } + if in.StorageVolumes != nil { + in, out := &in.StorageVolumes, &out.StorageVolumes + *out = make([]StorageVolume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -3421,6 +3433,21 @@ func (in *PDMSStatus) DeepCopyInto(out *PDMSStatus) { *out = new(appsv1.StatefulSetStatus) (*in).DeepCopyInto(*out) } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make(map[StorageVolumeName]*StorageVolumeStatus, len(*in)) + for key, val := range *in { + var outVal *StorageVolumeStatus + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(StorageVolumeStatus) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } if in.Members != nil { in, out := &in.Members, &out.Members *out = make([]string, len(*in)) diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 4eae0606d6..3aacf58d10 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -56,7 +56,7 @@ func NewController(deps *controller.Dependencies) *Controller { control: NewDefaultTidbClusterControl( deps.TiDBClusterControl, mm.NewPDMemberManager(deps, mm.NewPDScaler(deps), mm.NewPDUpgrader(deps), mm.NewPDFailover(deps), suspender, podVolumeModifier), - mm.NewPDMSMemberManager(deps, mm.NewPDMSScaler(deps), mm.NewPDMSUpgrader(deps), suspender), + mm.NewPDMSMemberManager(deps, mm.NewPDMSScaler(deps), mm.NewPDMSUpgrader(deps), suspender, podVolumeModifier), mm.NewTiKVMemberManager(deps, mm.NewTiKVFailover(deps), mm.NewTiKVScaler(deps), mm.NewTiKVUpgrader(deps, podVolumeModifier), suspender, podVolumeModifier), mm.NewTiDBMemberManager(deps, mm.NewTiDBScaler(deps), mm.NewTiDBUpgrader(deps), mm.NewTiDBFailover(deps), suspender, podVolumeModifier), mm.NewTiProxyMemberManager(deps, mm.NewTiProxyScaler(deps), mm.NewTiProxyUpgrader(deps), suspender), diff --git a/pkg/manager/member/constants/constants.go b/pkg/manager/member/constants/constants.go index 33d419d99a..1df08d0d69 100644 --- a/pkg/manager/member/constants/constants.go +++ b/pkg/manager/member/constants/constants.go @@ -23,9 +23,6 @@ const ( // PDDataVolumeMountPath is the mount path for pd data volume PDDataVolumeMountPath = "/var/lib/pd" - // PDMSDataVolumeMountPath is the mount path for pd microservice data volume - PDMSDataVolumeMountPath = "/var/lib/pdms" - // TiCDCCertPath is the path for ticdc cert in container TiCDCCertPath = "/var/lib/ticdc-tls" ) diff --git a/pkg/manager/member/pd_ms_member_manager.go b/pkg/manager/member/pd_ms_member_manager.go index e2fadb7e12..128df10a4e 100644 --- a/pkg/manager/member/pd_ms_member_manager.go +++ b/pkg/manager/member/pd_ms_member_manager.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/manager/member/startscript" "github.com/pingcap/tidb-operator/pkg/manager/suspender" mngerutils "github.com/pingcap/tidb-operator/pkg/manager/utils" + "github.com/pingcap/tidb-operator/pkg/manager/volumes" "github.com/pingcap/tidb-operator/pkg/metrics" "github.com/pingcap/tidb-operator/pkg/util" apps "k8s.io/api/apps/v1" @@ -38,19 +39,21 @@ import ( ) type pdMSMemberManager struct { - deps *controller.Dependencies - scaler Scaler - upgrader Upgrader - suspender suspender.Suspender + deps *controller.Dependencies + scaler Scaler + upgrader Upgrader + suspender suspender.Suspender + podVolumeModifier volumes.PodVolumeModifier } // NewPDMSMemberManager returns a *pdMSMemberManager -func NewPDMSMemberManager(dependencies *controller.Dependencies, pdMSScaler Scaler, pdMSUpgrader Upgrader, spder suspender.Suspender) manager.Manager { +func NewPDMSMemberManager(dependencies *controller.Dependencies, pdMSScaler Scaler, pdMSUpgrader Upgrader, spder suspender.Suspender, pvm volumes.PodVolumeModifier) manager.Manager { return &pdMSMemberManager{ - deps: dependencies, - scaler: pdMSScaler, - upgrader: pdMSUpgrader, - suspender: spder, + deps: dependencies, + scaler: pdMSScaler, + upgrader: pdMSUpgrader, + suspender: spder, + podVolumeModifier: pvm, } } @@ -331,15 +334,18 @@ func (m *pdMSMemberManager) syncStatus(tc *v1alpha1.TidbCluster, sts *apps.State } tc.Status.PDMS[curService].Members = members tc.Status.PDMS[curService].Synced = true + + // sync volumes + err = volumes.SyncVolumeStatus(m.podVolumeModifier, m.deps.PodLister, tc, v1alpha1.PDMSMemberType(curService)) + if err != nil { + return fmt.Errorf("failed to sync volume status for pd: %v", err) + } + return nil } // syncPDMSConfigMap syncs the configmap of PDMS func (m *pdMSMemberManager) syncPDMSConfigMap(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, curSpec *v1alpha1.PDMSSpec) (*corev1.ConfigMap, error) { - // For backward compatibility, only sync tidb configmap when .PDMS.config is non-nil - if curSpec.Config == nil { - return nil, nil - } newCm, err := m.getPDMSConfigMap(tc, curSpec) if err != nil { return nil, err @@ -517,6 +523,16 @@ func (m *pdMSMemberManager) getNewPDMSStatefulSet(tc *v1alpha1.TidbCluster, cm * } } + // handle StorageVolumes and AdditionalVolumeMounts in ComponentSpec + var additionalPVCs []corev1.PersistentVolumeClaim + // default in nil + if curSpec.StorageVolumes != nil { + storageVolMounts, addPVCs := util.BuildStorageVolumeAndVolumeMount(curSpec.StorageVolumes, curSpec.StorageClassName, v1alpha1.PDMSMemberType(curService)) + volMounts = append(volMounts, storageVolMounts...) + volMounts = append(volMounts, curSpec.AdditionalVolumeMounts...) + additionalPVCs = addPVCs + } + sysctls := "sysctl -w" var initContainers []corev1.Container if basePDMSSpec.Annotations() != nil { @@ -679,14 +695,23 @@ func (m *pdMSMemberManager) getNewPDMSStatefulSet(tc *v1alpha1.TidbCluster, cm * UpdateStrategy: updateStrategy, }, } + // default in nil + if curSpec.StorageVolumes != nil { + storageRequest, err := controller.ParseStorageRequest(curSpec.Requests) + if err != nil { + return nil, fmt.Errorf("cannot parse storage request for PD, tidbcluster %s/%s, error: %v", tc.Namespace, tc.Name, err) + } + dataVolumeName := string(v1alpha1.GetStorageVolumeName("", v1alpha1.PDMSMemberType(curService))) + pdMSSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + util.VolumeClaimTemplate(storageRequest, dataVolumeName, tc.Spec.PD.StorageClassName), + } + pdMSSet.Spec.VolumeClaimTemplates = append(pdMSSet.Spec.VolumeClaimTemplates, additionalPVCs...) + } return pdMSSet, nil } func (m *pdMSMemberManager) getPDMSConfigMap(tc *v1alpha1.TidbCluster, curSpec *v1alpha1.PDMSSpec) (*corev1.ConfigMap, error) { - if curSpec.Config == nil { - return nil, nil - } config := curSpec.Config.DeepCopy() // use copy to not update tc spec curService := curSpec.Name diff --git a/pkg/manager/member/pd_ms_member_manager_test.go b/pkg/manager/member/pd_ms_member_manager_test.go index b8345a8229..cd34981261 100644 --- a/pkg/manager/member/pd_ms_member_manager_test.go +++ b/pkg/manager/member/pd_ms_member_manager_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/manager/suspender" + "github.com/pingcap/tidb-operator/pkg/manager/volumes" "github.com/pingcap/tidb-operator/pkg/pdapi" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -495,10 +496,11 @@ func newFakePDMSMemberManager() (*pdMSMemberManager, cache.Indexer, cache.Indexe podIndexer := fakeDeps.KubeInformerFactory.Core().V1().Pods().Informer().GetIndexer() pvcIndexer := fakeDeps.KubeInformerFactory.Core().V1().PersistentVolumeClaims().Informer().GetIndexer() pdMSManager := &pdMSMemberManager{ - deps: fakeDeps, - scaler: NewFakePDMSScaler(), - upgrader: NewFakePDMSUpgrader(), - suspender: suspender.NewFakeSuspender(), + deps: fakeDeps, + scaler: NewFakePDMSScaler(), + upgrader: NewFakePDMSUpgrader(), + suspender: suspender.NewFakeSuspender(), + podVolumeModifier: &volumes.FakePodVolumeModifier{}, } return pdMSManager, podIndexer, pvcIndexer } @@ -1093,7 +1095,36 @@ func TestGetPDMSConfigMap(t *testing.T) { PDMS: []*v1alpha1.PDMSSpec{{Name: "tso"}}, }, }, - expected: nil, + expected: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-pdms-tso", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "tidb-cluster", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/component": "pdms-tso", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbCluster", + Name: "foo", + UID: "", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Data: map[string]string{ + "startup-script": "", + "config-file": "", + }, + }, }, { name: "basic", diff --git a/pkg/manager/member/pvc_resizer.go b/pkg/manager/member/pvc_resizer.go index f1ff3f34d7..295de0a629 100644 --- a/pkg/manager/member/pvc_resizer.go +++ b/pkg/manager/member/pvc_resizer.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" "time" "github.com/pingcap/tidb-operator/pkg/apis/label" @@ -202,6 +203,19 @@ func (p *pvcResizer) buildContextForTC(tc *v1alpha1.TidbCluster, status v1alpha1 ctx.desiredVolumeQuantity[v1alpha1.GetStorageVolumeName("", v1alpha1.PDMemberType)] = quantity } storageVolumes = tc.Spec.PD.StorageVolumes + // PD Micro Service + case v1alpha1.PDMSTSOMemberType, v1alpha1.PDMSSchedulingMemberType: + pdmsRequirement := util.MustNewRequirement(label.ComponentLabelKey, selection.Equals, []string{label.PDMSLabel(comp.String())}) + ctx.selector = selector.Add(*pdmsRequirement) + for _, component := range tc.Spec.PDMS { + if strings.Contains(comp.String(), component.Name) { + if tc.Status.PDMS[comp.String()].Volumes == nil { + tc.Status.PDMS[comp.String()].Volumes = map[v1alpha1.StorageVolumeName]*v1alpha1.StorageVolumeStatus{} + } + storageVolumes = component.StorageVolumes + break + } + } case v1alpha1.TiDBMemberType: ctx.selector = selector.Add(*tidbRequirement) if tc.Status.TiDB.Volumes == nil { diff --git a/pkg/manager/member/startscript/v1/render_script.go b/pkg/manager/member/startscript/v1/render_script.go index d686c8473d..767bf7f7ae 100644 --- a/pkg/manager/member/startscript/v1/render_script.go +++ b/pkg/manager/member/startscript/v1/render_script.go @@ -101,8 +101,7 @@ func renderPDMSStartScript(tc *v1alpha1.TidbCluster, name string) (string, error AcrossK8s: tc.AcrossK8s(), ClusterDomain: tc.Spec.ClusterDomain, }, - Scheme: tc.Scheme(), - DataDir: constants.PDMSDataVolumeMountPath, + Scheme: tc.Scheme(), } model.PDAddress = fmt.Sprintf("%s://%s-pd:%d", tc.Scheme(), tc.GetName(), v1alpha1.DefaultPDClientPort) diff --git a/pkg/manager/member/startscript/v1/template.go b/pkg/manager/member/startscript/v1/template.go index 8e8728894f..b9dc14e4d7 100644 --- a/pkg/manager/member/startscript/v1/template.go +++ b/pkg/manager/member/startscript/v1/template.go @@ -317,7 +317,6 @@ type PdMSStartScriptModel struct { CommonModel Scheme string - DataDir string PDAddress string } diff --git a/pkg/manager/member/startscript/v2/pd_start_script.go b/pkg/manager/member/startscript/v2/pd_start_script.go index d05e91b3b9..f0e51202ce 100644 --- a/pkg/manager/member/startscript/v2/pd_start_script.go +++ b/pkg/manager/member/startscript/v2/pd_start_script.go @@ -45,8 +45,6 @@ type PDMSStartScriptModel struct { PDMSDomain string PDMSName string - DataDir string - ListenAddr string AdvertiseListenAddr string BackendEndpoints string @@ -127,7 +125,6 @@ func renderPDMSStartScript(tc *v1alpha1.TidbCluster, name string) (string, error tcNS := tc.Namespace peerServiceName := controller.PDMSMemberName(tcName, name) - m.DataDir = constants.PDMSDataVolumeMountPath m.PDMSDomain = fmt.Sprintf("${PDMS_POD_NAME}.%s.%s.svc", peerServiceName, tcNS) if tc.Spec.ClusterDomain != "" { m.PDMSDomain = m.PDMSDomain + "." + tc.Spec.ClusterDomain diff --git a/pkg/manager/volumes/vol_compare_utils.go b/pkg/manager/volumes/vol_compare_utils.go index 62f8ca1c52..d6e01d5fd9 100644 --- a/pkg/manager/volumes/vol_compare_utils.go +++ b/pkg/manager/volumes/vol_compare_utils.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "sort" + "strings" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -234,7 +235,14 @@ func (u *volCompareUtils) GetDesiredVolumes(tc *v1alpha1.TidbCluster, mt v1alpha desiredVolumes = append(desiredVolumes, d) storageVolumes = tc.Spec.PD.StorageVolumes - + case v1alpha1.PDMSSchedulingMemberType, v1alpha1.PDMSTSOMemberType: + for _, component := range tc.Spec.PDMS { + if strings.Contains(mt.String(), component.Name) { + defaultScName = component.StorageClassName + storageVolumes = component.StorageVolumes + break + } + } case v1alpha1.TiDBMemberType: defaultScName = tc.Spec.TiDB.StorageClassName storageVolumes = tc.Spec.TiDB.StorageVolumes diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 2a11bcdd1d..829d126e7c 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -651,6 +651,42 @@ func TestBuildAdditionalVolumeAndVolumeMount(t *testing.T) { })) }, }, + { + name: "pdms spec storageVolumes", + storageVolumes: []v1alpha1.StorageVolume{ + { + Name: "log", + StorageSize: "2Gi", + MountPath: "/var/lib/log", + }}, + memberType: v1alpha1.PDMSTSOMemberType, + testResult: func(volMounts []corev1.VolumeMount, volumeClaims []corev1.PersistentVolumeClaim) { + g := NewGomegaWithT(t) + q, _ := resource.ParseQuantity("2Gi") + g.Expect(volumeClaims).To(Equal([]corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: v1alpha1.PDMSTSOMemberType.String() + "-log", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: q, + }, + }, + }, + }, + })) + g.Expect(volMounts).To(Equal([]corev1.VolumeMount{ + { + Name: fmt.Sprintf("%s-%s", v1alpha1.PDMSTSOMemberType, "log"), MountPath: "/var/lib/log", + }, + })) + }, + }, { name: "tikv spec multiple storageVolumes", storageClassName: pointer.StringPtr("ns2"),