From 24a2a0e9327a6a10b591042de5b26572e109e012 Mon Sep 17 00:00:00 2001 From: Dong Beiqing <350758787@qq.com> Date: Fri, 19 Jan 2024 09:28:41 +0800 Subject: [PATCH 1/2] configurable qos resources for clustermanager and klusterlet deployments Signed-off-by: Dong Beiqing <350758787@qq.com> --- ...cluster-management.io_klusterlets.crd.yaml | 40 +++++++++++- ...cluster-management.io_klusterlets.crd.yaml | 55 +++++++++++++++- ...ter-management.io_clustermanagers.crd.yaml | 63 +++++++++++++++++++ operator/v1/types_clustermanager.go | 5 ++ operator/v1/types_klusterlet.go | 17 +++-- operator/v1/zz_generated.deepcopy.go | 12 +++- .../v1/zz_generated.swagger_doc_generated.go | 6 +- 7 files changed, 188 insertions(+), 10 deletions(-) diff --git a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml index 9ccbdae5d..071da4342 100644 --- a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -156,15 +156,53 @@ spec: description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified. type: string resourceRequirement: - description: ResourceRequirement specify QoS classes of klusterlet deployment + description: ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments. type: object properties: + resourceRequirements: + description: ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: type: string default: Default enum: - Default - BestEffort + - ResourceRequirement workConfiguration: description: WorkConfiguration contains the configuration of work type: object diff --git a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml index df8d8324d..ad6e6434b 100644 --- a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -244,14 +244,65 @@ spec: will be used if unspecified. type: string resourceRequirement: - description: ResourceRequirement specify QoS classes of klusterlet - deployment + description: ResourceRequirement specify QoS classes of deployments + managed by klusterlet. It applies to all the containers in the deployments. properties: + resourceRequirements: + description: ResourceRequirements defines resource requests and + limits when Type is ResourceQosClassResourceRequirement + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object type: default: Default enum: - Default - BestEffort + - ResourceRequirement type: string type: object workConfiguration: diff --git a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml index b57e95f5f..8cc24234b 100644 --- a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml +++ b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml @@ -259,6 +259,69 @@ spec: description: RegistrationImagePullSpec represents the desired image of registration controller/webhook installed on hub. type: string + resourceRequirement: + description: ResourceRequirement specify QoS classes of deployments + managed by clustermanager. It applies to all the containers in the + deployments. + properties: + resourceRequirements: + description: ResourceRequirements defines resource requests and + limits when Type is ResourceQosClassResourceRequirement + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: + default: Default + enum: + - Default + - BestEffort + - ResourceRequirement + type: string + type: object workConfiguration: description: WorkConfiguration contains the configuration of work properties: diff --git a/operator/v1/types_clustermanager.go b/operator/v1/types_clustermanager.go index ed97ffd4c..33b0804f9 100644 --- a/operator/v1/types_clustermanager.go +++ b/operator/v1/types_clustermanager.go @@ -70,6 +70,11 @@ type ClusterManagerSpec struct { // AddOnManagerConfiguration contains the configuration of addon manager // +optional AddOnManagerConfiguration *AddOnManagerConfiguration `json:"addOnManagerConfiguration,omitempty"` + + // ResourceRequirement specify QoS classes of deployments managed by clustermanager. + // It applies to all the containers in the deployments. + // +optional + ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } // NodePlacement describes node scheduling configuration for the pods. diff --git a/operator/v1/types_klusterlet.go b/operator/v1/types_klusterlet.go index 2ae4bef9b..28bf07c04 100644 --- a/operator/v1/types_klusterlet.go +++ b/operator/v1/types_klusterlet.go @@ -1,6 +1,9 @@ package v1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // +genclient // +genclient:nonNamespaced @@ -87,7 +90,8 @@ type KlusterletSpec struct { // +optional HubApiServerHostAlias *HubApiServerHostAlias `json:"hubApiServerHostAlias,omitempty"` - // ResourceRequirement specify QoS classes of klusterlet deployment + // ResourceRequirement specify QoS classes of deployments managed by klusterlet. + // It applies to all the containers in the deployments. // +optional ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } @@ -95,17 +99,22 @@ type KlusterletSpec struct { type ResourceQosClass string const ( - // Default use resource setting in the template file + // Default use resource setting in the template file (with requests but no limits in the resources) ResourceQosClassDefault ResourceQosClass = "Default" // If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort. ResourceQosClassBestEffort ResourceQosClass = "BestEffort" + // Configurable resource requirements with requests and limits + ResourceQosClassResourceRequirement ResourceQosClass = "ResourceRequirement" ) // ResourceRequirement allow user override the default pod QoS classes type ResourceRequirement struct { - // +kubebuilder:validation:Enum=Default;BestEffort + // +kubebuilder:validation:Enum=Default;BestEffort;ResourceRequirement // +kubebuilder:default:=Default Type ResourceQosClass `json:"type"` + // ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement + // +optional + ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"` } // ServerURL represents the apiserver url and ca bundle that is accessible externally diff --git a/operator/v1/zz_generated.deepcopy.go b/operator/v1/zz_generated.deepcopy.go index 0de27326b..aaddf5b36 100644 --- a/operator/v1/zz_generated.deepcopy.go +++ b/operator/v1/zz_generated.deepcopy.go @@ -134,6 +134,11 @@ func (in *ClusterManagerSpec) DeepCopyInto(out *ClusterManagerSpec) { *out = new(AddOnManagerConfiguration) (*in).DeepCopyInto(*out) } + if in.ResourceRequirement != nil { + in, out := &in.ResourceRequirement, &out.ResourceRequirement + *out = new(ResourceRequirement) + (*in).DeepCopyInto(*out) + } return } @@ -353,7 +358,7 @@ func (in *KlusterletSpec) DeepCopyInto(out *KlusterletSpec) { if in.ResourceRequirement != nil { in, out := &in.ResourceRequirement, &out.ResourceRequirement *out = new(ResourceRequirement) - **out = **in + (*in).DeepCopyInto(*out) } return } @@ -504,6 +509,11 @@ func (in *RelatedResourceMeta) DeepCopy() *RelatedResourceMeta { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceRequirement) DeepCopyInto(out *ResourceRequirement) { *out = *in + if in.ResourceRequirements != nil { + in, out := &in.ResourceRequirements, &out.ResourceRequirements + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } return } diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 330df7997..03dd6add1 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -60,6 +60,7 @@ var map_ClusterManagerSpec = map[string]string{ "registrationConfiguration": "RegistrationConfiguration contains the configuration of registration", "workConfiguration": "WorkConfiguration contains the configuration of work", "addOnManagerConfiguration": "AddOnManagerConfiguration contains the configuration of addon manager", + "resourceRequirement": "ResourceRequirement specify QoS classes of deployments managed by clustermanager. It applies to all the containers in the deployments.", } func (ClusterManagerSpec) SwaggerDoc() map[string]string { @@ -213,7 +214,7 @@ var map_KlusterletSpec = map[string]string{ "registrationConfiguration": "RegistrationConfiguration contains the configuration of registration", "workConfiguration": "WorkConfiguration contains the configuration of work", "hubApiServerHostAlias": "HubApiServerHostAlias contains the host alias for hub api server. registration-agent and work-agent will use it to communicate with hub api server.", - "resourceRequirement": "ResourceRequirement specify QoS classes of klusterlet deployment", + "resourceRequirement": "ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments.", } func (KlusterletSpec) SwaggerDoc() map[string]string { @@ -245,7 +246,8 @@ func (RegistrationConfiguration) SwaggerDoc() map[string]string { } var map_ResourceRequirement = map[string]string{ - "": "ResourceRequirement allow user override the default pod QoS classes", + "": "ResourceRequirement allow user override the default pod QoS classes", + "resourceRequirements": "ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement", } func (ResourceRequirement) SwaggerDoc() map[string]string { From 25e246a5a4bbe6dad8b81d3390deccb7781281ff Mon Sep 17 00:00:00 2001 From: Dong Beiqing <350758787@qq.com> Date: Fri, 19 Jan 2024 16:00:23 +0800 Subject: [PATCH 2/2] move ResourceRequirement to a separate file Signed-off-by: Dong Beiqing <350758787@qq.com> --- operator/v1/funcs_clustermanager.go | 5 ++++ operator/v1/funcs_klusterlet.go | 5 ++++ operator/v1/type_resourcerequirement.go | 28 +++++++++++++++++++ operator/v1/types_klusterlet.go | 22 --------------- .../v1/zz_generated.swagger_doc_generated.go | 9 ------ 5 files changed, 38 insertions(+), 31 deletions(-) create mode 100644 operator/v1/funcs_clustermanager.go create mode 100644 operator/v1/funcs_klusterlet.go create mode 100644 operator/v1/type_resourcerequirement.go diff --git a/operator/v1/funcs_clustermanager.go b/operator/v1/funcs_clustermanager.go new file mode 100644 index 000000000..8ed923b8c --- /dev/null +++ b/operator/v1/funcs_clustermanager.go @@ -0,0 +1,5 @@ +package v1 + +func (cm *ClusterManager) GetResourceRequirement() *ResourceRequirement { + return cm.Spec.ResourceRequirement +} diff --git a/operator/v1/funcs_klusterlet.go b/operator/v1/funcs_klusterlet.go new file mode 100644 index 000000000..3bd56472f --- /dev/null +++ b/operator/v1/funcs_klusterlet.go @@ -0,0 +1,5 @@ +package v1 + +func (k *Klusterlet) GetResourceRequirement() *ResourceRequirement { + return k.Spec.ResourceRequirement +} diff --git a/operator/v1/type_resourcerequirement.go b/operator/v1/type_resourcerequirement.go new file mode 100644 index 000000000..e2f9ba6b3 --- /dev/null +++ b/operator/v1/type_resourcerequirement.go @@ -0,0 +1,28 @@ +package v1 + +import corev1 "k8s.io/api/core/v1" + +type ResourceRequirementAcquirer interface { + GetResourceRequirement() *ResourceRequirement +} + +// ResourceRequirement allow user override the default pod QoS classes +type ResourceRequirement struct { + // +kubebuilder:validation:Enum=Default;BestEffort;ResourceRequirement + // +kubebuilder:default:=Default + Type ResourceQosClass `json:"type"` + // ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement + // +optional + ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"` +} + +type ResourceQosClass string + +const ( + // Default use resource setting in the template file (with requests but no limits in the resources) + ResourceQosClassDefault ResourceQosClass = "Default" + // If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort. + ResourceQosClassBestEffort ResourceQosClass = "BestEffort" + // Configurable resource requirements with requests and limits + ResourceQosClassResourceRequirement ResourceQosClass = "ResourceRequirement" +) diff --git a/operator/v1/types_klusterlet.go b/operator/v1/types_klusterlet.go index 28bf07c04..bef65984f 100644 --- a/operator/v1/types_klusterlet.go +++ b/operator/v1/types_klusterlet.go @@ -1,7 +1,6 @@ package v1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -96,27 +95,6 @@ type KlusterletSpec struct { ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } -type ResourceQosClass string - -const ( - // Default use resource setting in the template file (with requests but no limits in the resources) - ResourceQosClassDefault ResourceQosClass = "Default" - // If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort. - ResourceQosClassBestEffort ResourceQosClass = "BestEffort" - // Configurable resource requirements with requests and limits - ResourceQosClassResourceRequirement ResourceQosClass = "ResourceRequirement" -) - -// ResourceRequirement allow user override the default pod QoS classes -type ResourceRequirement struct { - // +kubebuilder:validation:Enum=Default;BestEffort;ResourceRequirement - // +kubebuilder:default:=Default - Type ResourceQosClass `json:"type"` - // ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement - // +optional - ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"` -} - // ServerURL represents the apiserver url and ca bundle that is accessible externally type ServerURL struct { // URL is the url of apiserver endpoint of the managed cluster. diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 03dd6add1..8d8a14f20 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -245,15 +245,6 @@ func (RegistrationConfiguration) SwaggerDoc() map[string]string { return map_RegistrationConfiguration } -var map_ResourceRequirement = map[string]string{ - "": "ResourceRequirement allow user override the default pod QoS classes", - "resourceRequirements": "ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement", -} - -func (ResourceRequirement) SwaggerDoc() map[string]string { - return map_ResourceRequirement -} - var map_ServerURL = map[string]string{ "": "ServerURL represents the apiserver url and ca bundle that is accessible externally", "url": "URL is the url of apiserver endpoint of the managed cluster.",