From 9fa5b1529ec6d40ae5f2036084b08c0626b34239 Mon Sep 17 00:00:00 2001 From: Dong Beiqing <350758787@qq.com> Date: Thu, 18 Jan 2024 14:18:36 +0800 Subject: [PATCH] configurable qos resources for clustermanager and klusterlet deployments --- ...cluster-management.io_klusterlets.crd.yaml | 40 +++++++++++- ...cluster-management.io_klusterlets.crd.yaml | 55 +++++++++++++++- ...ter-management.io_clustermanagers.crd.yaml | 63 +++++++++++++++++++ operator/v1/types_clustermanager.go | 5 ++ operator/v1/types_klusterlet.go | 17 +++-- .../v1/zz_generated.swagger_doc_generated.go | 6 +- 6 files changed, 177 insertions(+), 9 deletions(-) diff --git a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml index 9ccbdae5d..7fc2f85cb 100644 --- a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -156,15 +156,53 @@ spec: description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified. type: string resourceRequirement: - description: ResourceRequirement specify QoS classes of klusterlet deployment + description: ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments. type: object properties: + resources: + description: Resources defines resource requests and limits when Type is ResourceQosClassConfigurable + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: type: string default: Default enum: - Default - BestEffort + - Configurable workConfiguration: description: WorkConfiguration contains the configuration of work type: object diff --git a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml index df8d8324d..f70d24693 100644 --- a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -244,14 +244,65 @@ spec: will be used if unspecified. type: string resourceRequirement: - description: ResourceRequirement specify QoS classes of klusterlet - deployment + description: ResourceRequirement specify QoS classes of deployments + managed by klusterlet. It applies to all the containers in the deployments. properties: + resources: + description: Resources defines resource requests and limits when + Type is ResourceQosClassConfigurable + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object type: default: Default enum: - Default - BestEffort + - Configurable type: string type: object workConfiguration: diff --git a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml index b57e95f5f..097183842 100644 --- a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml +++ b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml @@ -259,6 +259,69 @@ spec: description: RegistrationImagePullSpec represents the desired image of registration controller/webhook installed on hub. type: string + resourceRequirement: + description: ResourceRequirement specify QoS classes of deployments + managed by clustermanager. It applies to all the containers in the + deployments. + properties: + resources: + description: Resources defines resource requests and limits when + Type is ResourceQosClassConfigurable + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: + default: Default + enum: + - Default + - BestEffort + - Configurable + type: string + type: object workConfiguration: description: WorkConfiguration contains the configuration of work properties: diff --git a/operator/v1/types_clustermanager.go b/operator/v1/types_clustermanager.go index ed97ffd4c..33b0804f9 100644 --- a/operator/v1/types_clustermanager.go +++ b/operator/v1/types_clustermanager.go @@ -70,6 +70,11 @@ type ClusterManagerSpec struct { // AddOnManagerConfiguration contains the configuration of addon manager // +optional AddOnManagerConfiguration *AddOnManagerConfiguration `json:"addOnManagerConfiguration,omitempty"` + + // ResourceRequirement specify QoS classes of deployments managed by clustermanager. + // It applies to all the containers in the deployments. + // +optional + ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } // NodePlacement describes node scheduling configuration for the pods. diff --git a/operator/v1/types_klusterlet.go b/operator/v1/types_klusterlet.go index 2ae4bef9b..9311781a4 100644 --- a/operator/v1/types_klusterlet.go +++ b/operator/v1/types_klusterlet.go @@ -1,6 +1,9 @@ package v1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // +genclient // +genclient:nonNamespaced @@ -87,7 +90,8 @@ type KlusterletSpec struct { // +optional HubApiServerHostAlias *HubApiServerHostAlias `json:"hubApiServerHostAlias,omitempty"` - // ResourceRequirement specify QoS classes of klusterlet deployment + // ResourceRequirement specify QoS classes of deployments managed by klusterlet. + // It applies to all the containers in the deployments. // +optional ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } @@ -95,17 +99,22 @@ type KlusterletSpec struct { type ResourceQosClass string const ( - // Default use resource setting in the template file + // Default use resource setting in the template file (with requests but no limits in the resources) ResourceQosClassDefault ResourceQosClass = "Default" // If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort. ResourceQosClassBestEffort ResourceQosClass = "BestEffort" + // Configurable resources with requests and limits + ResourceQosClassConfigurable ResourceQosClass = "Configurable" ) // ResourceRequirement allow user override the default pod QoS classes type ResourceRequirement struct { - // +kubebuilder:validation:Enum=Default;BestEffort + // +kubebuilder:validation:Enum=Default;BestEffort;Configurable // +kubebuilder:default:=Default Type ResourceQosClass `json:"type"` + // Resources defines resource requests and limits when Type is ResourceQosClassConfigurable + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } // ServerURL represents the apiserver url and ca bundle that is accessible externally diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 330df7997..8adce498b 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -60,6 +60,7 @@ var map_ClusterManagerSpec = map[string]string{ "registrationConfiguration": "RegistrationConfiguration contains the configuration of registration", "workConfiguration": "WorkConfiguration contains the configuration of work", "addOnManagerConfiguration": "AddOnManagerConfiguration contains the configuration of addon manager", + "resourceRequirement": "ResourceRequirement specify QoS classes of deployments managed by clustermanager. It applies to all the containers in the deployments.", } func (ClusterManagerSpec) SwaggerDoc() map[string]string { @@ -213,7 +214,7 @@ var map_KlusterletSpec = map[string]string{ "registrationConfiguration": "RegistrationConfiguration contains the configuration of registration", "workConfiguration": "WorkConfiguration contains the configuration of work", "hubApiServerHostAlias": "HubApiServerHostAlias contains the host alias for hub api server. registration-agent and work-agent will use it to communicate with hub api server.", - "resourceRequirement": "ResourceRequirement specify QoS classes of klusterlet deployment", + "resourceRequirement": "ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments.", } func (KlusterletSpec) SwaggerDoc() map[string]string { @@ -245,7 +246,8 @@ func (RegistrationConfiguration) SwaggerDoc() map[string]string { } var map_ResourceRequirement = map[string]string{ - "": "ResourceRequirement allow user override the default pod QoS classes", + "": "ResourceRequirement allow user override the default pod QoS classes", + "resources": "Resources defines resource requests and limits when Type is ResourceQosClassConfigurable", } func (ResourceRequirement) SwaggerDoc() map[string]string {