✨ Configurable qos resources for deployments managed by clustermanage…

…r and klusterlet (#316) * configurable qos resources for clustermanager and klusterlet deployments Signed-off-by: Dong Beiqing <[email protected]> * move ResourceRequirement to a separate file Signed-off-by: Dong Beiqing <[email protected]> --------- Signed-off-by: Dong Beiqing <[email protected]>
open-cluster-management-io · Jan 22, 2024 · e7bd1bd · e7bd1bd
1 parent cb76224
commit e7bd1bd
Show file tree

Hide file tree

Showing 10 changed files with 216 additions and 31 deletions.
diff --git a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml
@@ -156,15 +156,53 @@ spec:
               description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified.
               type: string
             resourceRequirement:
-              description: ResourceRequirement specify QoS classes of klusterlet deployment
+              description: ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments.
               type: object
               properties:
+                resourceRequirements:
+                  description: ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement
+                  type: object
+                  properties:
+                    claims:
+                      description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
+                      type: array
+                      items:
+                        description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                        type: object
+                        required:
+                          - name
+                        properties:
+                          name:
+                            description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
+                            type: string
+                      x-kubernetes-list-map-keys:
+                        - name
+                      x-kubernetes-list-type: map
+                    limits:
+                      description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                      type: object
+                      additionalProperties:
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
+                    requests:
+                      description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                      type: object
+                      additionalProperties:
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
                 type:
                   type: string
                   default: Default
                   enum:
                     - Default
                     - BestEffort
+                    - ResourceRequirement
             workConfiguration:
               description: WorkConfiguration contains the configuration of work
               type: object

diff --git a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml
@@ -244,14 +244,65 @@ spec:
                   will be used if unspecified.
                 type: string
               resourceRequirement:
-                description: ResourceRequirement specify QoS classes of klusterlet
-                  deployment
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by klusterlet. It applies to all the containers in the deployments.
                 properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                   type:
                     default: Default
                     enum:
                     - Default
                     - BestEffort
+                    - ResourceRequirement
                     type: string
                 type: object
               workConfiguration:

diff --git a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml
@@ -259,6 +259,69 @@ spec:
                 description: RegistrationImagePullSpec represents the desired image
                   of registration controller/webhook installed on hub.
                 type: string
+              resourceRequirement:
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by clustermanager. It applies to all the containers in the
+                  deployments.
+                properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
+                  type:
+                    default: Default
+                    enum:
+                    - Default
+                    - BestEffort
+                    - ResourceRequirement
+                    type: string
+                type: object
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:

diff --git a/operator/v1/funcs_clustermanager.go b/operator/v1/funcs_clustermanager.go
@@ -0,0 +1,5 @@
+package v1
+
+func (cm *ClusterManager) GetResourceRequirement() *ResourceRequirement {
+	return cm.Spec.ResourceRequirement
+}
diff --git a/operator/v1/funcs_klusterlet.go b/operator/v1/funcs_klusterlet.go
@@ -0,0 +1,5 @@
+package v1
+
+func (k *Klusterlet) GetResourceRequirement() *ResourceRequirement {
+	return k.Spec.ResourceRequirement
+}
diff --git a/operator/v1/type_resourcerequirement.go b/operator/v1/type_resourcerequirement.go
@@ -0,0 +1,28 @@
+package v1
+
+import corev1 "k8s.io/api/core/v1"
+
+type ResourceRequirementAcquirer interface {
+	GetResourceRequirement() *ResourceRequirement
+}
+
+// ResourceRequirement allow user override the default pod QoS classes
+type ResourceRequirement struct {
+	// +kubebuilder:validation:Enum=Default;BestEffort;ResourceRequirement
+	// +kubebuilder:default:=Default
+	Type ResourceQosClass `json:"type"`
+	// ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement
+	// +optional
+	ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"`
+}
+
+type ResourceQosClass string
+
+const (
+	// Default use resource setting in the template file (with requests but no limits in the resources)
+	ResourceQosClassDefault ResourceQosClass = "Default"
+	// If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort.
+	ResourceQosClassBestEffort ResourceQosClass = "BestEffort"
+	// Configurable resource requirements with requests and limits
+	ResourceQosClassResourceRequirement ResourceQosClass = "ResourceRequirement"
+)
diff --git a/operator/v1/types_clustermanager.go b/operator/v1/types_clustermanager.go
@@ -70,6 +70,11 @@ type ClusterManagerSpec struct {
 	// AddOnManagerConfiguration contains the configuration of addon manager
 	// +optional
 	AddOnManagerConfiguration *AddOnManagerConfiguration `json:"addOnManagerConfiguration,omitempty"`
+
+	// ResourceRequirement specify QoS classes of deployments managed by clustermanager.
+	// It applies to all the containers in the deployments.
+	// +optional
+	ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"`
 }
 
 // NodePlacement describes node scheduling configuration for the pods.

diff --git a/operator/v1/types_klusterlet.go b/operator/v1/types_klusterlet.go
@@ -1,6 +1,8 @@
 package v1
 
-import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
 
 // +genclient
 // +genclient:nonNamespaced
@@ -87,27 +89,12 @@ type KlusterletSpec struct {
 	// +optional
 	HubApiServerHostAlias *HubApiServerHostAlias `json:"hubApiServerHostAlias,omitempty"`
 
-	// ResourceRequirement specify QoS classes of klusterlet deployment
+	// ResourceRequirement specify QoS classes of deployments managed by klusterlet.
+	// It applies to all the containers in the deployments.
 	// +optional
 	ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"`
 }
 
-type ResourceQosClass string
-
-const (
-	// Default use resource setting in the template file
-	ResourceQosClassDefault ResourceQosClass = "Default"
-	// If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort.
-	ResourceQosClassBestEffort ResourceQosClass = "BestEffort"
-)
-
-// ResourceRequirement allow user override the default pod QoS classes
-type ResourceRequirement struct {
-	// +kubebuilder:validation:Enum=Default;BestEffort
-	// +kubebuilder:default:=Default
-	Type ResourceQosClass `json:"type"`
-}
-
 // ServerURL represents the apiserver url and ca bundle that is accessible externally
 type ServerURL struct {
 	// URL is the url of apiserver endpoint of the managed cluster.

diff --git a/operator/v1/zz_generated.deepcopy.go b/operator/v1/zz_generated.deepcopy.go
diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go