configurable qos resources for clustermanager and klusterlet deployments

crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml

@@ -156,15 +156,53 @@ spec:
               description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified.
               type: string
             resourceRequirement:
-              description: ResourceRequirement specify QoS classes of klusterlet deployment
+              description: ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments.
               type: object
               properties:
+                resources:
+                  description: Resources defines resource requests and limits when Type is ResourceQosClassConfigurable
+                  type: object
+                  properties:
+                    claims:
+                      description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
+                      type: array
+                      items:
+                        description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                        type: object
+                        required:
+                          - name
+                        properties:
+                          name:
+                            description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
+                            type: string
+                      x-kubernetes-list-map-keys:
+                        - name
+                      x-kubernetes-list-type: map
+                    limits:
+                      description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                      type: object
+                      additionalProperties:
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
+                    requests:
+                      description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                      type: object
+                      additionalProperties:
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
                 type:
                   type: string
                   default: Default
                   enum:
                     - Default
                     - BestEffort
+                    - Configurable
             workConfiguration:
               description: WorkConfiguration contains the configuration of work
               type: object

operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

@@ -244,14 +244,65 @@ spec:
                   will be used if unspecified.
                 type: string
               resourceRequirement:
-                description: ResourceRequirement specify QoS classes of klusterlet
-                  deployment
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by klusterlet. It applies to all the containers in the deployments.
                 properties:
+                  resources:
+                    description: Resources defines resource requests and limits when
+                      Type is ResourceQosClassConfigurable
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                   type:
                     default: Default
                     enum:
                     - Default
                     - BestEffort
+                    - Configurable
                     type: string
                 type: object
               workConfiguration:

operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml

@@ -259,6 +259,69 @@ spec:
                 description: RegistrationImagePullSpec represents the desired image
                   of registration controller/webhook installed on hub.
                 type: string
+              resourceRequirement:
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by clustermanager. It applies to all the containers in the
+                  deployments.
+                properties:
+                  resources:
+                    description: Resources defines resource requests and limits when
+                      Type is ResourceQosClassConfigurable
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
+                  type:
+                    default: Default
+                    enum:
+                    - Default
+                    - BestEffort
+                    - Configurable
+                    type: string
+                type: object
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:

operator/v1/types_clustermanager.go

@@ -70,6 +70,11 @@ type ClusterManagerSpec struct {
 	// AddOnManagerConfiguration contains the configuration of addon manager
 	// +optional
 	AddOnManagerConfiguration *AddOnManagerConfiguration `json:"addOnManagerConfiguration,omitempty"`
+
+	// ResourceRequirement specify QoS classes of deployments managed by clustermanager.
+	// It applies to all the containers in the deployments.
+	// +optional
+	ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"`
 }
 
 // NodePlacement describes node scheduling configuration for the pods.

operator/v1/types_klusterlet.go

@@ -1,6 +1,9 @@
 package v1
 
-import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
 
 // +genclient
 // +genclient:nonNamespaced
@@ -87,25 +90,31 @@ type KlusterletSpec struct {
 	// +optional
 	HubApiServerHostAlias *HubApiServerHostAlias `json:"hubApiServerHostAlias,omitempty"`
 
-	// ResourceRequirement specify QoS classes of klusterlet deployment
+	// ResourceRequirement specify QoS classes of deployments managed by klusterlet.
+	// It applies to all the containers in the deployments.
 	// +optional
 	ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"`
 }
 
 type ResourceQosClass string
 
 const (
-	// Default use resource setting in the template file
+	// Default use resource setting in the template file (with requests but no limits in the resources)
 	ResourceQosClassDefault ResourceQosClass = "Default"
 	// If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort.
 	ResourceQosClassBestEffort ResourceQosClass = "BestEffort"
+	// Configurable resources with requests and limits
+	ResourceQosClassConfigurable ResourceQosClass = "Configurable"
 )
 
 // ResourceRequirement allow user override the default pod QoS classes
 type ResourceRequirement struct {
-	// +kubebuilder:validation:Enum=Default;BestEffort
+	// +kubebuilder:validation:Enum=Default;BestEffort;Configurable
 	// +kubebuilder:default:=Default
 	Type ResourceQosClass `json:"type"`
+	// Resources defines resource requests and limits when Type is ResourceQosClassConfigurable
+	// +optional
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
 }
 
 // ServerURL represents the apiserver url and ca bundle that is accessible externally