configurable qos resources for containers managed by cluster-manager …

…and klusterlet Signed-off-by: Dong Beiqing <[email protected]>
open-cluster-management-io · Jan 22, 2024 · 813421c · 813421c
1 parent bede3ed
commit 813421c
Show file tree

Hide file tree

Showing 31 changed files with 838 additions and 52 deletions.
diff --git a/...-manager/config/crds/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml b/...-manager/config/crds/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml
@@ -259,6 +259,69 @@ spec:
                 description: RegistrationImagePullSpec represents the desired image
                   of registration controller/webhook installed on hub.
                 type: string
+              resourceRequirement:
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by clustermanager. It applies to all the containers in the
+                  deployments.
+                properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
+                  type:
+                    default: Default
+                    enum:
+                    - Default
+                    - BestEffort
+                    - ResourceRequirement
+                    type: string
+                type: object
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:

diff --git a/...-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml b/...-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml
@@ -59,7 +59,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2024-01-15T16:05:25Z"
+    createdAt: "2024-01-22T02:10:08Z"
     description: Manages the installation and upgrade of the ClusterManager.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

diff --git a/...atalog/cluster-manager/manifests/operator.open-cluster-management.io_clustermanagers.yaml b/...atalog/cluster-manager/manifests/operator.open-cluster-management.io_clustermanagers.yaml
@@ -259,6 +259,69 @@ spec:
                 description: RegistrationImagePullSpec represents the desired image
                   of registration controller/webhook installed on hub.
                 type: string
+              resourceRequirement:
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by clustermanager. It applies to all the containers in the
+                  deployments.
+                properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
+                  type:
+                    default: Default
+                    enum:
+                    - Default
+                    - BestEffort
+                    - ResourceRequirement
+                    type: string
+                type: object
               workConfiguration:
                 description: WorkConfiguration contains the configuration of work
                 properties:

diff --git a/...y/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/...y/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml
@@ -244,14 +244,65 @@ spec:
                   will be used if unspecified.
                 type: string
               resourceRequirement:
-                description: ResourceRequirement specify QoS classes of klusterlet
-                  deployment
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by klusterlet. It applies to all the containers in the deployments.
                 properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                   type:
                     default: Default
                     enum:
                     - Default
                     - BestEffort
+                    - ResourceRequirement
                     type: string
                 type: object
               workConfiguration:

diff --git a/deploy/klusterlet/olm-catalog/klusterlet/manifests/klusterlet.clusterserviceversion.yaml b/deploy/klusterlet/olm-catalog/klusterlet/manifests/klusterlet.clusterserviceversion.yaml
@@ -31,7 +31,7 @@ metadata:
     categories: Integration & Delivery,OpenShift Optional
     certified: "false"
     containerImage: quay.io/open-cluster-management/registration-operator:latest
-    createdAt: "2024-01-15T16:05:25Z"
+    createdAt: "2024-01-22T02:10:08Z"
     description: Manages the installation and upgrade of the Klusterlet.
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

diff --git a/...let/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml b/...let/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml
@@ -244,14 +244,65 @@ spec:
                   will be used if unspecified.
                 type: string
               resourceRequirement:
-                description: ResourceRequirement specify QoS classes of klusterlet
-                  deployment
+                description: ResourceRequirement specify QoS classes of deployments
+                  managed by klusterlet. It applies to all the containers in the deployments.
                 properties:
+                  resourceRequirements:
+                    description: ResourceRequirements defines resource requests and
+                      limits when Type is ResourceQosClassResourceRequirement
+                    properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable. It can only be
+                          set for containers."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. Requests cannot exceed
+                          Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                   type:
                     default: Default
                     enum:
                     - Default
                     - BestEffort
+                    - ResourceRequirement
                     type: string
                 type: object
               workConfiguration:

diff --git a/go.mod b/go.mod
@@ -152,3 +152,6 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+// TODO remove the following once https://github.com/open-cluster-management-io/api/pull/316 is merged
+replace open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499 => github.com/promid/ocm-api v0.0.0-20240122015439-15ad4cea8533
diff --git a/go.sum b/go.sum
@@ -211,6 +211,8 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/promid/ocm-api v0.0.0-20240122015439-15ad4cea8533 h1:T30iHKynmZu9CWnBOs4Ulnm8WRznU3vO4pnW5+Qxudw=
+github.com/promid/ocm-api v0.0.0-20240122015439-15ad4cea8533/go.mod h1:K3Rza3qN/W1+E1a+wbLtFatvdI8UlQWkSqBlpeRHMPw=
 github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
 github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -452,8 +454,6 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.8.1-0.20240119025526-d2afcef1ff66 h1:d6vkhLLdam7/DLmZ4onPSbsRNHP3Rze0/0OkmjJgVjk=
 open-cluster-management.io/addon-framework v0.8.1-0.20240119025526-d2afcef1ff66/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw=
-open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499 h1:Odh200i57Z9V44eJak98VqdKMCAlj3m9dtnUp8285nE=
-open-cluster-management.io/api v0.12.1-0.20240115071352-3d94ce8f3499/go.mod h1:K3Rza3qN/W1+E1a+wbLtFatvdI8UlQWkSqBlpeRHMPw=
 open-cluster-management.io/sdk-go v0.0.0-20240118073603-6da392d39be0 h1:FCGRz0ZdCWCq6WebGJyr0XJfAsjEwYFJF6lXY4tBPFM=
 open-cluster-management.io/sdk-go v0.0.0-20240118073603-6da392d39be0/go.mod h1:LfUmo1hhhlQdui8CiNO8aUq050dbJTQ2nrE97DRFREs=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

diff --git a/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml
@@ -74,10 +74,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp