✨ Configurable qos resources for containers managed by cluster-manage…

…r and klusterlet (#351) * configurable qos resources for containers managed by cluster-manager and klusterlet Signed-off-by: Dong Beiqing <[email protected]> * run make fmt-imports Signed-off-by: Dong Beiqing <[email protected]> --------- Signed-off-by: Dong Beiqing <[email protected]>
open-cluster-management-io · Jan 23, 2024 · 5b9b566 · 5b9b566
1 parent c04ab36
commit 5b9b566
Show file tree

Hide file tree

Showing 15 changed files with 385 additions and 13 deletions.
diff --git a/manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml
@@ -78,10 +78,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

diff --git a/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml
@@ -75,10 +75,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

diff --git a/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml
@@ -74,10 +74,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

diff --git a/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml
@@ -82,10 +82,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         volumeMounts:
         - name: tmpdir
           mountPath: /tmp

diff --git a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml
@@ -56,10 +56,19 @@ spec:
         {{ if .HostedMode }}
         - "--kubeconfig=/var/run/secrets/hub/kubeconfig"
         {{ end }}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

diff --git a/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml
@@ -75,10 +75,19 @@ spec:
             scheme: HTTP
             port: 8000
           initialDelaySeconds: 2
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
         ports:
         - containerPort: 9443
           protocol: TCP

diff --git a/manifests/config.go b/manifests/config.go
@@ -1,5 +1,7 @@
 package manifests
 
+import operatorapiv1 "open-cluster-management.io/api/operator/v1"
+
 type HubConfig struct {
 	ClusterManagerName             string
 	ClusterManagerNamespace        string
@@ -18,6 +20,11 @@ type HubConfig struct {
 	AddOnManagerEnabled            bool
 	MWReplicaSetEnabled            bool
 	AutoApproveUsers               string
+	// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.
+	ResourceRequirementResourceType operatorapiv1.ResourceQosClass
+	// ResourceRequirements is the resource requirements for the cluster manager managed containers.
+	// The type has to be []byte to use "indent" template function.
+	ResourceRequirements []byte
 }
 
 type Webhook struct {

diff --git a/manifests/klusterlet/management/klusterlet-agent-deployment.yaml b/manifests/klusterlet/management/klusterlet-agent-deployment.yaml
@@ -129,12 +129,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
-        {{if ne .ResourceRequirement "BestEffort"}}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
-        {{end}}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
       volumes:
       - name: bootstrap-secret
         secret:

diff --git a/manifests/klusterlet/management/klusterlet-registration-deployment.yaml b/manifests/klusterlet/management/klusterlet-registration-deployment.yaml
@@ -120,12 +120,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
-        {{if ne .ResourceRequirement "BestEffort"}}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
-        {{end}}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
       volumes:
       - name: bootstrap-secret
         secret:

diff --git a/manifests/klusterlet/management/klusterlet-work-deployment.yaml b/manifests/klusterlet/management/klusterlet-work-deployment.yaml
@@ -113,12 +113,19 @@ spec:
             scheme: HTTPS
             port: 8443
           initialDelaySeconds: 2
-        {{if ne .ResourceRequirement "BestEffort"}}
+        {{- if or (eq .ResourceRequirementResourceType "Default") (eq .ResourceRequirementResourceType "") }}
         resources:
           requests:
             cpu: 2m
             memory: 16Mi
-        {{end}}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "BestEffort" }}
+        resources: {}
+        {{- end }}
+        {{- if eq .ResourceRequirementResourceType "ResourceRequirement" }}
+        resources:
+          {{ .ResourceRequirements | indent 10 }}
+        {{- end }}
       volumes:
       - name: hub-kubeconfig-secret
         secret:

diff --git a/pkg/operator/helpers/helpers.go b/pkg/operator/helpers/helpers.go
@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/ghodss/yaml"
 	"github.com/openshift/api"
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
@@ -605,11 +606,26 @@ func AgentNamespace(klusterlet *operatorapiv1.Klusterlet) string {
 }
 
 // ResourceType set default and return resource requirements override by user
-func ResourceType(klusterlet *operatorapiv1.Klusterlet) operatorapiv1.ResourceQosClass {
-	if klusterlet.Spec.ResourceRequirement == nil {
+func ResourceType(resourceRequirementAcquirer operatorapiv1.ResourceRequirementAcquirer) operatorapiv1.ResourceQosClass {
+	r := resourceRequirementAcquirer.GetResourceRequirement()
+	if r == nil {
 		return operatorapiv1.ResourceQosClassDefault
 	}
-	return klusterlet.Spec.ResourceRequirement.Type
+	return r.Type
+}
+
+// ResourceRequirements get resource requirements overridden by user for ResourceQosClassResourceRequirement type
+func ResourceRequirements(resourceRequirementAcquirer operatorapiv1.ResourceRequirementAcquirer) ([]byte, error) {
+	r := resourceRequirementAcquirer.GetResourceRequirement()
+	if r == nil || r.Type == operatorapiv1.ResourceQosClassBestEffort {
+		return nil, nil
+	}
+	marshal, err := yaml.Marshal(r.ResourceRequirements)
+	if err != nil {
+		klog.Errorf("failed to marshal resource requirement: %v", err)
+		return nil, err
+	}
+	return marshal, nil
 }
 
 // SyncSecret forked from:

diff --git a/...perators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go b/...perators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go
@@ -128,6 +128,12 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 	clusterManagerMode := clusterManager.Spec.DeployOption.Mode
 	clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManagerName, clusterManagerMode)
 
+	resourceRequirements, err := helpers.ResourceRequirements(clusterManager)
+	if err != nil {
+		klog.Errorf("failed to parse resource requirements for cluster manager %s: %v", clusterManager.Name, err)
+		return err
+	}
+
 	// This config is used to render template of manifests.
 	config := manifests.HubConfig{
 		ClusterManagerName:      clusterManager.Name,
@@ -144,6 +150,8 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
 		WorkWebhook: manifests.Webhook{
 			Port: defaultWebhookPort,
 		},
+		ResourceRequirementResourceType: helpers.ResourceType(clusterManager),
+		ResourceRequirements:            resourceRequirements,
 	}
 
 	var registrationFeatureMsgs, workFeatureMsgs, addonFeatureMsgs string