Skip to content

Commit

Permalink
Configurable QoS resources for cluster-manager and klusterlet operato…
Browse files Browse the repository at this point in the history
…rs and their managed pods (#400)

Signed-off-by: Dong Beiqing <[email protected]>
  • Loading branch information
dongbeiqing91 authored Feb 8, 2024
1 parent 41564d4 commit c384319
Show file tree
Hide file tree
Showing 13 changed files with 338 additions and 15 deletions.
4 changes: 4 additions & 0 deletions pkg/cmd/init/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
"If set, the command will initialize the OCM control plan in foreground.")
cmd.Flags().StringVarP(&o.output, "output", "o", "text", "output foramt, should be json or text")
cmd.Flags().BoolVar(&o.singleton, "singleton", false, "If true, deploy singleton controlplane instead of cluster-manager. This is an alpha stage flag.")
cmd.Flags().StringVar(&o.resourceQosClass, "resource-qos-class", "Default", "the resource QoS class of all the containers managed by the cluster manager and the cluster manager operator. Can be one of Default, BestEffort or ResourceRequirement.")
cmd.Flags().StringToStringVar(&o.resourceLimits, "resource-limits", nil, "the resource limits of all the containers managed by the cluster manager and the cluster manager operator, for example: cpu=800m,memory=800Mi")
cmd.Flags().StringToStringVar(&o.resourceRequests, "resource-requests", nil, "the resource requests of all the containers managed by the cluster manager and the cluster manager operator, for example: cpu=500m,memory=500Mi")
cmd.Flags().BoolVar(&o.createNamespace, "create-namespace", true, "If true, create open-cluster-management namespace, otherwise use existing one")

//clusterManagetSet contains the flags for deploy cluster-manager
clusterManagerSet := pflag.NewFlagSet("clusterManagerSet", pflag.ExitOnError)
Expand Down
14 changes: 12 additions & 2 deletions pkg/cmd/init/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
clusteradmjson "open-cluster-management.io/clusteradm/pkg/helpers/json"
preflightinterface "open-cluster-management.io/clusteradm/pkg/helpers/preflight"
"open-cluster-management.io/clusteradm/pkg/helpers/reader"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
"open-cluster-management.io/clusteradm/pkg/helpers/version"
helperwait "open-cluster-management.io/clusteradm/pkg/helpers/wait"
)
Expand Down Expand Up @@ -67,6 +68,11 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
WorkFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubWorkFeatureGates),
AddonFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubAddonManagerFeatureGates),
}
resourceRequirement, err := resourcerequirement.NewResourceRequirement(o.resourceQosClass, o.resourceLimits, o.resourceRequests)
if err != nil {
return err
}
o.values.ResourceRequirement = *resourceRequirement
} else {
o.Helm.WithNamespace(o.SingletonName)
}
Expand Down Expand Up @@ -150,9 +156,13 @@ func (o *Options) run() error {
} else {
token := fmt.Sprintf("%s.%s", o.values.Hub.TokenID, o.values.Hub.TokenSecret)

files := []string{
"init/namespace.yaml",
files := []string{}
if o.createNamespace {
files = append(files, "init/namespace.yaml")
} else {
fmt.Fprintf(o.Streams.Out, "skip creating namespace\n")
}

if o.useBootstrapToken {
files = append(files,
"init/bootstrap-token-secret.yaml",
Expand Down
13 changes: 13 additions & 0 deletions pkg/cmd/init/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
operatorv1 "open-cluster-management.io/api/operator/v1"
genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
"open-cluster-management.io/clusteradm/pkg/helpers/helm"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
)

// Options is holding all the command-line options
Expand All @@ -29,6 +30,14 @@ type Options struct {
SingletonName string
Helm *helm.Helm

// Resource requirement for the containers managed by the cluster manager and the cluster manager operator
resourceQosClass string
resourceLimits map[string]string
resourceRequests map[string]string

// If create ns or use existing ns
createNamespace bool

//If set, will be persisting the generated join command to a local file
outputJoinCommandFile string
//If set, the command will hold until the OCM control plane initialized
Expand Down Expand Up @@ -70,6 +79,10 @@ type Values struct {

// Features is the slice of feature for addon manager
AddonFeatures []operatorv1.FeatureGate

// ResourceRequirement is the resource requirement setting for the containers managed by the cluster manager
// and the cluster manager operator
ResourceRequirement resourcerequirement.ResourceRequirement
}

// Hub: The hub values for the template
Expand Down
6 changes: 6 additions & 0 deletions pkg/cmd/init/scenario/init/clustermanager.cr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ spec:
deployOption:
mode: Default
{{if .RegistrationFeatures}}
resourceRequirement:
type: {{ .ResourceRequirement.Type }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resourceRequirements:
{{ .ResourceRequirement.ResourceRequirements | indent 6 }}
{{- end }}
registrationConfiguration:
{{if .AutoApprove}}
autoApproveUsers:
Expand Down
15 changes: 15 additions & 0 deletions pkg/cmd/init/scenario/init/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ spec:
labels:
app: cluster-manager
spec:
volumes:
- emptyDir: {}
name: tmpdir
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
Expand Down Expand Up @@ -56,16 +59,28 @@ spec:
initialDelaySeconds: 2
periodSeconds: 10
name: registration-operator
volumeMounts:
- mountPath: /tmp
name: tmpdir
readinessProbe:
httpGet:
path: /healthz
port: 8443
scheme: HTTPS
initialDelaySeconds: 2
{{- if or (eq .ResourceRequirement.Type "Default") (eq .ResourceRequirement.Type "") }}
resources:
requests:
cpu: 100m
memory: 128Mi
{{- end }}
{{- if eq .ResourceRequirement.Type "BestEffort" }}
resources: {}
{{- end }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resources:
{{ .ResourceRequirement.ResourceRequirements | indent 10 }}
{{- end }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
Expand Down
4 changes: 3 additions & 1 deletion pkg/cmd/join/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,9 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
cmd.Flags().BoolVar(&o.singleton, "singleton", false, "If true, deploy singleton mode of klusterlet to have registration and work agents run in a single pod. This is an alpha stage flag.")
cmd.Flags().StringVar(&o.proxyURL, "proxy-url", "", "the URL of a forward proxy server that will be used by agents to connect to the hub cluster.")
cmd.Flags().StringVar(&o.proxyCAFile, "proxy-ca-file", "", "the file path to proxy ca, optional")
cmd.Flags().StringVar(&o.resourceQosClass, "resource-qos-class", "Default", "the resource QoS class of the klusterlet pod. Can be one of Default or BestEffort")
cmd.Flags().StringVar(&o.resourceQosClass, "resource-qos-class", "Default", "the resource QoS class of all the containers managed by the klusterlet and the klusterlet operator. Can be one of Default, BestEffort or ResourceRequirement.")
cmd.Flags().StringToStringVar(&o.resourceLimits, "resource-limits", nil, "the resource limits of all the containers managed by the klusterlet and the klusterlet operator, for example: cpu=800m,memory=800Mi")
cmd.Flags().StringToStringVar(&o.resourceRequests, "resource-requests", nil, "the resource requests of all the containers managed by the klusterlet and the klusterlet operator, for example: cpu=500m,memory=500Mi")
cmd.Flags().BoolVar(&o.createNameSpace, "create-namespace", true, "If true, create open-cluster-management namespace, otherwise use existing one")

return cmd
Expand Down
9 changes: 7 additions & 2 deletions pkg/cmd/join/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
preflightinterface "open-cluster-management.io/clusteradm/pkg/helpers/preflight"
"open-cluster-management.io/clusteradm/pkg/helpers/printer"
"open-cluster-management.io/clusteradm/pkg/helpers/reader"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
"open-cluster-management.io/clusteradm/pkg/helpers/version"
"open-cluster-management.io/clusteradm/pkg/helpers/wait"
)
Expand Down Expand Up @@ -119,9 +120,13 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
Name: klusterletName,
KlusterletNamespace: klusterletNamespace,
}
o.values.ResourceRequirement = ResourceRequirement{
Type: o.resourceQosClass,

resourceRequirement, err := resourcerequirement.NewResourceRequirement(o.resourceQosClass, o.resourceLimits, o.resourceRequests)
if err != nil {
return err
}
o.values.ResourceRequirement = *resourceRequirement

o.values.ManagedKubeconfig = o.managedKubeconfigFile
o.values.RegistrationFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeRegistrationFeatureGates)
o.values.WorkFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeWorkFeatureGates)
Expand Down
16 changes: 8 additions & 8 deletions pkg/cmd/join/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
operatorv1 "open-cluster-management.io/api/operator/v1"
genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
"open-cluster-management.io/clusteradm/pkg/helpers/resourcerequirement"
)

// Options: The structure holding all the command-line options
Expand Down Expand Up @@ -62,8 +63,10 @@ type Options struct {
//The proxy server ca-file(optional)
proxyCAFile string

// Resource requirement
// Resource requirement for the containers managed by klusterlet and the klusterlet operator
resourceQosClass string
resourceLimits map[string]string
resourceRequests map[string]string

// If create ns or use existing ns
createNameSpace bool
Expand All @@ -84,8 +87,6 @@ type Values struct {
Hub Hub
//Klusterlet is the klusterlet related configuration
Klusterlet Klusterlet
//ResourceRequirement is the resource requirement
ResourceRequirement ResourceRequirement
//Registry is the image registry related configuration
Registry string
//bundle version
Expand All @@ -98,6 +99,10 @@ type Values struct {

// Features is the slice of feature for work
WorkFeatures []operatorv1.FeatureGate

// ResourceRequirement is the resource requirement setting for the containers managed by the klusterlet
// and the klusterlet operator
ResourceRequirement resourcerequirement.ResourceRequirement
}

// Hub: The hub values for the template
Expand All @@ -117,11 +122,6 @@ type Klusterlet struct {
KlusterletNamespace string
}

// ResourceRequirement is for templating resource requirement
type ResourceRequirement struct {
Type string
}

type BundleVersion struct {
// registration image version
RegistrationImageVersion string
Expand Down
4 changes: 4 additions & 0 deletions pkg/cmd/join/scenario/join/klusterlets.cr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ spec:
{{ end }}
resourceRequirement:
type: {{ .ResourceRequirement.Type }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resourceRequirements:
{{ .ResourceRequirement.ResourceRequirements | indent 6 }}
{{- end }}
{{if .RegistrationFeatures}}
registrationConfiguration:
featureGates:
Expand Down
11 changes: 9 additions & 2 deletions pkg/cmd/join/scenario/join/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,19 @@ spec:
scheme: HTTPS
port: 8443
initialDelaySeconds: 2
{{ if ne .ResourceRequirement.Type "BestEffort" }}
{{- if or (eq .ResourceRequirement.Type "Default") (eq .ResourceRequirement.Type "") }}
resources:
requests:
cpu: 100m
memory: 128Mi
{{ end }}
{{- end }}
{{- if eq .ResourceRequirement.Type "BestEffort" }}
resources: {}
{{- end }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resources:
{{ .ResourceRequirement.ResourceRequirements | indent 10 }}
{{- end }}
volumeMounts:
- name: tmpdir
mountPath: /tmp
Expand Down
34 changes: 34 additions & 0 deletions pkg/helpers/resourcerequirement/fake_deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright Contributors to the Open Cluster Management project
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
a: b
name: fake-deployment
namespace: fake-namespace
spec:
replicas: 1
selector:
matchLabels:
a: b
template:
metadata:
labels:
a: b
spec:
containers:
- name: my-container
image: nginx
{{- if or (eq .ResourceRequirement.Type "Default") (eq .ResourceRequirement.Type "") }}
resources:
requests:
cpu: 100m
memory: 128Mi
{{- end }}
{{- if eq .ResourceRequirement.Type "BestEffort" }}
resources: {}
{{- end }}
{{- if eq .ResourceRequirement.Type "ResourceRequirement" }}
resources:
{{ .ResourceRequirement.ResourceRequirements | indent 10 }}
{{- end }}
78 changes: 78 additions & 0 deletions pkg/helpers/resourcerequirement/resourcerequirement.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright Contributors to the Open Cluster Management project

package resourcerequirement

import (
"fmt"

"github.com/ghodss/yaml"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
operatorv1 "open-cluster-management.io/api/operator/v1"
)

// ResourceRequirement is for templating resource requirement
type ResourceRequirement struct {
Type string
ResourceRequirements []byte
}

func NewResourceRequirement(resourceType string, limits, requests map[string]string) (*ResourceRequirement, error) {
if len(limits)+len(requests) == 0 {
if resourceType == string(operatorv1.ResourceQosClassResourceRequirement) {
return nil, fmt.Errorf("resource type is %s but both limits and requests are not set", resourceType)
}
return &ResourceRequirement{
Type: resourceType,
}, nil
}
if resourceType == "" {
resourceType = string(operatorv1.ResourceQosClassResourceRequirement)
} else if resourceType != string(operatorv1.ResourceQosClassResourceRequirement) {
return nil, fmt.Errorf("resource type must be %s when resource limits or requests are set", string(operatorv1.ResourceQosClassResourceRequirement))
}
rr := &corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
}
for rsc, quantityStr := range limits {
quantity, err := resource.ParseQuantity(quantityStr)
if err != nil {
return nil, err
}
rr.Limits[corev1.ResourceName(rsc)] = quantity
}
for rsc, quantityStr := range requests {
quantity, err := resource.ParseQuantity(quantityStr)
if err != nil {
return nil, err
}
rr.Requests[corev1.ResourceName(rsc)] = quantity
}
if err := ensureQuantity(rr); err != nil {
return nil, err
}
marshal, err := yaml.Marshal(rr)
if err != nil {
return nil, err
}
return &ResourceRequirement{
Type: resourceType,
ResourceRequirements: marshal,
}, nil
}

func ensureQuantity(r *corev1.ResourceRequirements) error {
for rsc, limitsQuantity := range r.Limits {
requestsQuantity, ok := r.Requests[rsc]
if !ok {
continue
}
if requestsQuantity.Cmp(limitsQuantity) <= 0 {
continue
}
return fmt.Errorf("requests %s must be less than or equal to limits %s",
requestsQuantity.String(), limitsQuantity.String())
}
return nil
}
Loading

0 comments on commit c384319

Please sign in to comment.