From 7ec86d7670585837b7c105c8fc8279054a41d62e Mon Sep 17 00:00:00 2001 From: ZhiweiYin Date: Mon, 15 Jan 2024 12:42:06 +0800 Subject: [PATCH] add QPS and burst for the agent components Signed-off-by: ZhiweiYin --- ...cluster-management.io_klusterlets.crd.yaml | 20 ++++++ ...cluster-management.io_klusterlets.crd.yaml | 28 +++++++++ operator/v1/types_klusterlet.go | 40 +++++++++++- operator/v1/zz_generated.deepcopy.go | 23 ++++++- .../v1/zz_generated.swagger_doc_generated.go | 12 ++++ test/integration/api/klusterlet_test.go | 61 ++++++++++++++++++- 6 files changed, 179 insertions(+), 5 deletions(-) diff --git a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml index 0d94d812f..e48e627a5 100644 --- a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -142,6 +142,16 @@ spec: enum: - Enable - Disable + kubeAPIBurst: + description: 'KubeAPIBurst indicates the maximum burst of the throttle while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 100' + type: integer + format: int32 + default: 100 + kubeAPIQPS: + description: 'KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50' + type: string + default: "50" + pattern: ^(-?\d+)(.\d+)?$ registrationImagePullSpec: description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified. type: string @@ -177,6 +187,16 @@ spec: enum: - Enable - Disable + kubeAPIBurst: + description: 'KubeAPIBurst indicates the maximum burst of the throttle while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 100' + type: integer + format: int32 + default: 100 + kubeAPIQPS: + description: 'KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50' + type: string + default: "50" + pattern: ^(-?\d+)(.\d+)?$ workImagePullSpec: description: WorkImagePullSpec represents the desired image configuration of work agent. quay.io/open-cluster-management.io/work:latest will be used if unspecified. type: string diff --git a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml index 11b91da28..057da136b 100644 --- a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -223,6 +223,20 @@ spec: - feature type: object type: array + kubeAPIBurst: + default: 100 + description: 'KubeAPIBurst indicates the maximum burst of the + throttle while talking with apiserver of hub cluster from the + spoke cluster. If it is set empty, use the default value: 100' + format: int32 + type: integer + kubeAPIQPS: + default: "50" + description: 'KubeAPIQPS indicates the maximum QPS while talking + with apiserver of hub cluster from the spoke cluster. If it + is set empty, use the default value: 50' + pattern: ^(-?\d+)(.\d+)?$ + type: string type: object registrationImagePullSpec: description: RegistrationImagePullSpec represents the desired image @@ -274,6 +288,20 @@ spec: - feature type: object type: array + kubeAPIBurst: + default: 100 + description: 'KubeAPIBurst indicates the maximum burst of the + throttle while talking with apiserver of hub cluster from the + spoke cluster. If it is set empty, use the default value: 100' + format: int32 + type: integer + kubeAPIQPS: + default: "50" + description: 'KubeAPIQPS indicates the maximum QPS while talking + with apiserver of hub cluster from the spoke cluster. If it + is set empty, use the default value: 50' + pattern: ^(-?\d+)(.\d+)?$ + type: string type: object workImagePullSpec: description: WorkImagePullSpec represents the desired image configuration diff --git a/operator/v1/types_klusterlet.go b/operator/v1/types_klusterlet.go index 31014edca..2974a6e1a 100644 --- a/operator/v1/types_klusterlet.go +++ b/operator/v1/types_klusterlet.go @@ -80,7 +80,7 @@ type KlusterletSpec struct { // WorkConfiguration contains the configuration of work // +optional - WorkConfiguration *WorkConfiguration `json:"workConfiguration,omitempty"` + WorkConfiguration *WorkAgentConfiguration `json:"workConfiguration,omitempty"` // HubApiServerHostAlias contains the host alias for hub api server. // registration-agent and work-agent will use it to communicate with hub api server. @@ -157,6 +157,44 @@ type RegistrationConfiguration struct { // ManagedCluster when creating only, other actors can update it afterwards. // +optional ClusterAnnotations map[string]string `json:"clusterAnnotations,omitempty"` + + // KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. + // If it is set empty, use the default value: 50 + // +optional + // +kubebuilder:default:="50" + // +kubebuilder:validation:Pattern=^(-?\d+)(.\d+)?$ + KubeAPIQPS string `json:"kubeAPIQPS,omitempty"` + + // KubeAPIBurst indicates the maximum burst of the throttle while talking with apiserver of hub cluster from the spoke cluster. + // If it is set empty, use the default value: 100 + // +optional + // +kubebuilder:default:=100 + KubeAPIBurst int32 `json:"kubeAPIBurst,omitempty"` +} + +type WorkAgentConfiguration struct { + // FeatureGates represents the list of feature gates for work + // If it is set empty, default feature gates will be used. + // If it is set, featuregate/Foo is an example of one item in FeatureGates: + // 1. If featuregate/Foo does not exist, registration-operator will discard it + // 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true] + // 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false, + // he can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false. + // +optional + FeatureGates []FeatureGate `json:"featureGates,omitempty"` + + // KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. + // If it is set empty, use the default value: 50 + // +optional + // +kubebuilder:default:="50" + // +kubebuilder:validation:Pattern=^(-?\d+)(.\d+)?$ + KubeAPIQPS string `json:"kubeAPIQPS,omitempty"` + + // KubeAPIBurst indicates the maximum burst of the throttle while talking with apiserver of hub cluster from the spoke cluster. + // If it is set empty, use the default value: 100 + // +optional + // +kubebuilder:default:=100 + KubeAPIBurst int32 `json:"kubeAPIBurst,omitempty"` } const ( diff --git a/operator/v1/zz_generated.deepcopy.go b/operator/v1/zz_generated.deepcopy.go index 8df30a39d..0de27326b 100644 --- a/operator/v1/zz_generated.deepcopy.go +++ b/operator/v1/zz_generated.deepcopy.go @@ -342,7 +342,7 @@ func (in *KlusterletSpec) DeepCopyInto(out *KlusterletSpec) { } if in.WorkConfiguration != nil { in, out := &in.WorkConfiguration, &out.WorkConfiguration - *out = new(WorkConfiguration) + *out = new(WorkAgentConfiguration) (*in).DeepCopyInto(*out) } if in.HubApiServerHostAlias != nil { @@ -554,6 +554,27 @@ func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkAgentConfiguration) DeepCopyInto(out *WorkAgentConfiguration) { + *out = *in + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make([]FeatureGate, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkAgentConfiguration. +func (in *WorkAgentConfiguration) DeepCopy() *WorkAgentConfiguration { + if in == nil { + return nil + } + out := new(WorkAgentConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkConfiguration) DeepCopyInto(out *WorkConfiguration) { *out = *in diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 752ffe2c2..330df7997 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -236,6 +236,8 @@ var map_RegistrationConfiguration = map[string]string{ "clientCertExpirationSeconds": "clientCertExpirationSeconds represents the seconds of a client certificate to expire. If it is not set or 0, the default duration seconds will be set by the hub cluster. If the value is larger than the max signing duration seconds set on the hub cluster, the max signing duration seconds will be set.", "featureGates": "FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates:\n 1. If featuregate/Foo does not exist, registration-operator will discard it\n 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true]\n 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false,\n \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false.", "clusterAnnotations": "ClusterAnnotations is annotations with the reserve prefix \"agent.open-cluster-management.io\" set on ManagedCluster when creating only, other actors can update it afterwards.", + "kubeAPIQPS": "KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50", + "kubeAPIBurst": "KubeAPIBurst indicates the maximum burst of the throttle while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 100", } func (RegistrationConfiguration) SwaggerDoc() map[string]string { @@ -260,4 +262,14 @@ func (ServerURL) SwaggerDoc() map[string]string { return map_ServerURL } +var map_WorkAgentConfiguration = map[string]string{ + "featureGates": "FeatureGates represents the list of feature gates for work If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates:\n 1. If featuregate/Foo does not exist, registration-operator will discard it\n 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true]\n 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false,\n \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false.", + "kubeAPIQPS": "KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50", + "kubeAPIBurst": "KubeAPIBurst indicates the maximum burst of the throttle while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 100", +} + +func (WorkAgentConfiguration) SwaggerDoc() map[string]string { + return map_WorkAgentConfiguration +} + // AUTO-GENERATED FUNCTIONS END HERE diff --git a/test/integration/api/klusterlet_test.go b/test/integration/api/klusterlet_test.go index 1aa34304c..8753e3d9c 100644 --- a/test/integration/api/klusterlet_test.go +++ b/test/integration/api/klusterlet_test.go @@ -3,6 +3,7 @@ package api import ( "context" "fmt" + "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -118,7 +119,7 @@ var _ = Describe("Klusterlet API test with WorkConfiguration", func() { Name: klusterletName, }, Spec: operatorv1.KlusterletSpec{ - WorkConfiguration: &operatorv1.WorkConfiguration{ + WorkConfiguration: &operatorv1.WorkAgentConfiguration{ FeatureGates: []operatorv1.FeatureGate{ { Feature: "Foo", @@ -138,7 +139,7 @@ var _ = Describe("Klusterlet API test with WorkConfiguration", func() { Name: klusterletName, }, Spec: operatorv1.KlusterletSpec{ - WorkConfiguration: &operatorv1.WorkConfiguration{ + WorkConfiguration: &operatorv1.WorkAgentConfiguration{ FeatureGates: []operatorv1.FeatureGate{ { Feature: "Foo", @@ -158,7 +159,7 @@ var _ = Describe("Klusterlet API test with WorkConfiguration", func() { Name: klusterletName, }, Spec: operatorv1.KlusterletSpec{ - WorkConfiguration: &operatorv1.WorkConfiguration{ + WorkConfiguration: &operatorv1.WorkAgentConfiguration{ FeatureGates: []operatorv1.FeatureGate{ { Feature: "Foo", @@ -177,4 +178,58 @@ var _ = Describe("Klusterlet API test with WorkConfiguration", func() { Expect(klusterlet.Spec.WorkConfiguration.FeatureGates[0].Mode).Should(Equal(operatorv1.FeatureGateModeTypeDisable)) Expect(klusterlet.Spec.WorkConfiguration.FeatureGates[1].Mode).Should(Equal(operatorv1.FeatureGateModeTypeEnable)) }) + + It("Create a klusterlet with right QPS", func() { + + cases := []struct { + qps string + expectErr bool + }{ + { + qps: "abc", + expectErr: true, + }, + { + qps: "123.abc", + expectErr: true, + }, + { + qps: "abc213", + expectErr: true, + }, + { + qps: "123", + expectErr: false, + }, + { + qps: "123.123", + expectErr: false, + }, + { + qps: "100.0", + expectErr: false, + }, + } + for _, c := range cases { + suffix := rand.String(5) + klusterletName = fmt.Sprintf("cm-%s", suffix) + klusterlet := &operatorv1.Klusterlet{ + ObjectMeta: metav1.ObjectMeta{ + Name: klusterletName, + }, + Spec: operatorv1.KlusterletSpec{ + WorkConfiguration: &operatorv1.WorkAgentConfiguration{ + KubeAPIQPS: c.qps, + }, + }, + } + _, err := operatorClient.OperatorV1().Klusterlets().Create(context.TODO(), klusterlet, metav1.CreateOptions{}) + if c.expectErr { + Expect(err).To(HaveOccurred()) + Expect(strings.Contains(err.Error(), "is invalid: spec.workConfiguration.kubeAPIQPS:")).To(BeTrue()) + } else { + Expect(err).To(BeNil()) + } + } + }) })