From cea6a14859ec68db80f828660c12d7c195c627a9 Mon Sep 17 00:00:00 2001 From: dtclxy64 <70486866+dtclxy64@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:19:15 -0500 Subject: [PATCH 1/4] Extract the logic to add cluster annotations to the driver interface and add unit and integration tests Signed-off-by: dtclxy64 <70486866+dtclxy64@users.noreply.github.com> --- .../register/aws_irsa/aws_irsa.go | 14 ++- pkg/registration/register/csr/csr.go | 4 + pkg/registration/register/interface.go | 4 +- pkg/registration/spoke/spokeagent.go | 20 +-- pkg/registration/spoke/spokeagent_test.go | 14 +++ .../clusterannotations_aws_test.go | 69 ++++++++++ .../registration/clusterannotations_test.go | 10 ++ .../spokecluster_aws_joining_test.go | 118 ++++++++++++++++++ 8 files changed, 235 insertions(+), 18 deletions(-) create mode 100644 test/integration/registration/clusterannotations_aws_test.go create mode 100644 test/integration/registration/spokecluster_aws_joining_test.go diff --git a/pkg/registration/register/aws_irsa/aws_irsa.go b/pkg/registration/register/aws_irsa/aws_irsa.go index d129d7985..bc66075ad 100644 --- a/pkg/registration/register/aws_irsa/aws_irsa.go +++ b/pkg/registration/register/aws_irsa/aws_irsa.go @@ -3,6 +3,7 @@ package aws_irsa import ( "context" "fmt" + operatorv1 "open-cluster-management.io/api/operator/v1" "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" @@ -22,7 +23,9 @@ const ( // TLSKeyFile is the name of tls key file in kubeconfigSecret TLSKeyFile = "tls.key" // TLSCertFile is the name of the tls cert file in kubeconfigSecret - TLSCertFile = "tls.crt" + TLSCertFile = "tls.crt" + ManagedClusterArn = "managed-cluster-arn" + ManagedClusterIAMRoleSuffix = "managed-cluster-iam-role-suffix" ) type AWSIRSADriver struct { @@ -95,6 +98,15 @@ func (c *AWSIRSADriver) IsHubKubeConfigValid(ctx context.Context, secretOption r return true, nil } +func (c *AWSIRSADriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { + if clusterAnnotations == nil { + clusterAnnotations = map[string]string{} + } + + clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterArn] = managedClusterArn + clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterIAMRoleSuffix] = managedClusterRoleSuffix +} + func NewAWSIRSADriver() register.RegisterDriver { return &AWSIRSADriver{} } diff --git a/pkg/registration/register/csr/csr.go b/pkg/registration/register/csr/csr.go index b7f63fda1..dd095cd1f 100644 --- a/pkg/registration/register/csr/csr.go +++ b/pkg/registration/register/csr/csr.go @@ -266,6 +266,10 @@ func (c *CSRDriver) IsHubKubeConfigValid(ctx context.Context, secretOption regis return isCertificateValid(logger, certData, nil) } +// AddClusterAnnotations noop for CSR driver +func (c *CSRDriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { +} + func NewCSRDriver() register.RegisterDriver { return &CSRDriver{} } diff --git a/pkg/registration/register/interface.go b/pkg/registration/register/interface.go index 2ec3b09e6..7283e74bb 100644 --- a/pkg/registration/register/interface.go +++ b/pkg/registration/register/interface.go @@ -2,7 +2,6 @@ package register import ( "context" - "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" corev1 "k8s.io/api/core/v1" @@ -71,6 +70,9 @@ type RegisterDriver interface { // InformerHandler returns informer of the related object. If no object needs to be watched, the func could // return nil, nil. InformerHandler(option any) (cache.SharedIndexInformer, factory.EventFilterFunc) + + // AddClusterAnnotations adds cluster annotations for non-CSR drivers + AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) } // Approvers is the inteface that each driver should implement on hub side. The hub controller will use this driver diff --git a/pkg/registration/spoke/spokeagent.go b/pkg/registration/spoke/spokeagent.go index 8b22f968d..de52fd9fb 100644 --- a/pkg/registration/spoke/spokeagent.go +++ b/pkg/registration/spoke/spokeagent.go @@ -26,8 +26,6 @@ import ( clusterv1informers "open-cluster-management.io/api/client/cluster/informers/externalversions" clusterv1 "open-cluster-management.io/api/cluster/v1" ocmfeature "open-cluster-management.io/api/feature" - operatorv1 "open-cluster-management.io/api/operator/v1" - "open-cluster-management.io/ocm/pkg/common/helpers" commonoptions "open-cluster-management.io/ocm/pkg/common/options" "open-cluster-management.io/ocm/pkg/features" @@ -191,24 +189,14 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context, // initiate registration driver var registerDriver register.RegisterDriver - if o.registrationOption.RegistrationAuth == AwsIrsaAuthType { - // TODO: may consider add additional validations - if o.registrationOption.HubClusterArn != "" { - registerDriver = awsIrsa.NewAWSIRSADriver() - if o.registrationOption.ClusterAnnotations == nil { - o.registrationOption.ClusterAnnotations = map[string]string{} - } - o.registrationOption.ClusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/managed-cluster-arn"] = o.registrationOption.ManagedClusterArn - o.registrationOption.ClusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/managed-cluster-iam-role-suffix"] = - o.registrationOption.ManagedClusterRoleSuffix - - } else { - panic("A valid EKS Hub Cluster ARN is required with awsirsa based authentication") - } + var registrationOption = o.registrationOption + if registrationOption.RegistrationAuth == AwsIrsaAuthType { + registerDriver = awsIrsa.NewAWSIRSADriver() } else { registerDriver = csr.NewCSRDriver() } + registerDriver.AddClusterAnnotations(registrationOption.ClusterAnnotations, registrationOption.ManagedClusterArn, registrationOption.ManagedClusterRoleSuffix) o.driver = registerDriver // get spoke cluster CA bundle diff --git a/pkg/registration/spoke/spokeagent_test.go b/pkg/registration/spoke/spokeagent_test.go index 2d510b1a6..110146fc9 100644 --- a/pkg/registration/spoke/spokeagent_test.go +++ b/pkg/registration/spoke/spokeagent_test.go @@ -45,6 +45,10 @@ func init() { func TestValidate(t *testing.T) { defaultCompletedOptions := NewSpokeAgentOptions() defaultCompletedOptions.BootstrapKubeconfig = "/spoke/bootstrap/kubeconfig" + awsCompletedOptionsHubArnMissing := *defaultCompletedOptions + awsCompletedOptionsHubArnMissing.RegistrationAuth = AwsIrsaAuthType + awsDefaultCompletedOptions := awsCompletedOptionsHubArnMissing + awsDefaultCompletedOptions.HubClusterArn = "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1" cases := []struct { name string @@ -78,6 +82,16 @@ func TestValidate(t *testing.T) { options: defaultCompletedOptions, expectedErr: "", }, + { + name: "default completed options for aws flow", + options: &awsDefaultCompletedOptions, + expectedErr: "", + }, + { + name: "default completed options without HubClusterArn for aws flow", + options: &awsCompletedOptionsHubArnMissing, + expectedErr: "EksHubClusterArn cannot be empty if RegistrationAuth is awsirsa", + }, { name: "default completed options", options: &SpokeAgentOptions{ diff --git a/test/integration/registration/clusterannotations_aws_test.go b/test/integration/registration/clusterannotations_aws_test.go new file mode 100644 index 000000000..2ea2a64aa --- /dev/null +++ b/test/integration/registration/clusterannotations_aws_test.go @@ -0,0 +1,69 @@ +package registration_test + +import ( + "fmt" + operatorv1 "open-cluster-management.io/api/operator/v1" + "path" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + commonoptions "open-cluster-management.io/ocm/pkg/common/options" + "open-cluster-management.io/ocm/pkg/registration/register/aws_irsa" + "open-cluster-management.io/ocm/pkg/registration/spoke" + "open-cluster-management.io/ocm/test/integration/util" +) + +var _ = ginkgo.Describe("Cluster Annotations for aws", func() { + ginkgo.It("Cluster Annotations for aws flow should be created on the managed cluster", func() { + managedClusterName := "clusterannotations-spokecluster-aws" + //#nosec G101 + hubKubeconfigSecret := "clusterannotations-hub-kubeconfig-secret" + hubKubeconfigDir := path.Join(util.TestDir, "clusterannotations", "hub-kubeconfig") + + managedClusterArn := "arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1" + managedClusterRoleSuffix := "7f8141296c75f2871e3d030f85c35692" + hubClusterArn := "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1" + agentOptions := &spoke.SpokeAgentOptions{ + RegistrationAuth: spoke.AwsIrsaAuthType, + HubClusterArn: hubClusterArn, + ManagedClusterArn: managedClusterArn, + ManagedClusterRoleSuffix: managedClusterRoleSuffix, + BootstrapKubeconfig: bootstrapKubeConfigFile, + HubKubeconfigSecret: hubKubeconfigSecret, + ClusterHealthCheckPeriod: 1 * time.Minute, + ClusterAnnotations: map[string]string{ + "agent.open-cluster-management.io/foo": "bar", + "foo": "bar", // this annotation should be filtered out + }, + } + + commOptions := commonoptions.NewAgentOptions() + commOptions.HubKubeconfigDir = hubKubeconfigDir + commOptions.SpokeClusterName = managedClusterName + + // run registration agent + cancel := runAgent("rotationtest", agentOptions, commOptions, spokeCfg) + defer cancel() + + // after bootstrap the spokecluster and csr should be created + gomega.Eventually(func() error { + mc, err := util.GetManagedCluster(clusterClient, managedClusterName) + if err != nil { + return err + } + + if mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn] != managedClusterArn { + return fmt.Errorf("expected annotation "+operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn+" to be "+managedClusterArn+", got %s", mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn]) + } + + if mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix] != managedClusterRoleSuffix { + return fmt.Errorf("expected annotation "+operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix+" to be "+managedClusterRoleSuffix+", got %s", mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix]) + } + + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed()) + + }) +}) diff --git a/test/integration/registration/clusterannotations_test.go b/test/integration/registration/clusterannotations_test.go index 2be6e7817..623115619 100644 --- a/test/integration/registration/clusterannotations_test.go +++ b/test/integration/registration/clusterannotations_test.go @@ -2,6 +2,8 @@ package registration_test import ( "fmt" + operatorv1 "open-cluster-management.io/api/operator/v1" + "open-cluster-management.io/ocm/pkg/registration/register/aws_irsa" "path" "time" @@ -49,6 +51,14 @@ var _ = ginkgo.Describe("Cluster Annotations", func() { return fmt.Errorf("unexpected annotations %v", mc.Annotations) } + if _, ok := mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn]; ok { + return fmt.Errorf("unexpected annotations %v", mc.Annotations) + } + + if _, ok := mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix]; ok { + return fmt.Errorf("unexpected annotations %v", mc.Annotations) + } + if mc.Annotations["agent.open-cluster-management.io/foo"] != "bar" { return fmt.Errorf("expected annotation agent.open-cluster-management.io/foo to be bar, got %s", mc.Annotations["agent.open-cluster-management.io/foo"]) } diff --git a/test/integration/registration/spokecluster_aws_joining_test.go b/test/integration/registration/spokecluster_aws_joining_test.go new file mode 100644 index 000000000..0eb54b4e5 --- /dev/null +++ b/test/integration/registration/spokecluster_aws_joining_test.go @@ -0,0 +1,118 @@ +package registration_test + +import ( + "fmt" + "path" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/rand" + commonoptions "open-cluster-management.io/ocm/pkg/common/options" + "open-cluster-management.io/ocm/pkg/registration/spoke" + "open-cluster-management.io/ocm/test/integration/util" +) + +var _ = ginkgo.Describe("Joining Process for aws flow", func() { + var bootstrapKubeconfig string + var managedClusterName string + var hubKubeconfigSecret string + var hubKubeconfigDir string + + ginkgo.BeforeEach(func() { + postfix := rand.String(5) + managedClusterName = fmt.Sprintf("joiningtest-managedcluster-%s", postfix) + hubKubeconfigSecret = fmt.Sprintf("joiningtest-hub-kubeconfig-secret-%s", postfix) + hubKubeconfigDir = path.Join(util.TestDir, fmt.Sprintf("joiningtest-%s", postfix), "hub-kubeconfig") + }) + + assertJoiningSucceed := func() { + ginkgo.It("managedcluster should join successfully for aws flow", func() { + var err error + + managedClusterArn := "arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1" + managedClusterRoleSuffix := "7f8141296c75f2871e3d030f85c35692" + hubClusterArn := "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1" + + // run registration agent + agentOptions := &spoke.SpokeAgentOptions{ + RegistrationAuth: spoke.AwsIrsaAuthType, + HubClusterArn: hubClusterArn, + ManagedClusterArn: managedClusterArn, + ManagedClusterRoleSuffix: managedClusterRoleSuffix, + BootstrapKubeconfig: bootstrapKubeconfig, + HubKubeconfigSecret: hubKubeconfigSecret, + ClusterHealthCheckPeriod: 1 * time.Minute, + } + commOptions := commonoptions.NewAgentOptions() + commOptions.HubKubeconfigDir = hubKubeconfigDir + commOptions.SpokeClusterName = managedClusterName + + cancel := runAgent("joiningtest", agentOptions, commOptions, spokeCfg) + defer cancel() + + // the ManagedCluster CR should be created after bootstrap + gomega.Eventually(func() error { + if _, err := util.GetManagedCluster(clusterClient, managedClusterName); err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + // the csr should not be created for aws flow after bootstrap + gomega.Eventually(func() error { + if _, err := util.FindUnapprovedSpokeCSR(kubeClient, managedClusterName); err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(gomega.HaveOccurred()) + + // simulate hub cluster admin to accept the managedcluster + err = util.AcceptManagedCluster(clusterClient, managedClusterName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = authn.ApproveSpokeClusterCSR(kubeClient, managedClusterName, time.Hour*24) + gomega.Expect(err).To(gomega.HaveOccurred()) + + // the hub kubeconfig secret should be filled after the ManagedCluster is accepted + // TODO: Revisit while implementing slice 3 + //gomega.Eventually(func() error { + // secret, err := util.GetFilledHubKubeConfigSecret(kubeClient, testNamespace, hubKubeconfigSecret) + // if err != nil { + // return err + // } + // + // // check if the proxyURL is set correctly + // proxyURL, err := getProxyURLFromKubeconfigData(secret.Data["kubeconfig"]) + // if err != nil { + // return err + // } + // if proxyURL != expectedProxyURL { + // return fmt.Errorf("expected proxy url %q, but got %q", expectedProxyURL, proxyURL) + // } + // return nil + //}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + // the spoke cluster should have joined condition finally + // TODO: Revisit while implementing slice 3 + //gomega.Eventually(func() error { + // spokeCluster, err := util.GetManagedCluster(clusterClient, managedClusterName) + // if err != nil { + // return err + // } + // if !meta.IsStatusConditionTrue(spokeCluster.Status.Conditions, clusterv1.ManagedClusterConditionJoined) { + // return fmt.Errorf("cluster should be joined") + // } + // return nil + //}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + }) + } + + ginkgo.Context("without proxy", func() { + ginkgo.BeforeEach(func() { + bootstrapKubeconfig = bootstrapKubeConfigFile + }) + assertJoiningSucceed() + }) + +}) From 79f4b72268cc7d5da60a3fc54a98f32ac0b554a7 Mon Sep 17 00:00:00 2001 From: dtclxy64 <70486866+dtclxy64@users.noreply.github.com> Date: Fri, 6 Dec 2024 18:04:10 -0500 Subject: [PATCH 2/4] Fix the make verify logic and linting errors. Signed-off-by: dtclxy64 <70486866+dtclxy64@users.noreply.github.com> --- pkg/registration/register/aws_irsa/aws_irsa.go | 3 ++- pkg/registration/register/interface.go | 1 + pkg/registration/register/secret_controller_test.go | 3 +++ pkg/registration/spoke/spokeagent.go | 1 + .../registration/clusterannotations_aws_test.go | 9 ++++++--- test/integration/registration/clusterannotations_test.go | 5 +++-- .../registration/spokecluster_aws_joining_test.go | 1 + 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pkg/registration/register/aws_irsa/aws_irsa.go b/pkg/registration/register/aws_irsa/aws_irsa.go index bc66075ad..76c55a3d3 100644 --- a/pkg/registration/register/aws_irsa/aws_irsa.go +++ b/pkg/registration/register/aws_irsa/aws_irsa.go @@ -3,7 +3,6 @@ package aws_irsa import ( "context" "fmt" - operatorv1 "open-cluster-management.io/api/operator/v1" "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" @@ -14,6 +13,8 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/klog/v2" + operatorv1 "open-cluster-management.io/api/operator/v1" + "open-cluster-management.io/ocm/pkg/registration/register" ) diff --git a/pkg/registration/register/interface.go b/pkg/registration/register/interface.go index 7283e74bb..56581da94 100644 --- a/pkg/registration/register/interface.go +++ b/pkg/registration/register/interface.go @@ -2,6 +2,7 @@ package register import ( "context" + "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" corev1 "k8s.io/api/core/v1" diff --git a/pkg/registration/register/secret_controller_test.go b/pkg/registration/register/secret_controller_test.go index 26a31aaf7..ef7fdbc80 100644 --- a/pkg/registration/register/secret_controller_test.go +++ b/pkg/registration/register/secret_controller_test.go @@ -167,6 +167,9 @@ type fakeDriver struct { cond *metav1.Condition } +func (f *fakeDriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { +} + func newFakeDriver(secret *corev1.Secret, cond *metav1.Condition, err error) *fakeDriver { return &fakeDriver{ secret: secret, diff --git a/pkg/registration/spoke/spokeagent.go b/pkg/registration/spoke/spokeagent.go index de52fd9fb..a23e28e95 100644 --- a/pkg/registration/spoke/spokeagent.go +++ b/pkg/registration/spoke/spokeagent.go @@ -26,6 +26,7 @@ import ( clusterv1informers "open-cluster-management.io/api/client/cluster/informers/externalversions" clusterv1 "open-cluster-management.io/api/cluster/v1" ocmfeature "open-cluster-management.io/api/feature" + "open-cluster-management.io/ocm/pkg/common/helpers" commonoptions "open-cluster-management.io/ocm/pkg/common/options" "open-cluster-management.io/ocm/pkg/features" diff --git a/test/integration/registration/clusterannotations_aws_test.go b/test/integration/registration/clusterannotations_aws_test.go index 2ea2a64aa..f855c83d7 100644 --- a/test/integration/registration/clusterannotations_aws_test.go +++ b/test/integration/registration/clusterannotations_aws_test.go @@ -2,13 +2,14 @@ package registration_test import ( "fmt" - operatorv1 "open-cluster-management.io/api/operator/v1" "path" "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + operatorv1 "open-cluster-management.io/api/operator/v1" + commonoptions "open-cluster-management.io/ocm/pkg/common/options" "open-cluster-management.io/ocm/pkg/registration/register/aws_irsa" "open-cluster-management.io/ocm/pkg/registration/spoke" @@ -55,11 +56,13 @@ var _ = ginkgo.Describe("Cluster Annotations for aws", func() { } if mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn] != managedClusterArn { - return fmt.Errorf("expected annotation "+operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn+" to be "+managedClusterArn+", got %s", mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn]) + return fmt.Errorf("expected annotation "+operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn+" to be "+ + ""+managedClusterArn+", got %s", mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterArn]) } if mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix] != managedClusterRoleSuffix { - return fmt.Errorf("expected annotation "+operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix+" to be "+managedClusterRoleSuffix+", got %s", mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix]) + return fmt.Errorf("expected annotation "+operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix+" "+ + "to be "+managedClusterRoleSuffix+", got %s", mc.Annotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+aws_irsa.ManagedClusterIAMRoleSuffix]) } return nil diff --git a/test/integration/registration/clusterannotations_test.go b/test/integration/registration/clusterannotations_test.go index 623115619..7d64c4f20 100644 --- a/test/integration/registration/clusterannotations_test.go +++ b/test/integration/registration/clusterannotations_test.go @@ -2,15 +2,16 @@ package registration_test import ( "fmt" - operatorv1 "open-cluster-management.io/api/operator/v1" - "open-cluster-management.io/ocm/pkg/registration/register/aws_irsa" "path" "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + operatorv1 "open-cluster-management.io/api/operator/v1" + commonoptions "open-cluster-management.io/ocm/pkg/common/options" + "open-cluster-management.io/ocm/pkg/registration/register/aws_irsa" "open-cluster-management.io/ocm/pkg/registration/spoke" "open-cluster-management.io/ocm/test/integration/util" ) diff --git a/test/integration/registration/spokecluster_aws_joining_test.go b/test/integration/registration/spokecluster_aws_joining_test.go index 0eb54b4e5..439928f44 100644 --- a/test/integration/registration/spokecluster_aws_joining_test.go +++ b/test/integration/registration/spokecluster_aws_joining_test.go @@ -8,6 +8,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/util/rand" + commonoptions "open-cluster-management.io/ocm/pkg/common/options" "open-cluster-management.io/ocm/pkg/registration/spoke" "open-cluster-management.io/ocm/test/integration/util" From 7373b404930a5eda9b3bd6d29148089951bcb723 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Mon, 9 Dec 2024 13:26:32 +0800 Subject: [PATCH 3/4] Add cluster decorator interface in register And refactor creating to controller to call decorators Signed-off-by: Jian Qiu --- .../register/aws_irsa/aws_irsa.go | 26 +++-- pkg/registration/register/csr/csr.go | 10 +- pkg/registration/register/interface.go | 7 +- .../register/secret_controller_test.go | 8 +- .../spoke/registration/creating_controller.go | 103 ++++++++++-------- .../registration/creating_controller_test.go | 13 ++- pkg/registration/spoke/spokeagent.go | 8 +- 7 files changed, 104 insertions(+), 71 deletions(-) diff --git a/pkg/registration/register/aws_irsa/aws_irsa.go b/pkg/registration/register/aws_irsa/aws_irsa.go index 76c55a3d3..1e85ef1f6 100644 --- a/pkg/registration/register/aws_irsa/aws_irsa.go +++ b/pkg/registration/register/aws_irsa/aws_irsa.go @@ -13,7 +13,9 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/klog/v2" - operatorv1 "open-cluster-management.io/api/operator/v1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + + //operatorv1 "open-cluster-management.io/api/operator/v1" "open-cluster-management.io/ocm/pkg/registration/register" ) @@ -25,8 +27,8 @@ const ( TLSKeyFile = "tls.key" // TLSCertFile is the name of the tls cert file in kubeconfigSecret TLSCertFile = "tls.crt" - ManagedClusterArn = "managed-cluster-arn" - ManagedClusterIAMRoleSuffix = "managed-cluster-iam-role-suffix" + //ManagedClusterArn = "managed-cluster-arn" + //ManagedClusterIAMRoleSuffix = "managed-cluster-iam-role-suffix" ) type AWSIRSADriver struct { @@ -99,15 +101,19 @@ func (c *AWSIRSADriver) IsHubKubeConfigValid(ctx context.Context, secretOption r return true, nil } -func (c *AWSIRSADriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { - if clusterAnnotations == nil { - clusterAnnotations = map[string]string{} - } - - clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterArn] = managedClusterArn - clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterIAMRoleSuffix] = managedClusterRoleSuffix +func (c *AWSIRSADriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + return cluster } +//func (c *AWSIRSADriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { +// if clusterAnnotations == nil { +// clusterAnnotations = map[string]string{} +// } +// +// clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterArn] = managedClusterArn +// clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterIAMRoleSuffix] = managedClusterRoleSuffix +//} + func NewAWSIRSADriver() register.RegisterDriver { return &AWSIRSADriver{} } diff --git a/pkg/registration/register/csr/csr.go b/pkg/registration/register/csr/csr.go index dd095cd1f..7b3627f43 100644 --- a/pkg/registration/register/csr/csr.go +++ b/pkg/registration/register/csr/csr.go @@ -22,6 +22,8 @@ import ( "k8s.io/client-go/util/keyutil" "k8s.io/klog/v2" + clusterv1 "open-cluster-management.io/api/cluster/v1" + "open-cluster-management.io/ocm/pkg/registration/register" ) @@ -266,10 +268,14 @@ func (c *CSRDriver) IsHubKubeConfigValid(ctx context.Context, secretOption regis return isCertificateValid(logger, certData, nil) } -// AddClusterAnnotations noop for CSR driver -func (c *CSRDriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { +func (c *CSRDriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + return cluster } +//// AddClusterAnnotations noop for CSR driver +//func (c *CSRDriver) AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) { +//} + func NewCSRDriver() register.RegisterDriver { return &CSRDriver{} } diff --git a/pkg/registration/register/interface.go b/pkg/registration/register/interface.go index 56581da94..1f2565a50 100644 --- a/pkg/registration/register/interface.go +++ b/pkg/registration/register/interface.go @@ -72,8 +72,11 @@ type RegisterDriver interface { // return nil, nil. InformerHandler(option any) (cache.SharedIndexInformer, factory.EventFilterFunc) - // AddClusterAnnotations adds cluster annotations for non-CSR drivers - AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) + // ManagedClusterDecorator is to change managed cluster metadata or spec during registration process. + ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster + + //// AddClusterAnnotations adds cluster annotations for non-CSR drivers + //AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) } // Approvers is the inteface that each driver should implement on hub side. The hub controller will use this driver diff --git a/pkg/registration/register/secret_controller_test.go b/pkg/registration/register/secret_controller_test.go index ef7fdbc80..a30d85da7 100644 --- a/pkg/registration/register/secret_controller_test.go +++ b/pkg/registration/register/secret_controller_test.go @@ -17,6 +17,8 @@ import ( "k8s.io/client-go/tools/cache" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + clusterv1 "open-cluster-management.io/api/cluster/v1" + testingcommon "open-cluster-management.io/ocm/pkg/common/testing" testinghelpers "open-cluster-management.io/ocm/pkg/registration/helpers/testing" ) @@ -133,7 +135,7 @@ func TestSync(t *testing.T) { for _, c := range testCases { t.Run(c.name, func(t *testing.T) { syncCtx := testingcommon.NewFakeSyncContext(t, "test") - kubeClient := kubefake.NewSimpleClientset(c.secrets...) + kubeClient := kubefake.NewClientset(c.secrets...) c.option.ManagementCoreClient = kubeClient.CoreV1() informerFactory := informers.NewSharedInformerFactory(kubeClient, 10*time.Minute) c.option.ManagementSecretInformer = informerFactory.Core().V1().Secrets().Informer() @@ -198,3 +200,7 @@ func (f *fakeDriver) Process( func (f *fakeDriver) InformerHandler(_ any) (cache.SharedIndexInformer, factory.EventFilterFunc) { return nil, nil } + +func (f *fakeDriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + return cluster +} diff --git a/pkg/registration/spoke/registration/creating_controller.go b/pkg/registration/spoke/registration/creating_controller.go index f37f3b175..9a6c6931d 100644 --- a/pkg/registration/spoke/registration/creating_controller.go +++ b/pkg/registration/spoke/registration/creating_controller.go @@ -23,28 +23,26 @@ var ( CreatingControllerSyncInterval = 60 * time.Minute ) +type ManagedClusterDecorator func(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster + // managedClusterCreatingController creates a ManagedCluster on hub cluster during the spoke agent bootstrap phase type managedClusterCreatingController struct { - clusterName string - spokeExternalServerURLs []string - spokeCABundle []byte - clusterAnnotations map[string]string - hubClusterClient clientset.Interface + clusterName string + clusterDecorators []ManagedClusterDecorator + hubClusterClient clientset.Interface } // NewManagedClusterCreatingController creates a new managedClusterCreatingController on the managed cluster. func NewManagedClusterCreatingController( - clusterName string, spokeExternalServerURLs []string, annotations map[string]string, - spokeCABundle []byte, + clusterName string, + decorators []ManagedClusterDecorator, hubClusterClient clientset.Interface, recorder events.Recorder) factory.Controller { c := &managedClusterCreatingController{ - clusterName: clusterName, - spokeExternalServerURLs: spokeExternalServerURLs, - spokeCABundle: spokeCABundle, - clusterAnnotations: commonhelpers.FilterClusterAnnotations(annotations), - hubClusterClient: hubClusterClient, + clusterName: clusterName, + hubClusterClient: hubClusterClient, + clusterDecorators: decorators, } return factory.New(). @@ -69,20 +67,12 @@ func (c *managedClusterCreatingController) sync(ctx context.Context, syncCtx fac if errors.IsNotFound(err) { managedCluster := &clusterv1.ManagedCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: c.clusterName, - Annotations: c.clusterAnnotations, + Name: c.clusterName, }, } - if len(c.spokeExternalServerURLs) != 0 { - var managedClusterClientConfigs []clusterv1.ClientConfig - for _, serverURL := range c.spokeExternalServerURLs { - managedClusterClientConfigs = append(managedClusterClientConfigs, clusterv1.ClientConfig{ - URL: serverURL, - CABundle: c.spokeCABundle, - }) - } - managedCluster.Spec.ManagedClusterClientConfigs = managedClusterClientConfigs + for _, decorator := range c.clusterDecorators { + managedCluster = decorator(managedCluster) } _, err = c.hubClusterClient.ClusterV1().ManagedClusters().Create(ctx, managedCluster, metav1.CreateOptions{}) @@ -94,37 +84,17 @@ func (c *managedClusterCreatingController) sync(ctx context.Context, syncCtx fac return nil } - // do not update ManagedClusterClientConfigs in ManagedCluster if spokeExternalServerURLs is empty - if len(c.spokeExternalServerURLs) == 0 { - return nil + managedCluster := existingCluster.DeepCopy() + for _, decorator := range c.clusterDecorators { + managedCluster = decorator(managedCluster) } - // merge ClientConfig - managedClusterClientConfigs := existingCluster.Spec.ManagedClusterClientConfigs - for _, serverURL := range c.spokeExternalServerURLs { - isIncludeByExisting := false - for _, existingClientConfig := range existingCluster.Spec.ManagedClusterClientConfigs { - if serverURL == existingClientConfig.URL { - isIncludeByExisting = true - break - } - } - - if !isIncludeByExisting { - managedClusterClientConfigs = append(managedClusterClientConfigs, clusterv1.ClientConfig{ - URL: serverURL, - CABundle: c.spokeCABundle, - }) - } - } - if len(existingCluster.Spec.ManagedClusterClientConfigs) == len(managedClusterClientConfigs) { + if len(existingCluster.Spec.ManagedClusterClientConfigs) == len(managedCluster.Spec.ManagedClusterClientConfigs) { return nil } // update ManagedClusterClientConfigs in ManagedCluster - clusterCopy := existingCluster.DeepCopy() - clusterCopy.Spec.ManagedClusterClientConfigs = managedClusterClientConfigs - _, err = c.hubClusterClient.ClusterV1().ManagedClusters().Update(ctx, clusterCopy, metav1.UpdateOptions{}) + _, err = c.hubClusterClient.ClusterV1().ManagedClusters().Update(ctx, managedCluster, metav1.UpdateOptions{}) // ManagedClusterClientConfigs in ManagedCluster is only allowed updated during bootstrap. // After bootstrap secret expired, an unauthorized error will be got, skip it if skipUnauthorizedError(err) != nil { @@ -141,3 +111,40 @@ func skipUnauthorizedError(err error) error { return err } + +func AnnotationDecorator(annotations map[string]string) ManagedClusterDecorator { + return func(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + filteredAnnotations := commonhelpers.FilterClusterAnnotations(annotations) + if cluster.Annotations == nil { + cluster.Annotations = make(map[string]string) + } + for key, value := range filteredAnnotations { + cluster.Annotations[key] = value + } + return cluster + } +} + +// ClientConfigDecorator merge ClientConfig +func ClientConfigDecorator(externalServerURLs []string, caBundle []byte) ManagedClusterDecorator { + return func(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + for _, serverURL := range externalServerURLs { + isIncludeByExisting := false + for _, existingClientConfig := range cluster.Spec.ManagedClusterClientConfigs { + if serverURL == existingClientConfig.URL { + isIncludeByExisting = true + break + } + } + + if !isIncludeByExisting { + cluster.Spec.ManagedClusterClientConfigs = append( + cluster.Spec.ManagedClusterClientConfigs, clusterv1.ClientConfig{ + URL: serverURL, + CABundle: caBundle, + }) + } + } + return cluster + } +} diff --git a/pkg/registration/spoke/registration/creating_controller_test.go b/pkg/registration/spoke/registration/creating_controller_test.go index 7201e2485..3ed52abd8 100644 --- a/pkg/registration/spoke/registration/creating_controller_test.go +++ b/pkg/registration/spoke/registration/creating_controller_test.go @@ -59,13 +59,14 @@ func TestCreateSpokeCluster(t *testing.T) { t.Run(c.name, func(t *testing.T) { clusterClient := clusterfake.NewSimpleClientset(c.startingObjects...) ctrl := managedClusterCreatingController{ - clusterName: testinghelpers.TestManagedClusterName, - spokeExternalServerURLs: []string{testSpokeExternalServerUrl}, - spokeCABundle: []byte("testcabundle"), - hubClusterClient: clusterClient, - clusterAnnotations: map[string]string{ - "agent.open-cluster-management.io/test": "true", + clusterName: testinghelpers.TestManagedClusterName, + clusterDecorators: []ManagedClusterDecorator{ + AnnotationDecorator(map[string]string{ + "agent.open-cluster-management.io/test": "true", + }), + ClientConfigDecorator([]string{testSpokeExternalServerUrl}, []byte("testcabundle")), }, + hubClusterClient: clusterClient, } syncErr := ctrl.sync(context.TODO(), testingcommon.NewFakeSyncContext(t, "")) diff --git a/pkg/registration/spoke/spokeagent.go b/pkg/registration/spoke/spokeagent.go index a23e28e95..16ad591f1 100644 --- a/pkg/registration/spoke/spokeagent.go +++ b/pkg/registration/spoke/spokeagent.go @@ -243,8 +243,12 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context, // start a SpokeClusterCreatingController to make sure there is a spoke cluster on hub cluster spokeClusterCreatingController := registration.NewManagedClusterCreatingController( - o.agentOptions.SpokeClusterName, o.registrationOption.SpokeExternalServerURLs, o.registrationOption.ClusterAnnotations, - spokeClusterCABundle, + o.agentOptions.SpokeClusterName, + []registration.ManagedClusterDecorator{ + registration.AnnotationDecorator(o.registrationOption.ClusterAnnotations), + registration.ClientConfigDecorator(o.registrationOption.SpokeExternalServerURLs, spokeClusterCABundle), + o.driver.ManagedClusterDecorator, + }, bootstrapClusterClient, recorder, ) From 69a47ed19e63b0aaadf084295e0fa8d6d6bfeb2c Mon Sep 17 00:00:00 2001 From: suvaanshkumar Date: Mon, 9 Dec 2024 17:57:03 -0500 Subject: [PATCH 4/4] updating ManagedClusterDecorator signature and updating clusterannotation --- pkg/registration/register/aws_irsa/aws_irsa.go | 12 +++++++++--- pkg/registration/register/csr/csr.go | 2 +- pkg/registration/register/interface.go | 2 +- pkg/registration/register/secret_controller_test.go | 2 +- .../spoke/registration/creating_controller.go | 10 +++++----- pkg/registration/spoke/spokeagent.go | 2 +- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg/registration/register/aws_irsa/aws_irsa.go b/pkg/registration/register/aws_irsa/aws_irsa.go index 1e85ef1f6..8e4a87efb 100644 --- a/pkg/registration/register/aws_irsa/aws_irsa.go +++ b/pkg/registration/register/aws_irsa/aws_irsa.go @@ -3,6 +3,7 @@ package aws_irsa import ( "context" "fmt" + operatorv1 "open-cluster-management.io/api/operator/v1" "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" @@ -27,8 +28,8 @@ const ( TLSKeyFile = "tls.key" // TLSCertFile is the name of the tls cert file in kubeconfigSecret TLSCertFile = "tls.crt" - //ManagedClusterArn = "managed-cluster-arn" - //ManagedClusterIAMRoleSuffix = "managed-cluster-iam-role-suffix" + ManagedClusterArn = "managed-cluster-arn" + ManagedClusterIAMRoleSuffix = "managed-cluster-iam-role-suffix" ) type AWSIRSADriver struct { @@ -101,7 +102,12 @@ func (c *AWSIRSADriver) IsHubKubeConfigValid(ctx context.Context, secretOption r return true, nil } -func (c *AWSIRSADriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { +func (c *AWSIRSADriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster { + if clusterAnnotations == nil { + clusterAnnotations = map[string]string{} + } + clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterArn] = managedClusterArn + clusterAnnotations[operatorv1.ClusterAnnotationsKeyPrefix+"/"+ManagedClusterIAMRoleSuffix] = managedClusterRoleSuffix return cluster } diff --git a/pkg/registration/register/csr/csr.go b/pkg/registration/register/csr/csr.go index 7b3627f43..73fba6d44 100644 --- a/pkg/registration/register/csr/csr.go +++ b/pkg/registration/register/csr/csr.go @@ -268,7 +268,7 @@ func (c *CSRDriver) IsHubKubeConfigValid(ctx context.Context, secretOption regis return isCertificateValid(logger, certData, nil) } -func (c *CSRDriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { +func (c *CSRDriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster { return cluster } diff --git a/pkg/registration/register/interface.go b/pkg/registration/register/interface.go index 1f2565a50..5f5eb6bfe 100644 --- a/pkg/registration/register/interface.go +++ b/pkg/registration/register/interface.go @@ -73,7 +73,7 @@ type RegisterDriver interface { InformerHandler(option any) (cache.SharedIndexInformer, factory.EventFilterFunc) // ManagedClusterDecorator is to change managed cluster metadata or spec during registration process. - ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster + ManagedClusterDecorator(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster //// AddClusterAnnotations adds cluster annotations for non-CSR drivers //AddClusterAnnotations(clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) diff --git a/pkg/registration/register/secret_controller_test.go b/pkg/registration/register/secret_controller_test.go index a30d85da7..4816e7ca9 100644 --- a/pkg/registration/register/secret_controller_test.go +++ b/pkg/registration/register/secret_controller_test.go @@ -201,6 +201,6 @@ func (f *fakeDriver) InformerHandler(_ any) (cache.SharedIndexInformer, factory. return nil, nil } -func (f *fakeDriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { +func (f *fakeDriver) ManagedClusterDecorator(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster { return cluster } diff --git a/pkg/registration/spoke/registration/creating_controller.go b/pkg/registration/spoke/registration/creating_controller.go index 9a6c6931d..cfc95cc6b 100644 --- a/pkg/registration/spoke/registration/creating_controller.go +++ b/pkg/registration/spoke/registration/creating_controller.go @@ -23,7 +23,7 @@ var ( CreatingControllerSyncInterval = 60 * time.Minute ) -type ManagedClusterDecorator func(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster +type ManagedClusterDecorator func(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster // managedClusterCreatingController creates a ManagedCluster on hub cluster during the spoke agent bootstrap phase type managedClusterCreatingController struct { @@ -72,7 +72,7 @@ func (c *managedClusterCreatingController) sync(ctx context.Context, syncCtx fac } for _, decorator := range c.clusterDecorators { - managedCluster = decorator(managedCluster) + managedCluster = decorator(managedCluster, nil, "", "") } _, err = c.hubClusterClient.ClusterV1().ManagedClusters().Create(ctx, managedCluster, metav1.CreateOptions{}) @@ -86,7 +86,7 @@ func (c *managedClusterCreatingController) sync(ctx context.Context, syncCtx fac managedCluster := existingCluster.DeepCopy() for _, decorator := range c.clusterDecorators { - managedCluster = decorator(managedCluster) + managedCluster = decorator(managedCluster, nil, "", "") } if len(existingCluster.Spec.ManagedClusterClientConfigs) == len(managedCluster.Spec.ManagedClusterClientConfigs) { @@ -113,7 +113,7 @@ func skipUnauthorizedError(err error) error { } func AnnotationDecorator(annotations map[string]string) ManagedClusterDecorator { - return func(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + return func(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster { filteredAnnotations := commonhelpers.FilterClusterAnnotations(annotations) if cluster.Annotations == nil { cluster.Annotations = make(map[string]string) @@ -127,7 +127,7 @@ func AnnotationDecorator(annotations map[string]string) ManagedClusterDecorator // ClientConfigDecorator merge ClientConfig func ClientConfigDecorator(externalServerURLs []string, caBundle []byte) ManagedClusterDecorator { - return func(cluster *clusterv1.ManagedCluster) *clusterv1.ManagedCluster { + return func(cluster *clusterv1.ManagedCluster, clusterAnnotations map[string]string, managedClusterArn string, managedClusterRoleSuffix string) *clusterv1.ManagedCluster { for _, serverURL := range externalServerURLs { isIncludeByExisting := false for _, existingClientConfig := range cluster.Spec.ManagedClusterClientConfigs { diff --git a/pkg/registration/spoke/spokeagent.go b/pkg/registration/spoke/spokeagent.go index 16ad591f1..ebe68fdbb 100644 --- a/pkg/registration/spoke/spokeagent.go +++ b/pkg/registration/spoke/spokeagent.go @@ -197,7 +197,7 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context, registerDriver = csr.NewCSRDriver() } - registerDriver.AddClusterAnnotations(registrationOption.ClusterAnnotations, registrationOption.ManagedClusterArn, registrationOption.ManagedClusterRoleSuffix) + //registerDriver.AddClusterAnnotations(registrationOption.ClusterAnnotations, registrationOption.ManagedClusterArn, registrationOption.ManagedClusterRoleSuffix) o.driver = registerDriver // get spoke cluster CA bundle