From 3bcc084394d496bfc1aff30613c71f84130c1a21 Mon Sep 17 00:00:00 2001 From: Jian Zhu <36154065+zhujian7@users.noreply.github.com> Date: Thu, 11 May 2023 16:45:06 +0800 Subject: [PATCH] replace the last applied managed cluster api server url when it changes (#605) * update vendor cluster-lifecycle-api Signed-off-by: zhujian * replace the last applied managed cluster api server url when it changes Signed-off-by: zhujian * check last applied url after updating client config Signed-off-by: zhujian --------- Signed-off-by: zhujian --- ...management.io_managedclusterinfos.crd.yaml | 3 + go.mod | 2 +- go.sum | 4 +- .../clusterca/clusterca_controller.go | 57 ++++++++----- .../clusterca/clusterca_controller_test.go | 80 ++++++++++++++++--- .../clusterinfo/distributionInfo_sync.go | 3 + .../clusterinfo/distributionInfo_sync_test.go | 53 +++++++----- ...management.io_managedclusterinfos.crd.yaml | 3 + .../clusterinfo/v1beta1/types.go | 6 ++ .../zz_generated.swagger_doc_generated.go | 1 + vendor/modules.txt | 2 +- 11 files changed, 160 insertions(+), 54 deletions(-) diff --git a/deploy/foundation/hub/resources/crds/internal.open-cluster-management.io_managedclusterinfos.crd.yaml b/deploy/foundation/hub/resources/crds/internal.open-cluster-management.io_managedclusterinfos.crd.yaml index 618a2e29d..3bd80d017 100644 --- a/deploy/foundation/hub/resources/crds/internal.open-cluster-management.io_managedclusterinfos.crd.yaml +++ b/deploy/foundation/hub/resources/crds/internal.open-cluster-management.io_managedclusterinfos.crd.yaml @@ -134,6 +134,9 @@ spec: desiredVersion: description: DesiredVersion is the version that the cluster is reconciling towards. Deprecated in release 2.3 and will be removed in the future. User Desired instead. type: string + lastAppliedAPIServerURL: + description: LastAppliedAPIServerURL is a valid URI with scheme 'https', address and optionally a port (defaulting to 443). it can be used by components like the web console to tell users where to find the Kubernetes API. This is the api server url that has been applied to the managedcluster resource successfully + type: string managedClusterClientConfig: description: Controller will sync this field to managedcluster's ManagedClusterClientConfigs type: object diff --git a/go.mod b/go.mod index 0e3fc486c..2a29ce452 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/prometheus/common v0.39.0 github.com/smartystreets/goconvey v1.7.2 github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace - github.com/stolostron/cluster-lifecycle-api v0.0.0-20230222063645-5b18b26381ff + github.com/stolostron/cluster-lifecycle-api v0.0.0-20230510064049-824d580bc143 github.com/stretchr/testify v1.8.1 golang.org/x/net v0.8.0 k8s.io/api v0.26.2 diff --git a/go.sum b/go.sum index 5c5199c8f..20a45ce20 100644 --- a/go.sum +++ b/go.sum @@ -437,8 +437,8 @@ github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlk github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stolostron/cluster-lifecycle-api v0.0.0-20230222063645-5b18b26381ff h1:psHMqHMefkFtr7y4JT5tz8oKwmChvLJe/cHUUM9tcMI= -github.com/stolostron/cluster-lifecycle-api v0.0.0-20230222063645-5b18b26381ff/go.mod h1:pNeVzujoHsTHDloNHVfp1QPYlQy8MkXMuuZme96/x8M= +github.com/stolostron/cluster-lifecycle-api v0.0.0-20230510064049-824d580bc143 h1:lPHc9Kaa3bZs4domKspWKyCZmN5uL4b0RWZXDhEiVgY= +github.com/stolostron/cluster-lifecycle-api v0.0.0-20230510064049-824d580bc143/go.mod h1:pNeVzujoHsTHDloNHVfp1QPYlQy8MkXMuuZme96/x8M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= diff --git a/pkg/controllers/clusterca/clusterca_controller.go b/pkg/controllers/clusterca/clusterca_controller.go index 9b88bc462..22b1bebd5 100644 --- a/pkg/controllers/clusterca/clusterca_controller.go +++ b/pkg/controllers/clusterca/clusterca_controller.go @@ -9,7 +9,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog" clusterv1 "open-cluster-management.io/api/cluster/v1" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -56,50 +55,70 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } func (r *ClusterCaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - clusterinfo := &clusterinfov1beta1.ManagedClusterInfo{} + clusterInfo := &clusterinfov1beta1.ManagedClusterInfo{} - err := r.client.Get(ctx, req.NamespacedName, clusterinfo) + err := r.client.Get(ctx, req.NamespacedName, clusterInfo) if err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } cluster := &clusterv1.ManagedCluster{} - err = r.client.Get(ctx, types.NamespacedName{Name: clusterinfo.Name}, cluster) + err = r.client.Get(ctx, types.NamespacedName{Name: clusterInfo.Name}, cluster) if err != nil { return ctrl.Result{}, err } - updatedClientConfig, needUpdate := updateClientConfig(cluster.Spec.ManagedClusterClientConfigs, clusterinfo.Status.DistributionInfo.OCP.ManagedClusterClientConfig) - if !needUpdate { - return ctrl.Result{}, nil - } + updatedClientConfig, needUpdate := updateClientConfig( + cluster.Spec.ManagedClusterClientConfigs, + clusterInfo.Status.DistributionInfo.OCP.ManagedClusterClientConfig, + clusterInfo.Status.DistributionInfo.OCP.LastAppliedAPIServerURL) - cluster.Spec.ManagedClusterClientConfigs = updatedClientConfig - err = r.client.Update(ctx, cluster, &client.UpdateOptions{}) + if needUpdate { + cluster.Spec.ManagedClusterClientConfigs = updatedClientConfig + err = r.client.Update(ctx, cluster, &client.UpdateOptions{}) + if err != nil { + return ctrl.Result{}, err + } + } - if err != nil { - return ctrl.Result{}, err + if clusterInfo.Status.DistributionInfo.OCP.LastAppliedAPIServerURL != + clusterInfo.Status.DistributionInfo.OCP.ManagedClusterClientConfig.URL { + clusterInfo.Status.DistributionInfo.OCP.LastAppliedAPIServerURL = + clusterInfo.Status.DistributionInfo.OCP.ManagedClusterClientConfig.URL + err = r.client.Status().Update(ctx, clusterInfo) + if err != nil { + return ctrl.Result{}, err + } } return ctrl.Result{}, nil } -// updateClientConfig merge config from clusterinfoconfigs to clusterconfigs -func updateClientConfig(clusterConfigs []clusterv1.ClientConfig, clusterinfoConfig clusterinfov1beta1.ClientConfig) ([]clusterv1.ClientConfig, bool) { +// updateClientConfig merge config from clusterinfoconfigs to clusterconfigs, it returns 2 parameters: +// - the updated cluster clientConfig +// - whether it is needed to update the managedCluster object +func updateClientConfig(clusterConfigs []clusterv1.ClientConfig, clusterinfoConfig clusterinfov1beta1.ClientConfig, + lastAppliedURL string) ([]clusterv1.ClientConfig, bool) { // If clusterinfo config is null return clusterconfigs if len(clusterinfoConfig.URL) == 0 { return clusterConfigs, false } + // The lastAppliedURL comes from the value of infrastructures config(status.apiServerURL), if the + // infrastructures config value changes, here we will replace the old clientConfig URL instead of + // prepending a new one. for i, cluConfig := range clusterConfigs { - if cluConfig.URL != clusterinfoConfig.URL { + if cluConfig.URL != clusterinfoConfig.URL && cluConfig.URL != lastAppliedURL { continue } - if !reflect.DeepEqual(cluConfig.CABundle, clusterinfoConfig.CABundle) { - clusterConfigs[i].CABundle = clusterinfoConfig.CABundle - return clusterConfigs, true + + if reflect.DeepEqual(cluConfig.CABundle, clusterinfoConfig.CABundle) && cluConfig.URL == clusterinfoConfig.URL { + return clusterConfigs, false } - return clusterConfigs, false + + clusterConfigs[i].URL = clusterinfoConfig.URL + clusterConfigs[i].CABundle = clusterinfoConfig.CABundle + return clusterConfigs, true } // do not have ca bundle in cluster config, *prepend* the clusterinfo config to the managed cluster config. diff --git a/pkg/controllers/clusterca/clusterca_controller_test.go b/pkg/controllers/clusterca/clusterca_controller_test.go index 503a0ee76..2fa943f3e 100644 --- a/pkg/controllers/clusterca/clusterca_controller_test.go +++ b/pkg/controllers/clusterca/clusterca_controller_test.go @@ -13,15 +13,16 @@ func Test_updateClientConfig(t *testing.T) { name string clusterConfig []clusterv1.ClientConfig clusterinfoConfig clusterinfov1beta1.ClientConfig + lastAppliedURL string wantConfig []clusterv1.ClientConfig - wantUpdate bool + wantUpdateMC bool }{ { name: "all nil", clusterConfig: []clusterv1.ClientConfig{}, clusterinfoConfig: clusterinfov1beta1.ClientConfig{}, wantConfig: []clusterv1.ClientConfig{}, - wantUpdate: false, + wantUpdateMC: false, }, { name: "clusterconfig is null", @@ -36,7 +37,7 @@ func Test_updateClientConfig(t *testing.T) { CABundle: []byte("ca data"), }, }, - wantUpdate: true, + wantUpdateMC: true, }, { name: "clusterinfoconfig is null", @@ -53,7 +54,7 @@ func Test_updateClientConfig(t *testing.T) { CABundle: []byte("ca data"), }, }, - wantUpdate: false, + wantUpdateMC: false, }, { name: "both of them is not null, and order matters", @@ -77,7 +78,7 @@ func Test_updateClientConfig(t *testing.T) { CABundle: []byte("ca data"), }, }, - wantUpdate: true, + wantUpdateMC: true, }, { name: "update cluster config", @@ -105,17 +106,78 @@ func Test_updateClientConfig(t *testing.T) { CABundle: []byte("new info data"), }, }, - wantUpdate: true, + wantUpdateMC: true, + }, + { + name: "replace the last applied url with new url", + clusterConfig: []clusterv1.ClientConfig{ + { + URL: "https:clusterurl.com:443", + CABundle: []byte("ca data"), + }, + }, + clusterinfoConfig: clusterinfov1beta1.ClientConfig{ + URL: "https:infourl.com:443", + CABundle: []byte("ca data"), + }, + lastAppliedURL: "https:clusterurl.com:443", + wantConfig: []clusterv1.ClientConfig{ + { + URL: "https:infourl.com:443", + CABundle: []byte("ca data"), + }, + }, + wantUpdateMC: true, + }, + { + name: "replace the last applied url with new url, order not change", + clusterConfig: []clusterv1.ClientConfig{ + { + URL: "https:clusterurl-1.com:443", + CABundle: []byte("new info data"), + }, + { + URL: "https:clusterurl.com:443", + CABundle: []byte("ca data"), + }, + { + URL: "https:clusterurl-2.com:443", + CABundle: []byte("new info data"), + }, + }, + clusterinfoConfig: clusterinfov1beta1.ClientConfig{ + URL: "https:infourl.com:443", + CABundle: []byte("ca data"), + }, + lastAppliedURL: "https:clusterurl.com:443", + wantConfig: []clusterv1.ClientConfig{ + { + URL: "https:clusterurl-1.com:443", + CABundle: []byte("new info data"), + }, + { + URL: "https:infourl.com:443", + CABundle: []byte("ca data"), + }, + { + URL: "https:clusterurl-2.com:443", + CABundle: []byte("new info data"), + }, + }, + wantUpdateMC: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - returnConfig, needUpdate := updateClientConfig(test.clusterConfig, test.clusterinfoConfig) - if needUpdate != test.wantUpdate { - t.Errorf("case: %v, needupdate is: %v, wantUpdate is : %v", test.name, needUpdate, test.wantUpdate) + returnConfig, needUpdate := updateClientConfig( + test.clusterConfig, test.clusterinfoConfig, test.lastAppliedURL) + if needUpdate != test.wantUpdateMC { + t.Errorf("case: %v, expected update managed cluster: %v, but got : %v", + test.name, test.wantUpdateMC, needUpdate) return } + if !reflect.DeepEqual(returnConfig, test.wantConfig) { t.Errorf("case:%v, returnConfig:%v, wantConfig:%v.", test.name, returnConfig, test.wantConfig) } diff --git a/pkg/klusterlet/clusterinfo/distributionInfo_sync.go b/pkg/klusterlet/clusterinfo/distributionInfo_sync.go index 4599271b9..c49256648 100644 --- a/pkg/klusterlet/clusterinfo/distributionInfo_sync.go +++ b/pkg/klusterlet/clusterinfo/distributionInfo_sync.go @@ -33,6 +33,7 @@ func (s *distributionInfoSyncer) sync(ctx context.Context, clusterInfo *clusterv } func (s *distributionInfoSyncer) syncOCPDistributionInfo(ctx context.Context, clusterInfoStatus *clusterv1beta1.ClusterInfoStatus) error { + lastAppliedAPIServerURL := clusterInfoStatus.DistributionInfo.OCP.LastAppliedAPIServerURL clusterInfoStatus.DistributionInfo = clusterv1beta1.DistributionInfo{ Type: clusterv1beta1.DistributionTypeOCP, } @@ -54,6 +55,8 @@ func (s *distributionInfoSyncer) syncOCPDistributionInfo(ctx context.Context, cl URL: string(clusterVersion.Status.Desired.URL), Channels: clusterVersion.Status.Desired.Channels, }, + // do not override the LastAppliedAPIServerURL to empty + LastAppliedAPIServerURL: lastAppliedAPIServerURL, } availableUpdates := clusterVersion.Status.AvailableUpdates diff --git a/pkg/klusterlet/clusterinfo/distributionInfo_sync_test.go b/pkg/klusterlet/clusterinfo/distributionInfo_sync_test.go index 14a35c2e7..2ea63fc49 100644 --- a/pkg/klusterlet/clusterinfo/distributionInfo_sync_test.go +++ b/pkg/klusterlet/clusterinfo/distributionInfo_sync_test.go @@ -197,19 +197,20 @@ func newConfigV1Client(version string, failingStatus bool) openshiftclientset.In func Test_distributionInfo_syncer(t *testing.T) { tests := []struct { - name string - managedClusterInfo *v1beta1.ManagedClusterInfo - configV1Client openshiftclientset.Interface - expectChannel string - expectDesiredVersion string - expectDesiredChannelLen int - expectUpgradeFail bool - expectAvailableUpdatesLen int - expectChannelAndURL bool - expectHistoryLen int - expectError string - expectVersion string - claims []runtime.Object + name string + managedClusterInfo *v1beta1.ManagedClusterInfo + configV1Client openshiftclientset.Interface + expectChannel string + expectDesiredVersion string + expectDesiredChannelLen int + expectUpgradeFail bool + expectAvailableUpdatesLen int + expectChannelAndURL bool + expectLastAppliedAPIServerURL string + expectHistoryLen int + expectError string + expectVersion string + claims []runtime.Object }{ { name: "OCP4.x", @@ -217,17 +218,24 @@ func Test_distributionInfo_syncer(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "c1", Namespace: "c1"}, Status: v1beta1.ClusterInfoStatus{ KubeVendor: v1beta1.KubeVendorOpenShift, + DistributionInfo: v1beta1.DistributionInfo{ + Type: v1beta1.DistributionTypeOCP, + OCP: v1beta1.OCPDistributionInfo{ + LastAppliedAPIServerURL: "http://test-last-applied-url", + }, + }, }, }, - configV1Client: newConfigV1Client("4.x", false), - expectChannel: "stable-4.5", - expectDesiredVersion: "4.6.8", - expectDesiredChannelLen: 7, - expectUpgradeFail: false, - expectAvailableUpdatesLen: 2, - expectChannelAndURL: true, - expectHistoryLen: 3, - expectError: "", + configV1Client: newConfigV1Client("4.x", false), + expectChannel: "stable-4.5", + expectDesiredVersion: "4.6.8", + expectDesiredChannelLen: 7, + expectUpgradeFail: false, + expectAvailableUpdatesLen: 2, + expectChannelAndURL: true, + expectHistoryLen: 3, + expectLastAppliedAPIServerURL: "http://test-last-applied-url", + expectError: "", claims: []runtime.Object{ newClaim(clusterclaim.ClaimOpenshiftVersion, "4.6.8"), newClaim(clusterclaim.ClaimOCMKubeVersion, "v1.20.0"), @@ -292,6 +300,7 @@ func Test_distributionInfo_syncer(t *testing.T) { assert.Equal(t, len(info.Desired.Channels), test.expectDesiredChannelLen) assert.Equal(t, info.UpgradeFailed, test.expectUpgradeFail) assert.Equal(t, len(info.VersionAvailableUpdates), test.expectAvailableUpdatesLen) + assert.Equal(t, info.LastAppliedAPIServerURL, test.expectLastAppliedAPIServerURL) //get the latest succeed version assert.Equal(t, test.expectVersion, info.Version) diff --git a/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/internal.open-cluster-management.io_managedclusterinfos.crd.yaml b/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/internal.open-cluster-management.io_managedclusterinfos.crd.yaml index 618a2e29d..3bd80d017 100644 --- a/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/internal.open-cluster-management.io_managedclusterinfos.crd.yaml +++ b/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/internal.open-cluster-management.io_managedclusterinfos.crd.yaml @@ -134,6 +134,9 @@ spec: desiredVersion: description: DesiredVersion is the version that the cluster is reconciling towards. Deprecated in release 2.3 and will be removed in the future. User Desired instead. type: string + lastAppliedAPIServerURL: + description: LastAppliedAPIServerURL is a valid URI with scheme 'https', address and optionally a port (defaulting to 443). it can be used by components like the web console to tell users where to find the Kubernetes API. This is the api server url that has been applied to the managedcluster resource successfully + type: string managedClusterClientConfig: description: Controller will sync this field to managedcluster's ManagedClusterClientConfigs type: object diff --git a/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/types.go b/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/types.go index e07d3f7c5..92da8d9e0 100644 --- a/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/types.go +++ b/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/types.go @@ -218,6 +218,12 @@ type OCPDistributionInfo struct { // Controller will sync this field to managedcluster's ManagedClusterClientConfigs // +optional ManagedClusterClientConfig ClientConfig `json:"managedClusterClientConfig,omitempty"` + + // LastAppliedAPIServerURL is a valid URI with scheme 'https', address and optionally + // a port (defaulting to 443). it can be used by components like the web console to + // tell users where to find the Kubernetes API. + // This is the api server url that has been applied to the managedcluster resource successfully + LastAppliedAPIServerURL string `json:"lastAppliedAPIServerURL"` } // DistributionType is type of distribution diff --git a/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/zz_generated.swagger_doc_generated.go b/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/zz_generated.swagger_doc_generated.go index eb4e8961c..1734fa67b 100644 --- a/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/zz_generated.swagger_doc_generated.go +++ b/vendor/github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1/zz_generated.swagger_doc_generated.go @@ -110,6 +110,7 @@ var map_OCPDistributionInfo = map[string]string{ "versionAvailableUpdates": "VersionAvailableUpdates contains the list of updates that are appropriate for this cluster. This list may be empty if no updates are recommended, if the update service is unavailable, or if an invalid channel has been specified.", "versionHistory": "VersionHistory contains a list of the most recent versions applied to the cluster. This value may be empty during cluster startup, and then will be updated when a new update is being applied. The newest update is first in the list and it is ordered by recency. Updates in the history have state Completed if the rollout completed - if an update was failing or halfway applied the state will be Partial. Only a limited amount of update history is preserved.", "managedClusterClientConfig": "Controller will sync this field to managedcluster's ManagedClusterClientConfigs", + "lastAppliedAPIServerURL": "LastAppliedAPIServerURL is a valid URI with scheme 'https', address and optionally a port (defaulting to 443). it can be used by components like the web console to tell users where to find the Kubernetes API. This is the api server url that has been applied to the managedcluster resource successfully", } func (OCPDistributionInfo) SwaggerDoc() map[string]string { diff --git a/vendor/modules.txt b/vendor/modules.txt index 526f337d7..5367dde64 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -405,7 +405,7 @@ github.com/spf13/pflag # github.com/stoewer/go-strcase v1.2.0 ## explicit; go 1.11 github.com/stoewer/go-strcase -# github.com/stolostron/cluster-lifecycle-api v0.0.0-20230222063645-5b18b26381ff +# github.com/stolostron/cluster-lifecycle-api v0.0.0-20230510064049-824d580bc143 ## explicit; go 1.18 github.com/stolostron/cluster-lifecycle-api/action/v1beta1 github.com/stolostron/cluster-lifecycle-api/clusterinfo/v1beta1