From dbe12e22b47666a2a98d49686fe19bb83ba0a41a Mon Sep 17 00:00:00 2001 From: Qing Hao Date: Thu, 5 Dec 2024 10:58:52 +0800 Subject: [PATCH] fix e2e test issues when sno leader election enabled (#434) Signed-off-by: Qing Hao --- test/e2e/autoimport_test.go | 3 ++ test/e2e/cleanup_test.go | 3 ++ test/e2e/e2e_suite_test.go | 65 +++++++++++++++++++++++++++++-- test/e2e/hostedcluster_test.go | 4 ++ test/e2e/klusterletconfig_test.go | 51 +++++++++++++++++++++--- test/e2e/util/util.go | 4 +- 6 files changed, 119 insertions(+), 11 deletions(-) diff --git a/test/e2e/autoimport_test.go b/test/e2e/autoimport_test.go index 3e630e10..fb68783d 100644 --- a/test/e2e/autoimport_test.go +++ b/test/e2e/autoimport_test.go @@ -258,6 +258,7 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f }) assertManagedClusterImportSecretApplied(managedClusterName) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) configName := "autoimport-config" testcluster := fmt.Sprintf("custom-%s", managedClusterName) @@ -292,6 +293,8 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f assertManagedClusterImportSecretCreated(testcluster, "other") assertManagedClusterImportSecretApplied(testcluster) assertManagedClusterAvailable(testcluster) + klusterletName := fmt.Sprintf("%s-klusterlet", testcluster) + assertManifestworkFinalizer(testcluster, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") AssertKlusterletNamespace(testcluster, "klusterlet-local", "open-cluster-management-local") diff --git a/test/e2e/cleanup_test.go b/test/e2e/cleanup_test.go index cd41bfd7..e8b0b15d 100644 --- a/test/e2e/cleanup_test.go +++ b/test/e2e/cleanup_test.go @@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun _, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // check the work has added finalizer before detaching the cluster + assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup") + // detach the cluster err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d424646a..89455f50 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -476,6 +476,7 @@ func assertHostedManagedClusterImportSecret(managedClusterName string) { } func assertManagedClusterDeleted(clusterName string) { + ginkgo.By(fmt.Sprintf("Delete the managed cluster %s", clusterName), func() { err := hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { @@ -738,11 +739,15 @@ func assertManagedClusterManifestWorks(clusterName string) { func assertManagedClusterManifestWorksAvailable(clusterName string) { assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup") + klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName) + klusterletName := fmt.Sprintf("%s-klusterlet", clusterName) + + assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup") + assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") + ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() { start := time.Now() gomega.Eventually(func() error { - klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName) - klusterletName := fmt.Sprintf("%s-klusterlet", clusterName) manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName) klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{}) @@ -774,10 +779,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup") + klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName) + assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") + ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() { start := time.Now() gomega.Eventually(func() error { - klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName) manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName) klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{}) @@ -1065,3 +1072,55 @@ func getKubeConfigFile() (string, error) { return kubeConfigFile, nil } + +func assertManifestworkFinalizer(namespace, workName, expected string) { + ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() { + gomega.Eventually(func() error { + work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{}) + if err != nil { + return err + } + for _, finalizer := range work.Finalizers { + if finalizer == expected { + return nil + } + } + return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected) + }, 3*time.Minute, 10*time.Second).Should(gomega.Succeed()) + }) +} + +func assertAgentLeaderElection() { + start := time.Now() + ginkgo.By("Check if klusterlet agent is leader", func() { + gomega.Eventually(func() error { + namespace := "open-cluster-management-agent" + agentSelector := "app=klusterlet-agent" + leaseName := "klusterlet-lock" + // agent pod + pods, err := hubKubeClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: agentSelector, + }) + if err != nil { + return fmt.Errorf("could not get agent pod: %v", err) + } + if len(pods.Items) != 1 { + return fmt.Errorf("should be only one agent pod but get %d", len(pods.Items)) + } + + // agent lease + lease, err := hubKubeClient.CoordinationV1().Leases(namespace).Get(context.TODO(), leaseName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get Lease: %v", err) + } + + // Check if the HolderIdentity field is present and if it has the prefix of the podName + if lease.Spec.HolderIdentity != nil && strings.HasPrefix(*lease.Spec.HolderIdentity, pods.Items[0].Name) { + return nil + } + + return fmt.Errorf("klusterlet agent leader is still %s not %s", *lease.Spec.HolderIdentity, pods.Items[0].Name) + }, 180*time.Second, 1*time.Second).Should(gomega.Succeed()) + }) + util.Logf("spending time: %.2f seconds", time.Since(start).Seconds()) +} diff --git a/test/e2e/hostedcluster_test.go b/test/e2e/hostedcluster_test.go index d9786134..b90bf6c3 100644 --- a/test/e2e/hostedcluster_test.go +++ b/test/e2e/hostedcluster_test.go @@ -79,6 +79,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted) assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) assertManagedClusterPriorityClassHosted(managedClusterName) }) }) @@ -133,6 +134,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) assertManagedClusterPriorityClassHosted(managedClusterName) }) @@ -173,6 +175,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) assertManagedClusterPriorityClassHosted(managedClusterName) }) }) @@ -249,6 +252,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted) assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) }) ginkgo.JustAfterEach(func() { diff --git a/test/e2e/klusterletconfig_test.go b/test/e2e/klusterletconfig_test.go index cec868c1..f763a693 100644 --- a/test/e2e/klusterletconfig_test.go +++ b/test/e2e/klusterletconfig_test.go @@ -13,6 +13,7 @@ import ( "time" . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" . "github.com/onsi/gomega" klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1" "github.com/stolostron/managedcluster-import-controller/pkg/bootstrap" @@ -128,6 +129,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( ) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should deploy the klusterlet with proxy config", func() { @@ -155,6 +157,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // klusterletconfig is missing and it will be ignored assertBootstrapKubeconfigWithProxyConfig("", nil, nil) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Create KlusterletConfig with http proxy", func() { _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ @@ -216,6 +219,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // cluster should become available because no proxy is used assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should ignore the proxy config for self managed cluster", func() { @@ -284,6 +288,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Expect(err).ToNot(HaveOccurred()) assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) customServerURL := "https://invalid.server.url:6443" customCAData, _, err := newCert("custom CA for hub Kube API server") @@ -338,6 +343,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, false) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) defaultServerUrl, err := bootstrap.GetKubeAPIServerAddress(context.TODO(), hubRuntimeClient, nil) Expect(err).ToNot(HaveOccurred()) @@ -359,10 +365,8 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( RuntimeClient: hubRuntimeClient, }, defaultServerUrl, managedClusterName, nil) Expect(err).ToNot(HaveOccurred()) - assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) - // here to restart agent pods to trigger bootstrap secret update to save time. - restartAgentPods() + assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) assertManagedClusterAvailable(managedClusterName) By("Delete Klusterletconfig", func() { @@ -372,11 +376,23 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, false) - - // here to restart agent pods to trigger bootstrap secret update to save time. - restartAgentPods() // cluster should become available because custom server URL and CA bundle is removed assertManagedClusterAvailable(managedClusterName) + + // The hubhash changes in this case, update EvictionGracePeriod to 10s to cleanup the unmanaged AppliedWork + By("Setup KlusterletConfig EvictionGracePeriod to clean up resource", func() { + _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: klusterletConfigName, + }, + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + AppliedManifestWorkEvictionGracePeriod: "10s", + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + // here to restart agent pods to trigger update to save time. + restartAgentPods() }) It("Should deploy the klusterlet with customized namespace", func() { @@ -393,6 +409,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // klusterletconfig is missing and it will be ignored assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Create KlusterletConfig with customized namespace", func() { _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ @@ -414,6 +431,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( AssertKlusterletNamespace(managedClusterName, "klusterlet-local", "open-cluster-management-local") assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Delete Klusterletconfig", func() { err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) @@ -423,6 +441,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( AssertKlusterletNamespace(managedClusterName, "klusterlet", "open-cluster-management-agent") assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should deploy the klusterlet with custom AppliedManifestWork eviction grace period", func() { @@ -440,6 +459,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // klusterletconfig is missing and it will be ignored assertAppliedManifestWorkEvictionGracePeriod(nil) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Create KlusterletConfig with custom AppliedManifestWork eviction grace period", func() { _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ @@ -457,6 +477,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Duration: 120 * time.Minute, }) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Delete Klusterletconfig", func() { err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) @@ -465,6 +486,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( assertAppliedManifestWorkEvictionGracePeriod(nil) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) }) @@ -519,13 +541,30 @@ func restartAgentPods(namespaces ...string) { if len(namespaces) == 0 { namespaces = []string{"open-cluster-management-agent"} } + nspodsnum := map[string]int{} for _, ns := range namespaces { pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"}) Expect(err).ToNot(HaveOccurred()) + nspodsnum[ns] = len(pods.Items) for _, pod := range pods.Items { err = hubKubeClient.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) } } + gomega.Eventually(func() error { + for _, ns := range namespaces { + pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"}) + if err != nil { + return err + } + if len(pods.Items) != nspodsnum[ns] { + return fmt.Errorf("waiting for pods restart in namespace %s", ns) + } + } + + return nil + }, 120*time.Second, 1*time.Second).Should(gomega.Succeed()) + + assertAgentLeaderElection() } diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 9f432e76..951136b4 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -104,7 +104,7 @@ func CreateHostedManagedClusterWithShortLeaseDuration(clusterClient clusterclien }, Spec: clusterv1.ManagedClusterSpec{ HubAcceptsClient: true, - LeaseDurationSeconds: 5, + LeaseDurationSeconds: 10, }, }, metav1.CreateOptions{}, @@ -205,7 +205,7 @@ func CreateManagedClusterWithShortLeaseDuration(clusterClient clusterclient.Inte }, Spec: clusterv1.ManagedClusterSpec{ HubAcceptsClient: true, - LeaseDurationSeconds: 5, + LeaseDurationSeconds: 10, }, }, metav1.CreateOptions{},