fix e2e test issues when sno leader election enabled (#434)

Signed-off-by: Qing Hao <[email protected]>

test/e2e/autoimport_test.go

@@ -258,6 +258,7 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f
 		})
 		assertManagedClusterImportSecretApplied(managedClusterName)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		configName := "autoimport-config"
 		testcluster := fmt.Sprintf("custom-%s", managedClusterName)
@@ -292,6 +293,8 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f
 		assertManagedClusterImportSecretCreated(testcluster, "other")
 		assertManagedClusterImportSecretApplied(testcluster)
 		assertManagedClusterAvailable(testcluster)
+		klusterletName := fmt.Sprintf("%s-klusterlet", testcluster)
+		assertManifestworkFinalizer(testcluster, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
 
 		AssertKlusterletNamespace(testcluster, "klusterlet-local", "open-cluster-management-local")
 

test/e2e/cleanup_test.go

@@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun
 			_, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{})
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
+			// check the work has added finalizer before detaching the cluster
+			assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 			// detach the cluster
 			err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{})
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())

test/e2e/e2e_suite_test.go

@@ -476,6 +476,7 @@ func assertHostedManagedClusterImportSecret(managedClusterName string) {
 }
 
 func assertManagedClusterDeleted(clusterName string) {
+
 	ginkgo.By(fmt.Sprintf("Delete the managed cluster %s", clusterName), func() {
 		err := hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{})
 		if err != nil && !errors.IsNotFound(err) {
@@ -738,11 +739,15 @@ func assertManagedClusterManifestWorks(clusterName string) {
 func assertManagedClusterManifestWorksAvailable(clusterName string) {
 	assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")
 
+	klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
+	klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
+
+	assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+	assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 	ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() {
 		start := time.Now()
 		gomega.Eventually(func() error {
-			klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
-			klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
 			manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName)
 
 			klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{})
@@ -774,10 +779,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste
 	assertManagedClusterFinalizer(clusterName,
 		"managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")
 
+	klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
+	assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 	ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() {
 		start := time.Now()
 		gomega.Eventually(func() error {
-			klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
 			manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName)
 
 			klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{})
@@ -1065,3 +1072,55 @@ func getKubeConfigFile() (string, error) {
 
 	return kubeConfigFile, nil
 }
+
+func assertManifestworkFinalizer(namespace, workName, expected string) {
+	ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() {
+		gomega.Eventually(func() error {
+			work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			for _, finalizer := range work.Finalizers {
+				if finalizer == expected {
+					return nil
+				}
+			}
+			return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected)
+		}, 3*time.Minute, 10*time.Second).Should(gomega.Succeed())
+	})
+}
+
+func assertAgentLeaderElection() {
+	start := time.Now()
+	ginkgo.By("Check if klusterlet agent is leader", func() {
+		gomega.Eventually(func() error {
+			namespace := "open-cluster-management-agent"
+			agentSelector := "app=klusterlet-agent"
+			leaseName := "klusterlet-lock"
+			// agent pod
+			pods, err := hubKubeClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
+				LabelSelector: agentSelector,
+			})
+			if err != nil {
+				return fmt.Errorf("could not get agent pod: %v", err)
+			}
+			if len(pods.Items) != 1 {
+				return fmt.Errorf("should be only one agent pod but get %d", len(pods.Items))
+			}
+
+			// agent lease
+			lease, err := hubKubeClient.CoordinationV1().Leases(namespace).Get(context.TODO(), leaseName, metav1.GetOptions{})
+			if err != nil {
+				return fmt.Errorf("could not get Lease: %v", err)
+			}
+
+			// Check if the HolderIdentity field is present and if it has the prefix of the podName
+			if lease.Spec.HolderIdentity != nil && strings.HasPrefix(*lease.Spec.HolderIdentity, pods.Items[0].Name) {
+				return nil
+			}
+
+			return fmt.Errorf("klusterlet agent leader is still %s not %s", *lease.Spec.HolderIdentity, pods.Items[0].Name)
+		}, 180*time.Second, 1*time.Second).Should(gomega.Succeed())
+	})
+	util.Logf("spending time: %.2f seconds", time.Since(start).Seconds())
+}

test/e2e/hostedcluster_test.go

@@ -79,6 +79,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 			assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted)
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 			assertManagedClusterPriorityClassHosted(managedClusterName)
 		})
 	})
@@ -133,6 +134,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 			assertManagedClusterPriorityClassHosted(managedClusterName)
 		})
 
@@ -173,6 +175,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 			assertManagedClusterPriorityClassHosted(managedClusterName)
 		})
 	})
@@ -249,6 +252,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 			assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted)
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 		})
 
 		ginkgo.JustAfterEach(func() {

test/e2e/klusterletconfig_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
 	. "github.com/onsi/gomega"
 	klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1"
 	"github.com/stolostron/managedcluster-import-controller/pkg/bootstrap"
@@ -128,6 +129,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		)
 
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should deploy the klusterlet with proxy config", func() {
@@ -155,6 +157,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// klusterletconfig is missing and it will be ignored
 		assertBootstrapKubeconfigWithProxyConfig("", nil, nil)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Create KlusterletConfig with http proxy", func() {
 			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
@@ -216,6 +219,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		// cluster should become available because no proxy is used
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should ignore the proxy config for self managed cluster", func() {
@@ -284,6 +288,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		Expect(err).ToNot(HaveOccurred())
 		assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		customServerURL := "https://invalid.server.url:6443"
 		customCAData, _, err := newCert("custom CA for hub Kube API server")
@@ -338,6 +343,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "",
 			"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, false)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		defaultServerUrl, err := bootstrap.GetKubeAPIServerAddress(context.TODO(), hubRuntimeClient, nil)
 		Expect(err).ToNot(HaveOccurred())
@@ -359,10 +365,8 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 			RuntimeClient: hubRuntimeClient,
 		}, defaultServerUrl, managedClusterName, nil)
 		Expect(err).ToNot(HaveOccurred())
-		assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false)
 
-		// here to restart agent pods to trigger bootstrap secret update to save time.
-		restartAgentPods()
+		assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false)
 		assertManagedClusterAvailable(managedClusterName)
 
 		By("Delete Klusterletconfig", func() {
@@ -372,11 +376,23 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "",
 			"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, false)
-
-		// here to restart agent pods to trigger bootstrap secret update to save time.
-		restartAgentPods()
 		// cluster should become available because custom server URL and CA bundle is removed
 		assertManagedClusterAvailable(managedClusterName)
+
+		// The hubhash changes in this case, update EvictionGracePeriod to 10s to cleanup the unmanaged AppliedWork
+		By("Setup KlusterletConfig EvictionGracePeriod to clean up resource", func() {
+			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: klusterletConfigName,
+				},
+				Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{
+					AppliedManifestWorkEvictionGracePeriod: "10s",
+				},
+			}, metav1.CreateOptions{})
+			Expect(err).ToNot(HaveOccurred())
+		})
+		// here to restart agent pods to trigger update to save time.
+		restartAgentPods()
 	})
 
 	It("Should deploy the klusterlet with customized namespace", func() {
@@ -393,6 +409,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		// klusterletconfig is missing and it will be ignored
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Create KlusterletConfig with customized namespace", func() {
 			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
@@ -414,6 +431,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		AssertKlusterletNamespace(managedClusterName, "klusterlet-local", "open-cluster-management-local")
 
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Delete Klusterletconfig", func() {
 			err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
@@ -423,6 +441,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		AssertKlusterletNamespace(managedClusterName, "klusterlet", "open-cluster-management-agent")
 
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should deploy the klusterlet with custom AppliedManifestWork eviction grace period", func() {
@@ -440,6 +459,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// klusterletconfig is missing and it will be ignored
 		assertAppliedManifestWorkEvictionGracePeriod(nil)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Create KlusterletConfig with custom AppliedManifestWork eviction grace period", func() {
 			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
@@ -457,6 +477,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 			Duration: 120 * time.Minute,
 		})
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Delete Klusterletconfig", func() {
 			err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
@@ -465,6 +486,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		assertAppliedManifestWorkEvictionGracePeriod(nil)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 })
 
@@ -519,13 +541,30 @@ func restartAgentPods(namespaces ...string) {
 	if len(namespaces) == 0 {
 		namespaces = []string{"open-cluster-management-agent"}
 	}
+	nspodsnum := map[string]int{}
 	for _, ns := range namespaces {
 		pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"})
 		Expect(err).ToNot(HaveOccurred())
 
+		nspodsnum[ns] = len(pods.Items)
 		for _, pod := range pods.Items {
 			err = hubKubeClient.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
 			Expect(err).ToNot(HaveOccurred())
 		}
 	}
+	gomega.Eventually(func() error {
+		for _, ns := range namespaces {
+			pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"})
+			if err != nil {
+				return err
+			}
+			if len(pods.Items) != nspodsnum[ns] {
+				return fmt.Errorf("waiting for pods restart in namespace %s", ns)
+			}
+		}
+
+		return nil
+	}, 120*time.Second, 1*time.Second).Should(gomega.Succeed())
+
+	assertAgentLeaderElection()
 }

test/e2e/util/util.go

@@ -104,7 +104,7 @@ func CreateHostedManagedClusterWithShortLeaseDuration(clusterClient clusterclien
 				},
 				Spec: clusterv1.ManagedClusterSpec{
 					HubAcceptsClient:     true,
-					LeaseDurationSeconds: 5,
+					LeaseDurationSeconds: 10,
 				},
 			},
 			metav1.CreateOptions{},
@@ -205,7 +205,7 @@ func CreateManagedClusterWithShortLeaseDuration(clusterClient clusterclient.Inte
 				},
 				Spec: clusterv1.ManagedClusterSpec{
 					HubAcceptsClient:     true,
-					LeaseDurationSeconds: 5,
+					LeaseDurationSeconds: 10,
 				},
 			},
 			metav1.CreateOptions{},