From 5ff5999a25247742ee5b780dda18e93d0719eb90 Mon Sep 17 00:00:00 2001 From: Maciej Zimnoch Date: Fri, 20 Dec 2024 16:53:28 +0100 Subject: [PATCH] Extend ScyllaDBCluster controller with finalizer Finalizer makes sure that before ScyllaDBCluster deletion, all dependant remote Namespaces (and other resources dependant on Namespace) are removed before ScyllaDBCluster is removed. --- pkg/controller/scylladbcluster/conditions.go | 2 + pkg/controller/scylladbcluster/sync.go | 20 +++ .../scylladbcluster/sync_finalizer.go | 160 ++++++++++++++++++ pkg/controllerhelpers/finalizers.go | 82 +++++++++ pkg/controllerhelpers/finalizers_test.go | 137 +++++++++++++++ pkg/naming/constants.go | 4 + ...ltidatacenter_scylladbcluster_finalizer.go | 119 +++++++++++++ 7 files changed, 524 insertions(+) create mode 100644 pkg/controller/scylladbcluster/sync_finalizer.go create mode 100644 pkg/controllerhelpers/finalizers.go create mode 100644 pkg/controllerhelpers/finalizers_test.go create mode 100644 test/e2e/set/scylladbcluster/multidatacenter_scylladbcluster_finalizer.go diff --git a/pkg/controller/scylladbcluster/conditions.go b/pkg/controller/scylladbcluster/conditions.go index 86db1639cb..6888013160 100644 --- a/pkg/controller/scylladbcluster/conditions.go +++ b/pkg/controller/scylladbcluster/conditions.go @@ -13,4 +13,6 @@ const ( remoteEndpointSliceControllerDegradedCondition = "RemoteEndpointSliceControllerDegraded" remoteEndpointsControllerProgressingCondition = "RemoteEndpointsControllerProgressing" remoteEndpointsControllerDegradedCondition = "RemoteEndpointsControllerDegraded" + scyllaDBClusterFinalizerProgressingCondition = "ScyllaDBClusterFinalizerProgressing" + scyllaDBClusterFinalizerDegradedCondition = "ScyllaDBClusterFinalizerDegraded" ) diff --git a/pkg/controller/scylladbcluster/sync.go b/pkg/controller/scylladbcluster/sync.go index 95515908cb..7a268f8aed 100644 --- a/pkg/controller/scylladbcluster/sync.go +++ b/pkg/controller/scylladbcluster/sync.go @@ -189,6 +189,18 @@ func (scc *Controller) sync(ctx context.Context, key string) error { status := scc.calculateStatus(sc, remoteScyllaDBDatacenterMap) if sc.DeletionTimestamp != nil { + err = controllerhelpers.RunSync( + &status.Conditions, + scyllaDBClusterFinalizerProgressingCondition, + scyllaDBClusterFinalizerDegradedCondition, + sc.Generation, + func() ([]metav1.Condition, error) { + return scc.syncFinalizer(ctx, sc, remoteNamespaces) + }, + ) + if err != nil { + return fmt.Errorf("can't finalize: %w", err) + } return scc.updateStatus(ctx, sc, status) } @@ -198,6 +210,14 @@ func (scc *Controller) sync(ctx context.Context, key string) error { } managingClusterDomain := *soc.Status.ClusterDomain + if !slices.ContainsItem(sc.GetFinalizers(), naming.ScyllaDBClusterFinalizer) { + err = scc.addFinalizer(ctx, sc) + if err != nil { + return fmt.Errorf("can't add finalizer: %w", err) + } + return nil + } + var errs []error err = controllerhelpers.RunSync( diff --git a/pkg/controller/scylladbcluster/sync_finalizer.go b/pkg/controller/scylladbcluster/sync_finalizer.go new file mode 100644 index 0000000000..e7557ef087 --- /dev/null +++ b/pkg/controller/scylladbcluster/sync_finalizer.go @@ -0,0 +1,160 @@ +// Copyright (c) 2024 ScyllaDB. + +package scylladbcluster + +import ( + "context" + "fmt" + + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/helpers/slices" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/pkg/pointer" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" +) + +func (scc *Controller) syncFinalizer(ctx context.Context, sc *scyllav1alpha1.ScyllaDBCluster, remoteNamespaces map[string]*corev1.Namespace) ([]metav1.Condition, error) { + var progressingConditions []metav1.Condition + var err error + + if !slices.ContainsItem(sc.GetFinalizers(), naming.ScyllaDBClusterFinalizer) { + klog.V(4).InfoS("Object is already finalized", "ScyllaDBCluster", klog.KObj(sc), "UID", sc.UID) + return progressingConditions, nil + } + + // Delete remote Namespace for every ScyllaDBCluster's datacenter. + // As all remotely reconciled objects are namespaced, except Namespace, + // it's enough to remove the Namespace and rely on remote GC to clear the rest. + // This may need to be adjusted when the Operator will reconcile other cluster-wide resources. + klog.V(4).InfoS("Finalizing object", "ScyllaDBCluster", klog.KObj(sc), "UID", sc.UID) + + informerRemoteNamespaces := map[string][]*corev1.Namespace{} + for _, dc := range sc.Spec.Datacenters { + remoteNamespace, ok := remoteNamespaces[dc.RemoteKubernetesClusterName] + if ok { + informerRemoteNamespaces[dc.RemoteKubernetesClusterName] = append(informerRemoteNamespaces[dc.RemoteKubernetesClusterName], remoteNamespace) + } + } + + deletionProgressingCondition, err := scc.finalizeRemoteNamespaces(ctx, sc, informerRemoteNamespaces) + if err != nil { + return progressingConditions, fmt.Errorf("can't finalize remote Namespaces: %w", err) + } + + progressingConditions = append(progressingConditions, deletionProgressingCondition...) + + // Wait until all Namespaces from Informers are gone. + if len(informerRemoteNamespaces) != 0 { + return progressingConditions, nil + } + + // Live list remote Namespaces to be 100% sure before we delete. Informer cache might not be updated yet. + var errs []error + clientRemoteNamespaces := map[string][]*corev1.Namespace{} + + for _, dc := range sc.Spec.Datacenters { + remoteClient, err := scc.kubeRemoteClient.Cluster(dc.RemoteKubernetesClusterName) + if err != nil { + errs = append(errs, fmt.Errorf("can't get remote kube client for %q cluster: %w", dc.RemoteKubernetesClusterName, err)) + continue + } + + rnss, err := remoteClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(naming.ScyllaDBClusterDatacenterSelectorLabels(sc, &dc)).String(), + }) + if err != nil { + errs = append(errs, fmt.Errorf("can't list remote Namespaces via %q cluster client: %w", dc.RemoteKubernetesClusterName, err)) + continue + } + + clientRemoteNamespaces[dc.RemoteKubernetesClusterName] = slices.ConvertSlice(rnss.Items, pointer.Ptr[corev1.Namespace]) + } + + deletionProgressingCondition, err = scc.finalizeRemoteNamespaces(ctx, sc, clientRemoteNamespaces) + if err != nil { + return progressingConditions, fmt.Errorf("can't finalize remote Namespaces: %w", err) + } + + progressingConditions = append(progressingConditions, deletionProgressingCondition...) + + // Wait until all Namespaces from clients are gone. + if len(clientRemoteNamespaces) != 0 { + return progressingConditions, nil + } + + klog.V(2).InfoS("ScyllaDBCluster no longer has dependant objects, removing finalizer") + err = scc.removeFinalizer(ctx, sc) + if err != nil { + return progressingConditions, fmt.Errorf("can't remove finalizer from ScyllaDBCluster %q: %w", naming.ObjRef(sc), err) + } + + return progressingConditions, nil +} + +func (scc *Controller) finalizeRemoteNamespaces(ctx context.Context, sc *scyllav1alpha1.ScyllaDBCluster, namespacesToDelete map[string][]*corev1.Namespace) ([]metav1.Condition, error) { + var progressingConditions []metav1.Condition + var errs []error + + for remoteCluster, remoteNamespaces := range namespacesToDelete { + remoteClient, err := scc.kubeRemoteClient.Cluster(remoteCluster) + if err != nil { + errs = append(errs, fmt.Errorf("can't get remote kube client for %q cluster: %w", remoteCluster, err)) + continue + } + + for _, remoteNamespace := range remoteNamespaces { + controllerhelpers.AddGenericProgressingStatusCondition(&progressingConditions, scyllaDBClusterFinalizerProgressingCondition, remoteNamespace, "delete", sc.Generation) + err = remoteClient.CoreV1().Namespaces().Delete(ctx, remoteNamespace.Name, metav1.DeleteOptions{ + Preconditions: metav1.NewUIDPreconditions(string(remoteNamespace.UID)), + PropagationPolicy: pointer.Ptr(metav1.DeletePropagationForeground), + }) + if err != nil { + errs = append(errs, fmt.Errorf("can't delete remote Namespace %q from %q cluster: %w", naming.ObjRef(remoteNamespace), remoteCluster, err)) + continue + } + } + } + + err := errors.NewAggregate(errs) + if err != nil { + return progressingConditions, fmt.Errorf("can't delete remote namespaces: %w", err) + } + + return progressingConditions, nil +} + +func (scc *Controller) addFinalizer(ctx context.Context, sc *scyllav1alpha1.ScyllaDBCluster) error { + patch, err := controllerhelpers.AddFinalizerPatch(sc, naming.ScyllaDBClusterFinalizer) + if err != nil { + return fmt.Errorf("can't create add finalizer patch: %w", err) + } + + _, err = scc.scyllaClient.ScyllaV1alpha1().ScyllaDBClusters(sc.Namespace).Patch(ctx, sc.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("can't patch ScyllaDBCluster %q: %w", naming.ObjRef(sc), err) + } + + klog.V(2).InfoS("Added finalizer to ScyllaDBCluster", "ScyllaDBCluster", klog.KObj(sc)) + return nil +} + +func (scc *Controller) removeFinalizer(ctx context.Context, sc *scyllav1alpha1.ScyllaDBCluster) error { + patch, err := controllerhelpers.RemoveFinalizerPatch(sc, naming.ScyllaDBClusterFinalizer) + if err != nil { + return fmt.Errorf("can't create remove finalizer patch: %w", err) + } + + _, err = scc.scyllaClient.ScyllaV1alpha1().ScyllaDBClusters(sc.Namespace).Patch(ctx, sc.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("can't patch ScyllaDBCluster %q: %w", naming.ObjRef(sc), err) + } + + klog.V(2).InfoS("Removed finalizer from ScyllaDBCluster", "ScyllaDBCluster", klog.KObj(sc)) + return nil +} diff --git a/pkg/controllerhelpers/finalizers.go b/pkg/controllerhelpers/finalizers.go new file mode 100644 index 0000000000..03f26047f4 --- /dev/null +++ b/pkg/controllerhelpers/finalizers.go @@ -0,0 +1,82 @@ +// Copyright (c) 2024 ScyllaDB. + +package controllerhelpers + +import ( + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type objectForFinalizersPatch struct { + objectMetaForFinalizersPatch `json:"metadata"` +} + +// objectMetaForFinalizersPatch defines object meta struct for finalizers patch operation. +type objectMetaForFinalizersPatch struct { + ResourceVersion string `json:"resourceVersion"` + Finalizers []string `json:"finalizers"` +} + +func RemoveFinalizerPatch(obj metav1.Object, finalizer string) ([]byte, error) { + if !HasFinalizer(obj, finalizer) { + return nil, nil + } + + finalizers := obj.GetFinalizers() + var newFinalizers []string + + for _, f := range finalizers { + if f == finalizer { + continue + } + newFinalizers = append(newFinalizers, f) + } + + patch, err := json.Marshal(&objectForFinalizersPatch{ + objectMetaForFinalizersPatch: objectMetaForFinalizersPatch{ + ResourceVersion: obj.GetResourceVersion(), + Finalizers: newFinalizers, + }, + }) + if err != nil { + return nil, fmt.Errorf("can't marshal finalizer remove patch: %w", err) + } + + return patch, nil +} + +func AddFinalizerPatch(obj metav1.Object, finalizer string) ([]byte, error) { + if HasFinalizer(obj, finalizer) { + return nil, nil + } + newFinalizers := make([]string, 0, len(obj.GetFinalizers())+1) + for _, f := range obj.GetFinalizers() { + newFinalizers = append(newFinalizers, f) + } + newFinalizers = append(newFinalizers, finalizer) + + patch, err := json.Marshal(&objectForFinalizersPatch{ + objectMetaForFinalizersPatch: objectMetaForFinalizersPatch{ + ResourceVersion: obj.GetResourceVersion(), + Finalizers: newFinalizers, + }, + }) + if err != nil { + return nil, fmt.Errorf("can't marshal finalizer add patch: %w", err) + } + + return patch, nil +} + +func HasFinalizer(obj metav1.Object, finalizer string) bool { + found := false + for _, f := range obj.GetFinalizers() { + if f == finalizer { + found = true + break + } + } + return found +} diff --git a/pkg/controllerhelpers/finalizers_test.go b/pkg/controllerhelpers/finalizers_test.go new file mode 100644 index 0000000000..d209048fe9 --- /dev/null +++ b/pkg/controllerhelpers/finalizers_test.go @@ -0,0 +1,137 @@ +// Copyright (c) 2024 ScyllaDB. + +package controllerhelpers + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAddFinalizerPatch(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + obj metav1.Object + finalizer string + expectedPatch []byte + expectedError error + }{ + { + name: "object has empty finalizers", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{}}, + finalizer: "my-finalizer", + expectedPatch: []byte(`{"metadata":{"resourceVersion":"123","finalizers":["my-finalizer"]}}`), + expectedError: nil, + }, + { + name: "duplicate finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{"a", "my-finalizer", "c"}}, + finalizer: "my-finalizer", + expectedPatch: nil, + expectedError: nil, + }, + } + + for i := range tt { + tc := tt[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + patch, err := AddFinalizerPatch(tc.obj, tc.finalizer) + if err != tc.expectedError { + t.Errorf("expected error %s, got %s", tc.expectedError, err) + } + if !equality.Semantic.DeepEqual(patch, tc.expectedPatch) { + t.Errorf("expected patch %s, got %s", string(tc.expectedPatch), string(patch)) + } + }) + } +} + +func TestRemoveFinalizerPatch(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + obj metav1.Object + finalizer string + expectedPatch []byte + expectedError error + }{ + { + name: "object is missing given finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{}}, + finalizer: "my-finalizer", + expectedPatch: nil, + expectedError: nil, + }, + { + name: "patch removes finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{"a", "my-finalizer", "c"}}, + finalizer: "my-finalizer", + expectedPatch: []byte(`{"metadata":{"resourceVersion":"123","finalizers":["a","c"]}}`), + expectedError: nil, + }, + { + name: "patch removes last finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{"my-finalizer"}}, + finalizer: "my-finalizer", + expectedPatch: []byte(`{"metadata":{"resourceVersion":"123","finalizers":null}}`), + expectedError: nil, + }, + } + + for i := range tt { + tc := tt[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + patch, err := RemoveFinalizerPatch(tc.obj, tc.finalizer) + if err != tc.expectedError { + t.Errorf("expected error %s, got %s", tc.expectedError, err) + } + if !equality.Semantic.DeepEqual(patch, tc.expectedPatch) { + t.Errorf("expected patch %s, got %s", string(tc.expectedPatch), string(patch)) + } + }) + } +} + +func TestHasFinalizer(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + obj metav1.Object + finalizer string + expectedResult bool + }{ + { + name: "false when empty finalizer list", + obj: &metav1.ObjectMeta{Finalizers: []string{}}, + finalizer: "my-finalizer", + expectedResult: false, + }, + { + name: "true when object contains finalizer", + obj: &metav1.ObjectMeta{Finalizers: []string{"a", "b", "my-finalizer", "c"}}, + finalizer: "my-finalizer", + expectedResult: true, + }, + } + + for i := range tt { + tc := tt[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := HasFinalizer(tc.obj, tc.finalizer) + if got != tc.expectedResult { + t.Errorf("expected %v, got %v", tc.expectedResult, got) + } + }) + } +} diff --git a/pkg/naming/constants.go b/pkg/naming/constants.go index e771052c96..d948ccf29a 100644 --- a/pkg/naming/constants.go +++ b/pkg/naming/constants.go @@ -232,3 +232,7 @@ const ( const ( OperatorAppNameWithDomain = "scylla-operator.scylladb.com" ) + +const ( + ScyllaDBClusterFinalizer = "scylla-operator.scylladb.com/scylladbcluster-protection" +) diff --git a/test/e2e/set/scylladbcluster/multidatacenter_scylladbcluster_finalizer.go b/test/e2e/set/scylladbcluster/multidatacenter_scylladbcluster_finalizer.go new file mode 100644 index 0000000000..0501079ce6 --- /dev/null +++ b/test/e2e/set/scylladbcluster/multidatacenter_scylladbcluster_finalizer.go @@ -0,0 +1,119 @@ +// Copyright (c) 2024 ScyllaDB. + +package scylladbcluster + +import ( + "context" + "fmt" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/naming" + "github.com/scylladb/scylla-operator/test/e2e/framework" + "github.com/scylladb/scylla-operator/test/e2e/utils" + v1alpha1utils "github.com/scylladb/scylla-operator/test/e2e/utils/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" +) + +var _ = g.Describe("ScyllaDBCluster finalizer", func() { + f := framework.NewFramework("scylladbcluster") + + g.It("should delete remote Namespaces when ScyllaDBCluster is deleted", func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + availableClusters := len(framework.TestContext.RestConfigs) + + framework.By("Creating RemoteKubernetesClusters") + rkcs := make([]*scyllav1alpha1.RemoteKubernetesCluster, 0, availableClusters) + rkcClusterMap := make(map[string]framework.ClusterInterface, availableClusters) + + metaCluster := f.Cluster(0) + for idx := range availableClusters { + cluster := f.Cluster(idx) + userNs, _ := cluster.CreateUserNamespace(ctx) + + clusterName := fmt.Sprintf("%s-%d", f.Namespace(), idx) + + framework.By("Creating SA having Operator ClusterRole in #%d cluster", idx) + adminKubeconfig, err := utils.GetKubeConfigHavingOperatorRemoteClusterRole(ctx, cluster.KubeAdminClient(), cluster.AdminClientConfig(), clusterName, userNs.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + + kubeconfig, err := clientcmd.Write(adminKubeconfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + rkc, err := utils.GetRemoteKubernetesClusterWithKubeconfig(ctx, metaCluster.KubeAdminClient(), kubeconfig, clusterName, f.Namespace()) + o.Expect(err).NotTo(o.HaveOccurred()) + + rc := framework.NewRestoringCleaner( + ctx, + f.KubeAdminClient(), + f.DynamicAdminClient(), + remoteKubernetesClusterResourceInfo, + rkc.Namespace, + rkc.Name, + framework.RestoreStrategyRecreate, + ) + f.AddCleaners(rc) + rc.DeleteObject(ctx, true) + + framework.By("Creating RemoteKubernetesCluster %q with credentials to cluster #%d", clusterName, idx) + rkc, err = metaCluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters().Create(ctx, rkc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + rkcs = append(rkcs, rkc) + rkcClusterMap[rkc.Name] = cluster + } + + for _, rkc := range rkcs { + func() { + framework.By("Waiting for the RemoteKubernetesCluster %q to roll out (RV=%s)", rkc.Name, rkc.ResourceVersion) + waitCtx1, waitCtx1Cancel := utils.ContextForRemoteKubernetesClusterRollout(ctx, rkc) + defer waitCtx1Cancel() + + _, err := controllerhelpers.WaitForRemoteKubernetesClusterState(waitCtx1, metaCluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters(), rkc.Name, controllerhelpers.WaitForStateOptions{}, utils.IsRemoteKubernetesClusterRolledOut) + o.Expect(err).NotTo(o.HaveOccurred()) + }() + } + + framework.By("Creating ScyllaDBCluster") + var err error + sc := f.GetDefaultScyllaDBCluster(rkcs) + sc, err = metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(f.Namespace()).Create(ctx, sc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for the ScyllaDBCluster %q roll out (RV=%s)", sc.Name, sc.ResourceVersion) + waitCtx2, waitCtx2Cancel := utils.ContextForMultiDatacenterScyllaDBClusterRollout(ctx, sc) + defer waitCtx2Cancel() + sc, err = controllerhelpers.WaitForScyllaDBClusterState(waitCtx2, metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(sc.Namespace), sc.Name, controllerhelpers.WaitForStateOptions{}, utils.IsScyllaDBClusterRolledOut) + o.Expect(err).NotTo(o.HaveOccurred()) + + verifyScyllaDBCluster(ctx, sc, rkcClusterMap) + err = v1alpha1utils.WaitForFullScyllaDBClusterQuorum(ctx, rkcClusterMap, sc) + o.Expect(err).NotTo(o.HaveOccurred()) + + const expectedFinalizer = "scylla-operator.scylladb.com/scylladbcluster-protection" + o.Expect(sc.Finalizers).To(o.ContainElement(expectedFinalizer)) + + framework.By("Deleting ScyllaDBCluster") + err = metaCluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(f.Namespace()).Delete(ctx, sc.Name, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for ScyllaDBCluster %q to be removed.", sc.Name) + err = framework.WaitForObjectDeletion(ctx, f.DynamicClient(), scyllav1alpha1.SchemeGroupVersion.WithResource("scylladbclusters"), sc.Namespace, sc.Name, &sc.UID) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Verifying if all remote Namespaces having ScyllaDBCluster %q selector are gone.", sc.Name) + o.Expect(sc.Spec.Datacenters).ToNot(o.BeEmpty()) + for i := range rkcs { + namespaces, err := f.Cluster(i).KubeAdminClient().CoreV1().Namespaces().List(ctx, metav1.ListOptions{ + LabelSelector: naming.ScyllaDBClusterSelector(sc).String(), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(namespaces.Items).To(o.BeEmpty()) + } + }) +})