diff --git a/pkg/cmd/operator/operator.go b/pkg/cmd/operator/operator.go index 9ca7675f51..13210d6f6c 100644 --- a/pkg/cmd/operator/operator.go +++ b/pkg/cmd/operator/operator.go @@ -436,6 +436,7 @@ func (o *OperatorOptions) run(ctx context.Context, streams genericclioptions.IOS o.kubeClient, o.scyllaClient.ScyllaV1alpha1(), scyllaInformers.Scylla().V1alpha1().RemoteKubernetesClusters(), + scyllaInformers.Scylla().V1alpha1().ScyllaDBClusters(), kubeInformers.Core().V1().Secrets(), []remoteclient.DynamicClusterInterface{ &o.clusterKubeClient, diff --git a/pkg/controller/remotekubernetescluster/conditions.go b/pkg/controller/remotekubernetescluster/conditions.go index 69c787ee6b..91ac97b7e4 100644 --- a/pkg/controller/remotekubernetescluster/conditions.go +++ b/pkg/controller/remotekubernetescluster/conditions.go @@ -7,4 +7,7 @@ const ( clientHealthcheckControllerAvailableCondition = "ClientHealthcheckControllerAvailable" clientHealthcheckControllerProgressingCondition = "ClientHealthcheckControllerProgressing" clientHealthcheckControllerDegradedCondition = "ClientHealthcheckControllerDegraded" + + remoteKubernetesClusterFinalizerProgressingCondition = "RemoteKubernetesClusterFinalizerProgressing" + remoteKubernetesClusterFinalizerDegradedCondition = "RemoteKubernetesClusterFinalizerDegraded" ) diff --git a/pkg/controller/remotekubernetescluster/controller.go b/pkg/controller/remotekubernetescluster/controller.go index cb7cf34fcb..26f69f38bb 100644 --- a/pkg/controller/remotekubernetescluster/controller.go +++ b/pkg/controller/remotekubernetescluster/controller.go @@ -47,6 +47,7 @@ type Controller struct { scyllaClient scyllav1alpha1client.ScyllaV1alpha1Interface remoteKubernetesClusterLister scyllav1alpha1listers.RemoteKubernetesClusterLister + scyllaDBClusterLister scyllav1alpha1listers.ScyllaDBClusterLister secretLister corev1listers.SecretLister clusterKubeClient remoteclient.ClusterClientInterface[kubernetes.Interface] @@ -65,6 +66,7 @@ func NewController( kubeClient kubernetes.Interface, scyllaClient scyllav1alpha1client.ScyllaV1alpha1Interface, remoteKubernetesClusterInformer scyllav1alpha1informers.RemoteKubernetesClusterInformer, + scyllaDBClusterInformer scyllav1alpha1informers.ScyllaDBClusterInformer, secretInformer corev1informers.SecretInformer, dynamicClusterHandlers []remoteclient.DynamicClusterInterface, clusterKubeClient remoteclient.ClusterClientInterface[kubernetes.Interface], @@ -79,6 +81,7 @@ func NewController( scyllaClient: scyllaClient, remoteKubernetesClusterLister: remoteKubernetesClusterInformer.Lister(), + scyllaDBClusterLister: scyllaDBClusterInformer.Lister(), secretLister: secretInformer.Lister(), dynamicClusterHandlers: dynamicClusterHandlers, @@ -88,6 +91,7 @@ func NewController( cachesToSync: []cache.InformerSynced{ remoteKubernetesClusterInformer.Informer().HasSynced, + scyllaDBClusterInformer.Informer().HasSynced, secretInformer.Informer().HasSynced, }, @@ -123,6 +127,12 @@ func NewController( DeleteFunc: rkcc.deleteRemoteKubernetesCluster, }) + scyllaDBClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: rkcc.addScyllaDBCluster, + UpdateFunc: rkcc.updateScyllaDBCluster, + DeleteFunc: rkcc.deleteScyllaDBCluster, + }) + secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: rkcc.addSecret, UpdateFunc: rkcc.updateSecret, @@ -263,3 +273,39 @@ func (rkcc *Controller) enqueueRemoteKubernetesClusterUsingSecret(secret *corev1 return rkc.Spec.KubeconfigSecretRef.Namespace == secret.Namespace && rkc.Spec.KubeconfigSecretRef.Name == secret.Name })) } + +func (rkcc *Controller) addScyllaDBCluster(obj interface{}) { + rkcc.handlers.HandleAdd( + obj.(*scyllav1alpha1.ScyllaDBCluster), + rkcc.enqueueScyllaDBCluster, + ) +} + +func (rkcc *Controller) updateScyllaDBCluster(old, cur interface{}) { + rkcc.handlers.HandleUpdate( + old.(*scyllav1alpha1.ScyllaDBCluster), + cur.(*scyllav1alpha1.ScyllaDBCluster), + rkcc.enqueueScyllaDBCluster, + rkcc.deleteScyllaDBCluster, + ) +} + +func (rkcc *Controller) deleteScyllaDBCluster(obj interface{}) { + rkcc.handlers.HandleDelete( + obj, + rkcc.enqueueScyllaDBCluster, + ) +} + +func (rkcc *Controller) enqueueScyllaDBCluster(depth int, obj kubeinterfaces.ObjectInterface, op controllerhelpers.HandlerOperationType) { + sc := obj.(*scyllav1alpha1.ScyllaDBCluster) + + for _, dc := range sc.Spec.Datacenters { + rkc, err := rkcc.remoteKubernetesClusterLister.Get(dc.RemoteKubernetesClusterName) + if err != nil { + utilruntime.HandleError(fmt.Errorf("couldn't find RemoteKubernetesCluster with name %q", dc.RemoteKubernetesClusterName)) + return + } + rkcc.handlers.Enqueue(depth+1, rkc, op) + } +} diff --git a/pkg/controller/remotekubernetescluster/sync.go b/pkg/controller/remotekubernetescluster/sync.go index a01531613d..137cc4969e 100644 --- a/pkg/controller/remotekubernetescluster/sync.go +++ b/pkg/controller/remotekubernetescluster/sync.go @@ -8,6 +8,8 @@ import ( "time" "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/helpers/slices" + "github.com/scylladb/scylla-operator/pkg/naming" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" @@ -42,9 +44,30 @@ func (rkcc *Controller) sync(ctx context.Context, key string) error { status := rkcc.calculateStatus(rkc) if rkc.DeletionTimestamp != nil { + err = controllerhelpers.RunSync( + &status.Conditions, + remoteKubernetesClusterFinalizerProgressingCondition, + remoteKubernetesClusterFinalizerDegradedCondition, + rkc.Generation, + func() ([]metav1.Condition, error) { + return rkcc.syncFinalizer(ctx, rkc) + }, + ) + if err != nil { + return fmt.Errorf("can't finalize: %w", err) + } + return rkcc.updateStatus(ctx, rkc, status) } + if !slices.ContainsItem(rkc.GetFinalizers(), naming.RemoteKubernetesClusterFinalizer) { + err = rkcc.addFinalizer(ctx, rkc) + if err != nil { + return fmt.Errorf("can't add finalizer: %w", err) + } + return nil + } + var errs []error err = controllerhelpers.RunSync( diff --git a/pkg/controller/remotekubernetescluster/sync_finalizer.go b/pkg/controller/remotekubernetescluster/sync_finalizer.go new file mode 100644 index 0000000000..7a0790ef77 --- /dev/null +++ b/pkg/controller/remotekubernetescluster/sync_finalizer.go @@ -0,0 +1,135 @@ +// Copyright (c) 2024 ScyllaDB. + +package remotekubernetescluster + +import ( + "context" + "fmt" + "strings" + + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/helpers/slices" + "github.com/scylladb/scylla-operator/pkg/naming" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" +) + +func (rkcc *Controller) syncFinalizer(ctx context.Context, rkc *scyllav1alpha1.RemoteKubernetesCluster) ([]metav1.Condition, error) { + var progressingConditions []metav1.Condition + var err error + + if !slices.ContainsItem(rkc.GetFinalizers(), naming.RemoteKubernetesClusterFinalizer) { + klog.V(4).InfoS("Object is already finalized", "RemoteKubernetesCluster", klog.KObj(rkc), "UID", rkc.UID) + return progressingConditions, nil + } + + klog.V(4).InfoS("Finalizing object", "RemoteKubernetesCluster", klog.KObj(rkc), "UID", rkc.UID) + + isUsed, users, err := rkcc.isBeingUsed(ctx, rkc) + if err != nil { + return progressingConditions, fmt.Errorf("can't check if RemoteKubernetesCluster %q is being used: %w", naming.ObjRef(rkc), err) + } + + if isUsed { + klog.V(2).InfoS("Keeping RemoteKubernetesCluster because it's being used", "RemoteKubernetesCluster", klog.KObj(rkc)) + + progressingConditions = append(progressingConditions, metav1.Condition{ + Type: remoteKubernetesClusterFinalizerProgressingCondition, + Status: metav1.ConditionTrue, + Reason: "IsBeingUsed", + Message: fmt.Sprintf("Object is being used by following ScyllaDBCluster(s): %s", strings.Join(users, ",")), + ObservedGeneration: rkc.Generation, + }) + + return progressingConditions, nil + } + + err = rkcc.removeFinalizer(ctx, rkc) + if err != nil { + return progressingConditions, fmt.Errorf("can't remove finalizer from RemoteKubernetesCluster %q: %w", naming.ObjRef(rkc), err) + } + return progressingConditions, nil +} + +func (rkcc *Controller) addFinalizer(ctx context.Context, rkc *scyllav1alpha1.RemoteKubernetesCluster) error { + patch, err := controllerhelpers.AddFinalizerPatch(rkc, naming.RemoteKubernetesClusterFinalizer) + if err != nil { + return fmt.Errorf("can't create add finalizer patch: %w", err) + } + + _, err = rkcc.scyllaClient.RemoteKubernetesClusters().Patch(ctx, rkc.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("can't patch RemoteKubernetesCluster %q: %w", naming.ObjRef(rkc), err) + } + + klog.V(2).InfoS("Added finalizer to RemoteKubernetesCluster", "RemoteKubernetesCluster", klog.KObj(rkc)) + return nil +} + +func (rkcc *Controller) removeFinalizer(ctx context.Context, rkc *scyllav1alpha1.RemoteKubernetesCluster) error { + patch, err := controllerhelpers.RemoveFinalizerPatch(rkc, naming.RemoteKubernetesClusterFinalizer) + if err != nil { + return fmt.Errorf("can't create remove finalizer patch: %w", err) + } + + _, err = rkcc.scyllaClient.RemoteKubernetesClusters().Patch(ctx, rkc.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("can't patch RemoteKubernetesCluster %q: %w", naming.ObjRef(rkc), err) + } + + klog.V(2).InfoS("Removed finalizer from RemoteKubernetesCluster", "RemoteKubernetesCluster", klog.KObj(rkc)) + return nil +} + +func (rkcc *Controller) isBeingUsed(ctx context.Context, rkc *scyllav1alpha1.RemoteKubernetesCluster) (bool, []string, error) { + scs, err := rkcc.scyllaDBClusterLister.List(labels.Everything()) + if err != nil { + return false, nil, fmt.Errorf("can't list all ScyllaClusters using lister: %w", err) + } + + var scyllaDBClusterReferents []string + for _, sc := range scs { + for _, dc := range sc.Spec.Datacenters { + if dc.RemoteKubernetesClusterName == rkc.Name { + scyllaDBClusterReferents = append(scyllaDBClusterReferents, naming.ObjRef(sc)) + } + } + } + + if len(scyllaDBClusterReferents) != 0 { + klog.V(4).InfoS("Listed ScyllaClusters using Informer and found ScyllaDBCluster's referencing it", "RemoteKubernetesCluster", klog.KObj(rkc), "ScyllaDBClusters", scyllaDBClusterReferents) + return true, scyllaDBClusterReferents, nil + } + + klog.V(4).InfoS("No ScyllaClusters referencing RemoteKubernetesCluster found in the Informer cache", "RemoteKubernetesCluster", klog.KObj(rkc)) + + // Live list ScyllaClusters to be 100% sure before we delete. Informer cache might not be updated yet. + scList, err := rkcc.scyllaClient.ScyllaDBClusters(corev1.NamespaceAll).List(ctx, metav1.ListOptions{ + LabelSelector: labels.Everything().String(), + }) + if err != nil { + return false, nil, fmt.Errorf("list all ScyllaClusters using lister: %w", err) + } + + scyllaDBClusterReferents = scyllaDBClusterReferents[:0] + for _, sc := range scList.Items { + for _, dc := range sc.Spec.Datacenters { + if dc.RemoteKubernetesClusterName == rkc.Name { + scyllaDBClusterReferents = append(scyllaDBClusterReferents, naming.ObjRef(&sc)) + } + } + } + + if len(scyllaDBClusterReferents) != 0 { + klog.V(4).InfoS("Listed ScyllaClusters using live call and found ScyllaCluster's referencing it", "RemoteKubernetesCluster", klog.KObj(rkc), "ScyllaClusters", scyllaDBClusterReferents) + return true, scyllaDBClusterReferents, nil + } + + klog.V(2).InfoS("RemoteKubernetesCluster doesn't have any referents", "RemoteKubernetesCluster", klog.KObj(rkc)) + + return false, nil, nil +} diff --git a/pkg/controllerhelpers/finalizers.go b/pkg/controllerhelpers/finalizers.go new file mode 100644 index 0000000000..03f26047f4 --- /dev/null +++ b/pkg/controllerhelpers/finalizers.go @@ -0,0 +1,82 @@ +// Copyright (c) 2024 ScyllaDB. + +package controllerhelpers + +import ( + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type objectForFinalizersPatch struct { + objectMetaForFinalizersPatch `json:"metadata"` +} + +// objectMetaForFinalizersPatch defines object meta struct for finalizers patch operation. +type objectMetaForFinalizersPatch struct { + ResourceVersion string `json:"resourceVersion"` + Finalizers []string `json:"finalizers"` +} + +func RemoveFinalizerPatch(obj metav1.Object, finalizer string) ([]byte, error) { + if !HasFinalizer(obj, finalizer) { + return nil, nil + } + + finalizers := obj.GetFinalizers() + var newFinalizers []string + + for _, f := range finalizers { + if f == finalizer { + continue + } + newFinalizers = append(newFinalizers, f) + } + + patch, err := json.Marshal(&objectForFinalizersPatch{ + objectMetaForFinalizersPatch: objectMetaForFinalizersPatch{ + ResourceVersion: obj.GetResourceVersion(), + Finalizers: newFinalizers, + }, + }) + if err != nil { + return nil, fmt.Errorf("can't marshal finalizer remove patch: %w", err) + } + + return patch, nil +} + +func AddFinalizerPatch(obj metav1.Object, finalizer string) ([]byte, error) { + if HasFinalizer(obj, finalizer) { + return nil, nil + } + newFinalizers := make([]string, 0, len(obj.GetFinalizers())+1) + for _, f := range obj.GetFinalizers() { + newFinalizers = append(newFinalizers, f) + } + newFinalizers = append(newFinalizers, finalizer) + + patch, err := json.Marshal(&objectForFinalizersPatch{ + objectMetaForFinalizersPatch: objectMetaForFinalizersPatch{ + ResourceVersion: obj.GetResourceVersion(), + Finalizers: newFinalizers, + }, + }) + if err != nil { + return nil, fmt.Errorf("can't marshal finalizer add patch: %w", err) + } + + return patch, nil +} + +func HasFinalizer(obj metav1.Object, finalizer string) bool { + found := false + for _, f := range obj.GetFinalizers() { + if f == finalizer { + found = true + break + } + } + return found +} diff --git a/pkg/controllerhelpers/finalizers_test.go b/pkg/controllerhelpers/finalizers_test.go new file mode 100644 index 0000000000..d209048fe9 --- /dev/null +++ b/pkg/controllerhelpers/finalizers_test.go @@ -0,0 +1,137 @@ +// Copyright (c) 2024 ScyllaDB. + +package controllerhelpers + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAddFinalizerPatch(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + obj metav1.Object + finalizer string + expectedPatch []byte + expectedError error + }{ + { + name: "object has empty finalizers", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{}}, + finalizer: "my-finalizer", + expectedPatch: []byte(`{"metadata":{"resourceVersion":"123","finalizers":["my-finalizer"]}}`), + expectedError: nil, + }, + { + name: "duplicate finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{"a", "my-finalizer", "c"}}, + finalizer: "my-finalizer", + expectedPatch: nil, + expectedError: nil, + }, + } + + for i := range tt { + tc := tt[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + patch, err := AddFinalizerPatch(tc.obj, tc.finalizer) + if err != tc.expectedError { + t.Errorf("expected error %s, got %s", tc.expectedError, err) + } + if !equality.Semantic.DeepEqual(patch, tc.expectedPatch) { + t.Errorf("expected patch %s, got %s", string(tc.expectedPatch), string(patch)) + } + }) + } +} + +func TestRemoveFinalizerPatch(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + obj metav1.Object + finalizer string + expectedPatch []byte + expectedError error + }{ + { + name: "object is missing given finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{}}, + finalizer: "my-finalizer", + expectedPatch: nil, + expectedError: nil, + }, + { + name: "patch removes finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{"a", "my-finalizer", "c"}}, + finalizer: "my-finalizer", + expectedPatch: []byte(`{"metadata":{"resourceVersion":"123","finalizers":["a","c"]}}`), + expectedError: nil, + }, + { + name: "patch removes last finalizer", + obj: &metav1.ObjectMeta{ResourceVersion: "123", Finalizers: []string{"my-finalizer"}}, + finalizer: "my-finalizer", + expectedPatch: []byte(`{"metadata":{"resourceVersion":"123","finalizers":null}}`), + expectedError: nil, + }, + } + + for i := range tt { + tc := tt[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + patch, err := RemoveFinalizerPatch(tc.obj, tc.finalizer) + if err != tc.expectedError { + t.Errorf("expected error %s, got %s", tc.expectedError, err) + } + if !equality.Semantic.DeepEqual(patch, tc.expectedPatch) { + t.Errorf("expected patch %s, got %s", string(tc.expectedPatch), string(patch)) + } + }) + } +} + +func TestHasFinalizer(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + obj metav1.Object + finalizer string + expectedResult bool + }{ + { + name: "false when empty finalizer list", + obj: &metav1.ObjectMeta{Finalizers: []string{}}, + finalizer: "my-finalizer", + expectedResult: false, + }, + { + name: "true when object contains finalizer", + obj: &metav1.ObjectMeta{Finalizers: []string{"a", "b", "my-finalizer", "c"}}, + finalizer: "my-finalizer", + expectedResult: true, + }, + } + + for i := range tt { + tc := tt[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := HasFinalizer(tc.obj, tc.finalizer) + if got != tc.expectedResult { + t.Errorf("expected %v, got %v", tc.expectedResult, got) + } + }) + } +} diff --git a/pkg/naming/constants.go b/pkg/naming/constants.go index e771052c96..9030dacbfd 100644 --- a/pkg/naming/constants.go +++ b/pkg/naming/constants.go @@ -232,3 +232,7 @@ const ( const ( OperatorAppNameWithDomain = "scylla-operator.scylladb.com" ) + +const ( + RemoteKubernetesClusterFinalizer = "scylla-operator.scylladb.com/remotekubernetescluster-protection" +) diff --git a/test/e2e/set/remotekubernetescluster/config.go b/test/e2e/set/remotekubernetescluster/config.go index 8ca4ade9b6..2b9c54eab1 100644 --- a/test/e2e/set/remotekubernetescluster/config.go +++ b/test/e2e/set/remotekubernetescluster/config.go @@ -1,9 +1,26 @@ package remotekubernetescluster -import "time" +import ( + "time" + + "github.com/scylladb/scylla-operator/pkg/gather/collect" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" +) const ( testSetupTimeout = 1 * time.Minute testTeardownTimeout = 1 * time.Minute - testTimeout = 3 * time.Minute + testTimeout = 30 * time.Minute +) + +var ( + remoteKubernetesClusterResourceInfo = collect.ResourceInfo{ + Resource: schema.GroupVersionResource{ + Group: "scylla.scylladb.com", + Version: "v1alpha1", + Resource: "remotekuberneteclusters", + }, + Scope: meta.RESTScopeRoot, + } ) diff --git a/test/e2e/set/remotekubernetescluster/remotekubernetescluster_finalizer.go b/test/e2e/set/remotekubernetescluster/remotekubernetescluster_finalizer.go new file mode 100644 index 0000000000..db7e66ded1 --- /dev/null +++ b/test/e2e/set/remotekubernetescluster/remotekubernetescluster_finalizer.go @@ -0,0 +1,136 @@ +// Copyright (c) 2024 ScyllaDB. + +package remotekubernetescluster + +import ( + "context" + "fmt" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" + "github.com/scylladb/scylla-operator/pkg/controllerhelpers" + "github.com/scylladb/scylla-operator/pkg/helpers/slices" + "github.com/scylladb/scylla-operator/test/e2e/framework" + "github.com/scylladb/scylla-operator/test/e2e/utils" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = g.Describe("RemoteKubernetesCluster finalizer", func() { + f := framework.NewFramework("remotekubernetescluster") + + g.It("should block deletion when there are referencing ScyllaDBClusters", func() { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + idx := 0 + cluster := f.Cluster(idx) + userNs, _ := cluster.CreateUserNamespace(ctx) + + rkcName := fmt.Sprintf("%s-%d", f.Namespace(), idx) + + framework.By("Creating RemoteKubernetesCluster %q with credentials to cluster #%d", rkcName, idx) + originalRKC, err := utils.GetRemoteKubernetesClusterWithOperatorClusterRole(ctx, cluster.KubeAdminClient(), cluster.AdminClientConfig(), rkcName, userNs.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + + rc := framework.NewRestoringCleaner( + ctx, + f.KubeAdminClient(), + f.DynamicAdminClient(), + remoteKubernetesClusterResourceInfo, + originalRKC.Namespace, + originalRKC.Name, + framework.RestoreStrategyRecreate, + ) + f.AddCleaners(rc) + + rkc, err := cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters().Create(ctx, originalRKC, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for the RemoteKubernetesCluster %q to rollout (RV=%s)", rkc.Name, rkc.ResourceVersion) + waitCtx1, waitCtx1Cancel := utils.ContextForRemoteKubernetesClusterRollout(ctx, rkc) + defer waitCtx1Cancel() + + const expectedFinalizer = "scylla-operator.scylladb.com/remotekubernetescluster-protection" + hasRKCFinalizer := func(rkc *scyllav1alpha1.RemoteKubernetesCluster) (bool, error) { + return slices.ContainsItem(rkc.Finalizers, expectedFinalizer), nil + } + + rkc, err = controllerhelpers.WaitForRemoteKubernetesClusterState(waitCtx1, cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters(), rkc.Name, controllerhelpers.WaitForStateOptions{}, + utils.IsRemoteKubernetesClusterRolledOut, + hasRKCFinalizer, + ) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Deleting RemoteKubernetesCluster %q", rkc.Name) + err = cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters().Delete(ctx, rkcName, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Awaiting RemoteKubernetesCluster %q deletion", rkc.Name) + err = framework.WaitForObjectDeletion(ctx, f.DynamicAdminClient(), scyllav1alpha1.GroupVersion.WithResource("remotekubernetesclusters"), rkc.Namespace, rkc.Name, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Recreating RemoteKubernetesCluster %q", rkcName) + rkc = originalRKC.DeepCopy() + rkc, err = cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters().Create(ctx, rkc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for the RemoteKubernetesCluster %q to rollout (RV=%s)", rkc.Name, rkc.ResourceVersion) + waitCtx2, waitCtx2Cancel := utils.ContextForRemoteKubernetesClusterRollout(ctx, rkc) + defer waitCtx2Cancel() + + rkc, err = controllerhelpers.WaitForRemoteKubernetesClusterState(waitCtx2, cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters(), rkc.Name, controllerhelpers.WaitForStateOptions{}, + utils.IsRemoteKubernetesClusterRolledOut, + hasRKCFinalizer, + ) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Creating ScyllaDBCluster using RemoteKubernetesCluster %q", rkcName) + sc1 := f.GetDefaultScyllaDBCluster([]*scyllav1alpha1.RemoteKubernetesCluster{rkc}) + + sc1, err = cluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(f.Namespace()).Create(ctx, sc1, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Waiting for the ScyllaDBCluster %q to rollout (RV=%s)", sc1.Name, sc1.ResourceVersion) + waitCtx3, waitCtx3Cancel := utils.ContextForMultiDatacenterScyllaDBClusterRollout(ctx, sc1) + defer waitCtx3Cancel() + sc1, err = controllerhelpers.WaitForScyllaDBClusterState(waitCtx3, cluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(f.Namespace()), sc1.Name, controllerhelpers.WaitForStateOptions{}, + utils.IsScyllaDBClusterRolledOut, + ) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Deleting RemoteKubernetesCluster %q", rkc.Name) + err = cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters().Delete(ctx, rkcName, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + rkc, err = cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters().Get(ctx, rkcName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(rkc.DeletionTimestamp).NotTo(o.BeNil()) + + hasIsBeingUsedCondition := func(rkc *scyllav1alpha1.RemoteKubernetesCluster) (bool, error) { + cond := meta.FindStatusCondition(rkc.Status.Conditions, "RemoteKubernetesClusterFinalizerProgressing") + return cond != nil && cond.Reason == "RemoteKubernetesClusterFinalizerProgressing_IsBeingUsed", nil + } + framework.By("Awaiting is being used condition on RemoteKubernetesCluster %q", rkc.Name) + waitCtx4, waitCtx4Cancel := utils.ContextForRemoteKubernetesClusterRollout(ctx, rkc) + defer waitCtx4Cancel() + rkc, err = controllerhelpers.WaitForRemoteKubernetesClusterState(waitCtx4, cluster.ScyllaAdminClient().ScyllaV1alpha1().RemoteKubernetesClusters(), rkc.Name, controllerhelpers.WaitForStateOptions{}, + hasRKCFinalizer, + hasIsBeingUsedCondition, + ) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Deleting ScyllaDBCluster %q", sc1.Name) + err = cluster.ScyllaAdminClient().ScyllaV1alpha1().ScyllaDBClusters(sc1.Namespace).Delete(ctx, sc1.Name, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Awaiting ScyllaDBCluster %q deletion", sc1.Name) + err = framework.WaitForObjectDeletion(ctx, f.DynamicAdminClient(), scyllav1alpha1.GroupVersion.WithResource("scylladbclusters"), sc1.Namespace, sc1.Name, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + + framework.By("Awaiting RemoteKubernetesCluster %q deletion", rkc.Name) + err = framework.WaitForObjectDeletion(ctx, f.DynamicAdminClient(), scyllav1alpha1.GroupVersion.WithResource("remotekubernetesclusters"), rkc.Namespace, rkc.Name, nil) + o.Expect(err).NotTo(o.HaveOccurred()) + }) +})