Only local Medusaconfigs allowed (#1267)

* Add webhook failure on resource creation for MedusaConfigRef in non-local ns.
* Switch out the current medusa bucket secret copying logic for a ReplicatedSecret.
* Allow labels to be added or dropped when replicating a secret via a ReplicatedSecret. This is available per target.

CHANGELOG/CHANGELOG-1.15.md

@@ -14,3 +14,7 @@ Changelog for the K8ssandra Operator, new PRs should update the `unreleased` sec
 When cutting a new release, update the `unreleased` heading to the tag being generated and date, like `## vX.Y.Z - YYYY-MM-DD` and create a new placeholder section for  `unreleased` entries.
 
 ## unreleased
+* [BUGFIX] [#1266](https://github.com/k8ssandra/k8ssandra-operator/issues/1266) MedusaConfigurations must now be namespace local to the K8ssandraCluster they are attached to, a webhook error will be thrown otherwise (for new clusters only). Additionally, ReplicatedSecrets should only pick up secrets from their local namespace to replicate.
+* [BUGFIX] [#1217](https://github.com/k8ssandra/k8ssandra-operator/issues/1217) Medusa storage secrets now use a ReplicatedSecret for synchronization, fixing an issue where changes to the secrets were not propagating. Additionally, fix a number of issues with local testing on ARM Macs.
+* [BUGFIX] [#1253](https://github.com/k8ssandra/k8ssandra-operator/issues/1253) Medusa storage secrets are now labelled with a unique label.
+* [FEATURE] [#1260](https://github.com/k8ssandra/k8ssandra-operator/issues/1260) Update controller-gen to version 0.14.0.

CHANGELOG/RELEASE-NOTES.md

@@ -1,5 +1,16 @@
 # k8ssandra-operator - Release Notes
 
+## v1.15.0
+
+### Deprecation of non-namespace-local MedusaConfigRef
+
+The previous version introduced functionality whereby a K8ssandraCluster could reference a MedusaConfiguration (via MedusaConfigRef) in a remote namespace within the same k8s cluster. This functionality is deprecated. Existing clusters will continue to reconcile, but new clusters (or updates to existing clusters) will be rejected at the webhook.
+
+To update an existing cluster, or create a new one, ensure that the `namespace` field is left unset in the `medusaConfigRef`, and ensure that the MedusaConfiguration you are referencing exists within the K8ssandraCluster's local namespace. 
+
+If this functionality is critical to your use case, please raise an issue on Github and describe why it is important to you.
+
+
 ## v1.12.0
 
 It is now possible to disable Reaper front end authentication by adding either `spec.reaper.uiUserSecretRef: {}` or `spec.reaper.uiUserSecretRef: ""`. 

Makefile

@@ -177,7 +177,8 @@ DATE  ?= $(shell date)
 COMMIT ?= $(shell git rev-parse --short HEAD)
 
 build: generate fmt vet ## Build manager binary.
-	go build --ldflags "-X \"main.version=${VERSION}\" -X \"main.date=${DATE}\" -X \"main.commit=${COMMIT}\"" -o bin/manager main.go
+	echo "\"main.version=${VERSION}\" -X \"main.date=$(DATE)\" -X \"main.commit=$(COMMIT)\"" 
+	go build -ldflags "-X \"main.version=${VERSION}\" -X \"main.date=$(DATE)\" -X \"main.commit=$(COMMIT)\"" -o bin/manager main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go

apis/k8ssandra/v1alpha1/constants.go

@@ -61,6 +61,9 @@ const (
 	DatacenterLabel = "k8ssandra.io/datacenter"
 	// Forces refresh of secrets which relate to roles and authn in Cassandra.
 	RefreshAnnotation = "k8ssandra.io/refresh"
+
+	// Annotation to indicate the purpose of a given resource.
+	PurposeAnnotation = "k8ssandra.io/purpose"
 )
 
 var (

apis/k8ssandra/v1alpha1/k8ssandracluster_types.go

@@ -543,6 +543,6 @@ func (kc *K8ssandraCluster) DefaultNumTokens(serverVersion *semver.Version) floa
 // this is to be used to name resources that are cluster specific in preference of concatenations
 // of the namespaced name, as the latter are becoming too long and causing issues due to DNS name length
 // constraints within k8s
-func (kc *K8ssandraCluster) GetClusterIdHash(nchars int) string {
+func (kc *K8ssandraCluster) GetClusterIdHash() string {
 	return utils.HashNameNamespace(kc.Name, kc.Namespace)
 }

apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go

@@ -18,9 +18,10 @@ package v1alpha1
 
 import (
 	"fmt"
-	"k8s.io/apimachinery/pkg/util/validation"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/util/validation"
+
 	"github.com/Masterminds/semver/v3"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/clientcache"
 	"github.com/pkg/errors"
@@ -98,14 +99,8 @@ func (r *K8ssandraCluster) validateK8ssandraCluster() error {
 		}
 	}
 
-	// Verify the Medusa storage prefix is explicitly set
-	// only relevant if Medusa is enabled and the MedusaConfiguration object is referenced
-	if r.Spec.Medusa != nil {
-		if r.Spec.Medusa.MedusaConfigurationRef.Name != "" {
-			if r.Spec.Medusa.StorageProperties.Prefix == "" {
-				return ErrNoStoragePrefix
-			}
-		}
+	if err := r.ValidateMedusa(); err != nil {
+		return err
 	}
 
 	if err := r.validateStatefulsetNameSize(); err != nil {
@@ -201,3 +196,30 @@ func (r *K8ssandraCluster) ValidateDelete() error {
 	webhookLog.Info("validate K8ssandraCluster delete", "name", r.Name)
 	return nil
 }
+
+func (r *K8ssandraCluster) ValidateMedusa() error {
+	if r.Spec.Medusa == nil {
+		return nil
+	}
+
+	// Verify the Medusa storage prefix is explicitly set
+	// only relevant if Medusa is enabled and the MedusaConfiguration object is referenced
+	if r.Spec.Medusa.MedusaConfigurationRef.Name != "" {
+		if r.Spec.Medusa.StorageProperties.Prefix == "" {
+			return ErrNoStoragePrefix
+		}
+		// Verify that any referenced MedusaConfig is NS-local
+		if r.Spec.Medusa.MedusaConfigurationRef.Namespace != "" {
+			return errors.New("Medusa config must be namespace local")
+		}
+		if r.Spec.Medusa.MedusaConfigurationRef.APIVersion != "" ||
+			r.Spec.Medusa.MedusaConfigurationRef.Kind != "" ||
+			r.Spec.Medusa.MedusaConfigurationRef.FieldPath != "" ||
+			r.Spec.Medusa.MedusaConfigurationRef.ResourceVersion != "" ||
+			r.Spec.Medusa.MedusaConfigurationRef.UID != "" {
+			return errors.New("Medusa config invalid, invalid field used")
+		}
+	}
+
+	return nil
+}

apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go

@@ -148,6 +148,7 @@ func TestWebhook(t *testing.T) {
 	t.Run("StsNameTooLong", testStsNameTooLong)
 	t.Run("MedusaPrefixMissing", testMedusaPrefixMissing)
 	t.Run("InvalidDcName", testInvalidDcName)
+	t.Run("MedusaConfigNonLocalNamespace", testMedusaNonLocalNamespace)
 }
 
 func testContextValidation(t *testing.T) {
@@ -465,3 +466,20 @@ func testInvalidDcName(t *testing.T) {
 	required.Error(err)
 	required.Contains(err.Error(), "invalid DC name")
 }
+
+func testMedusaNonLocalNamespace(t *testing.T) {
+	required := require.New(t)
+	badCluster := createMinimalClusterObj("medusaconfig-nonlocal", "ns")
+	badCluster.Spec.Medusa = &medusaapi.MedusaClusterTemplate{
+		MedusaConfigurationRef: corev1.ObjectReference{
+			Namespace: "nonlocal-ns",
+			Name:      "medusa-config",
+		},
+		StorageProperties: medusaapi.Storage{
+			Prefix: "some-prefix",
+		},
+	}
+	err := badCluster.validateK8ssandraCluster()
+	required.Error(err)
+	required.Contains(err.Error(), "Medusa config must be namespace local")
+}

apis/medusa/v1alpha1/medusaconfiguration_types.go

@@ -20,6 +20,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	MedusaStorageSecretIdentifierLabel = "k8ssandra.io/medusa-storage-secret"
+)
+
 // MedusaConfigurationSpec defines the desired state of MedusaConfiguration
 type MedusaConfigurationSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster

apis/replication/v1alpha1/replicatedsecret_types.go

@@ -56,6 +56,12 @@ type ReplicationTarget struct {
 	// as the original secret.
 	// +optional
 	TargetPrefix string `json:"targetPrefix,omitempty"`
+
+	// DropLabels defines the labels to be dropped from the secret before replication, this is sometimes neccessary to avoid infinite replication.
+	DropLabels []string `json:"dropLabels,omitempty"`
+
+	// AddLabels adds labels to the target secret.
+	AddLabels map[string]string `json:"addLabels,omitempty"`
 }
 
 // ReplicatedSecretStatus defines the observed state of ReplicatedSecret

charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml

@@ -33387,6 +33387,18 @@ spec:
                   the secrets are replicated to. If empty, no clusters are targeted
                 items:
                   properties:
+                    addLabels:
+                      additionalProperties:
+                        type: string
+                      description: AddLabels adds labels to the target secret.
+                      type: object
+                    dropLabels:
+                      description: DropLabels defines the labels to be dropped from
+                        the secret before replication, this is sometimes neccessary
+                        to avoid infinite replication.
+                      items:
+                        type: string
+                      type: array
                     k8sContextName:
                       description: K8sContextName defines the target cluster name
                         as set in the ClientConfig. If left empty, current cluster

config/crd/bases/replication.k8ssandra.io_replicatedsecrets.yaml

@@ -44,6 +44,18 @@ spec:
                   the secrets are replicated to. If empty, no clusters are targeted
                 items:
                   properties:
+                    addLabels:
+                      additionalProperties:
+                        type: string
+                      description: AddLabels adds labels to the target secret.
+                      type: object
+                    dropLabels:
+                      description: DropLabels defines the labels to be dropped from
+                        the secret before replication, this is sometimes neccessary
+                        to avoid infinite replication.
+                      items:
+                        type: string
+                      type: array
                     k8sContextName:
                       description: K8sContextName defines the target cluster name
                         as set in the ClientConfig. If left empty, current cluster

controllers/k8ssandra/add_dc_test.go

@@ -744,32 +744,39 @@ func addStargateAndReaperToCluster(ctx context.Context, t *testing.T, f *framewo
 }
 
 func addDcToCluster(ctx context.Context, t *testing.T, f *framework.Framework, kc *api.K8ssandraCluster, dcKey framework.ClusterKey) {
-	t.Logf("add %s to cluster", dcKey.Name)
+	require.Eventually(t, func() bool {
+		t.Logf("add %s to cluster", dcKey.Name)
 
-	key := utils.GetKey(kc)
-	err := f.Client.Get(ctx, key, kc)
-	require.NoError(t, err, "failed to get K8ssandraCluster")
+		key := utils.GetKey(kc)
+		err := f.Client.Get(ctx, key, kc)
+		require.NoError(t, err, "failed to get K8ssandraCluster")
 
-	kc.Spec.Cassandra.Datacenters = append(kc.Spec.Cassandra.Datacenters, api.CassandraDatacenterTemplate{
-		Meta: api.EmbeddedObjectMeta{
-			Name:      dcKey.Name,
-			Namespace: dcKey.Namespace,
-		},
-		K8sContext: dcKey.K8sContext,
-		Size:       3,
-		DatacenterOptions: api.DatacenterOptions{
-			ServerVersion: "4.0.1",
-			StorageConfig: &cassdcapi.StorageConfig{
-				CassandraDataVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{
-					StorageClassName: &defaultStorageClass,
+		kc.Spec.Cassandra.Datacenters = append(kc.Spec.Cassandra.Datacenters, api.CassandraDatacenterTemplate{
+			Meta: api.EmbeddedObjectMeta{
+				Name:      dcKey.Name,
+				Namespace: dcKey.Namespace,
+			},
+			K8sContext: dcKey.K8sContext,
+			Size:       3,
+			DatacenterOptions: api.DatacenterOptions{
+				ServerVersion: "4.0.1",
+				StorageConfig: &cassdcapi.StorageConfig{
+					CassandraDataVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{
+						StorageClassName: &defaultStorageClass,
+					},
 				},
 			},
-		},
-	})
-	annotations.AddAnnotation(kc, api.DcReplicationAnnotation, fmt.Sprintf(`{"%s": {"ks1": 3, "ks2": 3}}`, dcKey.Name))
+		})
+		annotations.AddAnnotation(kc, api.DcReplicationAnnotation, fmt.Sprintf(`{"%s": {"ks1": 3, "ks2": 3}}`, dcKey.Name))
+
+		err = f.Client.Update(ctx, kc)
+		if err != nil {
+			t.Logf("failed to add %s to cluster: %v", dcKey.Name, err)
+			return false
+		}
+		return err == nil
+	}, timeout, interval)
 
-	err = f.Client.Update(ctx, kc)
-	require.NoError(t, err, "failed to add dc to K8ssandraCluster")
 }
 
 func verifyReplicationOfSystemKeyspacesUpdated(t *testing.T, mockMgmtApi *testutils.FakeManagementApiFacade, replication, updatedReplication map[string]int) {