Refactor Reaper Deployment creation to support STSs as well

k8ssandra · Jul 3, 2024 · f8bdaff · f8bdaff
1 parent fbc32af
commit f8bdaff
Show file tree

Hide file tree

Showing 7 changed files with 188 additions and 115 deletions.
diff --git a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml
@@ -28674,6 +28674,10 @@ spec:
                     type: object
                   storageType:
                     default: cassandra
+                    description: |-
+                      The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store
+                      its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster).
+                      The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters.
                     enum:
                     - cassandra
                     - memory
@@ -34568,6 +34572,10 @@ spec:
                 type: boolean
               storageType:
                 default: cassandra
+                description: |-
+                  The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store
+                  its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster).
+                  The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters.
                 enum:
                 - cassandra
                 - memory

diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
@@ -28612,6 +28612,10 @@ spec:
                     type: object
                   storageType:
                     default: cassandra
+                    description: |-
+                      The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store
+                      its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster).
+                      The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters.
                     enum:
                     - cassandra
                     - memory

diff --git a/config/crd/bases/reaper.k8ssandra.io_reapers.yaml b/config/crd/bases/reaper.k8ssandra.io_reapers.yaml
@@ -2259,6 +2259,10 @@ spec:
                 type: boolean
               storageType:
                 default: cassandra
+                description: |-
+                  The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store
+                  its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster).
+                  The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters.
                 enum:
                 - cassandra
                 - memory

diff --git a/controllers/reaper/reaper_controller.go b/controllers/reaper/reaper_controller.go
@@ -192,6 +192,8 @@ func (r *ReaperReconciler) reconcileDeployment(
 	}
 
 	logger.Info("Reconciling reaper deployment", "actualReaper", actualReaper)
+
+	// todo depending on reaper's storage type, make a StatefulSet instead
 	desiredDeployment := reaper.NewDeployment(actualReaper, actualDc, keystorePassword, truststorePassword, logger, authVars...)
 
 	actualDeployment := &appsv1.Deployment{}

diff --git a/pkg/reaper/deployment.go b/pkg/reaper/deployment.go
@@ -2,6 +2,8 @@ package reaper
 
 import (
 	"fmt"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/cassandra"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/encryption"
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
@@ -10,8 +12,6 @@ import (
 	"github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/reaper/v1alpha1"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/annotations"
-	"github.com/k8ssandra/k8ssandra-operator/pkg/cassandra"
-	"github.com/k8ssandra/k8ssandra-operator/pkg/encryption"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/images"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/meta"
 	appsv1 "k8s.io/api/apps/v1"
@@ -42,27 +42,7 @@ var defaultImage = images.Image{
 	Tag:        DefaultVersion,
 }
 
-func NewDeployment(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, keystorePassword *string, truststorePassword *string, logger logr.Logger, authVars ...*corev1.EnvVar) *appsv1.Deployment {
-	selector := metav1.LabelSelector{
-		MatchExpressions: []metav1.LabelSelectorRequirement{
-			// Note: managed-by shouldn't be used here, but we're keeping it for backwards compatibility, since changing
-			// a deployment's selector is a breaking change.
-			{
-				Key:      v1alpha1.ManagedByLabel,
-				Operator: metav1.LabelSelectorOpIn,
-				Values:   []string{v1alpha1.NameLabelValue},
-			},
-			{
-				Key:      api.ReaperLabel,
-				Operator: metav1.LabelSelectorOpIn,
-				Values:   []string{reaper.Name},
-			},
-		},
-	}
-
-	readinessProbe := computeProbe(reaper.Spec.ReadinessProbe)
-	livenessProbe := computeProbe(reaper.Spec.LivenessProbe)
-
+func computeEnvVars(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter) []corev1.EnvVar {
 	envVars := []corev1.EnvVar{
 		{
 			Name:  "REAPER_STORAGE_TYPE",
@@ -169,121 +149,198 @@ func NewDeployment(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, keysto
 		}
 	}
 
+	return envVars
+}
+
+func computeVolumes(reaper *api.Reaper) ([]corev1.Volume, []corev1.VolumeMount) {
+	volumes := []corev1.Volume{
+		{
+			Name: "conf",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
+	}
+
 	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "conf",
 			MountPath: "/etc/cassandra-reaper/config",
 		},
 	}
-	volumes := []corev1.Volume{
-		{
-			Name: "conf",
+
+	if reaper.Spec.HttpManagement.Enabled && reaper.Spec.HttpManagement.Keystores != nil {
+		volumes = append(volumes, corev1.Volume{
+			Name: "management-api-keystore",
 			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{},
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: reaper.Spec.HttpManagement.Keystores.Name,
+				},
+			},
+		})
+
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "management-api-keystore",
+			MountPath: "/etc/encryption/mgmt",
+		})
+	}
+
+	return volumes, volumeMounts
+}
+
+func makeSelector(reaper *api.Reaper) *metav1.LabelSelector {
+	return &metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			// Note: managed-by shouldn't be used here, but we're keeping it for backwards compatibility, since changing
+			// a deployment's selector is a breaking change.
+			{
+				Key:      v1alpha1.ManagedByLabel,
+				Operator: metav1.LabelSelectorOpIn,
+				Values:   []string{v1alpha1.NameLabelValue},
+			},
+			{
+				Key:      api.ReaperLabel,
+				Operator: metav1.LabelSelectorOpIn,
+				Values:   []string{reaper.Name},
 			},
 		},
 	}
+}
+
+func makeObjectMeta(reaper *api.Reaper) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Namespace:   reaper.Namespace,
+		Name:        reaper.Name,
+		Labels:      createServiceAndDeploymentLabels(reaper),
+		Annotations: map[string]string{},
+	}
+}
+
+func computePodMeta(reaper *api.Reaper) metav1.ObjectMeta {
+	podMeta := getPodMeta(reaper)
+	return metav1.ObjectMeta{
+		Labels:      podMeta.Labels,
+		Annotations: podMeta.Annotations,
+	}
+}
+
+func configureClientEncryption(reaper *api.Reaper, envVars *[]corev1.EnvVar, volumes *[]corev1.Volume, volumeMounts *[]corev1.VolumeMount, keystorePassword *string, truststorePassword *string) {
 	// if client encryption is turned on, we need to mount the keystore and truststore volumes
+	// by client we mean a C* client, so this is only relevant if we are making a Deployment which uses C* as storage backend
 	if reaper.Spec.ClientEncryptionStores != nil && keystorePassword != nil && truststorePassword != nil {
 		keystoreVolume, truststoreVolume := cassandra.EncryptionVolumes(encryption.StoreTypeClient, *reaper.Spec.ClientEncryptionStores)
-		volumes = append(volumes, *keystoreVolume)
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+		*volumes = append(*volumes, *keystoreVolume)
+		*volumeMounts = append(*volumeMounts, corev1.VolumeMount{
 			Name:      keystoreVolume.Name,
 			MountPath: cassandra.StoreMountFullPath(encryption.StoreTypeClient, encryption.StoreNameKeystore),
 		})
-		volumes = append(volumes, *truststoreVolume)
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+		*volumes = append(*volumes, *truststoreVolume)
+		*volumeMounts = append(*volumeMounts, corev1.VolumeMount{
 			Name:      truststoreVolume.Name,
 			MountPath: cassandra.StoreMountFullPath(encryption.StoreTypeClient, encryption.StoreNameTruststore),
 		})
 
 		javaOpts := fmt.Sprintf("-Djavax.net.ssl.keyStore=/mnt/client-keystore/keystore -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=/mnt/client-truststore/truststore -Djavax.net.ssl.trustStorePassword=%s -Dssl.enable=true", *keystorePassword, *truststorePassword)
-		envVars = append(envVars, corev1.EnvVar{
+		*envVars = append(*envVars, corev1.EnvVar{
 			Name:  "JAVA_OPTS",
 			Value: javaOpts,
 		})
-		envVars = append(envVars, corev1.EnvVar{
+		*envVars = append(*envVars, corev1.EnvVar{
 			Name:  "REAPER_CASS_NATIVE_PROTOCOL_SSL_ENCRYPTION_ENABLED",
 			Value: "true",
 		})
 	}
+}
 
-	if reaper.Spec.HttpManagement.Enabled && reaper.Spec.HttpManagement.Keystores != nil {
-		volumes = append(volumes, corev1.Volume{
-			Name: "management-api-keystore",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: reaper.Spec.HttpManagement.Keystores.Name,
+func computePodSpec(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, initContainerResources *corev1.ResourceRequirements, keystorePassword *string, truststorePassword *string) corev1.PodSpec {
+	envVars := computeEnvVars(reaper, dc)
+	volumes, volumeMounts := computeVolumes(reaper)
+	mainImage := reaper.Spec.ContainerImage.ApplyDefaults(defaultImage)
+	mainContainerResources := computeMainContainerResources(reaper.Spec.Resources)
+
+	if keystorePassword != nil && truststorePassword != nil {
+		configureClientEncryption(reaper, &envVars, &volumes, &volumeMounts, keystorePassword, truststorePassword)
+	}
+
+	var initContainers []corev1.Container
+	if initContainerResources != nil {
+		initContainers = computeInitContainers(reaper, mainImage, envVars, volumeMounts, initContainerResources)
+	} else {
+		initContainers = nil
+	}
+
+	return corev1.PodSpec{
+		Affinity:       reaper.Spec.Affinity,
+		InitContainers: initContainers,
+		Containers: []corev1.Container{
+			{
+				Name:            "reaper",
+				Image:           mainImage.String(),
+				ImagePullPolicy: mainImage.PullPolicy,
+				SecurityContext: reaper.Spec.SecurityContext,
+				Ports: []corev1.ContainerPort{
+					{
+						Name:          "app",
+						ContainerPort: 8080,
+						Protocol:      "TCP",
+					},
+					{
+						Name:          "admin",
+						ContainerPort: 8081,
+						Protocol:      "TCP",
+					},
 				},
+				ReadinessProbe: computeProbe(reaper.Spec.ReadinessProbe),
+				LivenessProbe:  computeProbe(reaper.Spec.LivenessProbe),
+				Env:            envVars,
+				VolumeMounts:   volumeMounts,
+				Resources:      *mainContainerResources,
 			},
-		})
+		},
+		ServiceAccountName: reaper.Spec.ServiceAccountName,
+		Tolerations:        reaper.Spec.Tolerations,
+		SecurityContext:    reaper.Spec.PodSecurityContext,
+		ImagePullSecrets:   computeImagePullSecrets(reaper, mainImage),
+		Volumes:            volumes,
+	}
+}
 
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      "management-api-keystore",
-			MountPath: "/etc/encryption/mgmt",
-		})
+func NewStatefulSet(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, logger logr.Logger, authVars ...*corev1.EnvVar) *appsv1.StatefulSet {
+
+	// todo add a volume for reapers data
+
+	statefulSet := &appsv1.StatefulSet{
+		ObjectMeta: makeObjectMeta(reaper),
+		Spec: appsv1.StatefulSetSpec{
+			Selector: makeSelector(reaper),
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: computePodMeta(reaper),
+				Spec:       computePodSpec(reaper, dc, nil, nil, nil),
+			},
+		},
 	}
+	addAuthEnvVars(&statefulSet.Spec.Template, authVars)
+	configureVector(reaper, &statefulSet.Spec.Template, dc, logger)
+	annotations.AddHashAnnotation(statefulSet)
+	return statefulSet
+}
 
-	mainImage := reaper.Spec.ContainerImage.ApplyDefaults(defaultImage)
+func NewDeployment(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, keystorePassword *string, truststorePassword *string, logger logr.Logger, authVars ...*corev1.EnvVar) *appsv1.Deployment {
 
 	initContainerResources := computeInitContainerResources(reaper.Spec.InitContainerResources)
-	mainContainerResources := computeMainContainerResources(reaper.Spec.Resources)
-
-	podMeta := getPodMeta(reaper)
 
 	deployment := &appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace:   reaper.Namespace,
-			Name:        reaper.Name,
-			Labels:      createServiceAndDeploymentLabels(reaper),
-			Annotations: map[string]string{},
-		},
+		ObjectMeta: makeObjectMeta(reaper),
 		Spec: appsv1.DeploymentSpec{
-			Selector: &selector,
+			Selector: makeSelector(reaper),
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels:      podMeta.Labels,
-					Annotations: podMeta.Annotations,
-				},
-				Spec: corev1.PodSpec{
-					Affinity:       reaper.Spec.Affinity,
-					InitContainers: computeInitContainers(reaper, mainImage, envVars, volumeMounts, initContainerResources),
-					Containers: []corev1.Container{
-						{
-							Name:            "reaper",
-							Image:           mainImage.String(),
-							ImagePullPolicy: mainImage.PullPolicy,
-							SecurityContext: reaper.Spec.SecurityContext,
-							Ports: []corev1.ContainerPort{
-								{
-									Name:          "app",
-									ContainerPort: 8080,
-									Protocol:      "TCP",
-								},
-								{
-									Name:          "admin",
-									ContainerPort: 8081,
-									Protocol:      "TCP",
-								},
-							},
-							ReadinessProbe: readinessProbe,
-							LivenessProbe:  livenessProbe,
-							Env:            envVars,
-							VolumeMounts:   volumeMounts,
-							Resources:      *mainContainerResources,
-						},
-					},
-					ServiceAccountName: reaper.Spec.ServiceAccountName,
-					Tolerations:        reaper.Spec.Tolerations,
-					SecurityContext:    reaper.Spec.PodSecurityContext,
-					ImagePullSecrets:   computeImagePullSecrets(reaper, mainImage),
-					Volumes:            volumes,
-				},
+				ObjectMeta: computePodMeta(reaper),
+				Spec:       computePodSpec(reaper, dc, initContainerResources, keystorePassword, truststorePassword),
 			},
 		},
 	}
-	addAuthEnvVars(deployment, authVars)
-	configureVector(reaper, deployment, dc, logger)
+	addAuthEnvVars(&deployment.Spec.Template, authVars)
+	configureVector(reaper, &deployment.Spec.Template, dc, logger)
 	annotations.AddHashAnnotation(deployment)
 	return deployment
 }
@@ -367,18 +424,18 @@ func computeProbe(probeTemplate *corev1.Probe) *corev1.Probe {
 	return probe
 }
 
-func addAuthEnvVars(deployment *appsv1.Deployment, vars []*corev1.EnvVar) {
-	envVars := deployment.Spec.Template.Spec.Containers[0].Env
+func addAuthEnvVars(template *corev1.PodTemplateSpec, vars []*corev1.EnvVar) {
+	envVars := template.Spec.Containers[0].Env
 	for _, v := range vars {
 		envVars = append(envVars, *v)
 	}
-	deployment.Spec.Template.Spec.Containers[0].Env = envVars
-	if len(deployment.Spec.Template.Spec.InitContainers) > 0 {
-		initEnvVars := deployment.Spec.Template.Spec.InitContainers[0].Env
+	template.Spec.Containers[0].Env = envVars
+	if len(template.Spec.InitContainers) > 0 {
+		initEnvVars := template.Spec.InitContainers[0].Env
 		for _, v := range vars {
 			initEnvVars = append(initEnvVars, *v)
 		}
-		deployment.Spec.Template.Spec.InitContainers[0].Env = initEnvVars
+		template.Spec.InitContainers[0].Env = initEnvVars
 	}
 }