From f8bdaff362bd38e93cb8b7468af94b259f186193 Mon Sep 17 00:00:00 2001 From: Radovan Zvoncek Date: Wed, 3 Jul 2024 15:22:07 +0300 Subject: [PATCH] Refactor Reaper Deployment creation to support STSs as well --- .../crds/k8ssandra-operator-crds.yaml | 8 + .../bases/k8ssandra.io_k8ssandraclusters.yaml | 4 + .../bases/reaper.k8ssandra.io_reapers.yaml | 4 + controllers/reaper/reaper_controller.go | 2 + pkg/reaper/deployment.go | 259 +++++++++++------- pkg/reaper/vector.go | 8 +- pkg/reaper/vector_test.go | 18 +- 7 files changed, 188 insertions(+), 115 deletions(-) diff --git a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml index bdcffd571..85c68eb1a 100644 --- a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml +++ b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml @@ -28674,6 +28674,10 @@ spec: type: object storageType: default: cassandra + description: |- + The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store + its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster). + The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters. enum: - cassandra - memory @@ -34568,6 +34572,10 @@ spec: type: boolean storageType: default: cassandra + description: |- + The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store + its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster). + The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters. enum: - cassandra - memory diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml index 3c0a56853..d5fe35699 100644 --- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml +++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml @@ -28612,6 +28612,10 @@ spec: type: object storageType: default: cassandra + description: |- + The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store + its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster). + The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters. enum: - cassandra - memory diff --git a/config/crd/bases/reaper.k8ssandra.io_reapers.yaml b/config/crd/bases/reaper.k8ssandra.io_reapers.yaml index ab85ce91d..1dc806efa 100644 --- a/config/crd/bases/reaper.k8ssandra.io_reapers.yaml +++ b/config/crd/bases/reaper.k8ssandra.io_reapers.yaml @@ -2259,6 +2259,10 @@ spec: type: boolean storageType: default: cassandra + description: |- + The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store + its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster). + The "memory" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters. enum: - cassandra - memory diff --git a/controllers/reaper/reaper_controller.go b/controllers/reaper/reaper_controller.go index 2ff49f715..a81f97365 100644 --- a/controllers/reaper/reaper_controller.go +++ b/controllers/reaper/reaper_controller.go @@ -192,6 +192,8 @@ func (r *ReaperReconciler) reconcileDeployment( } logger.Info("Reconciling reaper deployment", "actualReaper", actualReaper) + + // todo depending on reaper's storage type, make a StatefulSet instead desiredDeployment := reaper.NewDeployment(actualReaper, actualDc, keystorePassword, truststorePassword, logger, authVars...) actualDeployment := &appsv1.Deployment{} diff --git a/pkg/reaper/deployment.go b/pkg/reaper/deployment.go index b291ab3a4..678f397f5 100644 --- a/pkg/reaper/deployment.go +++ b/pkg/reaper/deployment.go @@ -2,6 +2,8 @@ package reaper import ( "fmt" + "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" + "github.com/k8ssandra/k8ssandra-operator/pkg/encryption" "strings" "github.com/Masterminds/semver/v3" @@ -10,8 +12,6 @@ import ( "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/reaper/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/annotations" - "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" - "github.com/k8ssandra/k8ssandra-operator/pkg/encryption" "github.com/k8ssandra/k8ssandra-operator/pkg/images" "github.com/k8ssandra/k8ssandra-operator/pkg/meta" appsv1 "k8s.io/api/apps/v1" @@ -42,27 +42,7 @@ var defaultImage = images.Image{ Tag: DefaultVersion, } -func NewDeployment(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, keystorePassword *string, truststorePassword *string, logger logr.Logger, authVars ...*corev1.EnvVar) *appsv1.Deployment { - selector := metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - // Note: managed-by shouldn't be used here, but we're keeping it for backwards compatibility, since changing - // a deployment's selector is a breaking change. - { - Key: v1alpha1.ManagedByLabel, - Operator: metav1.LabelSelectorOpIn, - Values: []string{v1alpha1.NameLabelValue}, - }, - { - Key: api.ReaperLabel, - Operator: metav1.LabelSelectorOpIn, - Values: []string{reaper.Name}, - }, - }, - } - - readinessProbe := computeProbe(reaper.Spec.ReadinessProbe) - livenessProbe := computeProbe(reaper.Spec.LivenessProbe) - +func computeEnvVars(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter) []corev1.EnvVar { envVars := []corev1.EnvVar{ { Name: "REAPER_STORAGE_TYPE", @@ -169,121 +149,198 @@ func NewDeployment(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, keysto } } + return envVars +} + +func computeVolumes(reaper *api.Reaper) ([]corev1.Volume, []corev1.VolumeMount) { + volumes := []corev1.Volume{ + { + Name: "conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + volumeMounts := []corev1.VolumeMount{ { Name: "conf", MountPath: "/etc/cassandra-reaper/config", }, } - volumes := []corev1.Volume{ - { - Name: "conf", + + if reaper.Spec.HttpManagement.Enabled && reaper.Spec.HttpManagement.Keystores != nil { + volumes = append(volumes, corev1.Volume{ + Name: "management-api-keystore", VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, + Secret: &corev1.SecretVolumeSource{ + SecretName: reaper.Spec.HttpManagement.Keystores.Name, + }, + }, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "management-api-keystore", + MountPath: "/etc/encryption/mgmt", + }) + } + + return volumes, volumeMounts +} + +func makeSelector(reaper *api.Reaper) *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + // Note: managed-by shouldn't be used here, but we're keeping it for backwards compatibility, since changing + // a deployment's selector is a breaking change. + { + Key: v1alpha1.ManagedByLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{v1alpha1.NameLabelValue}, + }, + { + Key: api.ReaperLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{reaper.Name}, }, }, } +} + +func makeObjectMeta(reaper *api.Reaper) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Namespace: reaper.Namespace, + Name: reaper.Name, + Labels: createServiceAndDeploymentLabels(reaper), + Annotations: map[string]string{}, + } +} + +func computePodMeta(reaper *api.Reaper) metav1.ObjectMeta { + podMeta := getPodMeta(reaper) + return metav1.ObjectMeta{ + Labels: podMeta.Labels, + Annotations: podMeta.Annotations, + } +} + +func configureClientEncryption(reaper *api.Reaper, envVars *[]corev1.EnvVar, volumes *[]corev1.Volume, volumeMounts *[]corev1.VolumeMount, keystorePassword *string, truststorePassword *string) { // if client encryption is turned on, we need to mount the keystore and truststore volumes + // by client we mean a C* client, so this is only relevant if we are making a Deployment which uses C* as storage backend if reaper.Spec.ClientEncryptionStores != nil && keystorePassword != nil && truststorePassword != nil { keystoreVolume, truststoreVolume := cassandra.EncryptionVolumes(encryption.StoreTypeClient, *reaper.Spec.ClientEncryptionStores) - volumes = append(volumes, *keystoreVolume) - volumeMounts = append(volumeMounts, corev1.VolumeMount{ + *volumes = append(*volumes, *keystoreVolume) + *volumeMounts = append(*volumeMounts, corev1.VolumeMount{ Name: keystoreVolume.Name, MountPath: cassandra.StoreMountFullPath(encryption.StoreTypeClient, encryption.StoreNameKeystore), }) - volumes = append(volumes, *truststoreVolume) - volumeMounts = append(volumeMounts, corev1.VolumeMount{ + *volumes = append(*volumes, *truststoreVolume) + *volumeMounts = append(*volumeMounts, corev1.VolumeMount{ Name: truststoreVolume.Name, MountPath: cassandra.StoreMountFullPath(encryption.StoreTypeClient, encryption.StoreNameTruststore), }) javaOpts := fmt.Sprintf("-Djavax.net.ssl.keyStore=/mnt/client-keystore/keystore -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=/mnt/client-truststore/truststore -Djavax.net.ssl.trustStorePassword=%s -Dssl.enable=true", *keystorePassword, *truststorePassword) - envVars = append(envVars, corev1.EnvVar{ + *envVars = append(*envVars, corev1.EnvVar{ Name: "JAVA_OPTS", Value: javaOpts, }) - envVars = append(envVars, corev1.EnvVar{ + *envVars = append(*envVars, corev1.EnvVar{ Name: "REAPER_CASS_NATIVE_PROTOCOL_SSL_ENCRYPTION_ENABLED", Value: "true", }) } +} - if reaper.Spec.HttpManagement.Enabled && reaper.Spec.HttpManagement.Keystores != nil { - volumes = append(volumes, corev1.Volume{ - Name: "management-api-keystore", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: reaper.Spec.HttpManagement.Keystores.Name, +func computePodSpec(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, initContainerResources *corev1.ResourceRequirements, keystorePassword *string, truststorePassword *string) corev1.PodSpec { + envVars := computeEnvVars(reaper, dc) + volumes, volumeMounts := computeVolumes(reaper) + mainImage := reaper.Spec.ContainerImage.ApplyDefaults(defaultImage) + mainContainerResources := computeMainContainerResources(reaper.Spec.Resources) + + if keystorePassword != nil && truststorePassword != nil { + configureClientEncryption(reaper, &envVars, &volumes, &volumeMounts, keystorePassword, truststorePassword) + } + + var initContainers []corev1.Container + if initContainerResources != nil { + initContainers = computeInitContainers(reaper, mainImage, envVars, volumeMounts, initContainerResources) + } else { + initContainers = nil + } + + return corev1.PodSpec{ + Affinity: reaper.Spec.Affinity, + InitContainers: initContainers, + Containers: []corev1.Container{ + { + Name: "reaper", + Image: mainImage.String(), + ImagePullPolicy: mainImage.PullPolicy, + SecurityContext: reaper.Spec.SecurityContext, + Ports: []corev1.ContainerPort{ + { + Name: "app", + ContainerPort: 8080, + Protocol: "TCP", + }, + { + Name: "admin", + ContainerPort: 8081, + Protocol: "TCP", + }, }, + ReadinessProbe: computeProbe(reaper.Spec.ReadinessProbe), + LivenessProbe: computeProbe(reaper.Spec.LivenessProbe), + Env: envVars, + VolumeMounts: volumeMounts, + Resources: *mainContainerResources, }, - }) + }, + ServiceAccountName: reaper.Spec.ServiceAccountName, + Tolerations: reaper.Spec.Tolerations, + SecurityContext: reaper.Spec.PodSecurityContext, + ImagePullSecrets: computeImagePullSecrets(reaper, mainImage), + Volumes: volumes, + } +} - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "management-api-keystore", - MountPath: "/etc/encryption/mgmt", - }) +func NewStatefulSet(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, logger logr.Logger, authVars ...*corev1.EnvVar) *appsv1.StatefulSet { + + // todo add a volume for reapers data + + statefulSet := &appsv1.StatefulSet{ + ObjectMeta: makeObjectMeta(reaper), + Spec: appsv1.StatefulSetSpec{ + Selector: makeSelector(reaper), + Template: corev1.PodTemplateSpec{ + ObjectMeta: computePodMeta(reaper), + Spec: computePodSpec(reaper, dc, nil, nil, nil), + }, + }, } + addAuthEnvVars(&statefulSet.Spec.Template, authVars) + configureVector(reaper, &statefulSet.Spec.Template, dc, logger) + annotations.AddHashAnnotation(statefulSet) + return statefulSet +} - mainImage := reaper.Spec.ContainerImage.ApplyDefaults(defaultImage) +func NewDeployment(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, keystorePassword *string, truststorePassword *string, logger logr.Logger, authVars ...*corev1.EnvVar) *appsv1.Deployment { initContainerResources := computeInitContainerResources(reaper.Spec.InitContainerResources) - mainContainerResources := computeMainContainerResources(reaper.Spec.Resources) - - podMeta := getPodMeta(reaper) deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: reaper.Namespace, - Name: reaper.Name, - Labels: createServiceAndDeploymentLabels(reaper), - Annotations: map[string]string{}, - }, + ObjectMeta: makeObjectMeta(reaper), Spec: appsv1.DeploymentSpec{ - Selector: &selector, + Selector: makeSelector(reaper), Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podMeta.Labels, - Annotations: podMeta.Annotations, - }, - Spec: corev1.PodSpec{ - Affinity: reaper.Spec.Affinity, - InitContainers: computeInitContainers(reaper, mainImage, envVars, volumeMounts, initContainerResources), - Containers: []corev1.Container{ - { - Name: "reaper", - Image: mainImage.String(), - ImagePullPolicy: mainImage.PullPolicy, - SecurityContext: reaper.Spec.SecurityContext, - Ports: []corev1.ContainerPort{ - { - Name: "app", - ContainerPort: 8080, - Protocol: "TCP", - }, - { - Name: "admin", - ContainerPort: 8081, - Protocol: "TCP", - }, - }, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, - Env: envVars, - VolumeMounts: volumeMounts, - Resources: *mainContainerResources, - }, - }, - ServiceAccountName: reaper.Spec.ServiceAccountName, - Tolerations: reaper.Spec.Tolerations, - SecurityContext: reaper.Spec.PodSecurityContext, - ImagePullSecrets: computeImagePullSecrets(reaper, mainImage), - Volumes: volumes, - }, + ObjectMeta: computePodMeta(reaper), + Spec: computePodSpec(reaper, dc, initContainerResources, keystorePassword, truststorePassword), }, }, } - addAuthEnvVars(deployment, authVars) - configureVector(reaper, deployment, dc, logger) + addAuthEnvVars(&deployment.Spec.Template, authVars) + configureVector(reaper, &deployment.Spec.Template, dc, logger) annotations.AddHashAnnotation(deployment) return deployment } @@ -367,18 +424,18 @@ func computeProbe(probeTemplate *corev1.Probe) *corev1.Probe { return probe } -func addAuthEnvVars(deployment *appsv1.Deployment, vars []*corev1.EnvVar) { - envVars := deployment.Spec.Template.Spec.Containers[0].Env +func addAuthEnvVars(template *corev1.PodTemplateSpec, vars []*corev1.EnvVar) { + envVars := template.Spec.Containers[0].Env for _, v := range vars { envVars = append(envVars, *v) } - deployment.Spec.Template.Spec.Containers[0].Env = envVars - if len(deployment.Spec.Template.Spec.InitContainers) > 0 { - initEnvVars := deployment.Spec.Template.Spec.InitContainers[0].Env + template.Spec.Containers[0].Env = envVars + if len(template.Spec.InitContainers) > 0 { + initEnvVars := template.Spec.InitContainers[0].Env for _, v := range vars { initEnvVars = append(initEnvVars, *v) } - deployment.Spec.Template.Spec.InitContainers[0].Env = initEnvVars + template.Spec.InitContainers[0].Env = initEnvVars } } diff --git a/pkg/reaper/vector.go b/pkg/reaper/vector.go index dd9da7e13..41b27ab30 100644 --- a/pkg/reaper/vector.go +++ b/pkg/reaper/vector.go @@ -2,7 +2,6 @@ package reaper import ( "fmt" - "github.com/k8ssandra/k8ssandra-operator/pkg/labels" "github.com/k8ssandra/k8ssandra-operator/pkg/utils" "sigs.k8s.io/controller-runtime/pkg/client" @@ -13,7 +12,6 @@ import ( api "github.com/k8ssandra/k8ssandra-operator/apis/reaper/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "github.com/k8ssandra/k8ssandra-operator/pkg/vector" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -49,7 +47,7 @@ func CreateVectorConfigMap(namespace, vectorToml string, dc cassdcapi.CassandraD } } -func configureVector(reaper *api.Reaper, deployment *appsv1.Deployment, dc *cassdcapi.CassandraDatacenter, logger logr.Logger) { +func configureVector(reaper *api.Reaper, template *corev1.PodTemplateSpec, dc *cassdcapi.CassandraDatacenter, logger logr.Logger) { if reaper.Spec.Telemetry.IsVectorEnabled() { logger.Info("Injecting Vector agent into Reaper deployments") vectorImage := vector.DefaultVectorImage @@ -83,9 +81,9 @@ func configureVector(reaper *api.Reaper, deployment *appsv1.Deployment, dc *cass }, } // Add the container and volume to the deployment - cassandra.UpdateContainer(&deployment.Spec.Template, VectorContainerName, func(c *corev1.Container) { + cassandra.UpdateContainer(template, VectorContainerName, func(c *corev1.Container) { *c = vectorAgentContainer }) - cassandra.AddVolumesToPodTemplateSpec(&deployment.Spec.Template, vectorAgentVolume) + cassandra.AddVolumesToPodTemplateSpec(template, vectorAgentVolume) } } diff --git a/pkg/reaper/vector_test.go b/pkg/reaper/vector_test.go index a297a36a9..742a00aec 100644 --- a/pkg/reaper/vector_test.go +++ b/pkg/reaper/vector_test.go @@ -1,6 +1,7 @@ package reaper import ( + corev1 "k8s.io/api/core/v1" "testing" testlogr "github.com/go-logr/logr/testing" @@ -9,7 +10,6 @@ import ( telemetryapi "github.com/k8ssandra/k8ssandra-operator/apis/telemetry/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/vector" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/ptr" ) @@ -19,16 +19,16 @@ func TestConfigureVector(t *testing.T) { reaper := &api.Reaper{} reaper.Spec.Telemetry = telemetrySpec - deployment := &v1.Deployment{} + template := &corev1.PodTemplateSpec{} fakeDc := &cassdcapi.CassandraDatacenter{} logger := testlogr.NewTestLogger(t) - configureVector(reaper, deployment, fakeDc, logger) + configureVector(reaper, template, fakeDc, logger) - assert.Equal(t, 1, len(deployment.Spec.Template.Spec.Containers)) - assert.Equal(t, "reaper-vector-agent", deployment.Spec.Template.Spec.Containers[0].Name) - assert.Equal(t, resource.MustParse(vector.DefaultVectorCpuLimit), *deployment.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu()) - assert.Equal(t, resource.MustParse(vector.DefaultVectorCpuRequest), *deployment.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu()) - assert.Equal(t, resource.MustParse(vector.DefaultVectorMemoryLimit), *deployment.Spec.Template.Spec.Containers[0].Resources.Limits.Memory()) - assert.Equal(t, resource.MustParse(vector.DefaultVectorMemoryRequest), *deployment.Spec.Template.Spec.Containers[0].Resources.Requests.Memory()) + assert.Equal(t, 1, len(template.Spec.Containers)) + assert.Equal(t, "reaper-vector-agent", template.Spec.Containers[0].Name) + assert.Equal(t, resource.MustParse(vector.DefaultVectorCpuLimit), *template.Spec.Containers[0].Resources.Limits.Cpu()) + assert.Equal(t, resource.MustParse(vector.DefaultVectorCpuRequest), *template.Spec.Containers[0].Resources.Requests.Cpu()) + assert.Equal(t, resource.MustParse(vector.DefaultVectorMemoryLimit), *template.Spec.Containers[0].Resources.Limits.Memory()) + assert.Equal(t, resource.MustParse(vector.DefaultVectorMemoryRequest), *template.Spec.Containers[0].Resources.Requests.Memory()) }