From eca9d35d107ca93929e808abed826949ac3087f3 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 11 Sep 2024 10:44:04 +0300 Subject: [PATCH 01/10] If the ReadOnlyRootFilesystem is selected, always force the usage of k8ssandra-client for config building --- CHANGELOG.md | 1 + .../v1beta1/cassandradatacenter_types.go | 11 ++++ .../v1beta1/cassandradatacenter_types_test.go | 59 +++++++++++++++++++ .../construct_podtemplatespec.go | 12 ++-- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9acc5778..3d20251b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti ## v1.22.2 * [BUGFIX] [#703](https://github.com/k8ssandra/cass-operator/issues/703) Fix HCD config path from /etc/cassandra to /opt/hcd/resources/cassandra/conf +* [FEATURE] [#661](https://github.com/k8ssandra/cass-operator/issues/661) If ReadOnlyRootFilesystem is set, always use the new config builder instead of the old one. This change only applies to Cassandra 3.11, 4.0 and DSE 6.8, 6.9. Note that not all the versions mentioned here necessarily support ReadOnlyRootFilesystem. ## v1.22.1 diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index ec9aa20e..d3aa17c9 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -985,6 +985,10 @@ func (dc *CassandraDatacenter) DatacenterName() string { } func (dc *CassandraDatacenter) UseClientImage() bool { + if dc.ReadOnlyFs() { + return true + } + if dc.Spec.ServerType == "hcd" { return true } @@ -998,3 +1002,10 @@ func (dc *CassandraDatacenter) UseClientImage() bool { func (dc *CassandraDatacenter) GenerationChanged() bool { return dc.Status.ObservedGeneration < dc.Generation } + +func (dc *CassandraDatacenter) ReadOnlyFs() bool { + if dc.Spec.ReadOnlyRootFilesystem != nil { + return *dc.Spec.ReadOnlyRootFilesystem + } + return false +} diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go index 266897c2..ac928d46 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" ) var internodeEnabledAll = ` @@ -125,3 +126,61 @@ func TestUseClientImage(t *testing.T) { } } } + +func TestUseClientImageReadOnlyRootFilesystem(t *testing.T) { + assert := assert.New(t) + + tests := []struct { + serverType string + version string + }{ + { + serverType: "cassandra", + version: "4.1.0", + }, + { + serverType: "cassandra", + version: "4.1.2", + }, + { + serverType: "cassandra", + version: "5.0.0", + }, + { + serverType: "cassandra", + version: "3.11.17", + }, + { + serverType: "cassandra", + version: "4.0.8", + }, + { + serverType: "dse", + version: "6.8.39", + }, + { + serverType: "dse", + version: "6.9.0", + }, + { + serverType: "hcd", + version: "1.0.0", + }, + { + serverType: "dse", + version: "4.1.2", + }, + } + + for _, tt := range tests { + dc := CassandraDatacenter{ + Spec: CassandraDatacenterSpec{ + ServerVersion: tt.version, + ServerType: tt.serverType, + ReadOnlyRootFilesystem: ptr.To[bool](true), + }, + } + + assert.True(dc.UseClientImage()) + } +} diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index cbc4426c..aa2b0c3b 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -307,7 +307,7 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs} - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { tmp := corev1.Volume{ Name: "tmp", VolumeSource: corev1.VolumeSource{ @@ -649,7 +649,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { cassContainer.SecurityContext = &corev1.SecurityContext{ ReadOnlyRootFilesystem: ptr.To[bool](true), } @@ -680,7 +680,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}) } - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"}) } @@ -737,7 +737,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ Name: "tmp", MountPath: "/tmp", @@ -813,10 +813,6 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla return nil } -func readOnlyFs(dc *api.CassandraDatacenter) bool { - return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() -} - func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy() From 0076984004a2147482f778f2ef081aff3d2de6c5 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 11 Sep 2024 10:45:57 +0300 Subject: [PATCH 02/10] Fix issue number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d20251b..9e5bb76f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,11 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti ## unreleased * [FEATURE] [#651](https://github.com/k8ssandra/cass-operator/issues/651) Add tsreload task for DSE deployments and ability to check if sync operation is available on the mgmt-api side +* [FEATURE] [#701](https://github.com/k8ssandra/cass-operator/issues/701) If ReadOnlyRootFilesystem is set, always use the new config builder instead of the old one. This change only applies to Cassandra 3.11, 4.0 and DSE 6.8, 6.9. Note that not all the versions mentioned here necessarily support ReadOnlyRootFilesystem. ## v1.22.2 * [BUGFIX] [#703](https://github.com/k8ssandra/cass-operator/issues/703) Fix HCD config path from /etc/cassandra to /opt/hcd/resources/cassandra/conf -* [FEATURE] [#661](https://github.com/k8ssandra/cass-operator/issues/661) If ReadOnlyRootFilesystem is set, always use the new config builder instead of the old one. This change only applies to Cassandra 3.11, 4.0 and DSE 6.8, 6.9. Note that not all the versions mentioned here necessarily support ReadOnlyRootFilesystem. ## v1.22.1 From 368b775a814c349fdec8bdb14c3c6d0f4ad32567 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 11 Sep 2024 16:55:17 +0300 Subject: [PATCH 03/10] Add DSE specific cassandra base config path for readOnly --- pkg/reconciliation/construct_podtemplatespec.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index aa2b0c3b..306c6a97 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -476,6 +476,8 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl configContainer.Command = []string{"/bin/sh"} if dc.Spec.ServerType == "cassandra" { configContainer.Args = []string{"-c", "cp -rf /etc/cassandra/* /cassandra-base-config/"} + } else if dc.Spec.ServerType == "dse" { + configContainer.Args = []string{"-c", "cp -rf /opt/dse/resources/cassandra/conf/* /cassandra-base-config/"} } else if dc.Spec.ServerType == "hcd" { configContainer.Args = []string{"-c", "cp -rf /opt/hcd/resources/cassandra/conf/* /cassandra-base-config/"} } From 4ab222385eda79dcebd9dfa1803f522f4886750f Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 11 Sep 2024 17:56:57 +0300 Subject: [PATCH 04/10] Add different mounts for HCD & DSE --- pkg/reconciliation/construct_podtemplatespec.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 306c6a97..2ee149bb 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -750,6 +750,11 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla Name: "etc-cassandra", MountPath: "/opt/hcd/resources/cassandra/conf", }) + } else if dc.Spec.ServerType == "dse" { + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "etc-cassandra", + MountPath: "/opt/dse/resources/cassandra/conf", + }) } else { cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ Name: "etc-cassandra", From e52a0b18f2ec17e880c5a66cf674db1756ee5c22 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Mon, 16 Sep 2024 15:34:24 +0300 Subject: [PATCH 05/10] Add buildx options to docker-build --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e603f4a0..c67e0d45 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,8 @@ endif SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +BUILDX_OPTIONS ?= + .PHONY: all all: build @@ -160,7 +162,7 @@ run: manifests generate fmt vet ## Run a controller from your host. .PHONY: docker-build docker-build: ## Build docker image with the manager. - docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} . --load + docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} ${BUILDX_OPTIONS} . --load .PHONY: docker-kind docker-kind: docker-build ## Build docker image and load to kind cluster From fb135f1bbe8f56a4a8486c5f31e524758b5266bc Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Mon, 16 Sep 2024 16:45:52 +0300 Subject: [PATCH 06/10] Add annotation to control if k8ssandra-client is used for config building regardless if supported or not and add a /opt/dse/resources/dse/conf for dse.yaml output --- .../v1beta1/cassandradatacenter_types.go | 5 ++- .../v1beta1/cassandradatacenter_types_test.go | 8 ++++- .../construct_podtemplatespec.go | 31 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index d3aa17c9..351430b0 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -74,6 +74,9 @@ const ( // AllowStorageChangesAnnotation indicates the CassandraDatacenter StorageConfig can be modified for existing datacenters AllowStorageChangesAnnotation = "cassandra.datastax.com/allow-storage-changes" + // UseClientBuilderAnnotation enforces the usage of new config builder from k8ssandra-client for versions that would otherwise use the cass-config-builder + UseClientBuilderAnnotation = "cassandra.datastax.com/use-new-config-builder" + AllowUpdateAlways AllowUpdateType = "always" AllowUpdateOnce AllowUpdateType = "once" @@ -985,7 +988,7 @@ func (dc *CassandraDatacenter) DatacenterName() string { } func (dc *CassandraDatacenter) UseClientImage() bool { - if dc.ReadOnlyFs() { + if metav1.HasAnnotation(dc.ObjectMeta, UseClientBuilderAnnotation) && dc.Annotations[UseClientBuilderAnnotation] == "true" { return true } diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go index ac928d46..ed339317 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -127,7 +128,7 @@ func TestUseClientImage(t *testing.T) { } } -func TestUseClientImageReadOnlyRootFilesystem(t *testing.T) { +func TestUseClientImageEnforce(t *testing.T) { assert := assert.New(t) tests := []struct { @@ -174,6 +175,11 @@ func TestUseClientImageReadOnlyRootFilesystem(t *testing.T) { for _, tt := range tests { dc := CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + UseClientBuilderAnnotation: "true", + }, + }, Spec: CassandraDatacenterSpec{ ServerVersion: tt.version, ServerType: tt.serverType, diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 2ee149bb..bece9dae 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -323,6 +323,26 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe } volumeDefaults = append(volumeDefaults, tmp, etcCass) + + if dc.Spec.ServerType == "dse" { + dseConf := corev1.Volume{ + Name: "dse-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + volumeDefaults = append(volumeDefaults, dseConf) + + if !dc.UseClientImage() { + sparkConf := corev1.Volume{ + Name: "spark-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + volumeDefaults = append(volumeDefaults, sparkConf) + } + } } if dc.UseClientImage() { @@ -755,6 +775,17 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla Name: "etc-cassandra", MountPath: "/opt/dse/resources/cassandra/conf", }) + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "dse-conf", + MountPath: "/opt/dse/resources/dse/conf", + }) + + if !dc.UseClientImage() { + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "spark-conf", + MountPath: "/opt/dse/resources/spark/conf", + }) + } } else { cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ Name: "etc-cassandra", From 014dfd69f4992213846f85860c55ff517465b7e1 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Mon, 16 Sep 2024 17:07:10 +0300 Subject: [PATCH 07/10] Add collectd directory for readOnly --- pkg/reconciliation/construct_podtemplatespec.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index bece9dae..a581d5db 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -340,7 +340,15 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe EmptyDir: &corev1.EmptyDirVolumeSource{}, }, } - volumeDefaults = append(volumeDefaults, sparkConf) + + collectDConf := corev1.Volume{ + Name: "collectd-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + volumeDefaults = append(volumeDefaults, sparkConf, collectDConf) } } } @@ -785,6 +793,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla Name: "spark-conf", MountPath: "/opt/dse/resources/spark/conf", }) + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "collectd-conf", + MountPath: "/opt/dse/resources/dse/collectd/etc/collectd", + }) } } else { cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ From eb63666db434610de38e42cae00741d3f12503e8 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 17 Sep 2024 12:43:44 +0300 Subject: [PATCH 08/10] Add unit tests and change the CHANGELOG --- CHANGELOG.md | 2 +- .../construct_podtemplatespec_test.go | 245 +++++++++++++++++- 2 files changed, 243 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5bb76f..3739fd76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti ## unreleased * [FEATURE] [#651](https://github.com/k8ssandra/cass-operator/issues/651) Add tsreload task for DSE deployments and ability to check if sync operation is available on the mgmt-api side -* [FEATURE] [#701](https://github.com/k8ssandra/cass-operator/issues/701) If ReadOnlyRootFilesystem is set, always use the new config builder instead of the old one. This change only applies to Cassandra 3.11, 4.0 and DSE 6.8, 6.9. Note that not all the versions mentioned here necessarily support ReadOnlyRootFilesystem. +* [FEATURE] [#701](https://github.com/k8ssandra/cass-operator/issues/701) Allow ReadOnlyRootFilesystem for DSE also with extra mounts to provide support for cass-config-builder setups ## v1.22.2 diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index 98db5f8a..7335d675 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -1970,8 +1970,38 @@ func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) { podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) assert.NoError(err, "failed to build PodTemplateSpec") + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 2, "Unexpected number of init containers returned") + assert.Equal(ServerBaseConfigContainerName, initContainers[0].Name) + assert.Equal(ServerConfigContainerName, initContainers[1].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + assert.Equal(initContainers[0].Args[1], "cp -rf /etc/cassandra/* /cassandra-base-config/") + + assert.True(reflect.DeepEqual(initContainers[1].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + containers := podTemplateSpec.Spec.Containers - assert.NotNil(containers, "Unexpected containers containers received") + assert.NotNil(containers, "No containers were found") assert.NoError(err, "Unexpected error encountered") assert.Len(containers, 2, "Unexpected number of containers containers returned") @@ -2026,8 +2056,38 @@ func TestReadOnlyRootFilesystemVolumeChangesHCD(t *testing.T) { podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) assert.NoError(err, "failed to build PodTemplateSpec") + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 2, "Unexpected number of init containers returned") + assert.Equal(ServerBaseConfigContainerName, initContainers[0].Name) + assert.Equal(ServerConfigContainerName, initContainers[1].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + assert.Equal(initContainers[0].Args[1], "cp -rf /opt/hcd/resources/cassandra/conf/* /cassandra-base-config/") + + assert.True(reflect.DeepEqual(initContainers[1].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + containers := podTemplateSpec.Spec.Containers - assert.NotNil(containers, "Unexpected containers containers received") + assert.NotNil(containers, "No containers were found") assert.NoError(err, "Unexpected error encountered") assert.Len(containers, 2, "Unexpected number of containers containers returned") @@ -2058,7 +2118,186 @@ func TestReadOnlyRootFilesystemVolumeChangesHCD(t *testing.T) { }, }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) - // TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} + +func TestReadOnlyRootFilesystemVolumeChangesDSE(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "dse", + ServerVersion: "6.9.2", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 1, "Unexpected number of init containers returned") + assert.Equal(ServerConfigContainerName, initContainers[0].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "No containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/opt/dse/resources/cassandra/conf", + }, + { + Name: "dse-conf", + MountPath: "/opt/dse/resources/dse/conf", + }, + { + Name: "spark-conf", + MountPath: "/opt/dse/resources/spark/conf", + }, + { + Name: "collectd-conf", + MountPath: "/opt/dse/resources/dse/collectd/etc/collectd", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // Test that cassandra-base-config isn't here + + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} + +func TestReadOnlyRootFilesystemVolumeChangesDSEWithClient(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + api.UseClientBuilderAnnotation: "true", + }, + }, + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "dse", + ServerVersion: "6.9.2", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 2, "Unexpected number of init containers returned") + assert.Equal(ServerBaseConfigContainerName, initContainers[0].Name) + assert.Equal(ServerConfigContainerName, initContainers[1].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + assert.Equal(initContainers[0].Args[1], "cp -rf /opt/dse/resources/cassandra/conf/* /cassandra-base-config/") + + assert.True(reflect.DeepEqual(initContainers[1].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "No containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/opt/dse/resources/cassandra/conf", + }, + { + Name: "dse-conf", + MountPath: "/opt/dse/resources/dse/conf", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // Test that cassandra-base-config isn't here + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} assert.True(envVarsContains(containers[0].Env, mcacDisabled)) } From 06645d946690697e36849736029fa1de37650245 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 17 Sep 2024 12:46:35 +0300 Subject: [PATCH 09/10] Add smoke_test_read_only_fs for DSE 6.8.50 and 6.9.2 also --- .github/workflows/kindIntegTest.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index 6bf76f49..4af66bf4 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -155,7 +155,7 @@ jobs: strategy: matrix: version: - - "4.1.5" + - "4.1.6" integration_test: # Single worker tests: - additional_serviceoptions @@ -233,25 +233,31 @@ jobs: version: - "3.11.17" - "4.0.13" - - "4.1.5" + - "4.1.6" - "6.8.50" - - "6.9.0" + - "6.9.2" - "1.0.0" integration_test: - test_all_the_things + - smoke_test_read_only_fs include: - version: 6.8.50 serverImage: datastax/dse-mgmtapi-6_8:6.8.50-ubi8 # DSE 6.8.50 serverType: dse integration_test: "test_all_the_things" - - version: 6.9.0 - serverImage: datastax/dse-mgmtapi-6_8:6.9.0-ubi8 # DSE 6.9.0 + - version: 6.9.2 + serverImage: datastax/dse-mgmtapi-6_8:6.9.2-ubi # DSE 6.9.2 serverType: dse integration_test: "test_all_the_things" - version: 1.0.0 serverImage: datastax/hcd:1.0.0-ubi # HCD 1.0.0 serverType: hcd integration_test: "test_all_the_things" + exclude: + - version: 3.11.17 + integration_test: "smoke_test_read_only_fs" + - version: 4.0.13 + integration_test: "smoke_test_read_only_fs" fail-fast: true runs-on: ubuntu-latest env: @@ -282,7 +288,7 @@ jobs: strategy: matrix: version: - - "4.1.5" + - "4.1.6" integration_test: - pvc_expansion fail-fast: true From acf046341f7d5f21e96b6be96b667fcfff28d89b Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 17 Sep 2024 14:50:25 +0300 Subject: [PATCH 10/10] Fix smoke_test_read_only_fs --- .github/workflows/kindIntegTest.yml | 5 +---- ..._test_read_only_fs.go => smoke_test_read_only_fs_test.go} | 0 2 files changed, 1 insertion(+), 4 deletions(-) rename tests/smoke_test_read_only_fs/{smoke_test_read_only_fs.go => smoke_test_read_only_fs_test.go} (100%) diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index 4af66bf4..8595edaa 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -198,7 +198,7 @@ jobs: - scale_up - scale_up_stop_resume - seed_selection - - smoke_test_read_only_fs + # - smoke_test_read_only_fs #- config_fql # OSS only - decommission_dc # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA @@ -244,15 +244,12 @@ jobs: - version: 6.8.50 serverImage: datastax/dse-mgmtapi-6_8:6.8.50-ubi8 # DSE 6.8.50 serverType: dse - integration_test: "test_all_the_things" - version: 6.9.2 serverImage: datastax/dse-mgmtapi-6_8:6.9.2-ubi # DSE 6.9.2 serverType: dse - integration_test: "test_all_the_things" - version: 1.0.0 serverImage: datastax/hcd:1.0.0-ubi # HCD 1.0.0 serverType: hcd - integration_test: "test_all_the_things" exclude: - version: 3.11.17 integration_test: "smoke_test_read_only_fs" diff --git a/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs_test.go similarity index 100% rename from tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go rename to tests/smoke_test_read_only_fs/smoke_test_read_only_fs_test.go