ReadOnlyRootFilesystem support (#661)

* Add new annotation, cassandra.datastax.com/readonly-fs that makes the cassandra container to run in ReadOnlyRootFilesystem securityContext

* Move from annotation to .spec.readOnlyRootFilesystem property, add runAsNonRoot to default securityContext

* Fix existing tests

* Add unit tests, e2e test, modify readOnlyRootFilesystem to be *bool (so we can set it to true by default later), add verification that MCAC is disabled when used with readOnlyRootFilesystem

* Remove $ character

* Revert config changes

.github/workflows/kindIntegTest.yml

@@ -155,15 +155,13 @@ jobs:
     strategy:
       matrix:
         version:
-          - "4.1.4"
+          - "4.1.5"
         integration_test:
           # Single worker tests:
           - additional_serviceoptions
           - additional_volumes
           # - delete_node_terminated_container # This does not test any operator behavior
           - podspec_simple
-          # - smoke_test_oss # Converted to test_all_the_things, see below job
-          # - smoke_test_dse # Converted to test_all_the_things, see below job
           # - terminate
           # - timeout_prestop_termination
           # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now
@@ -200,10 +198,17 @@ jobs:
           - scale_up
           - scale_up_stop_resume
           - seed_selection
+          - smoke_test_read_only_fs
           #- config_fql # OSS only
           - decommission_dc
         # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA
-      # let other tests continue to run
+        include:
+          - version: 4.1.5
+            serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api
+            serverType: cassandra
+            integration_test: "smoke_test_read_only_fs"
+
+        # let other tests continue to run
       # even if one fails
       fail-fast: false
     runs-on: ubuntu-latest

.github/workflows/workflow-integration-tests.yaml

@@ -176,7 +176,6 @@ jobs:
           - additional_volumes
           # - delete_node_terminated_container # This does not test any operator behavior
           - podspec_simple
-          # - smoke_test_oss # Converted to test_all_the_things, see below job
           # - smoke_test_dse # Converted to test_all_the_things, see below job
           # - terminate # test_all_things
           # - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us

CHANGELOG.md

@@ -15,6 +15,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
 * [FEATURE] [#646](https://github.com/k8ssandra/cass-operator/issues/646) Allow starting multiple parallel pods if they have already previously bootstrapped and not planned for replacement. Set annotation ``cassandra.datastax.com/allow-parallel-starts: true`` to enable this feature.
 * [ENHANCEMENT] [#648](https://github.com/k8ssandra/cass-operator/issues/648) Make MinReadySeconds configurable value in the Spec.
 * [ENHANCEMENT] [#184](https://github.com/k8ssandra/cass-operator/issues/349) Add CassandraDatacenter.Status fields as metrics also
+* [ENHANCEMENT] [#199](https://github.com/k8ssandra/cass-operator/issues/199) If .spec.readOnlyRootFilesystem is set, run the cassandra container with readOnlyRootFilesystem. Also, modify the default SecurityContext to mention runAsNonRoot: true
 
 ## v1.21.1
 

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -268,6 +268,10 @@ type CassandraDatacenterSpec struct {
 	// MinReadySeconds sets the minimum number of seconds for which a newly created pod should be ready without any of its containers crashing, for it to be considered available. Defaults to 5 seconds and is set in the StatefulSet spec.
 	// Setting to 0 might cause multiple Cassandra pods to restart at the same time despite PodDisruptionBudget settings.
 	MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
+
+	// ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the
+	// new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD)
+	ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"`
 }
 
 type NetworkingConfig struct {

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -8820,6 +8820,11 @@ spec:
                   - name
                   type: object
                 type: array
+              readOnlyRootFilesystem:
+                description: |-
+                  ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the
+                  new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD)
+                type: boolean
               replaceNodes:
                 description: Deprecated Use CassandraTask replacenode to achieve correct
                   node replacement. A list of pod names that need to be replaced.

pkg/reconciliation/construct_podtemplatespec.go

@@ -23,6 +23,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -303,8 +304,27 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe
 			EmptyDir: &corev1.EmptyDirVolumeSource{},
 		},
 	}
+
 	volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs}
 
+	if readOnlyFs(dc) {
+		tmp := corev1.Volume{
+			Name: "tmp",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		}
+
+		etcCass := corev1.Volume{
+			Name: "etc-cassandra",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		}
+
+		volumeDefaults = append(volumeDefaults, tmp, etcCass)
+	}
+
 	if dc.UseClientImage() {
 		vBaseConfig := corev1.Volume{
 			Name: "server-config-base",
@@ -435,7 +455,7 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl
 
 		configMounts = append(configMounts, configBaseMount)
 
-		// Similar to k8ssandra 1.x, use config-container if use new config-builder replacement
+		// Similar to k8ssandra 1.x, use config-container if we use k8ssandra-client to build configs
 		if configContainerIndex < 0 {
 			configContainer = &corev1.Container{
 				Name: ServerBaseConfigContainerName,
@@ -629,13 +649,20 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		}
 	}
 
+	if readOnlyFs(dc) {
+		cassContainer.SecurityContext = &corev1.SecurityContext{
+			ReadOnlyRootFilesystem: ptr.To[bool](true),
+		}
+	}
+
 	// Combine env vars
 
 	envDefaults := []corev1.EnvVar{
 		{Name: "POD_NAME", ValueFrom: selectorFromFieldPath("metadata.name")},
 		{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")},
 		{Name: "DS_LICENSE", Value: "accept"},
 		{Name: "USE_MGMT_API", Value: "true"},
+		{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"},
 		{Name: "MGMT_API_EXPLICIT_START", Value: "true"},
 	}
 
@@ -653,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"})
 	}
 
+	if readOnlyFs(dc) {
+		envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"})
+	}
+
 	cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env)
 
 	// Combine ports
@@ -706,6 +737,17 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		}
 	}
 
+	if readOnlyFs(dc) {
+		cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
+			Name:      "tmp",
+			MountPath: "/tmp",
+		})
+		cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
+			Name:      "etc-cassandra",
+			MountPath: "/etc/cassandra",
+		})
+	}
+
 	volumeMounts = combineVolumeMountSlices(volumeMounts, cassContainer.VolumeMounts)
 	cassContainer.VolumeMounts = combineVolumeMountSlices(volumeMounts, generateStorageConfigVolumesMount(dc))
 
@@ -763,6 +805,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 	return nil
 }
 
+func readOnlyFs(dc *api.CassandraDatacenter) bool {
+	return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
+}
+
 func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) {
 
 	baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy()
@@ -795,9 +841,10 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI
 	if baseTemplate.Spec.SecurityContext == nil {
 		var userID int64 = 999
 		baseTemplate.Spec.SecurityContext = &corev1.PodSecurityContext{
-			RunAsUser:  &userID,
-			RunAsGroup: &userID,
-			FSGroup:    &userID,
+			RunAsUser:    &userID,
+			RunAsGroup:   &userID,
+			FSGroup:      &userID,
+			RunAsNonRoot: ptr.To[bool](true),
 		}
 	}
 

pkg/reconciliation/construct_podtemplatespec_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 
 	"k8s.io/apimachinery/pkg/api/resource"
 
@@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
 	nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
 	useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
 	explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
+	noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
 
 	templateSpec := &corev1.PodTemplateSpec{}
 	dc := &api.CassandraDatacenter{
@@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) {
 	assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
+	assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
 }
 
 func TestHCDContainerEnvVars(t *testing.T) {
@@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
 	useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
 	explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
 	hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}
+	noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
 
 	templateSpec := &corev1.PodTemplateSpec{}
 	dc := &api.CassandraDatacenter{
@@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) {
 	assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
+	assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, hcdAutoConf))
 }
 
@@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
 	nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}
 	useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"}
 	explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"}
+	noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}
 	dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"}
 	dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"}
 
@@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) {
 	assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar))
+	assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar))
 	assert.True(envVarsContains(cassContainer.Env, dseAutoConf))
 	assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar))
 }
@@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) {
 		assert.Equal(test.accountName, pds.Spec.ServiceAccountName)
 	}
 }
+
+func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) {
+	assert := assert.New(t)
+	dc := &api.CassandraDatacenter{
+		Spec: api.CassandraDatacenterSpec{
+			ClusterName:            "bob",
+			ServerType:             "cassandra",
+			ServerVersion:          "4.1.5",
+			ReadOnlyRootFilesystem: ptr.To[bool](true),
+			Racks: []api.Rack{
+				{
+					Name: "r1",
+				},
+			},
+		},
+	}
+
+	podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false)
+	assert.NoError(err, "failed to build PodTemplateSpec")
+
+	containers := podTemplateSpec.Spec.Containers
+	assert.NotNil(containers, "Unexpected containers containers received")
+	assert.NoError(err, "Unexpected error encountered")
+
+	assert.Len(containers, 2, "Unexpected number of containers containers returned")
+	assert.Equal("cassandra", containers[0].Name)
+	assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem)
+
+	assert.True(reflect.DeepEqual(containers[0].VolumeMounts,
+		[]corev1.VolumeMount{
+			{
+				Name:      "tmp",
+				MountPath: "/tmp",
+			},
+			{
+				Name:      "etc-cassandra",
+				MountPath: "/etc/cassandra",
+			},
+			{
+				Name:      "server-logs",
+				MountPath: "/var/log/cassandra",
+			},
+			{
+				Name:      "server-data",
+				MountPath: "/var/lib/cassandra",
+			},
+			{
+				Name:      "server-config",
+				MountPath: "/config",
+			},
+		}), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts))
+
+	// TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem
+	mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"}
+	assert.True(envVarsContains(containers[0].Env, mcacDisabled))
+}

pkg/reconciliation/construct_statefulset_test.go

@@ -436,9 +436,10 @@ func Test_newStatefulSetForCassandraPodSecurityContext(t *testing.T) {
 	}
 
 	defaultSecurityContext := &corev1.PodSecurityContext{
-		RunAsUser:  ptr.To(int64(999)),
-		RunAsGroup: ptr.To(int64(999)),
-		FSGroup:    ptr.To(int64(999)),
+		RunAsUser:    ptr.To(int64(999)),
+		RunAsGroup:   ptr.To(int64(999)),
+		FSGroup:      ptr.To(int64(999)),
+		RunAsNonRoot: ptr.To[bool](true),
 	}
 
 	tests := []struct {

tests/smoke_test_dse/smoke_test_dse_suite_test.go → tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go

@@ -1,29 +1,27 @@
 // Copyright DataStax, Inc.
 // Please see the included license file for details.
 
-package smoke_test_dse
+package smoke_test_read_only_fs
 
 import (
 	"fmt"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	corev1 "k8s.io/api/core/v1"
 
 	"github.com/k8ssandra/cass-operator/tests/kustomize"
 	ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo"
 	"github.com/k8ssandra/cass-operator/tests/util/kubectl"
 )
 
 var (
-	testName   = "Smoke test of basic functionality for one-node DSE cluster."
-	namespace  = "test-smoke-test-dse"
-	dcName     = "dc2"
-	dcYaml     = "../testdata/smoke-test-dse.yaml"
-	dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName)
-	dcLabel    = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
-	ns         = ginkgo_util.NewWrapper(testName, namespace)
+	testName  = "Smoke test of basic functionality for readOnlyRootFilesystem"
+	namespace = "test-smoke-test-read-only-fs"
+	dcName    = "dc1"
+	dcYaml    = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml"
+	dcLabel   = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName)
+	ns        = ginkgo_util.NewWrapper(testName, namespace)
 )
 
 func TestLifecycle(t *testing.T) {
@@ -68,20 +66,6 @@ var _ = Describe(testName, func() {
 			ns.WaitForDatacenterReady(dcName)
 			ns.ExpectDoneReconciling(dcName)
 
-			step = "scale up to 2 nodes"
-			json = "{\"spec\": {\"size\": 2}}"
-			k = kubectl.PatchMerge(dcResource, json)
-			ns.ExecAndLog(step, k)
-
-			ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue))
-			ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60)
-			ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse))
-
-			// Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running
-			Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2))
-
-			ns.WaitForDatacenterReady(dcName)
-
 			step = "deleting the dc"
 			k = kubectl.DeleteFromFiles(dcYaml)
 			ns.ExecAndLog(step, k)