Add CSI support

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -70,8 +70,12 @@ type CassandraUser struct {
 }
 
 type UserInfo struct {
-	Annotations    map[string]string `json:"annotations,omitempty"`
-	ServiceAccount string            `json:"serviceAccountName,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional
+	MountPath  string                  `json:"mountpath,omitempty"`
+	CSI        *corev1.CSIVolumeSource `json:"csi,omitempty"`
+	SecretName string                  `json:"secretName,omitempty"`
+	// ServiceAccount string            `json:"serviceAccountName,omitempty"`
 	// TODO Add CSI information here
 	/*
 		TODO Testing:
@@ -185,13 +189,13 @@ type CassandraDatacenterSpec struct {
 	// podAntiAffinity and requiredDuringSchedulingIgnoredDuringExecution.
 	AllowMultipleNodesPerWorker bool `json:"allowMultipleNodesPerWorker,omitempty"`
 
-	// This secret defines the username and password for the Cassandra server superuser.
+	// SuperuserSecretName is deprecated. Use UserInfo instead. This secret defines the username and password for the Cassandra server superuser.
 	// If it is omitted, we will generate a secret instead.
 	SuperuserSecretName string `json:"superuserSecretName,omitempty"`
 
 	UserInfo *UserInfo `json:"userInfo,omitempty"`
 
-	// The k8s service account to use for the server pods
+	// ServiceAccount is the k8s service account to use for the server pods
 	ServiceAccount string `json:"serviceAccount,omitempty"`
 
 	// Whether to do a rolling restart at the next opportunity. The operator will set this back

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -7420,7 +7420,8 @@ spec:
                 pattern: (6\.8\.\d+)|(3\.11\.\d+)|(4\.0\.\d+)
                 type: string
               serviceAccount:
-                description: The k8s service account to use for the server pods
+                description: ServiceAccount is the k8s service account to use for
+                  the server pods
                 type: string
               size:
                 description: Desired number of Cassandra server nodes
@@ -7796,9 +7797,9 @@ spec:
                     type: object
                 type: object
               superuserSecretName:
-                description: This secret defines the username and password for the
-                  Cassandra server superuser. If it is omitted, we will generate a
-                  secret instead.
+                description: SuperuserSecretName is deprecated. Use UserInfo instead.
+                  This secret defines the username and password for the Cassandra
+                  server superuser. If it is omitted, we will generate a secret instead.
                 type: string
               systemLoggerImage:
                 description: Container image for the log tailing sidecar container.
@@ -7878,7 +7879,54 @@ spec:
                     additionalProperties:
                       type: string
                     type: object
-                  serviceAccountName:
+                  csi:
+                    description: Represents a source location of a volume to mount,
+                      managed by an external CSI driver
+                    properties:
+                      driver:
+                        description: Driver is the name of the CSI driver that handles
+                          this volume. Consult with your admin for the correct name
+                          as registered in the cluster.
+                        type: string
+                      fsType:
+                        description: Filesystem type to mount. Ex. "ext4", "xfs",
+                          "ntfs". If not provided, the empty value is passed to the
+                          associated CSI driver which will determine the default filesystem
+                          to apply.
+                        type: string
+                      nodePublishSecretRef:
+                        description: NodePublishSecretRef is a reference to the secret
+                          object containing sensitive information to pass to the CSI
+                          driver to complete the CSI NodePublishVolume and NodeUnpublishVolume
+                          calls. This field is optional, and  may be empty if no secret
+                          is required. If the secret object contains more than one
+                          secret, all secret references are passed.
+                        properties:
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              TODO: Add other useful fields. apiVersion, kind, uid?'
+                            type: string
+                        type: object
+                      readOnly:
+                        description: Specifies a read-only configuration for the volume.
+                          Defaults to false (read/write).
+                        type: boolean
+                      volumeAttributes:
+                        additionalProperties:
+                          type: string
+                        description: VolumeAttributes stores driver-specific properties
+                          that are passed to the CSI driver. Consult your driver's
+                          documentation for supported values.
+                        type: object
+                    required:
+                    - driver
+                    type: object
+                  mountpath:
+                    description: MountPath tells the script where to read the user
+                      information. Required if annotation injection is used, otherwise
+                      optional
+                    type: string
+                  secretName:
                     type: string
                 type: object
               users:

pkg/reconciliation/reconcile_racks.go

@@ -912,6 +912,16 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
 	if dc.Spec.UserInfo != nil {
 		// Create the job
 		ttl := int32(86400)
+
+		// How to get this as input?
+		filePath := dc.Spec.UserInfo.MountPath
+		// filePath := "/vault/secrets/database-config.txt"
+
+		// We want to mount it as a directory and read the files as usernames
+		if dc.Spec.UserInfo.CSI != nil && filePath != "" {
+			filePath = "/mnt/secrets/users"
+		}
+
 		// TODO wait for it to complete before we continue..
 		job := batchv1.Job{
 			ObjectMeta: metav1.ObjectMeta{
@@ -920,17 +930,14 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
 			},
 			Spec: batchv1.JobSpec{
 				Template: corev1.PodTemplateSpec{
-					ObjectMeta: metav1.ObjectMeta{
-						Annotations: dc.Spec.UserInfo.Annotations,
-					},
 					Spec: corev1.PodSpec{
 						// TODO Add volumes here if CSI was used
 						Containers: []corev1.Container{
 							{
 								Name:            "client",
 								Image:           "burmanm/k8ssandra-client:latest", // k8ssandra/k8ssandra-client does not have this feature
 								ImagePullPolicy: corev1.PullIfNotPresent,
-								Args:            []string{"users", "add", "--path", "/vault/secrets/database-config.txt", "--dc", dc.Name}, // TODO This path must be configurable
+								Args:            []string{"users", "add", "--path", filePath, "--dc", dc.Name}, // TODO This path must be configurable
 							},
 						},
 						RestartPolicy: corev1.RestartPolicyNever,
@@ -941,7 +948,39 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
 			},
 		}
 
-		labels := dc.GetClusterLabels()
+		// job.Spec.Template.Spec.Volumes
+
+		if len(dc.Spec.UserInfo.Annotations) > 0 {
+			job.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations
+		}
+
+		// TODO Add verification that we can't have dual injection (CSI + annotations)
+
+		// TODO If Secret name is set, mount it just like the CSI
+
+		if dc.Spec.UserInfo.SecretName != "" {
+
+		}
+
+		if dc.Spec.UserInfo.CSI != nil {
+			vol := corev1.Volume{
+				Name: "user-source",
+				VolumeSource: corev1.VolumeSource{
+					// TODO Add .. something?
+					CSI: dc.Spec.UserInfo.CSI,
+				},
+			}
+			job.Spec.Template.Spec.Volumes = []corev1.Volume{vol}
+			job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+				{
+					Name:      "user-source",
+					ReadOnly:  true,
+					MountPath: filePath,
+				},
+			}
+		}
+
+		labels := dc.GetDatacenterLabels()
 		oplabels.AddOperatorLabels(labels, dc)
 		job.ObjectMeta.Labels = labels
 

tests/external_secret/README.md

@@ -1,15 +1,26 @@
+## Install Helm repositories
+
+```
+helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts
+helm repo add hashicorp https://helm.releases.hashicorp.com
+helm repo update
+```
+
 ## Install Vault
 
+```
 helm install vault hashicorp/vault --set "server.dev.enabled=true" --namespace cass-operator
 # --set "csi.enabled=true"
+```
 
 ## Go into Vault and exec certain commands..
 
+```
 kubectl exec -it vault-0 -- /bin/sh
 
 vault secrets enable -path=internal kv-v2
 
-vault kv put internal/database/config username="db-readonly-username" password="db-secret-password"
+vault kv put internal/database/config superuser="superpassword"
 
 vault auth enable kubernetes
 
@@ -27,7 +38,39 @@ vault write auth/kubernetes/role/internal-app \
     bound_service_account_namespaces=cass-operator \
     policies=internal-app \
     ttl=24h
+```
+
+## Install CSI driver:
+
+Not sure if syncSecret is needed, but Vault documentation wants it..
+
+```
+helm install csi secrets-store-csi-driver/secrets-store-csi-driver \
+    --set syncSecret.enabled=true --namespace cass-operator
+```
+
+Create the SecretProviderClass:
+
+```yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: vault-database
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.default:8200"
+    roleName: "internal-app"
+    objects: |
+      - objectName: "superuser"
+        secretPath: "internal/database/config"
+        secretKey: "superuser"
+```
+
+The objectName becomes the username and the secretKey's data becomes the password.
 
 ## Now create the DC:
 
+```
 kubectl apply -f tests/testdata/default-single-rack-single-node-dc-vault.yaml
+```

tests/external_secret/secret_provider.yaml

@@ -0,0 +1,13 @@
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: vault-database
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.default:8200"
+    roleName: "internal-app"
+    objects: |
+      - objectName: "superuser"
+        secretPath: "internal/database/config"
+        secretKey: "superuser"

tests/testdata/default-single-rack-single-node-dc-vault.yaml

@@ -18,6 +18,12 @@ spec:
           {{- end }}
           {{- end -}}
         vault.hashicorp.com/agent-pre-populate-only: 'true'
+    mountPath: /vault/secrets/database-config.txt
+    csi:
+      driver: secrets-store.csi.k8s.io
+      readOnly: true
+      volumeAttributes:
+        secretProviderClass: "vault-database"
   managementApiAuth:
     insecure: {}
   size: 1