WIP: Allow creation of users through a job

.github/workflows/operatorBuildAndDeploy.yml

@@ -71,7 +71,7 @@ jobs:
         with:
           file: Dockerfile
           context: .
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name != 'pull_request' && !env.ACT }}
           tags: k8ssandra/cass-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/cass-operator:latest, k8ssandra/cass-operator:${{ steps.vars.outputs.version }}
           platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
@@ -81,7 +81,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           file: logger.Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name != 'pull_request' && !env.ACT }}
           tags: k8ssandra/system-logger:${{ steps.vars.outputs.sha_short }}, k8ssandra/system-logger:latest
           platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
@@ -106,11 +106,12 @@ jobs:
           build-args: |
             VERSION=${{ steps.vars.outputs.version }}
           context: .
-          push: ${{ !env.ACT }}
+          push: ${{ github.event_name != 'pull_request' && !env.ACT }}
           tags: k8ssandra/cass-operator-bundle:v${{ steps.vars.outputs.version }}
           platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
       - name: Build and update cass-operator-catalog
         run: |
+          # Load the images here? It wants the bundle..
           make VERSION=${{ steps.vars.outputs.version }} CHANNEL=dev catalog-build catalog-push

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -77,6 +77,25 @@ type CassandraUser struct {
 	Superuser  bool   `json:"superuser"`
 }
 
+type UserInfo struct {
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional
+	MountPath  string                  `json:"mountPath,omitempty"`
+	CSI        *corev1.CSIVolumeSource `json:"csi,omitempty"`
+	SecretName string                  `json:"secretName,omitempty"`
+
+	// ServiceAccountName override job ServiceAccount, otherwise Spec.ServiceAccount (the one used to create the server pods) is used
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+	// TODO Add CSI information here
+	/*
+		TODO Testing:
+			* Add back the Helm utilities
+			* Install Vault + Vault server there
+			* Install CSI driver
+			* Try to create clusters with both methods and see that it succeeds (and uses Vault to fetch the user rights)
+	*/
+}
+
 // CassandraDatacenterSpec defines the desired state of a CassandraDatacenter
 // +k8s:openapi-gen=true
 // +kubebuilder:pruning:PreserveUnknownFields
@@ -180,11 +199,13 @@ type CassandraDatacenterSpec struct {
 	// podAntiAffinity and requiredDuringSchedulingIgnoredDuringExecution.
 	AllowMultipleNodesPerWorker bool `json:"allowMultipleNodesPerWorker,omitempty"`
 
-	// This secret defines the username and password for the Cassandra server superuser.
+	// SuperuserSecretName is deprecated. Use UserInfo instead. This secret defines the username and password for the Cassandra server superuser.
 	// If it is omitted, we will generate a secret instead.
 	SuperuserSecretName string `json:"superuserSecretName,omitempty"`
 
-	// The k8s service account to use for the server pods
+	UserInfo *UserInfo `json:"userInfo,omitempty"`
+
+	// ServiceAccount is the k8s service account to use for the server pods
 	ServiceAccount string `json:"serviceAccount,omitempty"`
 
 	// DEPRECATED. Use CassandraTask for rolling restarts. Whether to do a rolling restart at the next opportunity. The operator will set this back

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -7614,7 +7614,8 @@ spec:
                 pattern: (6\.8\.\d+)|(3\.11\.\d+)|(4\.\d+\.\d+)
                 type: string
               serviceAccount:
-                description: The k8s service account to use for the server pods
+                description: ServiceAccount is the k8s service account to use for
+                  the server pods
                 type: string
               size:
                 description: Desired number of Cassandra server nodes
@@ -7996,9 +7997,9 @@ spec:
                     type: object
                 type: object
               superuserSecretName:
-                description: This secret defines the username and password for the
-                  Cassandra server superuser. If it is omitted, we will generate a
-                  secret instead.
+                description: SuperuserSecretName is deprecated. Use UserInfo instead.
+                  This secret defines the username and password for the Cassandra
+                  server superuser. If it is omitted, we will generate a secret instead.
                 type: string
               systemLoggerImage:
                 description: Container image for the log tailing sidecar container.
@@ -8072,6 +8073,68 @@ spec:
                       type: string
                   type: object
                 type: array
+              userInfo:
+                properties:
+                  annotations:
+                    additionalProperties:
+                      type: string
+                    type: object
+                  csi:
+                    description: Represents a source location of a volume to mount,
+                      managed by an external CSI driver
+                    properties:
+                      driver:
+                        description: driver is the name of the CSI driver that handles
+                          this volume. Consult with your admin for the correct name
+                          as registered in the cluster.
+                        type: string
+                      fsType:
+                        description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If
+                          not provided, the empty value is passed to the associated
+                          CSI driver which will determine the default filesystem to
+                          apply.
+                        type: string
+                      nodePublishSecretRef:
+                        description: nodePublishSecretRef is a reference to the secret
+                          object containing sensitive information to pass to the CSI
+                          driver to complete the CSI NodePublishVolume and NodeUnpublishVolume
+                          calls. This field is optional, and  may be empty if no secret
+                          is required. If the secret object contains more than one
+                          secret, all secret references are passed.
+                        properties:
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              TODO: Add other useful fields. apiVersion, kind, uid?'
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      readOnly:
+                        description: readOnly specifies a read-only configuration
+                          for the volume. Defaults to false (read/write).
+                        type: boolean
+                      volumeAttributes:
+                        additionalProperties:
+                          type: string
+                        description: volumeAttributes stores driver-specific properties
+                          that are passed to the CSI driver. Consult your driver's
+                          documentation for supported values.
+                        type: object
+                    required:
+                    - driver
+                    type: object
+                  mountPath:
+                    description: MountPath tells the script where to read the user
+                      information. Required if annotation injection is used, otherwise
+                      optional
+                    type: string
+                  secretName:
+                    type: string
+                  serviceAccountName:
+                    description: ServiceAccountName override job ServiceAccount, otherwise
+                      Spec.ServiceAccount (the one used to create the server pods)
+                      is used
+                    type: string
+                type: object
               users:
                 description: Cassandra users to bootstrap
                 items:

config/manager/kustomization.yaml

@@ -14,4 +14,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: k8ssandra/cass-operator
-  newTag: latest
+  newTag: v1.12.0-dev.28b8ea7-20220629

config/rbac/role.yaml

@@ -5,6 +5,18 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

controllers/cassandra/cassandradatacenter_controller.go

@@ -57,6 +57,7 @@ import (
 // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=pods;endpoints;services;configmaps;secrets;persistentvolumeclaims;events,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=namespaces,verbs=get
 // +kubebuilder:rbac:groups=core,resources=persistentvolumes;nodes,verbs=get;list;watch
+// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=policy,namespace=cass-operator,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
 
 // CassandraDatacenterReconciler reconciles a cassandraDatacenter object

pkg/reconciliation/reconcile_racks.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -105,6 +106,14 @@ func (rc *ReconciliationContext) CalculateRackInformation() error {
 func (rc *ReconciliationContext) CheckSuperuserSecretCreation() result.ReconcileResult {
 	rc.ReqLogger.Info("reconcile_racks::CheckSuperuserSecretCreation")
 
+	if rc.Datacenter.Spec.UserInfo != nil {
+		return result.Continue()
+	}
+
+	if rc.IsInitialized() {
+		return result.Continue()
+	}
+
 	_, err := rc.retrieveSuperuserSecretOrCreateDefault()
 	if err != nil {
 		rc.ReqLogger.Error(err, "error retrieving SuperuserSecret for CassandraDatacenter.")
@@ -876,6 +885,10 @@ func (rc *ReconciliationContext) UpdateSecretWatches() error {
 func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
 	dc := rc.Datacenter
 
+	if rc.IsInitialized() {
+		return result.Continue()
+	}
+
 	if val, found := dc.Annotations[api.SkipUserCreationAnnotation]; found && val == "true" {
 		rc.ReqLogger.Info(api.SkipUserCreationAnnotation + " is set, skipping CreateUser")
 		return result.Continue()
@@ -888,15 +901,117 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult {
 
 	rc.ReqLogger.Info("reconcile_racks::CreateUsers")
 
+	// TODO We should check if we've already created this ..
+	// TODO This should be cleaned up after a while (TTL)
+	if dc.Spec.UserInfo != nil {
+		// Create the job
+		ttl := int32(86400)
+
+		// TODO Instead of this, I should probably have a "SecretRef" generic structure. That way we could use it for all Secrets (like mgmt-api auth), supporting
+		// 		both legacy as well as new structure
+
+		// How to get this as input?
+		filePath := dc.Spec.UserInfo.MountPath
+
+		// We want to mount it as a directory and read the files as usernames
+		if dc.Spec.UserInfo.CSI != nil || dc.Spec.UserInfo.SecretName != "" {
+			filePath = "/mnt/secrets/users"
+		}
+
+		// TODO wait for it to complete before we continue..
+
+		job := batchv1.Job{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: dc.Namespace,
+				Name:      fmt.Sprintf("usercreate-%s", dc.Name),
+			},
+			Spec: batchv1.JobSpec{
+				Template: corev1.PodTemplateSpec{
+					Spec: corev1.PodSpec{
+						Containers: []corev1.Container{
+							{
+								Name:            "client",
+								Image:           "k8ssandra/k8ssandra-client:latest",
+								ImagePullPolicy: corev1.PullIfNotPresent,
+								Args:            []string{"users", "add", "--path", filePath, "--dc", dc.Name},
+							},
+						},
+						RestartPolicy: corev1.RestartPolicyNever,
+					},
+				},
+				TTLSecondsAfterFinished: &ttl,
+			},
+		}
+
+		if len(dc.Spec.UserInfo.Annotations) > 0 {
+			job.Spec.Template.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations
+		}
+
+		// TODO Add verification that we can't have dual injection (CSI + annotations)
+		if dc.Spec.UserInfo.SecretName != "" || dc.Spec.UserInfo.CSI != nil {
+			vol := corev1.Volume{
+				Name: "user-source",
+			}
+			if dc.Spec.UserInfo.SecretName != "" {
+				vol.VolumeSource = corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: dc.Spec.UserInfo.SecretName,
+					},
+				}
+			}
+
+			if dc.Spec.UserInfo.CSI != nil {
+				vol.VolumeSource = corev1.VolumeSource{
+					CSI: dc.Spec.UserInfo.CSI,
+				}
+			}
+
+			job.Spec.Template.Spec.Volumes = []corev1.Volume{vol}
+			job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+				{
+					Name:      "user-source",
+					ReadOnly:  true,
+					MountPath: filePath,
+				},
+			}
+		}
+
+		// labels := dc.GetDatacenterLabels()
+		// oplabels.AddOperatorLabels(labels, dc)
+
+		// Set CassandraDatacenter dc as the owner and controller
+		err := setControllerReference(dc, &job, rc.Scheme)
+		if err != nil {
+			return result.Error(err)
+		}
+
+		if dc.Spec.UserInfo.ServiceAccountName != "" {
+			job.Spec.Template.Spec.ServiceAccountName = dc.Spec.UserInfo.ServiceAccountName
+		} else {
+			job.Spec.Template.Spec.ServiceAccountName = dc.Spec.ServiceAccount
+		}
+
+		if err := rc.Client.Create(rc.Ctx, &job); err != nil {
+			if errors.IsAlreadyExists(err) {
+				return result.Continue()
+			}
+			return result.Error(err)
+		}
+		return result.RequeueSoon(5)
+	}
+
 	err := rc.UpdateSecretWatches()
 	if err != nil {
 		rc.ReqLogger.Error(err, "Failed to update dynamic watches on secrets")
 	}
 
 	// make sure the default superuser secret exists
-	_, err = rc.retrieveSuperuserSecretOrCreateDefault()
-	if err != nil {
-		rc.ReqLogger.Error(err, "Failed to verify superuser secret status")
+	// TODO Nope..
+	if dc.Spec.UserInfo == nil {
+		_, err = rc.retrieveSuperuserSecretOrCreateDefault()
+		if err != nil {
+			rc.ReqLogger.Error(err, "Failed to verify superuser secret status")
+		}
 	}
 
 	users := rc.GetUsers()