diff --git a/.github/workflows/operatorBuildAndDeploy.yml b/.github/workflows/operatorBuildAndDeploy.yml index ec313359..ca13acc0 100644 --- a/.github/workflows/operatorBuildAndDeploy.yml +++ b/.github/workflows/operatorBuildAndDeploy.yml @@ -71,7 +71,7 @@ jobs: with: file: Dockerfile context: . - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.event_name != 'pull_request' && !env.ACT }} tags: k8ssandra/cass-operator:${{ steps.vars.outputs.sha_short }}, k8ssandra/cass-operator:latest, k8ssandra/cass-operator:${{ steps.vars.outputs.version }} platforms: linux/amd64 cache-from: type=local,src=/tmp/.buildx-cache @@ -81,7 +81,7 @@ jobs: uses: docker/build-push-action@v3 with: file: logger.Dockerfile - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.event_name != 'pull_request' && !env.ACT }} tags: k8ssandra/system-logger:${{ steps.vars.outputs.sha_short }}, k8ssandra/system-logger:latest platforms: linux/amd64 cache-from: type=local,src=/tmp/.buildx-cache @@ -106,11 +106,12 @@ jobs: build-args: | VERSION=${{ steps.vars.outputs.version }} context: . - push: ${{ !env.ACT }} + push: ${{ github.event_name != 'pull_request' && !env.ACT }} tags: k8ssandra/cass-operator-bundle:v${{ steps.vars.outputs.version }} platforms: linux/amd64 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: Build and update cass-operator-catalog run: | + # Load the images here? It wants the bundle.. make VERSION=${{ steps.vars.outputs.version }} CHANNEL=dev catalog-build catalog-push diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 0e9f7dfd..ae60b123 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -77,6 +77,25 @@ type CassandraUser struct { Superuser bool `json:"superuser"` } +type UserInfo struct { + Annotations map[string]string `json:"annotations,omitempty"` + // MountPath tells the script where to read the user information. Required if annotation injection is used, otherwise optional + MountPath string `json:"mountPath,omitempty"` + CSI *corev1.CSIVolumeSource `json:"csi,omitempty"` + SecretName string `json:"secretName,omitempty"` + + // ServiceAccountName override job ServiceAccount, otherwise Spec.ServiceAccount (the one used to create the server pods) is used + ServiceAccountName string `json:"serviceAccountName,omitempty"` + // TODO Add CSI information here + /* + TODO Testing: + * Add back the Helm utilities + * Install Vault + Vault server there + * Install CSI driver + * Try to create clusters with both methods and see that it succeeds (and uses Vault to fetch the user rights) + */ +} + // CassandraDatacenterSpec defines the desired state of a CassandraDatacenter // +k8s:openapi-gen=true // +kubebuilder:pruning:PreserveUnknownFields @@ -180,11 +199,13 @@ type CassandraDatacenterSpec struct { // podAntiAffinity and requiredDuringSchedulingIgnoredDuringExecution. AllowMultipleNodesPerWorker bool `json:"allowMultipleNodesPerWorker,omitempty"` - // This secret defines the username and password for the Cassandra server superuser. + // SuperuserSecretName is deprecated. Use UserInfo instead. This secret defines the username and password for the Cassandra server superuser. // If it is omitted, we will generate a secret instead. SuperuserSecretName string `json:"superuserSecretName,omitempty"` - // The k8s service account to use for the server pods + UserInfo *UserInfo `json:"userInfo,omitempty"` + + // ServiceAccount is the k8s service account to use for the server pods ServiceAccount string `json:"serviceAccount,omitempty"` // DEPRECATED. Use CassandraTask for rolling restarts. Whether to do a rolling restart at the next opportunity. The operator will set this back diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index 9cb1ed33..cf00e11c 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -285,6 +285,11 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.UserInfo != nil { + in, out := &in.UserInfo, &out.UserInfo + *out = new(UserInfo) + (*in).DeepCopyInto(*out) + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -667,3 +672,30 @@ func (in *StorageConfig) DeepCopy() *StorageConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserInfo) DeepCopyInto(out *UserInfo) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.CSI != nil { + in, out := &in.CSI, &out.CSI + *out = new(v1.CSIVolumeSource) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserInfo. +func (in *UserInfo) DeepCopy() *UserInfo { + if in == nil { + return nil + } + out := new(UserInfo) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index b42060c4..4e6c1247 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -7614,7 +7614,8 @@ spec: pattern: (6\.8\.\d+)|(3\.11\.\d+)|(4\.\d+\.\d+) type: string serviceAccount: - description: The k8s service account to use for the server pods + description: ServiceAccount is the k8s service account to use for + the server pods type: string size: description: Desired number of Cassandra server nodes @@ -7996,9 +7997,9 @@ spec: type: object type: object superuserSecretName: - description: This secret defines the username and password for the - Cassandra server superuser. If it is omitted, we will generate a - secret instead. + description: SuperuserSecretName is deprecated. Use UserInfo instead. + This secret defines the username and password for the Cassandra + server superuser. If it is omitted, we will generate a secret instead. type: string systemLoggerImage: description: Container image for the log tailing sidecar container. @@ -8072,6 +8073,68 @@ spec: type: string type: object type: array + userInfo: + properties: + annotations: + additionalProperties: + type: string + type: object + csi: + description: Represents a source location of a volume to mount, + managed by an external CSI driver + properties: + driver: + description: driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If + not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem to + apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the secret + object containing sensitive information to pass to the CSI + driver to complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may be empty if no secret + is required. If the secret object contains more than one + secret, all secret references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + mountPath: + description: MountPath tells the script where to read the user + information. Required if annotation injection is used, otherwise + optional + type: string + secretName: + type: string + serviceAccountName: + description: ServiceAccountName override job ServiceAccount, otherwise + Spec.ServiceAccount (the one used to create the server pods) + is used + type: string + type: object users: description: Cassandra users to bootstrap items: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 7621efda..ab5cc780 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -14,4 +14,4 @@ kind: Kustomization images: - name: controller newName: k8ssandra/cass-operator - newTag: latest + newTag: v1.12.0-dev.28b8ea7-20220629 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8ee7de45..4d7e1a52 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/cassandra/cassandradatacenter_controller.go b/controllers/cassandra/cassandradatacenter_controller.go index 16d00c37..62e94b29 100644 --- a/controllers/cassandra/cassandradatacenter_controller.go +++ b/controllers/cassandra/cassandradatacenter_controller.go @@ -57,6 +57,7 @@ import ( // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=pods;endpoints;services;configmaps;secrets;persistentvolumeclaims;events,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=namespaces,verbs=get // +kubebuilder:rbac:groups=core,resources=persistentvolumes;nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=policy,namespace=cass-operator,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete // CassandraDatacenterReconciler reconciles a cassandraDatacenter object diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index 38d6da6f..7716e7e5 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -12,6 +12,7 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -105,6 +106,14 @@ func (rc *ReconciliationContext) CalculateRackInformation() error { func (rc *ReconciliationContext) CheckSuperuserSecretCreation() result.ReconcileResult { rc.ReqLogger.Info("reconcile_racks::CheckSuperuserSecretCreation") + if rc.Datacenter.Spec.UserInfo != nil { + return result.Continue() + } + + if rc.IsInitialized() { + return result.Continue() + } + _, err := rc.retrieveSuperuserSecretOrCreateDefault() if err != nil { rc.ReqLogger.Error(err, "error retrieving SuperuserSecret for CassandraDatacenter.") @@ -876,6 +885,10 @@ func (rc *ReconciliationContext) UpdateSecretWatches() error { func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { dc := rc.Datacenter + if rc.IsInitialized() { + return result.Continue() + } + if val, found := dc.Annotations[api.SkipUserCreationAnnotation]; found && val == "true" { rc.ReqLogger.Info(api.SkipUserCreationAnnotation + " is set, skipping CreateUser") return result.Continue() @@ -888,15 +901,117 @@ func (rc *ReconciliationContext) CreateUsers() result.ReconcileResult { rc.ReqLogger.Info("reconcile_racks::CreateUsers") + // TODO We should check if we've already created this .. + // TODO This should be cleaned up after a while (TTL) + if dc.Spec.UserInfo != nil { + // Create the job + ttl := int32(86400) + + // TODO Instead of this, I should probably have a "SecretRef" generic structure. That way we could use it for all Secrets (like mgmt-api auth), supporting + // both legacy as well as new structure + + // How to get this as input? + filePath := dc.Spec.UserInfo.MountPath + + // We want to mount it as a directory and read the files as usernames + if dc.Spec.UserInfo.CSI != nil || dc.Spec.UserInfo.SecretName != "" { + filePath = "/mnt/secrets/users" + } + + // TODO wait for it to complete before we continue.. + + job := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dc.Namespace, + Name: fmt.Sprintf("usercreate-%s", dc.Name), + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "client", + Image: "k8ssandra/k8ssandra-client:latest", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"users", "add", "--path", filePath, "--dc", dc.Name}, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + TTLSecondsAfterFinished: &ttl, + }, + } + + if len(dc.Spec.UserInfo.Annotations) > 0 { + job.Spec.Template.ObjectMeta.Annotations = dc.Spec.UserInfo.Annotations + } + + // TODO Add verification that we can't have dual injection (CSI + annotations) + if dc.Spec.UserInfo.SecretName != "" || dc.Spec.UserInfo.CSI != nil { + vol := corev1.Volume{ + Name: "user-source", + } + if dc.Spec.UserInfo.SecretName != "" { + vol.VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: dc.Spec.UserInfo.SecretName, + }, + } + } + + if dc.Spec.UserInfo.CSI != nil { + vol.VolumeSource = corev1.VolumeSource{ + CSI: dc.Spec.UserInfo.CSI, + } + } + + job.Spec.Template.Spec.Volumes = []corev1.Volume{vol} + job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: "user-source", + ReadOnly: true, + MountPath: filePath, + }, + } + } + + // labels := dc.GetDatacenterLabels() + // oplabels.AddOperatorLabels(labels, dc) + + // Set CassandraDatacenter dc as the owner and controller + err := setControllerReference(dc, &job, rc.Scheme) + if err != nil { + return result.Error(err) + } + + if dc.Spec.UserInfo.ServiceAccountName != "" { + job.Spec.Template.Spec.ServiceAccountName = dc.Spec.UserInfo.ServiceAccountName + } else { + job.Spec.Template.Spec.ServiceAccountName = dc.Spec.ServiceAccount + } + + if err := rc.Client.Create(rc.Ctx, &job); err != nil { + if errors.IsAlreadyExists(err) { + return result.Continue() + } + return result.Error(err) + } + return result.RequeueSoon(5) + } + err := rc.UpdateSecretWatches() if err != nil { rc.ReqLogger.Error(err, "Failed to update dynamic watches on secrets") } // make sure the default superuser secret exists - _, err = rc.retrieveSuperuserSecretOrCreateDefault() - if err != nil { - rc.ReqLogger.Error(err, "Failed to verify superuser secret status") + // TODO Nope.. + if dc.Spec.UserInfo == nil { + _, err = rc.retrieveSuperuserSecretOrCreateDefault() + if err != nil { + rc.ReqLogger.Error(err, "Failed to verify superuser secret status") + } } users := rc.GetUsers() diff --git a/tests/external_secret/README.md b/tests/external_secret/README.md new file mode 100644 index 00000000..1a74ca96 --- /dev/null +++ b/tests/external_secret/README.md @@ -0,0 +1,83 @@ +## Install Helm repositories + +``` +helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts +helm repo add hashicorp https://helm.releases.hashicorp.com +helm repo update +``` + +## Install Vault + +``` +helm install vault hashicorp/vault --set "server.dev.enabled=true" --namespace cass-operator +# --set "csi.enabled=true" +``` + +## Go into Vault and exec certain commands.. + +``` +kubectl exec -it vault-0 -- /bin/sh + +vault secrets enable -path=internal kv-v2 + +vault kv put internal/database/config username="superuser" password="superpassword" + +vault auth enable kubernetes + +vault write auth/kubernetes/config \ + kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" + +vault policy write internal-app - <