controller: OpenShift support

README.md

@@ -1 +1 @@
-# coroot-operator
+Coroot Kubernetes/Openshift operator.

config/rbac/role.yaml

@@ -105,6 +105,8 @@ rules:
   resources:
   - clusterrolebindings
   - clusterroles
+  - rolebindings
+  - roles
   verbs:
   - create
   - delete
@@ -113,6 +115,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use
 - apiGroups:
   - storage.k8s.io
   resources:

controller/clickhouse.go

@@ -177,11 +177,12 @@ func (r *CorootReconciler) clickhouseStatefulSets(cr *corootv1.Coroot) []*appsv1
 					Labels: ls,
 				},
 				Spec: corev1.PodSpec{
-					SecurityContext: nonRootSecurityContext,
-					Affinity:        cr.Spec.Clickhouse.Affinity,
+					ServiceAccountName: cr.Name + "-clickhouse",
+					SecurityContext:    nonRootSecurityContext,
+					Affinity:           cr.Spec.Clickhouse.Affinity,
 					InitContainers: []corev1.Container{
 						{
-							Image:        BusyboxImage,
+							Image:        UBIMinimalImage,
 							Name:         "config",
 							Command:      []string{"/bin/sh", "-c"},
 							Args:         []string{clickhouseConfigCmd("/config/config.xml", cr, shards, int(replicas), ClickhouseKeeperReplicas)},

controller/keeper.go → controller/clickhouse_keeper.go

@@ -80,7 +80,7 @@ func (r *CorootReconciler) clickhouseKeeperStatefulSet(cr *corootv1.Coroot) *app
 	ls := Labels(cr, "clickhouse-keeper")
 	ss := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-clickhouse-keeper", cr.Name),
+			Name:      cr.Name + "-clickhouse-keeper",
 			Namespace: cr.Namespace,
 			Labels:    ls,
 		},
@@ -102,27 +102,18 @@ func (r *CorootReconciler) clickhouseKeeperStatefulSet(cr *corootv1.Coroot) *app
 				Name:      "data",
 				Namespace: cr.Namespace,
 			},
-			//Spec: corev1.PersistentVolumeClaimSpec{
-			//	VolumeMode:  ptr.To(corev1.PersistentVolumeFilesystem),
-			//	AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
-			//	Resources: corev1.VolumeResourceRequirements{
-			//		Requests: corev1.ResourceList{
-			//			corev1.ResourceStorage: storageSize,
-			//		},
-			//	},
-			//	StorageClassName: cr.Spec.Clickhouse.Keeper.Storage.ClassName,
-			//},
 		}},
 		Template: corev1.PodTemplateSpec{
 			ObjectMeta: metav1.ObjectMeta{
 				Labels: ls,
 			},
 			Spec: corev1.PodSpec{
-				SecurityContext: nonRootSecurityContext,
-				Affinity:        cr.Spec.Clickhouse.Keeper.Affinity,
+				ServiceAccountName: cr.Name + "-clickhouse-keeper",
+				SecurityContext:    nonRootSecurityContext,
+				Affinity:           cr.Spec.Clickhouse.Keeper.Affinity,
 				InitContainers: []corev1.Container{
 					{
-						Image:        BusyboxImage,
+						Image:        UBIMinimalImage,
 						Name:         "config",
 						Command:      []string{"/bin/sh", "-c"},
 						Args:         []string{clickhouseKeeperConfigCmd("/config/config.xml", cr, int(replicas))},
@@ -185,7 +176,7 @@ func clickhouseKeeperConfigCmd(filename string, cr *corootv1.Coroot, replicas in
 	}
 	var out bytes.Buffer
 	_ = clickhouseKeeperConfigTemplate.Execute(&out, params)
-	return "cat <<EOF | sed s/SERVER_ID/$(hostname -s | sed -E 's/.*-([0-9]+)$/\\1/')/ > " + filename + out.String() + "EOF"
+	return "cat <<EOF | sed s/SERVER_ID/$(echo $HOSTNAME | sed -E 's/.*-([0-9]+)$/\\1/')/ > " + filename + out.String() + "EOF"
 }
 
 var clickhouseKeeperConfigTemplate = template.Must(template.New("").Parse(`

controller/cluster_agent.go

@@ -104,8 +104,9 @@ func (r *CorootReconciler) clusterAgentDeployment(cr *corootv1.Coroot) *appsv1.D
 				Labels: ls,
 			},
 			Spec: corev1.PodSpec{
-				SecurityContext: nonRootSecurityContext,
-				Affinity:        cr.Spec.ClusterAgent.Affinity,
+				ServiceAccountName: cr.Name + "-cluster-agent",
+				SecurityContext:    nonRootSecurityContext,
+				Affinity:           cr.Spec.ClusterAgent.Affinity,
 				Containers: []corev1.Container{
 					{
 						Image: r.getAppImage(cr, AppClusterAgent),

controller/controller.go

@@ -22,7 +22,7 @@ import (
 
 const (
 	AppVersionsUpdateInterval = time.Hour
-	BusyboxImage              = "busybox:1.36"
+	UBIMinimalImage           = "registry.access.redhat.com/ubi9/ubi-minimal"
 )
 
 type CorootReconciler struct {
@@ -71,7 +71,8 @@ func NewCorootReconciler(mgr ctrl.Manager) *CorootReconciler {
 // +kubebuilder:rbac:groups=apps,resources=deployments;replicasets;daemonsets;statefulsets;cronjobs,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses;volumeattachments,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings;roles;rolebindings,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use
 
 func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
@@ -97,34 +98,43 @@ func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 	r.instances[req] = true
 	r.instancesLock.Unlock()
 
+	r.CreateOrUpdateRole(ctx, cr, r.openshiftSCCRole(cr, sccNonroot))
+	r.CreateOrUpdateRole(ctx, cr, r.openshiftSCCRole(cr, sccPrivileged))
+
+	r.CreateOrUpdateServiceAccount(ctx, cr, "node-agent", sccPrivileged)
 	r.CreateOrUpdateDaemonSet(ctx, cr, r.nodeAgentDaemonSet(cr))
 
-	r.CreateOrUpdateDeployment(ctx, cr, r.clusterAgentDeployment(cr))
+	r.CreateOrUpdateServiceAccount(ctx, cr, "cluster-agent", sccNonroot)
 	r.CreateOrUpdateClusterRole(ctx, cr, r.clusterAgentClusterRole(cr))
 	r.CreateOrUpdateClusterRoleBinding(ctx, cr, r.clusterAgentClusterRoleBinding(cr))
+	r.CreateOrUpdateDeployment(ctx, cr, r.clusterAgentDeployment(cr))
 
 	if cr.Spec.AgentsOnly != nil {
 		// TODO: delete
 		return ctrl.Result{}, nil
 	}
 
+	r.CreateOrUpdateServiceAccount(ctx, cr, "coroot", sccNonroot)
 	r.CreateOrUpdatePVC(ctx, cr, r.corootPVC(cr))
 	r.CreateOrUpdateDeployment(ctx, cr, r.corootDeployment(cr))
 	r.CreateOrUpdateService(ctx, cr, r.corootService(cr))
 
+	r.CreateOrUpdateServiceAccount(ctx, cr, "prometheus", sccNonroot)
 	r.CreateOrUpdatePVC(ctx, cr, r.prometheusPVC(cr))
 	r.CreateOrUpdateDeployment(ctx, cr, r.prometheusDeployment(cr))
 	r.CreateOrUpdateService(ctx, cr, r.prometheusService(cr))
 
 	if cr.Spec.ExternalClickhouse == nil {
 		r.CreateSecret(ctx, cr, r.clickhouseSecret(cr))
 
+		r.CreateOrUpdateServiceAccount(ctx, cr, "clickhouse-keeper", sccNonroot)
 		for _, pvc := range r.clickhouseKeeperPVCs(cr) {
 			r.CreateOrUpdatePVC(ctx, cr, pvc)
 		}
 		r.CreateOrUpdateStatefulSet(ctx, cr, r.clickhouseKeeperStatefulSet(cr))
 		r.CreateOrUpdateService(ctx, cr, r.clickhouseKeeperServiceHeadless(cr))
 
+		r.CreateOrUpdateServiceAccount(ctx, cr, "clickhouse", sccNonroot)
 		r.CreateOrUpdateService(ctx, cr, r.clickhouseServiceHeadless(cr))
 		for _, pvc := range r.clickhousePVCs(cr) {
 			r.CreateOrUpdatePVC(ctx, cr, pvc)
@@ -163,26 +173,20 @@ func (r *CorootReconciler) CreateSecret(ctx context.Context, cr *corootv1.Coroot
 }
 
 func (r *CorootReconciler) CreateOrUpdateDeployment(ctx context.Context, cr *corootv1.Coroot, d *appsv1.Deployment) {
-	r.CreateOrUpdateServiceAccount(ctx, cr, d.ObjectMeta)
-	d.Spec.Template.Spec.ServiceAccountName = d.ObjectMeta.Name
 	spec := d.Spec
 	r.CreateOrUpdate(ctx, cr, d, func() error {
 		return Merge(&d.Spec, spec)
 	})
 }
 
 func (r *CorootReconciler) CreateOrUpdateDaemonSet(ctx context.Context, cr *corootv1.Coroot, ds *appsv1.DaemonSet) {
-	r.CreateOrUpdateServiceAccount(ctx, cr, ds.ObjectMeta)
-	ds.Spec.Template.Spec.ServiceAccountName = ds.ObjectMeta.Name
 	spec := ds.Spec
 	r.CreateOrUpdate(ctx, cr, ds, func() error {
 		return Merge(&ds.Spec, spec)
 	})
 }
 
 func (r *CorootReconciler) CreateOrUpdateStatefulSet(ctx context.Context, cr *corootv1.Coroot, ss *appsv1.StatefulSet) {
-	r.CreateOrUpdateServiceAccount(ctx, cr, ss.ObjectMeta)
-	ss.Spec.Template.Spec.ServiceAccountName = ss.ObjectMeta.Name
 	spec := ss.Spec
 	r.CreateOrUpdate(ctx, cr, ss, func() error {
 		volumeClaimTemplates := ss.Spec.VolumeClaimTemplates[:]
@@ -208,13 +212,22 @@ func (r *CorootReconciler) CreateOrUpdateService(ctx context.Context, cr *coroot
 	})
 }
 
-func (r *CorootReconciler) CreateOrUpdateServiceAccount(ctx context.Context, cr *corootv1.Coroot, om metav1.ObjectMeta) {
+func (r *CorootReconciler) CreateOrUpdateServiceAccount(ctx context.Context, cr *corootv1.Coroot, component, scc string) {
 	sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{
-		Name:      om.Name,
-		Namespace: om.Namespace,
-		Labels:    om.Labels,
+		Name:      cr.Name + "-" + component,
+		Namespace: cr.Namespace,
+		Labels:    Labels(cr, component),
 	}}
 	r.CreateOrUpdate(ctx, cr, sa, nil)
+	r.CreateOrUpdate(ctx, cr, r.openshiftSCCRoleBinding(cr, component, scc), nil)
+}
+
+func (r *CorootReconciler) CreateOrUpdateRole(ctx context.Context, cr *corootv1.Coroot, role *rbacv1.Role) {
+	rules := role.Rules
+	r.CreateOrUpdate(ctx, cr, role, func() error {
+		role.Rules = rules
+		return nil
+	})
 }
 
 func (r *CorootReconciler) CreateOrUpdateClusterRole(ctx context.Context, cr *corootv1.Coroot, role *rbacv1.ClusterRole) {

controller/coroot.go

@@ -122,8 +122,9 @@ func (r *CorootReconciler) corootDeployment(cr *corootv1.Coroot) *appsv1.Deploym
 				Labels: ls,
 			},
 			Spec: corev1.PodSpec{
-				SecurityContext: nonRootSecurityContext,
-				Affinity:        cr.Spec.Affinity,
+				ServiceAccountName: cr.Name + "-coroot",
+				SecurityContext:    nonRootSecurityContext,
+				Affinity:           cr.Spec.Affinity,
 				Containers: []corev1.Container{
 					{
 						Image: image,

controller/node_agent.go

@@ -45,10 +45,11 @@ func (r *CorootReconciler) nodeAgentDaemonSet(cr *corootv1.Coroot) *appsv1.Daemo
 				Labels: ls,
 			},
 			Spec: corev1.PodSpec{
-				HostPID:           true,
-				Tolerations:       []corev1.Toleration{{Operator: corev1.TolerationOpExists}},
-				PriorityClassName: cr.Spec.NodeAgent.PriorityClassName,
-				Affinity:          cr.Spec.NodeAgent.Affinity,
+				ServiceAccountName: cr.Name + "-node-agent",
+				HostPID:            true,
+				Tolerations:        []corev1.Toleration{{Operator: corev1.TolerationOpExists}},
+				PriorityClassName:  cr.Spec.NodeAgent.PriorityClassName,
+				Affinity:           cr.Spec.NodeAgent.Affinity,
 				Containers: []corev1.Container{
 					{
 						Name:  "node-agent",

controller/openshift.go

@@ -0,0 +1,52 @@
+package controller
+
+import (
+	corootv1 "github.io/coroot/operator/api/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	sccNonroot    = "nonroot"
+	sccPrivileged = "privileged"
+)
+
+func (r *CorootReconciler) openshiftSCCRole(cr *corootv1.Coroot, scc string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cr.Name + "-" + scc,
+			Namespace: cr.Namespace,
+			Labels:    Labels(cr, "roles"),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{"security.openshift.io"},
+				Resources:     []string{"securitycontextconstraints"},
+				ResourceNames: []string{scc},
+				Verbs:         []string{"use"},
+			},
+		},
+	}
+}
+
+func (r *CorootReconciler) openshiftSCCRoleBinding(cr *corootv1.Coroot, component, scc string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cr.Name + "-" + component + "-" + scc,
+			Namespace: cr.Namespace,
+			Labels:    Labels(cr, component),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      cr.Name + "-" + component,
+				Namespace: cr.Namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: rbacv1.GroupName,
+			Kind:     "Role",
+			Name:     cr.Name + "-" + scc,
+		},
+	}
+}

controller/prometheus.go

@@ -88,8 +88,9 @@ func (r *CorootReconciler) prometheusDeployment(cr *corootv1.Coroot) *appsv1.Dep
 				Labels: ls,
 			},
 			Spec: corev1.PodSpec{
-				SecurityContext: nonRootSecurityContext,
-				Affinity:        cr.Spec.Prometheus.Affinity,
+				ServiceAccountName: cr.Name + "-prometheus",
+				SecurityContext:    nonRootSecurityContext,
+				Affinity:           cr.Spec.Prometheus.Affinity,
 				Containers: []corev1.Container{
 					{
 						Image:   PrometheusImage,