From f77f55b0e5dc8ccaa03cca831753d6c5c18d7b73 Mon Sep 17 00:00:00 2001 From: Anton Petruhin Date: Fri, 15 Nov 2024 14:46:28 +0400 Subject: [PATCH] controller: OpenShift support --- README.md | 2 +- config/rbac/role.yaml | 8 +++ controller/clickhouse.go | 7 +-- .../{keeper.go => clickhouse_keeper.go} | 21 +++----- controller/cluster_agent.go | 5 +- controller/controller.go | 39 +++++++++----- controller/coroot.go | 5 +- controller/node_agent.go | 9 ++-- controller/openshift.go | 52 +++++++++++++++++++ controller/prometheus.go | 5 +- 10 files changed, 111 insertions(+), 42 deletions(-) rename controller/{keeper.go => clickhouse_keeper.go} (89%) create mode 100644 controller/openshift.go diff --git a/README.md b/README.md index edfb138..a7c9d4f 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# coroot-operator +Coroot Kubernetes/Openshift operator. \ No newline at end of file diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0b47856..cb5f59e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -105,6 +105,8 @@ rules: resources: - clusterrolebindings - clusterroles + - rolebindings + - roles verbs: - create - delete @@ -113,6 +115,12 @@ rules: - patch - update - watch +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use - apiGroups: - storage.k8s.io resources: diff --git a/controller/clickhouse.go b/controller/clickhouse.go index 3d23ed2..da66511 100644 --- a/controller/clickhouse.go +++ b/controller/clickhouse.go @@ -177,11 +177,12 @@ func (r *CorootReconciler) clickhouseStatefulSets(cr *corootv1.Coroot) []*appsv1 Labels: ls, }, Spec: corev1.PodSpec{ - SecurityContext: nonRootSecurityContext, - Affinity: cr.Spec.Clickhouse.Affinity, + ServiceAccountName: cr.Name + "-clickhouse", + SecurityContext: nonRootSecurityContext, + Affinity: cr.Spec.Clickhouse.Affinity, InitContainers: []corev1.Container{ { - Image: BusyboxImage, + Image: UBIMinimalImage, Name: "config", Command: []string{"/bin/sh", "-c"}, Args: []string{clickhouseConfigCmd("/config/config.xml", cr, shards, int(replicas), ClickhouseKeeperReplicas)}, diff --git a/controller/keeper.go b/controller/clickhouse_keeper.go similarity index 89% rename from controller/keeper.go rename to controller/clickhouse_keeper.go index 62dd67c..0290e4c 100644 --- a/controller/keeper.go +++ b/controller/clickhouse_keeper.go @@ -80,7 +80,7 @@ func (r *CorootReconciler) clickhouseKeeperStatefulSet(cr *corootv1.Coroot) *app ls := Labels(cr, "clickhouse-keeper") ss := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-clickhouse-keeper", cr.Name), + Name: cr.Name + "-clickhouse-keeper", Namespace: cr.Namespace, Labels: ls, }, @@ -102,27 +102,18 @@ func (r *CorootReconciler) clickhouseKeeperStatefulSet(cr *corootv1.Coroot) *app Name: "data", Namespace: cr.Namespace, }, - //Spec: corev1.PersistentVolumeClaimSpec{ - // VolumeMode: ptr.To(corev1.PersistentVolumeFilesystem), - // AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - // Resources: corev1.VolumeResourceRequirements{ - // Requests: corev1.ResourceList{ - // corev1.ResourceStorage: storageSize, - // }, - // }, - // StorageClassName: cr.Spec.Clickhouse.Keeper.Storage.ClassName, - //}, }}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: ls, }, Spec: corev1.PodSpec{ - SecurityContext: nonRootSecurityContext, - Affinity: cr.Spec.Clickhouse.Keeper.Affinity, + ServiceAccountName: cr.Name + "-clickhouse-keeper", + SecurityContext: nonRootSecurityContext, + Affinity: cr.Spec.Clickhouse.Keeper.Affinity, InitContainers: []corev1.Container{ { - Image: BusyboxImage, + Image: UBIMinimalImage, Name: "config", Command: []string{"/bin/sh", "-c"}, Args: []string{clickhouseKeeperConfigCmd("/config/config.xml", cr, int(replicas))}, @@ -185,7 +176,7 @@ func clickhouseKeeperConfigCmd(filename string, cr *corootv1.Coroot, replicas in } var out bytes.Buffer _ = clickhouseKeeperConfigTemplate.Execute(&out, params) - return "cat < " + filename + out.String() + "EOF" + return "cat < " + filename + out.String() + "EOF" } var clickhouseKeeperConfigTemplate = template.Must(template.New("").Parse(` diff --git a/controller/cluster_agent.go b/controller/cluster_agent.go index 4782e69..fcbacc7 100644 --- a/controller/cluster_agent.go +++ b/controller/cluster_agent.go @@ -104,8 +104,9 @@ func (r *CorootReconciler) clusterAgentDeployment(cr *corootv1.Coroot) *appsv1.D Labels: ls, }, Spec: corev1.PodSpec{ - SecurityContext: nonRootSecurityContext, - Affinity: cr.Spec.ClusterAgent.Affinity, + ServiceAccountName: cr.Name + "-cluster-agent", + SecurityContext: nonRootSecurityContext, + Affinity: cr.Spec.ClusterAgent.Affinity, Containers: []corev1.Container{ { Image: r.getAppImage(cr, AppClusterAgent), diff --git a/controller/controller.go b/controller/controller.go index d3b5c7b..54f829c 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -22,7 +22,7 @@ import ( const ( AppVersionsUpdateInterval = time.Hour - BusyboxImage = "busybox:1.36" + UBIMinimalImage = "registry.access.redhat.com/ubi9/ubi-minimal" ) type CorootReconciler struct { @@ -71,7 +71,8 @@ func NewCorootReconciler(mgr ctrl.Manager) *CorootReconciler { // +kubebuilder:rbac:groups=apps,resources=deployments;replicasets;daemonsets;statefulsets;cronjobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses;volumeattachments,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings;roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -97,21 +98,28 @@ func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr r.instances[req] = true r.instancesLock.Unlock() + r.CreateOrUpdateRole(ctx, cr, r.openshiftSCCRole(cr, sccNonroot)) + r.CreateOrUpdateRole(ctx, cr, r.openshiftSCCRole(cr, sccPrivileged)) + + r.CreateOrUpdateServiceAccount(ctx, cr, "node-agent", sccPrivileged) r.CreateOrUpdateDaemonSet(ctx, cr, r.nodeAgentDaemonSet(cr)) - r.CreateOrUpdateDeployment(ctx, cr, r.clusterAgentDeployment(cr)) + r.CreateOrUpdateServiceAccount(ctx, cr, "cluster-agent", sccNonroot) r.CreateOrUpdateClusterRole(ctx, cr, r.clusterAgentClusterRole(cr)) r.CreateOrUpdateClusterRoleBinding(ctx, cr, r.clusterAgentClusterRoleBinding(cr)) + r.CreateOrUpdateDeployment(ctx, cr, r.clusterAgentDeployment(cr)) if cr.Spec.AgentsOnly != nil { // TODO: delete return ctrl.Result{}, nil } + r.CreateOrUpdateServiceAccount(ctx, cr, "coroot", sccNonroot) r.CreateOrUpdatePVC(ctx, cr, r.corootPVC(cr)) r.CreateOrUpdateDeployment(ctx, cr, r.corootDeployment(cr)) r.CreateOrUpdateService(ctx, cr, r.corootService(cr)) + r.CreateOrUpdateServiceAccount(ctx, cr, "prometheus", sccNonroot) r.CreateOrUpdatePVC(ctx, cr, r.prometheusPVC(cr)) r.CreateOrUpdateDeployment(ctx, cr, r.prometheusDeployment(cr)) r.CreateOrUpdateService(ctx, cr, r.prometheusService(cr)) @@ -119,12 +127,14 @@ func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if cr.Spec.ExternalClickhouse == nil { r.CreateSecret(ctx, cr, r.clickhouseSecret(cr)) + r.CreateOrUpdateServiceAccount(ctx, cr, "clickhouse-keeper", sccNonroot) for _, pvc := range r.clickhouseKeeperPVCs(cr) { r.CreateOrUpdatePVC(ctx, cr, pvc) } r.CreateOrUpdateStatefulSet(ctx, cr, r.clickhouseKeeperStatefulSet(cr)) r.CreateOrUpdateService(ctx, cr, r.clickhouseKeeperServiceHeadless(cr)) + r.CreateOrUpdateServiceAccount(ctx, cr, "clickhouse", sccNonroot) r.CreateOrUpdateService(ctx, cr, r.clickhouseServiceHeadless(cr)) for _, pvc := range r.clickhousePVCs(cr) { r.CreateOrUpdatePVC(ctx, cr, pvc) @@ -163,8 +173,6 @@ func (r *CorootReconciler) CreateSecret(ctx context.Context, cr *corootv1.Coroot } func (r *CorootReconciler) CreateOrUpdateDeployment(ctx context.Context, cr *corootv1.Coroot, d *appsv1.Deployment) { - r.CreateOrUpdateServiceAccount(ctx, cr, d.ObjectMeta) - d.Spec.Template.Spec.ServiceAccountName = d.ObjectMeta.Name spec := d.Spec r.CreateOrUpdate(ctx, cr, d, func() error { return Merge(&d.Spec, spec) @@ -172,8 +180,6 @@ func (r *CorootReconciler) CreateOrUpdateDeployment(ctx context.Context, cr *cor } func (r *CorootReconciler) CreateOrUpdateDaemonSet(ctx context.Context, cr *corootv1.Coroot, ds *appsv1.DaemonSet) { - r.CreateOrUpdateServiceAccount(ctx, cr, ds.ObjectMeta) - ds.Spec.Template.Spec.ServiceAccountName = ds.ObjectMeta.Name spec := ds.Spec r.CreateOrUpdate(ctx, cr, ds, func() error { return Merge(&ds.Spec, spec) @@ -181,8 +187,6 @@ func (r *CorootReconciler) CreateOrUpdateDaemonSet(ctx context.Context, cr *coro } func (r *CorootReconciler) CreateOrUpdateStatefulSet(ctx context.Context, cr *corootv1.Coroot, ss *appsv1.StatefulSet) { - r.CreateOrUpdateServiceAccount(ctx, cr, ss.ObjectMeta) - ss.Spec.Template.Spec.ServiceAccountName = ss.ObjectMeta.Name spec := ss.Spec r.CreateOrUpdate(ctx, cr, ss, func() error { volumeClaimTemplates := ss.Spec.VolumeClaimTemplates[:] @@ -208,13 +212,22 @@ func (r *CorootReconciler) CreateOrUpdateService(ctx context.Context, cr *coroot }) } -func (r *CorootReconciler) CreateOrUpdateServiceAccount(ctx context.Context, cr *corootv1.Coroot, om metav1.ObjectMeta) { +func (r *CorootReconciler) CreateOrUpdateServiceAccount(ctx context.Context, cr *corootv1.Coroot, component, scc string) { sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ - Name: om.Name, - Namespace: om.Namespace, - Labels: om.Labels, + Name: cr.Name + "-" + component, + Namespace: cr.Namespace, + Labels: Labels(cr, component), }} r.CreateOrUpdate(ctx, cr, sa, nil) + r.CreateOrUpdate(ctx, cr, r.openshiftSCCRoleBinding(cr, component, scc), nil) +} + +func (r *CorootReconciler) CreateOrUpdateRole(ctx context.Context, cr *corootv1.Coroot, role *rbacv1.Role) { + rules := role.Rules + r.CreateOrUpdate(ctx, cr, role, func() error { + role.Rules = rules + return nil + }) } func (r *CorootReconciler) CreateOrUpdateClusterRole(ctx context.Context, cr *corootv1.Coroot, role *rbacv1.ClusterRole) { diff --git a/controller/coroot.go b/controller/coroot.go index 84e6bdd..b2509fa 100644 --- a/controller/coroot.go +++ b/controller/coroot.go @@ -122,8 +122,9 @@ func (r *CorootReconciler) corootDeployment(cr *corootv1.Coroot) *appsv1.Deploym Labels: ls, }, Spec: corev1.PodSpec{ - SecurityContext: nonRootSecurityContext, - Affinity: cr.Spec.Affinity, + ServiceAccountName: cr.Name + "-coroot", + SecurityContext: nonRootSecurityContext, + Affinity: cr.Spec.Affinity, Containers: []corev1.Container{ { Image: image, diff --git a/controller/node_agent.go b/controller/node_agent.go index dbe2211..4f47c96 100644 --- a/controller/node_agent.go +++ b/controller/node_agent.go @@ -45,10 +45,11 @@ func (r *CorootReconciler) nodeAgentDaemonSet(cr *corootv1.Coroot) *appsv1.Daemo Labels: ls, }, Spec: corev1.PodSpec{ - HostPID: true, - Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, - PriorityClassName: cr.Spec.NodeAgent.PriorityClassName, - Affinity: cr.Spec.NodeAgent.Affinity, + ServiceAccountName: cr.Name + "-node-agent", + HostPID: true, + Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, + PriorityClassName: cr.Spec.NodeAgent.PriorityClassName, + Affinity: cr.Spec.NodeAgent.Affinity, Containers: []corev1.Container{ { Name: "node-agent", diff --git a/controller/openshift.go b/controller/openshift.go new file mode 100644 index 0000000..f4216b4 --- /dev/null +++ b/controller/openshift.go @@ -0,0 +1,52 @@ +package controller + +import ( + corootv1 "github.io/coroot/operator/api/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + sccNonroot = "nonroot" + sccPrivileged = "privileged" +) + +func (r *CorootReconciler) openshiftSCCRole(cr *corootv1.Coroot, scc string) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name + "-" + scc, + Namespace: cr.Namespace, + Labels: Labels(cr, "roles"), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{scc}, + Verbs: []string{"use"}, + }, + }, + } +} + +func (r *CorootReconciler) openshiftSCCRoleBinding(cr *corootv1.Coroot, component, scc string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name + "-" + component + "-" + scc, + Namespace: cr.Namespace, + Labels: Labels(cr, component), + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: cr.Name + "-" + component, + Namespace: cr.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: cr.Name + "-" + scc, + }, + } +} diff --git a/controller/prometheus.go b/controller/prometheus.go index 0029013..02c8e79 100644 --- a/controller/prometheus.go +++ b/controller/prometheus.go @@ -88,8 +88,9 @@ func (r *CorootReconciler) prometheusDeployment(cr *corootv1.Coroot) *appsv1.Dep Labels: ls, }, Spec: corev1.PodSpec{ - SecurityContext: nonRootSecurityContext, - Affinity: cr.Spec.Prometheus.Affinity, + ServiceAccountName: cr.Name + "-prometheus", + SecurityContext: nonRootSecurityContext, + Affinity: cr.Spec.Prometheus.Affinity, Containers: []corev1.Container{ { Image: PrometheusImage,