diff --git a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go index e242cf5387..159eec4c84 100644 --- a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go +++ b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/hypershift" + "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/microshift" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/openshift" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/rendering" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/util" @@ -247,6 +248,21 @@ func (r *ObservabilityAddonReconciler) Reconcile(ctx context.Context, req ctrl.R if err != nil { return ctrl.Result{}, fmt.Errorf("failed to render prometheus templates: %w", err) } + + if !isHubMetricsCollector { + microshiftVersion, err := microshift.IsMicroshiftCluster(ctx, r.Client) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to check if the cluster is microshift: %w", err) + } + + if len(microshiftVersion) > 0 { + mcs := microshift.NewMicroshift(namespace) + if err := mcs.Render(ctx, toDeploy); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to render microshift templates: %w", err) + } + } + } + deployer := deploying.NewDeployer(r.Client) for _, res := range toDeploy { if res.GetNamespace() != namespace { diff --git a/operators/endpointmetrics/pkg/microshift/microshift.go b/operators/endpointmetrics/pkg/microshift/microshift.go new file mode 100644 index 0000000000..ccc2f02d85 --- /dev/null +++ b/operators/endpointmetrics/pkg/microshift/microshift.go @@ -0,0 +1,417 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package microshift + +import ( + "context" + "fmt" + "os" + + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + etcdClientCertSecretName = "etcd-client-cert" //nolint:gosec +) + +type Microshift struct { + // client client.Client + addonNamespace string +} + +func NewMicroshift(addonNs string) *Microshift { + return &Microshift{ + // client: client, + addonNamespace: addonNs, + } +} + +// Render renders the resources for the microshift cluster +// If the cluster is not a microshift cluster, it modifies the resources +// to adapt to the microshift cluster +func (m *Microshift) Render(ctx context.Context, resources []*unstructured.Unstructured) error { + jobRes, err := m.renderCronJobExposingMicroshiftSecrets() + if err != nil { + return fmt.Errorf("failed to render cronjob for secrets: %w", err) + } + resources = append(resources, jobRes...) + + etcdRes, err := m.renderEtcdResources() + if err != nil { + return fmt.Errorf("failed to render etcd resources: %w", err) + } + resources = append(resources, etcdRes...) + + if err := m.renderPrometheus(resources); err != nil { + return fmt.Errorf("failed to render prometheus: %w", err) + } + + return nil +} + +// renderPrometheus modifies the prometheus resource to adapt to the microshift cluster +// It adds the etcd client key and certificate secret to the prometheus pod +func (m *Microshift) renderPrometheus(res []*unstructured.Unstructured) error { + promRes, err := getResource(res, "Prometheus", "prometheus") + if err != nil { + return fmt.Errorf("failed to get prometheus resource: %w", err) + } + + prom := &promv1.Prometheus{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(promRes.Object, prom); err != nil { + return fmt.Errorf("failed to convert unstructured object to prometheus object: %w", err) + } + + prom.Spec.Secrets = append(prom.Spec.Secrets, etcdClientCertSecretName) + + return nil + +} + +// renderCronJobExposingMicroshiftSecrets creates a cronjob to expose Microshift's host secrets needed in Microshift itself. +// For example, Microshift clusters run etcd directly on the host. It exposes its metrics via a secured port. +// The job ensures that etcd client key and certificate are exposed as a secret in the addon namespace. +func (m *Microshift) renderCronJobExposingMicroshiftSecrets() ([]*unstructured.Unstructured, error) { + ret := []*unstructured.Unstructured{} + jobName := "microshift-secrets-exposer" + + // Create a cronjob to update the etcd client key and certificate secret + // every hour + cronJob := &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: m.addonNamespace, + }, + Spec: batchv1.CronJobSpec{ + Schedule: "0 * * * *", + JobTemplate: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "microshift-certs-updater", + Image: "registry.access.redhat.com/ubi9/ubi-minimal@sha256:ef6fb6b3b38ef6c85daebeabebc7ff3151b9dd1500056e6abc9c3295e4b78a51", + Command: []string{"/bin/sh", "-c"}, + Args: []string{fmt.Sprintf( + ` + kubectl create secret generic %s --from-file=key=/tmp/etcd-certs/ca.key --from-file=cert=/tmp/etcd-certs/ca.crt --dry-run=client -o yaml | kubectl apply -f - + `, etcdClientCertSecretName), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "etcd-certs", + MountPath: "/tmp/etcd-certs", + ReadOnly: true, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: new(int64), // 0 for root user + AllowPrivilegeEscalation: new(bool), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: jobName, + Volumes: []corev1.Volume{ + { + Name: "etcd-certs", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/microshift/certs/etcd-signer/", + Type: newHostPathType(corev1.HostPathDirectory), + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + unstructuredCronJob, err := convertToUnstructured(cronJob) + if err != nil { + return nil, fmt.Errorf("failed to convert cronjob to unstructured: %w", err) + } + ret = append(ret, unstructuredCronJob) + + // Add service account to the cronjob + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: m.addonNamespace, + }, + } + + unstructuredSA, err := convertToUnstructured(sa) + if err != nil { + return nil, fmt.Errorf("failed to convert service account to unstructured: %w", err) + } + ret = append(ret, unstructuredSA) + + // Add permissions to the service account to update the secret and run as root + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: m.addonNamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create", "update"}, + }, + }, + } + + unstructuredRole, err := convertToUnstructured(role) + if err != nil { + return nil, fmt.Errorf("failed to convert role to unstructured: %w", err) + } + ret = append(ret, unstructuredRole) + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: m.addonNamespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: jobName, + Namespace: m.addonNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: jobName, + APIGroup: "rbac.authorization.k8s.io", + }, + } + + unstructuredRoleBinding, err := convertToUnstructured(roleBinding) + if err != nil { + return nil, fmt.Errorf("failed to convert role binding to unstructured: %w", err) + } + ret = append(ret, unstructuredRoleBinding) + + // Add cluster role for hostmount and anyuid permissions- apiGroups: + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"hostmount-anyuid"}, + Verbs: []string{"use"}, + }, + }, + } + + unstructuredClusterRole, err := convertToUnstructured(clusterRole) + if err != nil { + return nil, fmt.Errorf("failed to convert cluster role to unstructured: %w", err) + } + ret = append(ret, unstructuredClusterRole) + + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: jobName, + Namespace: m.addonNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: jobName, + APIGroup: "rbac.authorization.k8s.io", + }, + } + + unstructuredClusterRoleBinding, err := convertToUnstructured(clusterRoleBinding) + if err != nil { + return nil, fmt.Errorf("failed to convert cluster role binding to unstructured: %w", err) + } + ret = append(ret, unstructuredClusterRoleBinding) + + return ret, nil +} + +// renderServiceMonitors modifies or creates the service monitors to adapt to the microshift cluster +func (m *Microshift) renderEtcdResources() ([]*unstructured.Unstructured, error) { + ret := []*unstructured.Unstructured{} + + hostIP := os.Getenv("HOST_IP") + if hostIP == "" { + return nil, fmt.Errorf("HOST_IP env var is not set") + } + + // Expose etcd endpoint in the addon namespace + endpoint := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd", + Namespace: m.addonNamespace, + Labels: map[string]string{ + "app": "etcd", + }, + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: hostIP, + }, + }, + Ports: []corev1.EndpointPort{ + { + Name: "metrics", + Port: 2381, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + } + + unstructuredEndpoint, err := convertToUnstructured(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to convert endpoint to unstructured: %w", err) + } + ret = append(ret, unstructuredEndpoint) + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd", + Namespace: m.addonNamespace, + Labels: map[string]string{ + "app": "etcd", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "metrics", + Port: 2381, + TargetPort: intstr.FromInt(2381), + }, + }, + Selector: map[string]string{ + "app": "etcd", + }, + }, + } + + unstructuredService, err := convertToUnstructured(service) + if err != nil { + return nil, fmt.Errorf("failed to convert service to unstructured: %w", err) + } + ret = append(ret, unstructuredService) + + // render service monitor for etcd + smon := &promv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd", + Namespace: m.addonNamespace, + }, + Spec: promv1.ServiceMonitorSpec{ + Endpoints: []promv1.Endpoint{ + { + Scheme: "https", + Path: "/metrics", + Interval: "15s", + // Use secret etcd-cert to scrape etcd metrics + TLSConfig: &promv1.TLSConfig{ + CertFile: "/etc/prometheus/secrets/etcd-cert/ca.crt", + KeyFile: "/etc/prometheus/secrets/etcd-cert/ca.key", + CAFile: "/etc/prometheus/secrets/etcd-cert/ca.crt", + }, + }, + }, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "etcd", + }, + }, + NamespaceSelector: promv1.NamespaceSelector{ + MatchNames: []string{m.addonNamespace}, + }, + }, + } + + unstructuredSMon, err := convertToUnstructured(smon) + if err != nil { + return nil, fmt.Errorf("failed to convert service monitor to unstructured: %w", err) + } + ret = append(ret, unstructuredSMon) + + return ret, nil +} + +// IsMicroshiftCluster checks if the cluster is a microshift cluster. +// It verifies the existence of the configmap microshift-version in namespace kube-public. +// If the configmap exists, it returns the version of the microshift cluster. +// If the configmap does not exist, it returns an empty string. +func IsMicroshiftCluster(ctx context.Context, client client.Client) (string, error) { + res := &corev1.ConfigMap{} + err := client.Get(ctx, types.NamespacedName{ + Name: "microshift-version", + Namespace: "kube-public", + }, res) + if err != nil { + if errors.IsNotFound(err) { + return "", nil + } + return "", err + } + + return res.Data["version"], nil +} + +func getResource(res []*unstructured.Unstructured, kind, name string) (*unstructured.Unstructured, error) { + for _, r := range res { + if r.GetKind() == kind && r.GetName() == name { + return r, nil + } + } + return nil, errors.NewNotFound(schema.GroupResource{ + Group: "", + Resource: kind, + }, name) +} + +func convertToUnstructured(obj interface{}) (*unstructured.Unstructured, error) { + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + + return &unstructured.Unstructured{Object: unstructuredObj}, nil +} + +func newHostPathType(pathType corev1.HostPathType) *corev1.HostPathType { + return &pathType +} diff --git a/operators/multiclusterobservability/controllers/placementrule/manifestwork.go b/operators/multiclusterobservability/controllers/placementrule/manifestwork.go index 0ca19672b5..6401c66a64 100644 --- a/operators/multiclusterobservability/controllers/placementrule/manifestwork.go +++ b/operators/multiclusterobservability/controllers/placementrule/manifestwork.go @@ -409,6 +409,16 @@ func createManifestWorks( Value: "true", }) + // Add host ip env for endpoint operator. It is needed for microshift to scrape host processes metrics + spec.Containers[0].Env = append(spec.Containers[0].Env, corev1.EnvVar{ + Name: "HOST_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.hostIP", + }, + }, + }) + dep.ObjectMeta.Name = config.HubEndpointOperatorName }