append new scrape config as text

Signed-off-by: Thibault Mange <[email protected]>

operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go

@@ -250,7 +250,7 @@ func (r *ObservabilityAddonReconciler) Reconcile(ctx context.Context, req ctrl.R
 			}
 
 			if len(microshiftVersion) > 0 {
-				mcs := microshift.NewMicroshift(r.Client, r.Namespace)
+				mcs := microshift.NewMicroshift(r.Client, r.Namespace, log)
 				toDeploy, err = mcs.Render(ctx, toDeploy)
 				if err != nil {
 					return ctrl.Result{}, fmt.Errorf("failed to render microshift templates: %w", err)

operators/endpointmetrics/pkg/microshift/microshift.go

@@ -6,9 +6,11 @@ package microshift
 
 import (
 	"context"
+	"strings"
 
 	"fmt"
 
+	"github.com/go-logr/logr"
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promcommon "github.com/prometheus/common/config"
 	batchv1 "k8s.io/api/batch/v1"
@@ -26,19 +28,21 @@ import (
 
 const (
 	etcdClientCertSecretName  = "etcd-client-cert" //nolint:gosec
-	prometheusScrapeCfgSecret = "prometheus-scrape-config"
-	scrapeConfigKey           = "scrape-config.yaml"
+	prometheusScrapeCfgSecret = "prometheus-scrape-targets"
+	scrapeConfigKey           = "scrape-targets.yaml"
 )
 
 type Microshift struct {
 	client         client.Client
 	addonNamespace string
+	logger         logr.Logger
 }
 
-func NewMicroshift(c client.Client, addonNs string) *Microshift {
+func NewMicroshift(c client.Client, addonNs string, logger logr.Logger) *Microshift {
 	return &Microshift{
 		addonNamespace: addonNs,
 		client:         c,
+		logger:         logger.WithName("microshift"),
 	}
 }
 
@@ -62,6 +66,10 @@ func (m *Microshift) Render(ctx context.Context, resources []*unstructured.Unstr
 		return nil, fmt.Errorf("failed to render prometheus: %w", err)
 	}
 
+	if err := m.renderScrapeConfig(resources); err != nil {
+		return nil, fmt.Errorf("failed to render scrape config: %w", err)
+	}
+
 	return resources, nil
 }
 
@@ -80,12 +88,10 @@ func (m *Microshift) renderPrometheus(res []*unstructured.Unstructured) error {
 
 	prom.Spec.Secrets = append(prom.Spec.Secrets, etcdClientCertSecretName)
 	prom.Spec.HostNetwork = true
-	// add scrape config for etcd that is running on the host
-	prom.Spec.AdditionalScrapeConfigs = &corev1.SecretKeySelector{
-		LocalObjectReference: corev1.LocalObjectReference{
-			Name: prometheusScrapeCfgSecret,
-		},
-		Key: scrapeConfigKey,
+
+	// check that additional scrape config is as expected
+	if prom.Spec.AdditionalScrapeConfigs == nil || prom.Spec.AdditionalScrapeConfigs.LocalObjectReference.Name != prometheusScrapeCfgSecret {
+		return fmt.Errorf(fmt.Sprintf("additional scrape config is not as expected, want %s, got %s", prometheusScrapeCfgSecret, prom.Spec.AdditionalScrapeConfigs.LocalObjectReference.Name))
 	}
 
 	promRes.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(prom)
@@ -97,6 +103,62 @@ func (m *Microshift) renderPrometheus(res []*unstructured.Unstructured) error {
 
 }
 
+func (m *Microshift) renderScrapeConfig(res []*unstructured.Unstructured) error {
+	secret, err := getResource(res, "Secret", prometheusScrapeCfgSecret)
+	if err != nil {
+		return fmt.Errorf("failed to get prometheus scrape secret resource: %w", err)
+	}
+
+	scrapeSecret := &corev1.Secret{}
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(secret.Object, scrapeSecret); err != nil {
+		return fmt.Errorf("failed to convert unstructured object to secret object: %w", err)
+	}
+
+	etcdScrapeCfg := ScrapeConfig{}
+	etcdScrapeCfg.JobName = "etcd"
+	etcdScrapeCfg.Scheme = "https"
+	etcdScrapeCfg.HTTPClientConfig = promcommon.HTTPClientConfig{
+		TLSConfig: promcommon.TLSConfig{
+			CertFile: fmt.Sprintf("/etc/prometheus/secrets/%s/ca.crt", etcdClientCertSecretName),
+			KeyFile:  fmt.Sprintf("/etc/prometheus/secrets/%s/ca.key", etcdClientCertSecretName),
+			CAFile:   fmt.Sprintf("/etc/prometheus/secrets/%s/ca.crt", etcdClientCertSecretName),
+		},
+	}
+	etcdScrapeCfg.StaticConfigs = []StaticConfig{
+		{
+			Targets: []string{"localhost:2381"},
+		},
+	}
+	newScrapeCfgs := &ScrapeConfigs{
+		ScrapeConfigs: []ScrapeConfig{etcdScrapeCfg},
+	}
+
+	// Append additional scrape config for etcd on the host
+	// We don't unmarshal the existing scrape config to avoid adding default values
+	// when marshalling the new scrape config. Instead, we append the new scrape config
+	// to the existing scrape config.
+	var ret strings.Builder
+	ret.WriteString(strings.TrimSpace(scrapeSecret.StringData[scrapeConfigKey]))
+
+	scrapeCfgsYaml, err := newScrapeCfgs.MarshalYAML()
+	if err != nil {
+		return fmt.Errorf("failed to marshal scrape config: %w", err)
+	}
+
+	ret.WriteString("\n")
+	ret.WriteString(strings.TrimSpace(string(scrapeCfgsYaml)))
+	newScrapeConfigYaml := ret.String()
+
+	scrapeSecret.StringData[scrapeConfigKey] = newScrapeConfigYaml
+
+	secret.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(scrapeSecret)
+	if err != nil {
+		return fmt.Errorf("failed to convert secret object to unstructured object: %w", err)
+	}
+
+	return nil
+}
+
 // renderCronJobExposingMicroshiftSecrets creates a cronjob to expose Microshift's host secrets needed in Microshift itself.
 // For example, Microshift clusters run etcd directly on the host. It exposes its metrics via a secured port.
 // The job ensures that etcd client key and certificate are exposed as a secret in the addon namespace.
@@ -462,6 +524,7 @@ func IsMicroshiftCluster(ctx context.Context, client client.Client) (string, err
 
 func getResource(res []*unstructured.Unstructured, kind, name string) (*unstructured.Unstructured, error) {
 	for _, r := range res {
+		fmt.Println("Resource: ", r.GetKind(), r.GetName())
 		if r.GetKind() == kind && r.GetName() == name {
 			return r, nil
 		}

operators/endpointmetrics/pkg/microshift/microshift_test.go

@@ -0,0 +1,67 @@
+package microshift
+
+import (
+	"os"
+	"testing"
+
+	"github.com/go-logr/logr"
+	promcfg "github.com/prometheus/prometheus/config"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/yaml"
+)
+
+func TestScrapeConfigUpdate(t *testing.T) {
+	client := fake.NewClientBuilder().Build()
+	secretFile, err := os.ReadFile("../../manifests/prometheus/prometheus-scrape-targets-secret.yaml")
+	if err != nil {
+		t.Fatalf("Failed to open file: %v", err)
+	}
+
+	secret := &corev1.Secret{}
+	if err := yaml.Unmarshal(secretFile, secret); err != nil {
+		t.Fatalf("Failed to unmarshal secret: %v", err)
+	}
+
+	// Transform secret to unstructured
+	secretUnstructured, err := convertToUnstructured(secret)
+	if err != nil {
+		t.Fatalf("Failed to convert secret to unstructured: %v", err)
+	}
+
+	mc := NewMicroshift(client, "ns", logr.Logger{})
+	resources := []*unstructured.Unstructured{secretUnstructured}
+	if err := mc.renderScrapeConfig(resources); err != nil {
+		t.Fatalf("Failed to render scrape config: %v", err)
+	}
+
+	newScrapeSecret := &corev1.Secret{}
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(secretUnstructured.Object, newScrapeSecret); err != nil {
+		t.Fatalf("Failed to convert unstructured to secret: %v", err)
+	}
+
+	sc := ScrapeConfigs{}
+	if err := sc.UnmarshalYAML([]byte(newScrapeSecret.StringData[scrapeConfigKey])); err != nil {
+		t.Fatalf("Failed to unmarshal scrape config: %v", err)
+	}
+
+	// Check if the scrape config has been updated
+	assert.Greater(t, len(sc.ScrapeConfigs), 1, "Scrape config not updated")
+	found := false
+	for _, scrapeConfig := range sc.ScrapeConfigs {
+		if scrapeConfig.JobName == "etcd" {
+			found = true
+			break
+		}
+	}
+	assert.True(t, found, "Scrape config not updated")
+
+	// validate configs
+	for _, scrapeConfig := range sc.ScrapeConfigs {
+		assert.NoError(t, scrapeConfig.Validate(promcfg.DefaultGlobalConfig))
+		assert.NoError(t, scrapeConfig.HTTPClientConfig.Validate())
+	}
+}

operators/endpointmetrics/pkg/microshift/scrape.go

@@ -6,7 +6,7 @@ import (
 )
 
 type ScrapeConfigs struct {
-	ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs"`
+	ScrapeConfigs []ScrapeConfig `yaml:",inline"`
 }
 
 type ScrapeConfig struct {
@@ -18,10 +18,17 @@ type StaticConfig struct {
 	Targets []string `yaml:"targets"`
 }
 
-func (sc ScrapeConfigs) MarshalYAML() ([]byte, error) {
-	ret, err := yaml.Marshal(sc)
+func (sc *ScrapeConfigs) MarshalYAML() ([]byte, error) {
+	ret, err := yaml.Marshal(sc.ScrapeConfigs)
 	if err != nil {
 		return nil, err
 	}
 	return ret, nil
 }
+
+func (sc *ScrapeConfigs) UnmarshalYAML(data []byte) error {
+	if sc.ScrapeConfigs == nil {
+		sc.ScrapeConfigs = []ScrapeConfig{}
+	}
+	return yaml.Unmarshal(data, &sc.ScrapeConfigs)
+}