diff --git a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go index c589dda0a5..14388fa486 100644 --- a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go +++ b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller.go @@ -250,7 +250,7 @@ func (r *ObservabilityAddonReconciler) Reconcile(ctx context.Context, req ctrl.R } if len(microshiftVersion) > 0 { - mcs := microshift.NewMicroshift(r.Client, r.Namespace) + mcs := microshift.NewMicroshift(r.Client, r.Namespace, log) toDeploy, err = mcs.Render(ctx, toDeploy) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to render microshift templates: %w", err) diff --git a/operators/endpointmetrics/pkg/microshift/microshift.go b/operators/endpointmetrics/pkg/microshift/microshift.go index f35be9e4b5..fc658dce40 100644 --- a/operators/endpointmetrics/pkg/microshift/microshift.go +++ b/operators/endpointmetrics/pkg/microshift/microshift.go @@ -6,9 +6,11 @@ package microshift import ( "context" + "strings" "fmt" + "github.com/go-logr/logr" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promcommon "github.com/prometheus/common/config" batchv1 "k8s.io/api/batch/v1" @@ -26,19 +28,21 @@ import ( const ( etcdClientCertSecretName = "etcd-client-cert" //nolint:gosec - prometheusScrapeCfgSecret = "prometheus-scrape-config" - scrapeConfigKey = "scrape-config.yaml" + prometheusScrapeCfgSecret = "prometheus-scrape-targets" + scrapeConfigKey = "scrape-targets.yaml" ) type Microshift struct { client client.Client addonNamespace string + logger logr.Logger } -func NewMicroshift(c client.Client, addonNs string) *Microshift { +func NewMicroshift(c client.Client, addonNs string, logger logr.Logger) *Microshift { return &Microshift{ addonNamespace: addonNs, client: c, + logger: logger.WithName("microshift"), } } @@ -62,6 +66,10 @@ func (m *Microshift) Render(ctx context.Context, resources []*unstructured.Unstr return nil, fmt.Errorf("failed to render prometheus: %w", err) } + if err := m.renderScrapeConfig(resources); err != nil { + return nil, fmt.Errorf("failed to render scrape config: %w", err) + } + return resources, nil } @@ -80,12 +88,10 @@ func (m *Microshift) renderPrometheus(res []*unstructured.Unstructured) error { prom.Spec.Secrets = append(prom.Spec.Secrets, etcdClientCertSecretName) prom.Spec.HostNetwork = true - // add scrape config for etcd that is running on the host - prom.Spec.AdditionalScrapeConfigs = &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: prometheusScrapeCfgSecret, - }, - Key: scrapeConfigKey, + + // check that additional scrape config is as expected + if prom.Spec.AdditionalScrapeConfigs == nil || prom.Spec.AdditionalScrapeConfigs.LocalObjectReference.Name != prometheusScrapeCfgSecret { + return fmt.Errorf(fmt.Sprintf("additional scrape config is not as expected, want %s, got %s", prometheusScrapeCfgSecret, prom.Spec.AdditionalScrapeConfigs.LocalObjectReference.Name)) } promRes.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(prom) @@ -97,6 +103,62 @@ func (m *Microshift) renderPrometheus(res []*unstructured.Unstructured) error { } +func (m *Microshift) renderScrapeConfig(res []*unstructured.Unstructured) error { + secret, err := getResource(res, "Secret", prometheusScrapeCfgSecret) + if err != nil { + return fmt.Errorf("failed to get prometheus scrape secret resource: %w", err) + } + + scrapeSecret := &corev1.Secret{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(secret.Object, scrapeSecret); err != nil { + return fmt.Errorf("failed to convert unstructured object to secret object: %w", err) + } + + etcdScrapeCfg := ScrapeConfig{} + etcdScrapeCfg.JobName = "etcd" + etcdScrapeCfg.Scheme = "https" + etcdScrapeCfg.HTTPClientConfig = promcommon.HTTPClientConfig{ + TLSConfig: promcommon.TLSConfig{ + CertFile: fmt.Sprintf("/etc/prometheus/secrets/%s/ca.crt", etcdClientCertSecretName), + KeyFile: fmt.Sprintf("/etc/prometheus/secrets/%s/ca.key", etcdClientCertSecretName), + CAFile: fmt.Sprintf("/etc/prometheus/secrets/%s/ca.crt", etcdClientCertSecretName), + }, + } + etcdScrapeCfg.StaticConfigs = []StaticConfig{ + { + Targets: []string{"localhost:2381"}, + }, + } + newScrapeCfgs := &ScrapeConfigs{ + ScrapeConfigs: []ScrapeConfig{etcdScrapeCfg}, + } + + // Append additional scrape config for etcd on the host + // We don't unmarshal the existing scrape config to avoid adding default values + // when marshalling the new scrape config. Instead, we append the new scrape config + // to the existing scrape config. + var ret strings.Builder + ret.WriteString(strings.TrimSpace(scrapeSecret.StringData[scrapeConfigKey])) + + scrapeCfgsYaml, err := newScrapeCfgs.MarshalYAML() + if err != nil { + return fmt.Errorf("failed to marshal scrape config: %w", err) + } + + ret.WriteString("\n") + ret.WriteString(strings.TrimSpace(string(scrapeCfgsYaml))) + newScrapeConfigYaml := ret.String() + + scrapeSecret.StringData[scrapeConfigKey] = newScrapeConfigYaml + + secret.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(scrapeSecret) + if err != nil { + return fmt.Errorf("failed to convert secret object to unstructured object: %w", err) + } + + return nil +} + // renderCronJobExposingMicroshiftSecrets creates a cronjob to expose Microshift's host secrets needed in Microshift itself. // For example, Microshift clusters run etcd directly on the host. It exposes its metrics via a secured port. // The job ensures that etcd client key and certificate are exposed as a secret in the addon namespace. @@ -462,6 +524,7 @@ func IsMicroshiftCluster(ctx context.Context, client client.Client) (string, err func getResource(res []*unstructured.Unstructured, kind, name string) (*unstructured.Unstructured, error) { for _, r := range res { + fmt.Println("Resource: ", r.GetKind(), r.GetName()) if r.GetKind() == kind && r.GetName() == name { return r, nil } diff --git a/operators/endpointmetrics/pkg/microshift/microshift_test.go b/operators/endpointmetrics/pkg/microshift/microshift_test.go new file mode 100644 index 0000000000..3c351b0d0f --- /dev/null +++ b/operators/endpointmetrics/pkg/microshift/microshift_test.go @@ -0,0 +1,67 @@ +package microshift + +import ( + "os" + "testing" + + "github.com/go-logr/logr" + promcfg "github.com/prometheus/prometheus/config" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" +) + +func TestScrapeConfigUpdate(t *testing.T) { + client := fake.NewClientBuilder().Build() + secretFile, err := os.ReadFile("../../manifests/prometheus/prometheus-scrape-targets-secret.yaml") + if err != nil { + t.Fatalf("Failed to open file: %v", err) + } + + secret := &corev1.Secret{} + if err := yaml.Unmarshal(secretFile, secret); err != nil { + t.Fatalf("Failed to unmarshal secret: %v", err) + } + + // Transform secret to unstructured + secretUnstructured, err := convertToUnstructured(secret) + if err != nil { + t.Fatalf("Failed to convert secret to unstructured: %v", err) + } + + mc := NewMicroshift(client, "ns", logr.Logger{}) + resources := []*unstructured.Unstructured{secretUnstructured} + if err := mc.renderScrapeConfig(resources); err != nil { + t.Fatalf("Failed to render scrape config: %v", err) + } + + newScrapeSecret := &corev1.Secret{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(secretUnstructured.Object, newScrapeSecret); err != nil { + t.Fatalf("Failed to convert unstructured to secret: %v", err) + } + + sc := ScrapeConfigs{} + if err := sc.UnmarshalYAML([]byte(newScrapeSecret.StringData[scrapeConfigKey])); err != nil { + t.Fatalf("Failed to unmarshal scrape config: %v", err) + } + + // Check if the scrape config has been updated + assert.Greater(t, len(sc.ScrapeConfigs), 1, "Scrape config not updated") + found := false + for _, scrapeConfig := range sc.ScrapeConfigs { + if scrapeConfig.JobName == "etcd" { + found = true + break + } + } + assert.True(t, found, "Scrape config not updated") + + // validate configs + for _, scrapeConfig := range sc.ScrapeConfigs { + assert.NoError(t, scrapeConfig.Validate(promcfg.DefaultGlobalConfig)) + assert.NoError(t, scrapeConfig.HTTPClientConfig.Validate()) + } +} diff --git a/operators/endpointmetrics/pkg/microshift/scrape.go b/operators/endpointmetrics/pkg/microshift/scrape.go index 5419bf33f3..c93b2d43a5 100644 --- a/operators/endpointmetrics/pkg/microshift/scrape.go +++ b/operators/endpointmetrics/pkg/microshift/scrape.go @@ -6,7 +6,7 @@ import ( ) type ScrapeConfigs struct { - ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs"` + ScrapeConfigs []ScrapeConfig `yaml:",inline"` } type ScrapeConfig struct { @@ -18,10 +18,17 @@ type StaticConfig struct { Targets []string `yaml:"targets"` } -func (sc ScrapeConfigs) MarshalYAML() ([]byte, error) { - ret, err := yaml.Marshal(sc) +func (sc *ScrapeConfigs) MarshalYAML() ([]byte, error) { + ret, err := yaml.Marshal(sc.ScrapeConfigs) if err != nil { return nil, err } return ret, nil } + +func (sc *ScrapeConfigs) UnmarshalYAML(data []byte) error { + if sc.ScrapeConfigs == nil { + sc.ScrapeConfigs = []ScrapeConfig{} + } + return yaml.Unmarshal(data, &sc.ScrapeConfigs) +}