fix: Optimize Prometheus scrape target discovery and prepare images f…

…or 1.29.1 release (#1669)
kyma-project · Dec 11, 2024 · 838c4e3 · 838c4e3
1 parent 375f649
commit 838c4e3
Show file tree

Hide file tree

Showing 9 changed files with 50 additions and 11 deletions.
diff --git a/.env b/.env
@@ -1,7 +1,7 @@
 ### Default Environment Variables
 ## General
 ENV_K3S_K8S_VERSION=1.30.5 # refers to the version of kubernetes used in K3s
-ENV_IMG=europe-docker.pkg.dev/kyma-project/prod/telemetry-manager:1.29.0 # Image URL to use all building/pushing image targets
+ENV_IMG=europe-docker.pkg.dev/kyma-project/prod/telemetry-manager:1.29.1 # Image URL to use all building/pushing image targets
 
 ## Gardener
 ENV_GARDENER_K8S_VERSION=1.30

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -8,4 +8,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: europe-docker.pkg.dev/kyma-project/prod/telemetry-manager
-  newTag: 1.29.0
+  newTag: 1.29.1
diff --git a/hack/load-tests/metric-agent-test-setup.yaml b/hack/load-tests/metric-agent-test-setup.yaml
@@ -50,7 +50,7 @@ spec:
             - podAffinityTerm:
                 labelSelector:
                   matchExpressions:
-                    - key: app
+                    - key: app.kubernetes.io/name
                       operator: In
                       values:
                         - metric-agent-load-generator

diff --git a/internal/otelcollector/config/metric/agent/config.go b/internal/otelcollector/config/metric/agent/config.go
@@ -187,11 +187,17 @@ type StaticDiscoveryConfig struct {
 }
 
 type KubernetesDiscoveryConfig struct {
-	Role Role `yaml:"role"`
+	Role      Role                   `yaml:"role"`
+	Selectors []K8SDiscoverySelector `yaml:"selectors,omitempty"`
 }
 
 type Role string
 
+type K8SDiscoverySelector struct {
+	Role  Role   `yaml:"role"`
+	Field string `yaml:"field"`
+}
+
 const (
 	RoleEndpoints Role = "endpoints"
 	RolePod       Role = "pod"

diff --git a/internal/otelcollector/config/metric/agent/prometheus_receiver.go b/internal/otelcollector/config/metric/agent/prometheus_receiver.go
@@ -18,8 +18,9 @@ const (
 type AnnotatedResource string
 
 const (
-	AnnotatedPod     AnnotatedResource = "pod"
-	AnnotatedService AnnotatedResource = "service"
+	AnnotatedPod                   AnnotatedResource = "pod"
+	AnnotatedService               AnnotatedResource = "service"
+	PodNodeSelectorFieldExpression string            = "spec.nodeName=${MY_NODE_NAME}"
 )
 
 const (
@@ -37,7 +38,7 @@ func makePrometheusConfigForPods() *PrometheusReceiver {
 	scrapeConfig := ScrapeConfig{
 		ScrapeInterval:             scrapeInterval,
 		SampleLimit:                sampleLimit,
-		KubernetesDiscoveryConfigs: []KubernetesDiscoveryConfig{{Role: RolePod}},
+		KubernetesDiscoveryConfigs: makeDiscoveryConfigWithNodeSelector(RolePod),
 		JobName:                    appPodsJobName,
 		RelabelConfigs:             makePrometheusPodsRelabelConfigs(),
 	}
@@ -57,7 +58,7 @@ func makePrometheusConfigForServices(opts BuildOptions) *PrometheusReceiver {
 	baseScrapeConfig := ScrapeConfig{
 		ScrapeInterval:             scrapeInterval,
 		SampleLimit:                sampleLimit,
-		KubernetesDiscoveryConfigs: []KubernetesDiscoveryConfig{{Role: RoleEndpoints}},
+		KubernetesDiscoveryConfigs: makeDiscoveryConfigWithNodeSelector(RoleEndpoints),
 	}
 
 	httpScrapeConfig := baseScrapeConfig
@@ -146,7 +147,7 @@ func makePrometheusIstioConfig() *PrometheusReceiver {
 					SampleLimit:                sampleLimit,
 					MetricsPath:                "/stats/prometheus",
 					ScrapeInterval:             scrapeInterval,
-					KubernetesDiscoveryConfigs: []KubernetesDiscoveryConfig{{Role: RolePod}},
+					KubernetesDiscoveryConfigs: makeDiscoveryConfigWithNodeSelector(RolePod),
 					RelabelConfigs: []RelabelConfig{
 						keepIfRunningOnSameNode(NodeAffiliatedPod),
 						keepIfIstioProxy(),
@@ -291,3 +292,17 @@ func dropIfSchemeHTTPS() RelabelConfig {
 		Regex:        "(https)",
 	}
 }
+
+func makeDiscoveryConfigWithNodeSelector(role Role) []KubernetesDiscoveryConfig {
+	return []KubernetesDiscoveryConfig{
+		{
+			Role: role,
+			Selectors: []K8SDiscoverySelector{
+				{
+					Role:  RolePod,
+					Field: PodNodeSelectorFieldExpression,
+				},
+			},
+		},
+	}
+}
diff --git a/internal/otelcollector/config/metric/agent/testdata/config_istio_enabled.yaml b/internal/otelcollector/config/metric/agent/testdata/config_istio_enabled.yaml
@@ -175,6 +175,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: pod
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
     prometheus/app-services:
         config:
             scrape_configs:
@@ -223,6 +226,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: endpoints
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
                 - job_name: app-services-secure
                   sample_limit: 50000
                   scrape_interval: 30s
@@ -268,6 +274,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: endpoints
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
                   tls_config:
                     ca_file: /etc/istio-output-certs/root-cert.pem
                     cert_file: /etc/istio-output-certs/cert-chain.pem
@@ -299,6 +308,9 @@ receivers:
                       action: keep
                   kubernetes_sd_configs:
                     - role: pod
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
 processors:
     batch:
         send_batch_size: 1024

diff --git a/internal/otelcollector/config/metric/agent/testdata/config_istio_not_enabled.yaml b/internal/otelcollector/config/metric/agent/testdata/config_istio_not_enabled.yaml
@@ -164,6 +164,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: pod
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
     prometheus/app-services:
         config:
             scrape_configs:
@@ -212,6 +215,9 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: endpoints
+                      selectors:
+                        - role: pod
+                          field: spec.nodeName=${MY_NODE_NAME}
 processors:
     batch:
         send_batch_size: 1024

diff --git a/main.go b/main.go
@@ -74,7 +74,7 @@ var (
 	setupLog           = ctrl.Log.WithName("setup")
 	telemetryNamespace string
 	// TODO: replace with build version based on git revision
-	version = "1.29.0"
+	version = "1.29.1"
 
 	// Operator flags
 	certDir                   string

diff --git a/sec-scanners-config.yaml b/sec-scanners-config.yaml
@@ -1,6 +1,6 @@
 module-name: telemetry
 protecode:
-  - europe-docker.pkg.dev/kyma-project/prod/telemetry-manager:1.29.0
+  - europe-docker.pkg.dev/kyma-project/prod/telemetry-manager:1.29.1
   - europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.114.0-1.29.0
   - europe-docker.pkg.dev/kyma-project/prod/external/fluent/fluent-bit:3.2.2
   - europe-docker.pkg.dev/kyma-project/prod/directory-size-exporter:v20241024-8bc3f6a8