fix(metrics): drop stale datapoints

(cherry picked from commit 70af83f)
SumoLogic · Oct 14, 2023 · 277d6db · 277d6db
1 parent 04a6dd5
commit 277d6db
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 0 deletions.
diff --git a/.changelog/3318.fixed.txt b/.changelog/3318.fixed.txt
@@ -0,0 +1 @@
+fix(metrics): drop stale datapoints
diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
@@ -27,6 +27,12 @@ processors:
     send_batch_size: 1000
     timeout: 1s
 
+  # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+  filter/drop_stale_datapoints:
+    metrics:
+      datapoint:
+      - 'flags == FLAG_NO_RECORDED_VALUE'
+
   transform/drop_unnecessary_attributes:
     error_mode: ignore
     metric_statements:
@@ -217,6 +223,7 @@ service:
       exporters: [otlphttp]
       processors:
         - batch
+        - filter/drop_stale_datapoints
 {{- if .Values.sumologic.metrics.dropHistogramBuckets }}
         - transform/extract_sum_count_from_histograms
 {{- end }}

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
@@ -91,6 +91,12 @@ spec:
         send_batch_size: 1000
         timeout: 1s
 
+      # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - 'flags == FLAG_NO_RECORDED_VALUE'
+
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -263,6 +269,7 @@ spec:
           exporters: [otlphttp]
           processors:
             - batch
+            - filter/drop_stale_datapoints
             - transform/extract_sum_count_from_histograms
             - filter/drop_unnecessary_metrics
             - transform/drop_unnecessary_attributes

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
@@ -110,6 +110,12 @@ spec:
         send_batch_size: 1000
         timeout: 1s
 
+      # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - 'flags == FLAG_NO_RECORDED_VALUE'
+
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -1071,6 +1077,7 @@ spec:
           exporters: [otlphttp]
           processors:
             - batch
+            - filter/drop_stale_datapoints
             - filter/drop_unnecessary_metrics
             - transform/drop_unnecessary_attributes
             - filter/app_metrics