diff --git a/.changelog/3318.fixed.txt b/.changelog/3318.fixed.txt new file mode 100644 index 0000000000..b847aebbd5 --- /dev/null +++ b/.changelog/3318.fixed.txt @@ -0,0 +1 @@ +fix(metrics): drop stale datapoints \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml index eadb184e3c..fef54fb6ab 100644 --- a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml @@ -27,6 +27,12 @@ processors: send_batch_size: 1000 timeout: 1s + # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good + filter/drop_stale_datapoints: + metrics: + datapoint: + - 'flags == FLAG_NO_RECORDED_VALUE' + transform/drop_unnecessary_attributes: error_mode: ignore metric_statements: @@ -203,6 +209,7 @@ service: exporters: [otlphttp] processors: - batch + - filter/drop_stale_datapoints {{- if .Values.sumologic.metrics.dropHistogramBuckets }} - transform/extract_sum_count_from_histograms {{- end }} diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml index 70ef273500..83d6c686f0 100644 --- a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml @@ -86,6 +86,10 @@ spec: send_batch_max_size: 2000 send_batch_size: 1000 timeout: 1s + filter/drop_stale_datapoints: + metrics: + datapoint: + - flags == FLAG_NO_RECORDED_VALUE transform/drop_unnecessary_attributes: error_mode: ignore metric_statements: @@ -242,6 +246,7 @@ spec: - otlphttp processors: - batch + - filter/drop_stale_datapoints - transform/extract_sum_count_from_histograms - transform/drop_unnecessary_attributes receivers: diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml index 9d74926572..baeff887f7 100644 --- a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml +++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml @@ -105,6 +105,10 @@ spec: send_batch_max_size: 2000 send_batch_size: 5000 timeout: 1s + filter/drop_stale_datapoints: + metrics: + datapoint: + - flags == FLAG_NO_RECORDED_VALUE transform/drop_unnecessary_attributes: error_mode: ignore metric_statements: @@ -136,6 +140,7 @@ spec: - otlphttp processors: - batch + - filter/drop_stale_datapoints - transform/drop_unnecessary_attributes receivers: - prometheus