diff --git a/.changelog/3318.fixed.txt b/.changelog/3318.fixed.txt new file mode 100644 index 0000000000..b847aebbd5 --- /dev/null +++ b/.changelog/3318.fixed.txt @@ -0,0 +1 @@ +fix(metrics): drop stale datapoints \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml index 3c2eb3738a..15ccd89cc4 100644 --- a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml @@ -27,6 +27,12 @@ processors: send_batch_size: 1000 timeout: 1s + # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good + filter/drop_stale_datapoints: + metrics: + datapoint: + - 'flags == FLAG_NO_RECORDED_VALUE' + transform/drop_unnecessary_attributes: error_mode: ignore metric_statements: @@ -217,6 +223,7 @@ service: exporters: [otlphttp] processors: - batch + - filter/drop_stale_datapoints {{- if .Values.sumologic.metrics.dropHistogramBuckets }} - transform/extract_sum_count_from_histograms {{- end }} diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml index bcebf2c2ad..9b4bcc0dec 100644 --- a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml @@ -91,6 +91,12 @@ spec: send_batch_size: 1000 timeout: 1s + # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good + filter/drop_stale_datapoints: + metrics: + datapoint: + - 'flags == FLAG_NO_RECORDED_VALUE' + transform/drop_unnecessary_attributes: error_mode: ignore metric_statements: @@ -263,6 +269,7 @@ spec: exporters: [otlphttp] processors: - batch + - filter/drop_stale_datapoints - transform/extract_sum_count_from_histograms - filter/drop_unnecessary_metrics - transform/drop_unnecessary_attributes diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml index 8e060ba16d..ab625721a9 100644 --- a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml +++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml @@ -110,6 +110,12 @@ spec: send_batch_size: 1000 timeout: 1s + # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good + filter/drop_stale_datapoints: + metrics: + datapoint: + - 'flags == FLAG_NO_RECORDED_VALUE' + transform/drop_unnecessary_attributes: error_mode: ignore metric_statements: @@ -1071,6 +1077,7 @@ spec: exporters: [otlphttp] processors: - batch + - filter/drop_stale_datapoints - filter/drop_unnecessary_metrics - transform/drop_unnecessary_attributes - filter/app_metrics