Merge pull request #97 from lightstep/jmacd/111

Update to v0.113.0; revise recommended configuration for new component versions

arrow/Dockerfile

@@ -1,12 +1,11 @@
-FROM golang:1.21 AS sandbox
+FROM golang:1.22 AS sandbox
 
 WORKDIR /otelarrowcol
 COPY . .
 ENV CGO_ENABLED=0
 
 # Future optimization - curl the release.
-RUN BUILDER_VERSION=$(grep 'otelcol_version' /otelarrowcol/otelcolarrow-build.yaml | awk '{print $2}') && \
-    go install go.opentelemetry.io/collector/cmd/builder@v${BUILDER_VERSION}
+RUN go install go.opentelemetry.io/collector/cmd/[email protected]
 
 # This command generates main.go, go.mod but does not update deps.
 RUN builder --config=/otelarrowcol/otelcolarrow-build.yaml

arrow/config/gateway-collector.yaml

@@ -23,7 +23,7 @@
 #       cpu: 32
 #       memory: 64Gi
 #
-# In the larger configuration tested, we used `max_in_flight_size_mib: 256`.
+# In the larger configuration tested, we set `admission::request_size_mib: 256`.
 # In the smaller configuration, relatively more memory is required due to
 # garbage collection costs.
 
@@ -47,6 +47,12 @@ receivers:
             max_connection_age: 30s
             max_connection_age_grace: 2m30s
 
+      # This receiver applies admission control to OTLP and OTel-Arrow data
+      # based on a count of uncompressed bytes.  For larger instance sizes,
+      # raise this parameter.
+      admission:
+        request_size_mib: 128
+
   # otlp is the core OTLP exporter, which we enable to receive
   # OTLP/HTTP data.
   otlp:
@@ -61,10 +67,6 @@ processors:
     send_batch_max_size: 1500
     send_batch_size: 1000
     timeout: 1s
-
-    # Use max_in_flight_size_mib=64 in a 2 cpu configuration;
-    # Use max_in_flight_size_mib=1024 in a 16 cpu configuration.
-    max_in_flight_size_mib: 64
 
 exporters:
   otelarrow:
@@ -109,7 +111,7 @@ exporters:
 
       # A stream lifetime limit is required to avoid spurious
       # disconnect error messages in the collector logs.
-      max_stream_lifetime: 4m
+      max_stream_lifetime: 30s
 
 service:
   pipelines:

arrow/otelcolarrow-build.yaml

@@ -37,42 +37,37 @@ dist:
   #   description: My Organization telemetry collector
   #
   name: otelarrowcol
-  version: 0.26.0
+  version: 0.30.0
   description: ServiceNow Cloud Observability OpenTelemetry Protocol with Apache Arrow gateway collector
 
-  # This indicates which version of the core collector components as
-  # well as the builder version.  At the time of this writing, we
-  # recommend using at least the current release.
-  otelcol_version: 0.107.0
-
 # We recommend building in both the OTLP exporter and the OTel-Arrow
 # exporter.
 exporters:
   # This is the core OpenTelemetry Protocol with Apache Arrow exporter,
   # recommended for exporting to ServiceNow Cloud Observability using
   # either OTel-Arrow or standard OTLP over gRPC.
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter v0.113.0
 
   # The following components may be useful for debugging.
 
   # As an alternate to the OTel-Arrow exporter, we recommend building
   # with the core OTLP exporter as a fallback.  This component and the
   # OTel-Arrow exporter use compatible configuration.
-  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.107.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.113.0
 
   # In case OTLP/HTTP export is required, as opposed to the two
   # gRPC options above.
-  - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.107.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.113.0
 
   # In case you want to record an OTLP telemetry session to a JSON
   # file, we recommend this additional utility.  See also the
   # obfuscation processor, listed below.  The OTel-Arrow provides
   # offline tools that can help explain poor compression performance
   # using inputs generated by this exporter, for example.
-  - gomod: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter v0.25.0
+  - gomod: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter v0.30.0
 
   # The debug exporter, useful for printing telemetry to the console.
-  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.107.0
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.113.0
 
 receivers:
   # The OTel-Arrow receiver supports multiple protocols including OTel-Arrow, OTLP gRPC,
@@ -82,69 +77,76 @@ receivers:
   # This enables other OTel Collectors or experimental SDKs that support OTel-Arrow to
   # send to this collector, such as this one for the OTel-Go Trace SDK:
   # https://github.com/lightstep/otel-launcher-go/tree/main/lightstep/sdk/trace/exporters/otlp/otelcol
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver v0.113.0
 
   # To support receiving OTLP/HTTP.
-  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.107.0
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.113.0
 
   # You may wish to enable other receivers from the Collector-Contrib repository or
   # elsewhere.  Here are some that might be useful:
   #
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.107.0
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.107.0
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.107.0
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.107.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.113.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.113.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.113.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.113.0
 
 processors:
   # We recommend use of the OTel-Arrow concurrent batch processor.
-  - gomod: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.25.0
+  - gomod: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.30.0
 
   # We recommend building with the follow processor for obfuscation, in case you want to
   # record telemetry sessions for offline analysis.
-  - gomod: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor v0.25.0
+  - gomod: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor v0.30.0
 
   # We emphatically DO NOT recommend use of the memory limiter
   # processor or the core batch processor, i.e., do not build with
   # either of:
   # - go.opentelemetry.io/collector/processor/memorylimiterprocessor.
-  # - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.107.0
+  # - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.113.0
 
   # These Collector-Contrib components are referred to in the charts in this package.
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.113.0
 
   # ServiceNow recommends the probabilistic sampler processor in order to support
   # probability sampling in the collector pipieline.  This sampler component supports
   # OpenTelemetry sampling specifications that enable correctly adjusting counts, in
   # metrics derived from sampled spans.
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.113.0
 
   # The tail sampling processor is included, by request.  Please note that this component
   # does not provide the necessary information to correctly adjust counts in metrics derived
   # from sampled spans.
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.113.0
 
   # You may be interested in other transform components in the
   # Collector-Contrib repository.  Here are some that might be useful:
   #
-  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.107.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.113.0
 
 extensions:
   # You may be interested in externsions from the Collector-Contrib repository.  None are
   # required.  Here are some that might be useful:
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.107.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.107.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.113.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.113.0
 
   # We do not support use of headersetterextension to apply the lightstep-access-token
   # header on a per-request basis.  Contact ServiceNow if you are interested in
   # multi-tenancy with OTel-Arrow.  Do not build with
   # github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension
 
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.17.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.17.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.17.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.17.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.17.0
+
 # If you are building this collector in an Golang monorepo, or to apply security patches
 # and other overrides to the go.mod file, use this section.
 replaces:

charts/collector-k8s/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: otelcollector
 description: Deprecated chart for using the OpenTelemetry Collector to scape static or dynamic metric targets.
 type: application
-version: 0.4.0
-appVersion: 0.107.0
+version: 0.5.0
+appVersion: 0.113.0
 deprecated: true
 dependencies: []
 # cert manager must be manually installed because it has CRDs

charts/kube-otel-stack/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kube-otel-stack
 description: Chart for sending Kubernetes metrics to Lightstep using the OpenTelemetry Operator.
 type: application
-version: 0.10.0
-appVersion: 0.107.0
+version: 0.11.0
+appVersion: 0.113.0
 dependencies:
   # cert manager must be manually installed because it has CRDs
   # https://github.com/kubernetes-sigs/security-profiles-operator/issues/1062

charts/kube-otel-stack/values.yaml

@@ -76,7 +76,7 @@ tracesCollector:
     # When scaling these values, raise or lower the following settings
     # proportionally:
     #
-    # - concurrentbatch::max_in_flight_size_mib
+    # - receiver::admission::request_size_mib
     # - exporters::otelarrow::arrow::num_streams
     requests:
       cpu: 2
@@ -107,10 +107,6 @@ tracesCollector:
         send_batch_size: 1000
         timeout: 1s
         send_batch_max_size: 1500
-
-        # Use max_in_flight_size_mib=64 in a 2 cpu configuration;
-        # Use max_in_flight_size_mib=1024 in a 16 cpu configuration.
-        max_in_flight_size_mib: 64
       k8sattributes:
         passthrough: false
         pod_association:

charts/otel-cloud-stack/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "0.9.0"
+version: "0.10.0"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.31.0"
+appVersion: "1.32.0"

charts/otel-cloud-stack/values.yaml

@@ -458,7 +458,7 @@ tracesCollector:
     # When scaling these values, raise or lower the following settings
     # proportionally:
     #
-    # - concurrentbatch::max_in_flight_size_mib
+    # - receiver::admission::request_size_mib
     # - exporters::otelarrow::arrow::num_streams
     requests:
       cpu: 2
@@ -489,10 +489,6 @@ tracesCollector:
         send_batch_size: 1000
         timeout: 1s
         send_batch_max_size: 1500
-
-        # Use max_in_flight_size_mib=64 in a 2 cpu configuration;
-        # Use max_in_flight_size_mib=1024 in a 16 cpu configuration.
-        max_in_flight_size_mib: 64
       k8sattributes:
         passthrough: false
         pod_association: