diff --git a/arrow/Dockerfile b/arrow/Dockerfile index 33ac460..9c485cc 100644 --- a/arrow/Dockerfile +++ b/arrow/Dockerfile @@ -1,12 +1,11 @@ -FROM golang:1.21 AS sandbox +FROM golang:1.22 AS sandbox WORKDIR /otelarrowcol COPY . . ENV CGO_ENABLED=0 # Future optimization - curl the release. -RUN BUILDER_VERSION=$(grep 'otelcol_version' /otelarrowcol/otelcolarrow-build.yaml | awk '{print $2}') && \ - go install go.opentelemetry.io/collector/cmd/builder@v${BUILDER_VERSION} +RUN go install go.opentelemetry.io/collector/cmd/builder@v0.113.0 # This command generates main.go, go.mod but does not update deps. RUN builder --config=/otelarrowcol/otelcolarrow-build.yaml diff --git a/arrow/config/gateway-collector.yaml b/arrow/config/gateway-collector.yaml index f1d2018..11a6335 100644 --- a/arrow/config/gateway-collector.yaml +++ b/arrow/config/gateway-collector.yaml @@ -23,7 +23,7 @@ # cpu: 32 # memory: 64Gi # -# In the larger configuration tested, we used `max_in_flight_size_mib: 256`. +# In the larger configuration tested, we set `admission::request_size_mib: 256`. # In the smaller configuration, relatively more memory is required due to # garbage collection costs. @@ -47,6 +47,12 @@ receivers: max_connection_age: 30s max_connection_age_grace: 2m30s + # This receiver applies admission control to OTLP and OTel-Arrow data + # based on a count of uncompressed bytes. For larger instance sizes, + # raise this parameter. + admission: + request_size_mib: 128 + # otlp is the core OTLP exporter, which we enable to receive # OTLP/HTTP data. otlp: @@ -61,10 +67,6 @@ processors: send_batch_max_size: 1500 send_batch_size: 1000 timeout: 1s - - # Use max_in_flight_size_mib=64 in a 2 cpu configuration; - # Use max_in_flight_size_mib=1024 in a 16 cpu configuration. - max_in_flight_size_mib: 64 exporters: otelarrow: @@ -109,7 +111,7 @@ exporters: # A stream lifetime limit is required to avoid spurious # disconnect error messages in the collector logs. - max_stream_lifetime: 4m + max_stream_lifetime: 30s service: pipelines: diff --git a/arrow/otelcolarrow-build.yaml b/arrow/otelcolarrow-build.yaml index 7bd2779..0642283 100644 --- a/arrow/otelcolarrow-build.yaml +++ b/arrow/otelcolarrow-build.yaml @@ -37,42 +37,37 @@ dist: # description: My Organization telemetry collector # name: otelarrowcol - version: 0.26.0 + version: 0.30.0 description: ServiceNow Cloud Observability OpenTelemetry Protocol with Apache Arrow gateway collector - # This indicates which version of the core collector components as - # well as the builder version. At the time of this writing, we - # recommend using at least the current release. - otelcol_version: 0.107.0 - # We recommend building in both the OTLP exporter and the OTel-Arrow # exporter. exporters: # This is the core OpenTelemetry Protocol with Apache Arrow exporter, # recommended for exporting to ServiceNow Cloud Observability using # either OTel-Arrow or standard OTLP over gRPC. - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter v0.107.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/otelarrowexporter v0.113.0 # The following components may be useful for debugging. # As an alternate to the OTel-Arrow exporter, we recommend building # with the core OTLP exporter as a fallback. This component and the # OTel-Arrow exporter use compatible configuration. - - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.107.0 + - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.113.0 # In case OTLP/HTTP export is required, as opposed to the two # gRPC options above. - - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.107.0 + - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.113.0 # In case you want to record an OTLP telemetry session to a JSON # file, we recommend this additional utility. See also the # obfuscation processor, listed below. The OTel-Arrow provides # offline tools that can help explain poor compression performance # using inputs generated by this exporter, for example. - - gomod: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter v0.25.0 + - gomod: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter v0.30.0 # The debug exporter, useful for printing telemetry to the console. - - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.107.0 + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.113.0 receivers: # The OTel-Arrow receiver supports multiple protocols including OTel-Arrow, OTLP gRPC, @@ -82,69 +77,76 @@ receivers: # This enables other OTel Collectors or experimental SDKs that support OTel-Arrow to # send to this collector, such as this one for the OTel-Go Trace SDK: # https://github.com/lightstep/otel-launcher-go/tree/main/lightstep/sdk/trace/exporters/otlp/otelcol - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver v0.107.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/otelarrowreceiver v0.113.0 # To support receiving OTLP/HTTP. - - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.107.0 + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.113.0 # You may wish to enable other receivers from the Collector-Contrib repository or # elsewhere. Here are some that might be useful: # - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.107.0 - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.107.0 - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.107.0 - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.107.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.113.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.113.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.113.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.113.0 processors: # We recommend use of the OTel-Arrow concurrent batch processor. - - gomod: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.25.0 + - gomod: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.30.0 # We recommend building with the follow processor for obfuscation, in case you want to # record telemetry sessions for offline analysis. - - gomod: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor v0.25.0 + - gomod: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor v0.30.0 # We emphatically DO NOT recommend use of the memory limiter # processor or the core batch processor, i.e., do not build with # either of: # - go.opentelemetry.io/collector/processor/memorylimiterprocessor. - # - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.107.0 + # - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.113.0 # These Collector-Contrib components are referred to in the charts in this package. - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.107.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.113.0 # ServiceNow recommends the probabilistic sampler processor in order to support # probability sampling in the collector pipieline. This sampler component supports # OpenTelemetry sampling specifications that enable correctly adjusting counts, in # metrics derived from sampled spans. - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.107.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.113.0 # The tail sampling processor is included, by request. Please note that this component # does not provide the necessary information to correctly adjust counts in metrics derived # from sampled spans. - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.107.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.113.0 # You may be interested in other transform components in the # Collector-Contrib repository. Here are some that might be useful: # - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.107.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.113.0 extensions: # You may be interested in externsions from the Collector-Contrib repository. None are # required. Here are some that might be useful: - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.107.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.107.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.113.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.113.0 # We do not support use of headersetterextension to apply the lightstep-access-token # header on a per-request basis. Contact ServiceNow if you are interested in # multi-tenancy with OTel-Arrow. Do not build with # github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension +providers: + - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.17.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.17.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.17.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.17.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.17.0 + # If you are building this collector in an Golang monorepo, or to apply security patches # and other overrides to the go.mod file, use this section. replaces: diff --git a/charts/collector-k8s/Chart.yaml b/charts/collector-k8s/Chart.yaml index c586e1d..1ca966b 100644 --- a/charts/collector-k8s/Chart.yaml +++ b/charts/collector-k8s/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: otelcollector description: Deprecated chart for using the OpenTelemetry Collector to scape static or dynamic metric targets. type: application -version: 0.4.0 -appVersion: 0.107.0 +version: 0.5.0 +appVersion: 0.113.0 deprecated: true dependencies: [] # cert manager must be manually installed because it has CRDs diff --git a/charts/kube-otel-stack/Chart.yaml b/charts/kube-otel-stack/Chart.yaml index 8fe7327..f5c4d9a 100644 --- a/charts/kube-otel-stack/Chart.yaml +++ b/charts/kube-otel-stack/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kube-otel-stack description: Chart for sending Kubernetes metrics to Lightstep using the OpenTelemetry Operator. type: application -version: 0.10.0 -appVersion: 0.107.0 +version: 0.11.0 +appVersion: 0.113.0 dependencies: # cert manager must be manually installed because it has CRDs # https://github.com/kubernetes-sigs/security-profiles-operator/issues/1062 diff --git a/charts/kube-otel-stack/values.yaml b/charts/kube-otel-stack/values.yaml index 9c9a21a..5bd83d2 100644 --- a/charts/kube-otel-stack/values.yaml +++ b/charts/kube-otel-stack/values.yaml @@ -76,7 +76,7 @@ tracesCollector: # When scaling these values, raise or lower the following settings # proportionally: # - # - concurrentbatch::max_in_flight_size_mib + # - receiver::admission::request_size_mib # - exporters::otelarrow::arrow::num_streams requests: cpu: 2 @@ -107,10 +107,6 @@ tracesCollector: send_batch_size: 1000 timeout: 1s send_batch_max_size: 1500 - - # Use max_in_flight_size_mib=64 in a 2 cpu configuration; - # Use max_in_flight_size_mib=1024 in a 16 cpu configuration. - max_in_flight_size_mib: 64 k8sattributes: passthrough: false pod_association: diff --git a/charts/otel-cloud-stack/Chart.yaml b/charts/otel-cloud-stack/Chart.yaml index 595ab79..eec90eb 100644 --- a/charts/otel-cloud-stack/Chart.yaml +++ b/charts/otel-cloud-stack/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.9.0" +version: "0.10.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.31.0" +appVersion: "1.32.0" diff --git a/charts/otel-cloud-stack/values.yaml b/charts/otel-cloud-stack/values.yaml index 2cd1cbe..af35934 100644 --- a/charts/otel-cloud-stack/values.yaml +++ b/charts/otel-cloud-stack/values.yaml @@ -458,7 +458,7 @@ tracesCollector: # When scaling these values, raise or lower the following settings # proportionally: # - # - concurrentbatch::max_in_flight_size_mib + # - receiver::admission::request_size_mib # - exporters::otelarrow::arrow::num_streams requests: cpu: 2 @@ -489,10 +489,6 @@ tracesCollector: send_batch_size: 1000 timeout: 1s send_batch_max_size: 1500 - - # Use max_in_flight_size_mib=64 in a 2 cpu configuration; - # Use max_in_flight_size_mib=1024 in a 16 cpu configuration. - max_in_flight_size_mib: 64 k8sattributes: passthrough: false pod_association: