Add autoconfigure support for OTLP gzip compression (#3589)

* Add autoconfigure support for OTLP gzip compression * Respond to PR feedback.
open-telemetry · Sep 10, 2021 · 2670fe2 · 2670fe2
1 parent 856a995
commit 2670fe2
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 13 deletions.
diff --git a/sdk-extensions/autoconfigure/README.md b/sdk-extensions/autoconfigure/README.md
@@ -59,6 +59,9 @@ The [OpenTelemetry Protocol (OTLP)](https://github.com/open-telemetry/openteleme
 | otel.exporter.otlp.headers   | OTEL_EXPORTER_OTLP_HEADERS  | Key-value pairs separated by commas to pass as request headers on OTLP trace and metrics requests.        |
 | otel.exporter.otlp.traces.headers   | OTEL_EXPORTER_OTLP_TRACES_HEADERS  | Key-value pairs separated by commas to pass as request headers on OTLP trace requests.        |
 | otel.exporter.otlp.metrics.headers   | OTEL_EXPORTER_OTLP_METRICS_HEADERS  | Key-value pairs separated by commas to pass as request headers on OTLP metrics requests.        |
+| otel.exporter.otlp.compression   | OTEL_EXPORTER_OTLP_COMPRESSION | The compression type to use on OTLP trace and metric requests. Options include `gzip`. By default no compression will be used. |
+| otel.exporter.otlp.traces.compression   | OTEL_EXPORTER_OTLP_TRACES_COMPRESSION | The compression type to use on OTLP trace requests. Options include `gzip`. By default no compression will be used. |
+| otel.exporter.otlp.metrics.compression   | OTEL_EXPORTER_OTLP_METRICS_COMPRESSION | The compression type to use on OTLP metric requests. Options include `gzip`. By default no compression will be used. |
 | otel.exporter.otlp.timeout   | OTEL_EXPORTER_OTLP_TIMEOUT  | The maximum waiting time, in milliseconds, allowed to send each OTLP trace and metric batch. Default is `10000`.  |
 | otel.exporter.otlp.traces.timeout   | OTEL_EXPORTER_OTLP_TRACES_TIMEOUT  | The maximum waiting time, in milliseconds, allowed to send each OTLP trace batch. Default is `10000`.  |
 | otel.exporter.otlp.metrics.timeout   | OTEL_EXPORTER_OTLP_METRICS_TIMEOUT  | The maximum waiting time, in milliseconds, allowed to send each OTLP metric batch. Default is `10000`.  |

diff --git a/...nfigure/src/main/java/io/opentelemetry/sdk/autoconfigure/MetricExporterConfiguration.java b/...nfigure/src/main/java/io/opentelemetry/sdk/autoconfigure/MetricExporterConfiguration.java
@@ -99,6 +99,7 @@ static MetricExporter configureOtlpMetrics(
           config,
           builder::setEndpoint,
           builder::addHeader,
+          builder::setCompression,
           builder::setTimeout,
           builder::setTrustedCertificates);
 
@@ -122,6 +123,7 @@ static MetricExporter configureOtlpMetrics(
           config,
           builder::setEndpoint,
           builder::addHeader,
+          builder::setCompression,
           builder::setTimeout,
           builder::setTrustedCertificates);
 

diff --git a/...nsions/autoconfigure/src/main/java/io/opentelemetry/sdk/autoconfigure/OtlpConfigUtil.java b/...nsions/autoconfigure/src/main/java/io/opentelemetry/sdk/autoconfigure/OtlpConfigUtil.java
@@ -34,6 +34,7 @@ static void configureOtlpExporterBuilder(
       ConfigProperties config,
       Consumer<String> setEndpoint,
       BiConsumer<String, String> addHeader,
+      Consumer<String> setCompression,
       Consumer<Duration> setTimeout,
       Consumer<byte[]> setTrustedCertificates) {
     String endpoint = config.getString("otel.exporter.otlp." + dataType + ".endpoint");
@@ -50,6 +51,14 @@ static void configureOtlpExporterBuilder(
     }
     headers.forEach(addHeader);
 
+    String compression = config.getString("otel.exporter.otlp." + dataType + ".compression");
+    if (compression == null) {
+      compression = config.getString("otel.exporter.otlp.compression");
+    }
+    if (compression != null) {
+      setCompression.accept(compression);
+    }
+
     Duration timeout = config.getDuration("otel.exporter.otlp." + dataType + ".timeout");
     if (timeout == null) {
       timeout = config.getDuration("otel.exporter.otlp.timeout");

diff --git a/...configure/src/main/java/io/opentelemetry/sdk/autoconfigure/SpanExporterConfiguration.java b/...configure/src/main/java/io/opentelemetry/sdk/autoconfigure/SpanExporterConfiguration.java
@@ -119,6 +119,7 @@ static SpanExporter configureOtlp(ConfigProperties config) {
           config,
           builder::setEndpoint,
           builder::addHeader,
+          builder::setCompression,
           builder::setTimeout,
           builder::setTrustedCertificates);
 
@@ -135,6 +136,7 @@ static SpanExporter configureOtlp(ConfigProperties config) {
           config,
           builder::setEndpoint,
           builder::addHeader,
+          builder::setCompression,
           builder::setTimeout,
           builder::setTrustedCertificates);
 

diff --git a/...onfigure/src/testOtlpGrpc/java/io/opentelemetry/sdk/autoconfigure/OtlpGrpcConfigTest.java b/...onfigure/src/testOtlpGrpc/java/io/opentelemetry/sdk/autoconfigure/OtlpGrpcConfigTest.java
@@ -135,6 +135,7 @@ void configureExportersGeneral() {
     props.put("otel.exporter.otlp.endpoint", "https://localhost:" + server.httpsPort());
     props.put("otel.exporter.otlp.certificate", certificate.certificateFile().getAbsolutePath());
     props.put("otel.exporter.otlp.headers", "header-key=header-value");
+    props.put("otel.exporter.otlp.compression", "gzip");
     props.put("otel.exporter.otlp.timeout", "15s");
     ConfigProperties properties = DefaultConfigProperties.createForTest(props);
     SpanExporter spanExporter =
@@ -156,7 +157,8 @@ void configureExportersGeneral() {
             headers ->
                 headers.contains(
                         ":path", "/opentelemetry.proto.collector.trace.v1.TraceService/Export")
-                    && headers.contains("header-key", "header-value"));
+                    && headers.contains("header-key", "header-value")
+                    && headers.contains("grpc-encoding", "gzip"));
 
     assertThat(metricExporter).extracting("timeoutNanos").isEqualTo(TimeUnit.SECONDS.toNanos(15));
     assertThat(
@@ -171,7 +173,8 @@ void configureExportersGeneral() {
             headers ->
                 headers.contains(
                         ":path", "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export")
-                    && headers.contains("header-key", "header-value"));
+                    && headers.contains("header-key", "header-value")
+                    && headers.contains("grpc-encoding", "gzip"));
   }
 
   @Test
@@ -182,11 +185,13 @@ void configureSpanExporter() {
     props.put("otel.exporter.otlp.endpoint", "http://foo.bar");
     props.put("otel.exporter.otlp.certificate", Paths.get("foo", "bar", "baz").toString());
     props.put("otel.exporter.otlp.headers", "header-key=dummy-value");
+    props.put("otel.exporter.otlp.compression", "foo");
     props.put("otel.exporter.otlp.timeout", "10s");
     props.put("otel.exporter.otlp.traces.endpoint", "https://localhost:" + server.httpsPort());
     props.put(
         "otel.exporter.otlp.traces.certificate", certificate.certificateFile().getAbsolutePath());
     props.put("otel.exporter.otlp.traces.headers", "header-key=header-value");
+    props.put("otel.exporter.otlp.traces.compression", "gzip");
     props.put("otel.exporter.otlp.traces.timeout", "15s");
     SpanExporter spanExporter =
         SpanExporterConfiguration.configureExporter(
@@ -216,11 +221,13 @@ public void configureMetricExporter() {
     props.put("otel.exporter.otlp.endpoint", "http://foo.bar");
     props.put("otel.exporter.otlp.certificate", Paths.get("foo", "bar", "baz").toString());
     props.put("otel.exporter.otlp.headers", "header-key=dummy-value");
+    props.put("otel.exporter.otlp.compression", "gzip");
     props.put("otel.exporter.otlp.timeout", "10s");
     props.put("otel.exporter.otlp.metrics.endpoint", "https://localhost:" + server.httpsPort());
     props.put(
         "otel.exporter.otlp.metrics.certificate", certificate.certificateFile().getAbsolutePath());
     props.put("otel.exporter.otlp.metrics.headers", "header-key=header-value");
+    props.put("otel.exporter.otlp.metrics.compression", "gzip");
     props.put("otel.exporter.otlp.metrics.timeout", "15s");
     MetricExporter metricExporter =
         MetricExporterConfiguration.configureOtlpMetrics(
@@ -239,7 +246,8 @@ public void configureMetricExporter() {
             headers ->
                 headers.contains(
                         ":path", "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export")
-                    && headers.contains("header-key", "header-value"));
+                    && headers.contains("header-key", "header-value")
+                    && headers.contains("grpc-encoding", "gzip"));
   }
 
   @Test

diff --git a/...onfigure/src/testOtlpHttp/java/io/opentelemetry/sdk/autoconfigure/OtlpHttpConfigTest.java b/...onfigure/src/testOtlpHttp/java/io/opentelemetry/sdk/autoconfigure/OtlpHttpConfigTest.java
@@ -13,7 +13,6 @@
 import static org.awaitility.Awaitility.await;
 
 import com.google.common.collect.Lists;
-import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.Message;
 import com.linecorp.armeria.common.HttpResponse;
 import com.linecorp.armeria.common.HttpStatus;
@@ -41,6 +40,8 @@
 import io.opentelemetry.sdk.trace.data.SpanData;
 import io.opentelemetry.sdk.trace.data.StatusData;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.net.InetAddress;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -54,6 +55,9 @@
 import java.util.concurrent.TimeUnit;
 import okhttp3.OkHttpClient;
 import okhttp3.tls.HeldCertificate;
+import okio.Buffer;
+import okio.GzipSource;
+import okio.Okio;
 import org.assertj.core.api.InstanceOfAssertFactories;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -120,18 +124,27 @@ private static <T extends Message> HttpService httpService(
                     aggReq -> {
                       requestHeaders.add(aggReq.headers());
                       try {
-                        queue.add(
-                            (T)
-                                defaultMessage
-                                    .getParserForType()
-                                    .parseFrom(aggReq.content().array()));
-                      } catch (InvalidProtocolBufferException e) {
+                        byte[] requestBody =
+                            maybeGzipInflate(aggReq.headers(), aggReq.content().array());
+                        queue.add((T) defaultMessage.getParserForType().parseFrom(requestBody));
+                      } catch (IOException e) {
                         return HttpResponse.of(HttpStatus.BAD_REQUEST);
                       }
                       return HttpResponse.of(HttpStatus.OK);
                     }));
   }
 
+  private static byte[] maybeGzipInflate(RequestHeaders requestHeaders, byte[] content)
+      throws IOException {
+    if (!requestHeaders.contains("content-encoding", "gzip")) {
+      return content;
+    }
+    Buffer buffer = new Buffer();
+    GzipSource gzipSource = new GzipSource(Okio.source(new ByteArrayInputStream(content)));
+    gzipSource.read(buffer, Integer.MAX_VALUE);
+    return buffer.readByteArray();
+  }
+
   @BeforeEach
   void setUp() {
     traceRequests.clear();
@@ -155,6 +168,7 @@ void configureExportersGeneral() {
     props.put("otel.exporter.otlp.metrics.endpoint", metricEndpoint());
     props.put("otel.exporter.otlp.certificate", certificateExtension.filePath);
     props.put("otel.exporter.otlp.headers", "header-key=header-value");
+    props.put("otel.exporter.otlp.compression", "gzip");
     props.put("otel.exporter.otlp.timeout", "15s");
     ConfigProperties properties = DefaultConfigProperties.createForTest(props);
     SpanExporter spanExporter =
@@ -178,7 +192,8 @@ void configureExportersGeneral() {
         .anyMatch(
             headers ->
                 headers.contains(":path", "/v1/traces")
-                    && headers.contains("header-key", "header-value"));
+                    && headers.contains("header-key", "header-value")
+                    && headers.contains("content-encoding", "gzip"));
 
     assertThat(metricExporter)
         .extracting("client", as(InstanceOfAssertFactories.type(OkHttpClient.class)))
@@ -195,7 +210,8 @@ void configureExportersGeneral() {
         .anyMatch(
             headers ->
                 headers.contains(":path", "/v1/metrics")
-                    && headers.contains("header-key", "header-value"));
+                    && headers.contains("header-key", "header-value")
+                    && headers.contains("content-encoding", "gzip"));
   }
 
   @Test
@@ -208,10 +224,12 @@ void configureSpanExporter() {
     props.put("otel.exporter.otlp.endpoint", "http://foo.bar");
     props.put("otel.exporter.otlp.certificate", Paths.get("foo", "bar", "baz").toString());
     props.put("otel.exporter.otlp.headers", "header-key=dummy-value");
+    props.put("otel.exporter.otlp.compression", "foo");
     props.put("otel.exporter.otlp.timeout", "10s");
     props.put("otel.exporter.otlp.traces.endpoint", traceEndpoint());
     props.put("otel.exporter.otlp.traces.certificate", certificateExtension.filePath);
     props.put("otel.exporter.otlp.traces.headers", "header-key=header-value");
+    props.put("otel.exporter.otlp.traces.compression", "gzip");
     props.put("otel.exporter.otlp.traces.timeout", "15s");
     SpanExporter spanExporter =
         SpanExporterConfiguration.configureExporter(
@@ -232,7 +250,8 @@ void configureSpanExporter() {
         .anyMatch(
             headers ->
                 headers.contains(":path", "/v1/traces")
-                    && headers.contains("header-key", "header-value"));
+                    && headers.contains("header-key", "header-value")
+                    && headers.contains("content-encoding", "gzip"));
   }
 
   @Test
@@ -245,10 +264,12 @@ public void configureMetricExporter() {
     props.put("otel.exporter.otlp.endpoint", "http://foo.bar");
     props.put("otel.exporter.otlp.certificate", Paths.get("foo", "bar", "baz").toString());
     props.put("otel.exporter.otlp.headers", "header-key=dummy-value");
+    props.put("otel.exporter.otlp.compression", "foo");
     props.put("otel.exporter.otlp.timeout", "10s");
     props.put("otel.exporter.otlp.metrics.endpoint", metricEndpoint());
     props.put("otel.exporter.otlp.metrics.certificate", certificateExtension.filePath);
     props.put("otel.exporter.otlp.metrics.headers", "header-key=header-value");
+    props.put("otel.exporter.otlp.metrics.compression", "gzip");
     props.put("otel.exporter.otlp.metrics.timeout", "15s");
     MetricExporter metricExporter =
         MetricExporterConfiguration.configureOtlpMetrics(