feat: Add gzip compression to Chronicle Exporter (#1405)

* Add gzip compression * Add default to doc Co-authored-by: Brandon Johnson <[email protected]> --------- Co-authored-by: Brandon Johnson <[email protected]>
observIQ · Jan 4, 2024 · 1d78771 · 1d78771
1 parent a759398
commit 1d78771
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 4 deletions.
diff --git a/exporter/chronicleexporter/README.md b/exporter/chronicleexporter/README.md
@@ -32,6 +32,7 @@ The exporter can be configured using the following fields:
 | `customer_id`       | string |                                                | `false`  | The customer ID used for sending logs.                                                      |
 | `override_log_type` | bool   | `true`                                         | `false`  | Whether or not to override the `log_type` in the config with `attributes["log_type"]`       |
 | `namespace`         | string |                                                | `false`  | User-configured environment namespace to identify the data domain the logs originated from. |
+| `compression`       | string | `none`                                         | `false`  | The compression type to use when sending logs. valid values are `none` and `gzip`           |
 
 ### Log Type
 

diff --git a/exporter/chronicleexporter/config.go b/exporter/chronicleexporter/config.go
@@ -24,6 +24,14 @@ import (
 	"go.uber.org/zap"
 )
 
+const (
+	// gzipCompression is the gzip compression type.
+	gzipCompression = "gzip"
+
+	// noCompression is the no compression type.
+	noCompression = "none"
+)
+
 // Config defines configuration for the Chronicle exporter.
 type Config struct {
 	exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
@@ -53,6 +61,9 @@ type Config struct {
 
 	// Namespace is the namespace that will be used to send logs to Chronicle.
 	Namespace string `mapstructure:"namespace"`
+
+	// Compression is the compression type that will be used to send logs to Chronicle.
+	Compression string `mapstructure:"compression"`
 }
 
 // Validate checks if the configuration is valid.
@@ -74,5 +85,9 @@ func (cfg *Config) Validate() error {
 		}
 	}
 
+	if cfg.Compression != gzipCompression && cfg.Compression != noCompression {
+		return fmt.Errorf("invalid compression type: %s", cfg.Compression)
+	}
+
 	return nil
 }
diff --git a/exporter/chronicleexporter/config_test.go b/exporter/chronicleexporter/config_test.go
@@ -32,22 +32,25 @@ func TestConfigValidate(t *testing.T) {
 				CredsFilePath: "/path/to/creds_file",
 				Creds:         "creds_example",
 				LogType:       "log_type_example",
+				Compression:   noCompression,
 			},
 			expectedErr: "can only specify creds_file_path or creds",
 		},
 		{
 			desc: "LogType is empty",
 			config: &Config{
-				Creds: "creds_example",
+				Creds:       "creds_example",
+				Compression: noCompression,
 			},
 			expectedErr: "log_type is required",
 		},
 
 		{
 			desc: "Valid config with creds",
 			config: &Config{
-				Creds:   "creds_example",
-				LogType: "log_type_example",
+				Creds:       "creds_example",
+				LogType:     "log_type_example",
+				Compression: noCompression,
 			},
 			expectedErr: "",
 		},
@@ -56,6 +59,7 @@ func TestConfigValidate(t *testing.T) {
 			config: &Config{
 				CredsFilePath: "/path/to/creds_file",
 				LogType:       "log_type_example",
+				Compression:   noCompression,
 			},
 			expectedErr: "",
 		},
@@ -65,9 +69,19 @@ func TestConfigValidate(t *testing.T) {
 				CredsFilePath: "/path/to/creds_file",
 				LogType:       "log_type_example",
 				RawLogField:   `body["field"]`,
+				Compression:   noCompression,
 			},
 			expectedErr: "",
 		},
+		{
+			desc: "Invalid compression type",
+			config: &Config{
+				CredsFilePath: "/path/to/creds_file",
+				LogType:       "log_type_example",
+				Compression:   "invalid",
+			},
+			expectedErr: "invalid compression type",
+		},
 	}
 
 	for _, tc := range testCases {

diff --git a/exporter/chronicleexporter/exporter.go b/exporter/chronicleexporter/exporter.go
@@ -16,6 +16,7 @@ package chronicleexporter
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"errors"
@@ -121,11 +122,31 @@ func (ce *chronicleExporter) logsDataPusher(ctx context.Context, ld plog.Logs) e
 }
 
 func (ce *chronicleExporter) uploadToChronicle(ctx context.Context, data []byte) error {
-	request, err := http.NewRequestWithContext(ctx, "POST", ce.endpoint, bytes.NewBuffer(data))
+	var body io.Reader
+
+	if ce.cfg.Compression == gzipCompression {
+		var b bytes.Buffer
+		gz := gzip.NewWriter(&b)
+		if _, err := gz.Write(data); err != nil {
+			return fmt.Errorf("gzip write: %w", err)
+		}
+		if err := gz.Close(); err != nil {
+			return fmt.Errorf("gzip close: %w", err)
+		}
+		body = &b
+	} else {
+		body = bytes.NewBuffer(data)
+	}
+
+	request, err := http.NewRequestWithContext(ctx, "POST", ce.endpoint, body)
 	if err != nil {
 		return fmt.Errorf("create request: %w", err)
 	}
 
+	if ce.cfg.Compression == gzipCompression {
+		request.Header.Set("Content-Encoding", "gzip")
+	}
+
 	request.Header.Set("Content-Type", "application/json")
 
 	resp, err := ce.httpClient.Do(request)

diff --git a/exporter/chronicleexporter/factory.go b/exporter/chronicleexporter/factory.go
@@ -40,6 +40,7 @@ func createDefaultConfig() component.Config {
 		RetrySettings:   exporterhelper.NewDefaultRetrySettings(),
 		OverrideLogType: true,
 		Endpoint:        baseEndpoint,
+		Compression:     noCompression,
 	}
 }
 

diff --git a/exporter/chronicleexporter/factory_test.go b/exporter/chronicleexporter/factory_test.go
@@ -28,6 +28,7 @@ func Test_createDefaultConfig(t *testing.T) {
 		RetrySettings:   exporterhelper.NewDefaultRetrySettings(),
 		OverrideLogType: true,
 		Endpoint:        "https://malachiteingestion-pa.googleapis.com",
+		Compression:     "none",
 	}
 
 	actual := createDefaultConfig()