From ce16a9e2fce25ba6adbbd48e0c37e3876383bad9 Mon Sep 17 00:00:00 2001 From: Ian Adams Date: Mon, 9 Dec 2024 15:01:07 +0000 Subject: [PATCH 1/8] fix: Shut down zombie goroutine in chronicleexporter (#2029) * Properly shut down chronicleexporter zombie goroutine * Fix lint * Fix the same problem for the GRPC workflow --- exporter/chronicleexporter/exporter.go | 8 ++++---- exporter/chronicleexporter/grpc_exporter_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/exporter/chronicleexporter/exporter.go b/exporter/chronicleexporter/exporter.go index 4df807910..535364ffb 100644 --- a/exporter/chronicleexporter/exporter.go +++ b/exporter/chronicleexporter/exporter.go @@ -79,6 +79,10 @@ func newExporter(cfg *Config, params exporter.Settings, exporterID string) (*chr }, nil } +func (ce *chronicleExporter) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: false} +} + func (ce *chronicleExporter) Start(ctx context.Context, _ component.Host) error { ts, err := tokenSource(ctx, ce.cfg) if err != nil { @@ -134,10 +138,6 @@ func (ce *chronicleExporter) Shutdown(context.Context) error { return nil } -func (ce *chronicleExporter) Capabilities() consumer.Capabilities { - return consumer.Capabilities{MutatesData: false} -} - func (ce *chronicleExporter) logsDataPusher(ctx context.Context, ld plog.Logs) error { payloads, err := ce.marshaler.MarshalRawLogs(ctx, ld) if err != nil { diff --git a/exporter/chronicleexporter/grpc_exporter_test.go b/exporter/chronicleexporter/grpc_exporter_test.go index f95e89ade..8bb8186ae 100644 --- a/exporter/chronicleexporter/grpc_exporter_test.go +++ b/exporter/chronicleexporter/grpc_exporter_test.go @@ -19,7 +19,7 @@ import ( "net" "testing" - "github.com/observiq/bindplane-agent/exporter/chronicleexporter/protos/api" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/consumer/consumererror" From 7853a76e8697b87c41e96f6483f2accc2a6ef311 Mon Sep 17 00:00:00 2001 From: Daniel Jaglowski Date: Mon, 16 Dec 2024 14:07:42 -0500 Subject: [PATCH 2/8] chore: Add new tests for chronicle exporter with http and grpc servers (#2049) --- exporter/chronicleexporter/exporter.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exporter/chronicleexporter/exporter.go b/exporter/chronicleexporter/exporter.go index 535364ffb..4df807910 100644 --- a/exporter/chronicleexporter/exporter.go +++ b/exporter/chronicleexporter/exporter.go @@ -79,10 +79,6 @@ func newExporter(cfg *Config, params exporter.Settings, exporterID string) (*chr }, nil } -func (ce *chronicleExporter) Capabilities() consumer.Capabilities { - return consumer.Capabilities{MutatesData: false} -} - func (ce *chronicleExporter) Start(ctx context.Context, _ component.Host) error { ts, err := tokenSource(ctx, ce.cfg) if err != nil { @@ -138,6 +134,10 @@ func (ce *chronicleExporter) Shutdown(context.Context) error { return nil } +func (ce *chronicleExporter) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: false} +} + func (ce *chronicleExporter) logsDataPusher(ctx context.Context, ld plog.Logs) error { payloads, err := ce.marshaler.MarshalRawLogs(ctx, ld) if err != nil { From b85c66833698043fd100e8b35e9df92cf2d779dc Mon Sep 17 00:00:00 2001 From: Ian Adams Date: Mon, 9 Dec 2024 13:15:11 -0500 Subject: [PATCH 3/8] feat: Enforce request maximum size and number of logs --- exporter/chronicleexporter/config.go | 2 +- exporter/chronicleexporter/config_test.go | 40 ++++++------ exporter/chronicleexporter/factory_test.go | 8 +-- exporter/chronicleexporter/marshal.go | 74 +++++++++++----------- 4 files changed, 62 insertions(+), 62 deletions(-) diff --git a/exporter/chronicleexporter/config.go b/exporter/chronicleexporter/config.go index c580d428e..a02072626 100644 --- a/exporter/chronicleexporter/config.go +++ b/exporter/chronicleexporter/config.go @@ -104,7 +104,7 @@ type Config struct { // BatchRequestSizeLimitHTTP is the maximum batch request size, in bytes, that can be sent to Chronicle via the HTTP protocol // This field is defaulted to 1048576 as that is the default Chronicle backend limit // Setting this option to a value above the Chronicle backend limit may result in rejected log batch requests - BatchRequestSizeLimitHTTP int `mapstructure:"batch_request_size_limit_http"` + BatchRequestSizeLimitHTTP int `mapstructure:"batch_request_size_limit_grpc"` } // Validate checks if the configuration is valid. diff --git a/exporter/chronicleexporter/config_test.go b/exporter/chronicleexporter/config_test.go index 43ec13a27..f23c99731 100644 --- a/exporter/chronicleexporter/config_test.go +++ b/exporter/chronicleexporter/config_test.go @@ -33,8 +33,8 @@ func TestConfigValidate(t *testing.T) { Creds: "creds_example", LogType: "log_type_example", Compression: noCompression, - BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, }, expectedErr: "can only specify creds_file_path or creds", }, @@ -45,8 +45,8 @@ func TestConfigValidate(t *testing.T) { LogType: "log_type_example", Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, }, expectedErr: "", }, @@ -57,8 +57,8 @@ func TestConfigValidate(t *testing.T) { LogType: "log_type_example", Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, }, expectedErr: "", }, @@ -70,8 +70,8 @@ func TestConfigValidate(t *testing.T) { RawLogField: `body["field"]`, Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, }, expectedErr: "", }, @@ -83,7 +83,7 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Protocol: protocolGRPC, BatchLogCountLimitGRPC: 0, - BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, + BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, }, expectedErr: "positive batch count log limit is required when protocol is grpc", }, @@ -94,7 +94,7 @@ func TestConfigValidate(t *testing.T) { LogType: "log_type_example", Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, + BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, BatchRequestSizeLimitGRPC: 0, }, expectedErr: "positive batch request size limit is required when protocol is grpc", @@ -117,8 +117,8 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Forwarder: "forwarder_example", Project: "project_example", - BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, }, expectedErr: "location is required when protocol is https", }, @@ -131,8 +131,8 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Project: "project_example", Location: "location_example", - BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, }, expectedErr: "forwarder is required when protocol is https", }, @@ -145,8 +145,8 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Location: "location_example", Forwarder: "forwarder_example", - BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, }, expectedErr: "project is required when protocol is https", }, @@ -160,7 +160,7 @@ func TestConfigValidate(t *testing.T) { Project: "project_example", Location: "location_example", Forwarder: "forwarder_example", - BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, + BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, BatchLogCountLimitHTTP: 0, }, expectedErr: "positive batch count log limit is required when protocol is https", @@ -176,7 +176,7 @@ func TestConfigValidate(t *testing.T) { Location: "location_example", Forwarder: "forwarder_example", BatchRequestSizeLimitHTTP: 0, - BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, + BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, }, expectedErr: "positive batch request size limit is required when protocol is https", }, @@ -190,8 +190,8 @@ func TestConfigValidate(t *testing.T) { Project: "project_example", Location: "location_example", Forwarder: "forwarder_example", - BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, }, }, } diff --git a/exporter/chronicleexporter/factory_test.go b/exporter/chronicleexporter/factory_test.go index a542324b2..55c01dd0f 100644 --- a/exporter/chronicleexporter/factory_test.go +++ b/exporter/chronicleexporter/factory_test.go @@ -32,10 +32,10 @@ func Test_createDefaultConfig(t *testing.T) { Compression: "none", CollectAgentMetrics: true, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, - BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, - BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, } actual := createDefaultConfig() diff --git a/exporter/chronicleexporter/marshal.go b/exporter/chronicleexporter/marshal.go index 832f769b3..722985db5 100644 --- a/exporter/chronicleexporter/marshal.go +++ b/exporter/chronicleexporter/marshal.go @@ -423,31 +423,31 @@ func (m *protoMarshaler) constructPayloads(rawLogs map[string][]*api.LogEntry, n func (m *protoMarshaler) enforceMaximumsGRPCRequest(request *api.BatchCreateLogsRequest) []*api.BatchCreateLogsRequest { size := proto.Size(request) - entries := request.Batch.Entries - if size <= m.cfg.BatchRequestSizeLimitGRPC && len(entries) <= m.cfg.BatchLogCountLimitGRPC { - return []*api.BatchCreateLogsRequest{ - request, + if size > m.cfg.BatchRequestSizeLimitGRPC || len(request.Batch.Entries) > m.cfg.BatchLogCountLimitGRPC { + if len(request.Batch.Entries) < 2 { + m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + return []*api.BatchCreateLogsRequest{} } - } - if len(entries) < 2 { - m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) - return []*api.BatchCreateLogsRequest{} - } + // split request into two + entries := request.Batch.Entries + mid := len(entries) / 2 + leftHalf := entries[:mid] + rightHalf := entries[mid:] - // split request into two - mid := len(entries) / 2 - leftHalf := entries[:mid] - rightHalf := entries[mid:] + request.Batch.Entries = leftHalf + otherHalfRequest := m.buildGRPCRequest(rightHalf, request.Batch.LogType, request.Batch.Source.Namespace, request.Batch.Source.Labels) - request.Batch.Entries = leftHalf - otherHalfRequest := m.buildGRPCRequest(rightHalf, request.Batch.LogType, request.Batch.Source.Namespace, request.Batch.Source.Labels) + // re-enforce max size restriction on each half + enforcedRequest := m.enforceMaximumsGRPCRequest(request) + enforcedOtherHalfRequest := m.enforceMaximumsGRPCRequest(otherHalfRequest) - // re-enforce max size restriction on each half - enforcedRequest := m.enforceMaximumsGRPCRequest(request) - enforcedOtherHalfRequest := m.enforceMaximumsGRPCRequest(otherHalfRequest) + return append(enforcedRequest, enforcedOtherHalfRequest...) + } - return append(enforcedRequest, enforcedOtherHalfRequest...) + return []*api.BatchCreateLogsRequest{ + request, + } } func (m *protoMarshaler) buildGRPCRequest(entries []*api.LogEntry, logType, namespace string, ingestionLabels []*api.Label) *api.BatchCreateLogsRequest { @@ -535,30 +535,30 @@ func (m *protoMarshaler) constructHTTPPayloads(rawLogs map[string][]*api.Log) ma func (m *protoMarshaler) enforceMaximumsHTTPRequest(request *api.ImportLogsRequest) []*api.ImportLogsRequest { size := proto.Size(request) logs := request.GetInlineSource().Logs - if size <= m.cfg.BatchRequestSizeLimitHTTP && len(logs) <= m.cfg.BatchLogCountLimitHTTP { - return []*api.ImportLogsRequest{ - request, + if size > m.cfg.BatchRequestSizeLimitHTTP || len(logs) > m.cfg.BatchLogCountLimitHTTP { + if len(logs) < 2 { + m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + return []*api.ImportLogsRequest{} } - } - if len(logs) < 2 { - m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) - return []*api.ImportLogsRequest{} - } + // split request into two + mid := len(logs) / 2 + leftHalf := logs[:mid] + rightHalf := logs[mid:] - // split request into two - mid := len(logs) / 2 - leftHalf := logs[:mid] - rightHalf := logs[mid:] + request.GetInlineSource().Logs = leftHalf + otherHalfRequest := m.buildHTTPRequest(rightHalf) - request.GetInlineSource().Logs = leftHalf - otherHalfRequest := m.buildHTTPRequest(rightHalf) + // re-enforce max size restriction on each half + enforcedRequest := m.enforceMaximumsHTTPRequest(request) + enforcedOtherHalfRequest := m.enforceMaximumsHTTPRequest(otherHalfRequest) - // re-enforce max size restriction on each half - enforcedRequest := m.enforceMaximumsHTTPRequest(request) - enforcedOtherHalfRequest := m.enforceMaximumsHTTPRequest(otherHalfRequest) + return append(enforcedRequest, enforcedOtherHalfRequest...) + } - return append(enforcedRequest, enforcedOtherHalfRequest...) + return []*api.ImportLogsRequest{ + request, + } } func (m *protoMarshaler) buildHTTPRequest(entries []*api.Log) *api.ImportLogsRequest { From e7e163a8db220d03936d1ce1bd972880829e5990 Mon Sep 17 00:00:00 2001 From: Ian Adams Date: Mon, 9 Dec 2024 13:28:28 -0500 Subject: [PATCH 4/8] Fix lint --- exporter/chronicleexporter/config_test.go | 40 +++++++++++----------- exporter/chronicleexporter/factory_test.go | 8 ++--- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/exporter/chronicleexporter/config_test.go b/exporter/chronicleexporter/config_test.go index f23c99731..43ec13a27 100644 --- a/exporter/chronicleexporter/config_test.go +++ b/exporter/chronicleexporter/config_test.go @@ -33,8 +33,8 @@ func TestConfigValidate(t *testing.T) { Creds: "creds_example", LogType: "log_type_example", Compression: noCompression, - BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, }, expectedErr: "can only specify creds_file_path or creds", }, @@ -45,8 +45,8 @@ func TestConfigValidate(t *testing.T) { LogType: "log_type_example", Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, }, expectedErr: "", }, @@ -57,8 +57,8 @@ func TestConfigValidate(t *testing.T) { LogType: "log_type_example", Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, }, expectedErr: "", }, @@ -70,8 +70,8 @@ func TestConfigValidate(t *testing.T) { RawLogField: `body["field"]`, Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, }, expectedErr: "", }, @@ -83,7 +83,7 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Protocol: protocolGRPC, BatchLogCountLimitGRPC: 0, - BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, + BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, }, expectedErr: "positive batch count log limit is required when protocol is grpc", }, @@ -94,7 +94,7 @@ func TestConfigValidate(t *testing.T) { LogType: "log_type_example", Compression: noCompression, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, + BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, BatchRequestSizeLimitGRPC: 0, }, expectedErr: "positive batch request size limit is required when protocol is grpc", @@ -117,8 +117,8 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Forwarder: "forwarder_example", Project: "project_example", - BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, }, expectedErr: "location is required when protocol is https", }, @@ -131,8 +131,8 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Project: "project_example", Location: "location_example", - BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, }, expectedErr: "forwarder is required when protocol is https", }, @@ -145,8 +145,8 @@ func TestConfigValidate(t *testing.T) { Compression: noCompression, Location: "location_example", Forwarder: "forwarder_example", - BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, }, expectedErr: "project is required when protocol is https", }, @@ -160,7 +160,7 @@ func TestConfigValidate(t *testing.T) { Project: "project_example", Location: "location_example", Forwarder: "forwarder_example", - BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, + BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, BatchLogCountLimitHTTP: 0, }, expectedErr: "positive batch count log limit is required when protocol is https", @@ -176,7 +176,7 @@ func TestConfigValidate(t *testing.T) { Location: "location_example", Forwarder: "forwarder_example", BatchRequestSizeLimitHTTP: 0, - BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, + BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, }, expectedErr: "positive batch request size limit is required when protocol is https", }, @@ -190,8 +190,8 @@ func TestConfigValidate(t *testing.T) { Project: "project_example", Location: "location_example", Forwarder: "forwarder_example", - BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, - BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, }, }, } diff --git a/exporter/chronicleexporter/factory_test.go b/exporter/chronicleexporter/factory_test.go index 55c01dd0f..a542324b2 100644 --- a/exporter/chronicleexporter/factory_test.go +++ b/exporter/chronicleexporter/factory_test.go @@ -32,10 +32,10 @@ func Test_createDefaultConfig(t *testing.T) { Compression: "none", CollectAgentMetrics: true, Protocol: protocolGRPC, - BatchLogCountLimitGRPC: DefaultBatchLogCountLimitGRPC, - BatchRequestSizeLimitGRPC: DefaultBatchRequestSizeLimitGRPC, - BatchLogCountLimitHTTP: DefaultBatchLogCountLimitHTTP, - BatchRequestSizeLimitHTTP: DefaultBatchRequestSizeLimitHTTP, + BatchLogCountLimitGRPC: defaultBatchLogCountLimitGRPC, + BatchRequestSizeLimitGRPC: defaultBatchRequestSizeLimitGRPC, + BatchLogCountLimitHTTP: defaultBatchLogCountLimitHTTP, + BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, } actual := createDefaultConfig() From f8e8b919be1af69651118ca44a500bf213eb8041 Mon Sep 17 00:00:00 2001 From: Ian Adams Date: Mon, 9 Dec 2024 14:59:42 -0500 Subject: [PATCH 5/8] Refactor to be more go-idiomatic --- exporter/chronicleexporter/marshal.go | 74 +++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/exporter/chronicleexporter/marshal.go b/exporter/chronicleexporter/marshal.go index 722985db5..832f769b3 100644 --- a/exporter/chronicleexporter/marshal.go +++ b/exporter/chronicleexporter/marshal.go @@ -423,31 +423,31 @@ func (m *protoMarshaler) constructPayloads(rawLogs map[string][]*api.LogEntry, n func (m *protoMarshaler) enforceMaximumsGRPCRequest(request *api.BatchCreateLogsRequest) []*api.BatchCreateLogsRequest { size := proto.Size(request) - if size > m.cfg.BatchRequestSizeLimitGRPC || len(request.Batch.Entries) > m.cfg.BatchLogCountLimitGRPC { - if len(request.Batch.Entries) < 2 { - m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) - return []*api.BatchCreateLogsRequest{} + entries := request.Batch.Entries + if size <= m.cfg.BatchRequestSizeLimitGRPC && len(entries) <= m.cfg.BatchLogCountLimitGRPC { + return []*api.BatchCreateLogsRequest{ + request, } + } - // split request into two - entries := request.Batch.Entries - mid := len(entries) / 2 - leftHalf := entries[:mid] - rightHalf := entries[mid:] + if len(entries) < 2 { + m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + return []*api.BatchCreateLogsRequest{} + } - request.Batch.Entries = leftHalf - otherHalfRequest := m.buildGRPCRequest(rightHalf, request.Batch.LogType, request.Batch.Source.Namespace, request.Batch.Source.Labels) + // split request into two + mid := len(entries) / 2 + leftHalf := entries[:mid] + rightHalf := entries[mid:] - // re-enforce max size restriction on each half - enforcedRequest := m.enforceMaximumsGRPCRequest(request) - enforcedOtherHalfRequest := m.enforceMaximumsGRPCRequest(otherHalfRequest) + request.Batch.Entries = leftHalf + otherHalfRequest := m.buildGRPCRequest(rightHalf, request.Batch.LogType, request.Batch.Source.Namespace, request.Batch.Source.Labels) - return append(enforcedRequest, enforcedOtherHalfRequest...) - } + // re-enforce max size restriction on each half + enforcedRequest := m.enforceMaximumsGRPCRequest(request) + enforcedOtherHalfRequest := m.enforceMaximumsGRPCRequest(otherHalfRequest) - return []*api.BatchCreateLogsRequest{ - request, - } + return append(enforcedRequest, enforcedOtherHalfRequest...) } func (m *protoMarshaler) buildGRPCRequest(entries []*api.LogEntry, logType, namespace string, ingestionLabels []*api.Label) *api.BatchCreateLogsRequest { @@ -535,30 +535,30 @@ func (m *protoMarshaler) constructHTTPPayloads(rawLogs map[string][]*api.Log) ma func (m *protoMarshaler) enforceMaximumsHTTPRequest(request *api.ImportLogsRequest) []*api.ImportLogsRequest { size := proto.Size(request) logs := request.GetInlineSource().Logs - if size > m.cfg.BatchRequestSizeLimitHTTP || len(logs) > m.cfg.BatchLogCountLimitHTTP { - if len(logs) < 2 { - m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) - return []*api.ImportLogsRequest{} + if size <= m.cfg.BatchRequestSizeLimitHTTP && len(logs) <= m.cfg.BatchLogCountLimitHTTP { + return []*api.ImportLogsRequest{ + request, } + } - // split request into two - mid := len(logs) / 2 - leftHalf := logs[:mid] - rightHalf := logs[mid:] + if len(logs) < 2 { + m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + return []*api.ImportLogsRequest{} + } - request.GetInlineSource().Logs = leftHalf - otherHalfRequest := m.buildHTTPRequest(rightHalf) + // split request into two + mid := len(logs) / 2 + leftHalf := logs[:mid] + rightHalf := logs[mid:] - // re-enforce max size restriction on each half - enforcedRequest := m.enforceMaximumsHTTPRequest(request) - enforcedOtherHalfRequest := m.enforceMaximumsHTTPRequest(otherHalfRequest) + request.GetInlineSource().Logs = leftHalf + otherHalfRequest := m.buildHTTPRequest(rightHalf) - return append(enforcedRequest, enforcedOtherHalfRequest...) - } + // re-enforce max size restriction on each half + enforcedRequest := m.enforceMaximumsHTTPRequest(request) + enforcedOtherHalfRequest := m.enforceMaximumsHTTPRequest(otherHalfRequest) - return []*api.ImportLogsRequest{ - request, - } + return append(enforcedRequest, enforcedOtherHalfRequest...) } func (m *protoMarshaler) buildHTTPRequest(entries []*api.Log) *api.ImportLogsRequest { From f1b0c430f177359131b1a2deff5bb7e7dc6bcd6b Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 11 Dec 2024 13:13:25 -0500 Subject: [PATCH 6/8] Split HTTP and GRPC exporters --- exporter/chronicleexporter/config.go | 2 +- exporter/chronicleexporter/exporter.go | 283 -------------------- exporter/chronicleexporter/exporter_test.go | 185 ------------- exporter/chronicleexporter/factory.go | 37 ++- exporter/chronicleexporter/factory_test.go | 2 +- exporter/chronicleexporter/grpc_exporter.go | 164 ++++++++++++ exporter/chronicleexporter/hostmetrics.go | 16 +- exporter/chronicleexporter/http_exporter.go | 178 ++++++++++++ exporter/chronicleexporter/marshal.go | 37 ++- exporter/chronicleexporter/marshal_test.go | 4 +- 10 files changed, 387 insertions(+), 521 deletions(-) delete mode 100644 exporter/chronicleexporter/exporter.go delete mode 100644 exporter/chronicleexporter/exporter_test.go create mode 100644 exporter/chronicleexporter/grpc_exporter.go create mode 100644 exporter/chronicleexporter/http_exporter.go diff --git a/exporter/chronicleexporter/config.go b/exporter/chronicleexporter/config.go index a02072626..c580d428e 100644 --- a/exporter/chronicleexporter/config.go +++ b/exporter/chronicleexporter/config.go @@ -104,7 +104,7 @@ type Config struct { // BatchRequestSizeLimitHTTP is the maximum batch request size, in bytes, that can be sent to Chronicle via the HTTP protocol // This field is defaulted to 1048576 as that is the default Chronicle backend limit // Setting this option to a value above the Chronicle backend limit may result in rejected log batch requests - BatchRequestSizeLimitHTTP int `mapstructure:"batch_request_size_limit_grpc"` + BatchRequestSizeLimitHTTP int `mapstructure:"batch_request_size_limit_http"` } // Validate checks if the configuration is valid. diff --git a/exporter/chronicleexporter/exporter.go b/exporter/chronicleexporter/exporter.go deleted file mode 100644 index 4df807910..000000000 --- a/exporter/chronicleexporter/exporter.go +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright observIQ, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chronicleexporter - -import ( - "bytes" - "compress/gzip" - "context" - "fmt" - "io" - "net/http" - - "github.com/google/uuid" - "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/consumer" - "go.opentelemetry.io/collector/consumer/consumererror" - "go.opentelemetry.io/collector/exporter" - "go.opentelemetry.io/collector/pdata/plog" - "go.uber.org/zap" - "golang.org/x/oauth2" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/oauth" - grpcgzip "google.golang.org/grpc/encoding/gzip" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/encoding/protojson" -) - -const ( - grpcScope = "https://www.googleapis.com/auth/malachite-ingestion" - httpScope = "https://www.googleapis.com/auth/cloud-platform" -) - -type chronicleExporter struct { - cfg *Config - set component.TelemetrySettings - marshaler logMarshaler - exporterID string - - // fields used for gRPC - grpcClient api.IngestionServiceV2Client - grpcConn *grpc.ClientConn - metrics *hostMetricsReporter - - // fields used for HTTP - httpClient *http.Client -} - -func newExporter(cfg *Config, params exporter.Settings, exporterID string) (*chronicleExporter, error) { - customerID, err := uuid.Parse(cfg.CustomerID) - if err != nil { - return nil, fmt.Errorf("parse customer ID: %w", err) - } - - marshaller, err := newProtoMarshaler(*cfg, params.TelemetrySettings, customerID[:]) - if err != nil { - return nil, fmt.Errorf("create proto marshaller: %w", err) - } - - return &chronicleExporter{ - cfg: cfg, - set: params.TelemetrySettings, - marshaler: marshaller, - exporterID: exporterID, - }, nil -} - -func (ce *chronicleExporter) Start(ctx context.Context, _ component.Host) error { - ts, err := tokenSource(ctx, ce.cfg) - if err != nil { - return fmt.Errorf("load Google credentials: %w", err) - } - - if ce.cfg.Protocol == protocolHTTPS { - ce.httpClient = oauth2.NewClient(context.Background(), ts) - return nil - } - - endpoint, dialOpts := grpcClientParams(ce.cfg.Endpoint, ts) - conn, err := grpc.NewClient(endpoint, dialOpts...) - if err != nil { - return fmt.Errorf("dial: %w", err) - } - ce.grpcConn = conn - ce.grpcClient = api.NewIngestionServiceV2Client(conn) - - if ce.cfg.CollectAgentMetrics { - f := func(ctx context.Context, request *api.BatchCreateEventsRequest) error { - _, err := ce.grpcClient.BatchCreateEvents(ctx, request) - return err - } - metrics, err := newHostMetricsReporter(ce.cfg, ce.set, ce.exporterID, f) - if err != nil { - return fmt.Errorf("create metrics reporter: %w", err) - } - ce.metrics = metrics - ce.metrics.start() - } - - return nil -} - -func (ce *chronicleExporter) Shutdown(context.Context) error { - defer http.DefaultTransport.(*http.Transport).CloseIdleConnections() - if ce.cfg.Protocol == protocolHTTPS { - t := ce.httpClient.Transport.(*oauth2.Transport) - if t.Base != nil { - t.Base.(*http.Transport).CloseIdleConnections() - } - return nil - } - if ce.metrics != nil { - ce.metrics.shutdown() - } - if ce.grpcConn != nil { - if err := ce.grpcConn.Close(); err != nil { - return fmt.Errorf("connection close: %s", err) - } - } - return nil -} - -func (ce *chronicleExporter) Capabilities() consumer.Capabilities { - return consumer.Capabilities{MutatesData: false} -} - -func (ce *chronicleExporter) logsDataPusher(ctx context.Context, ld plog.Logs) error { - payloads, err := ce.marshaler.MarshalRawLogs(ctx, ld) - if err != nil { - return fmt.Errorf("marshal logs: %w", err) - } - - for _, payload := range payloads { - if err := ce.uploadToChronicle(ctx, payload); err != nil { - return err - } - } - - return nil -} - -func (ce *chronicleExporter) uploadToChronicle(ctx context.Context, request *api.BatchCreateLogsRequest) error { - if ce.metrics != nil { - totalLogs := int64(len(request.GetBatch().GetEntries())) - defer ce.metrics.recordSent(totalLogs) - } - - _, err := ce.grpcClient.BatchCreateLogs(ctx, request, ce.buildOptions()...) - if err != nil { - errCode := status.Code(err) - switch errCode { - // These errors are potentially transient - case codes.Canceled, - codes.Unavailable, - codes.DeadlineExceeded, - codes.ResourceExhausted, - codes.Aborted: - return fmt.Errorf("upload logs to chronicle: %w", err) - default: - return consumererror.NewPermanent(fmt.Errorf("upload logs to chronicle: %w", err)) - } - } - - return nil -} - -func (ce *chronicleExporter) buildOptions() []grpc.CallOption { - opts := make([]grpc.CallOption, 0) - - if ce.cfg.Compression == grpcgzip.Name { - opts = append(opts, grpc.UseCompressor(grpcgzip.Name)) - } - - return opts -} - -func (ce *chronicleExporter) logsHTTPDataPusher(ctx context.Context, ld plog.Logs) error { - payloads, err := ce.marshaler.MarshalRawLogsForHTTP(ctx, ld) - if err != nil { - return fmt.Errorf("marshal logs: %w", err) - } - - for logType, logTypePayloads := range payloads { - for _, payload := range logTypePayloads { - if err := ce.uploadToChronicleHTTP(ctx, payload, logType); err != nil { - return err - } - } - } - - return nil -} - -func (ce *chronicleExporter) uploadToChronicleHTTP(ctx context.Context, logs *api.ImportLogsRequest, logType string) error { - - data, err := protojson.Marshal(logs) - if err != nil { - return fmt.Errorf("marshal protobuf logs to JSON: %w", err) - } - - var body io.Reader - - if ce.cfg.Compression == grpcgzip.Name { - var b bytes.Buffer - gz := gzip.NewWriter(&b) - if _, err := gz.Write(data); err != nil { - return fmt.Errorf("gzip write: %w", err) - } - if err := gz.Close(); err != nil { - return fmt.Errorf("gzip close: %w", err) - } - body = &b - } else { - body = bytes.NewBuffer(data) - } - - request, err := http.NewRequestWithContext(ctx, "POST", httpEndpoint(ce.cfg, logType), body) - if err != nil { - return fmt.Errorf("create request: %w", err) - } - - if ce.cfg.Compression == grpcgzip.Name { - request.Header.Set("Content-Encoding", "gzip") - } - - request.Header.Set("Content-Type", "application/json") - - resp, err := ce.httpClient.Do(request) - if err != nil { - return fmt.Errorf("send request to Chronicle: %w", err) - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err == nil && resp.StatusCode == http.StatusOK { - return nil - } - - if err != nil { - ce.set.Logger.Warn("Failed to read response body", zap.Error(err)) - } else { - ce.set.Logger.Warn("Received non-OK response from Chronicle", zap.String("status", resp.Status), zap.ByteString("response", respBody)) - } - - // TODO interpret with https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/internal/coreinternal/errorutil/http.go - statusErr := fmt.Errorf("upload logs to chronicle: %s", resp.Status) - switch resp.StatusCode { - case http.StatusInternalServerError, http.StatusServiceUnavailable: // potentially transient - return statusErr - default: - return consumererror.NewPermanent(statusErr) - } -} - -// Override for testing -var grpcClientParams = func(cfgEndpoint string, ts oauth2.TokenSource) (string, []grpc.DialOption) { - return cfgEndpoint + ":443", []grpc.DialOption{ - grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: ts}), - grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), - } -} - -// This uses the DataPlane URL for the request -// URL for the request: https://{region}-chronicle.googleapis.com/{version}/projects/{project}/location/{region}/instances/{customerID} -// Override for testing -var httpEndpoint = func(cfg *Config, logType string) string { - formatString := "https://%s-%s/v1alpha/projects/%s/locations/%s/instances/%s/logTypes/%s/logs:import" - return fmt.Sprintf(formatString, cfg.Location, cfg.Endpoint, cfg.Project, cfg.Location, cfg.CustomerID, logType) -} diff --git a/exporter/chronicleexporter/exporter_test.go b/exporter/chronicleexporter/exporter_test.go deleted file mode 100644 index 583c48f28..000000000 --- a/exporter/chronicleexporter/exporter_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright observIQ, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chronicleexporter - -import ( - "context" - "errors" - "testing" - - "github.com/golang/mock/gomock" - "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" - "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api/mocks" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component/componenttest" - "go.opentelemetry.io/collector/consumer/consumererror" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestLogsDataPusher(t *testing.T) { - - // Set up configuration, logger, and context - cfg := Config{Endpoint: defaultEndpoint} - ctx := context.Background() - - testCases := []struct { - desc string - setupExporter func() *chronicleExporter - setupMocks func(*mocks.MockIngestionServiceV2Client) - expectedErr string - permanentErr bool - }{ - { - desc: "successful push to Chronicle", - setupExporter: func() *chronicleExporter { - mockClient := mocks.NewMockIngestionServiceV2Client(gomock.NewController(t)) - marshaller := NewMockMarshaler(t) - marshaller.On("MarshalRawLogs", mock.Anything, mock.Anything).Return([]*api.BatchCreateLogsRequest{{}}, nil) - return &chronicleExporter{ - cfg: &cfg, - set: componenttest.NewNopTelemetrySettings(), - grpcClient: mockClient, - marshaler: marshaller, - } - }, - setupMocks: func(mockClient *mocks.MockIngestionServiceV2Client) { - mockClient.EXPECT().BatchCreateLogs(gomock.Any(), gomock.Any(), gomock.Any()).Return(&api.BatchCreateLogsResponse{}, nil) - }, - expectedErr: "", - }, - { - desc: "successful push to Chronicle with metrics", - setupExporter: func() *chronicleExporter { - mockClient := mocks.NewMockIngestionServiceV2Client(gomock.NewController(t)) - marshaller := NewMockMarshaler(t) - marshaller.On("MarshalRawLogs", mock.Anything, mock.Anything).Return([]*api.BatchCreateLogsRequest{{}}, nil) - cfg.CollectAgentMetrics = true - return &chronicleExporter{ - cfg: &cfg, - set: componenttest.NewNopTelemetrySettings(), - grpcClient: mockClient, - marshaler: marshaller, - } - }, - setupMocks: func(mockClient *mocks.MockIngestionServiceV2Client) { - mockClient.EXPECT().BatchCreateLogs(gomock.Any(), gomock.Any(), gomock.Any()).Return(&api.BatchCreateLogsResponse{}, nil) - }, - expectedErr: "", - }, - { - desc: "upload to Chronicle fails (transient)", - setupExporter: func() *chronicleExporter { - mockClient := mocks.NewMockIngestionServiceV2Client(gomock.NewController(t)) - marshaller := NewMockMarshaler(t) - marshaller.On("MarshalRawLogs", mock.Anything, mock.Anything).Return([]*api.BatchCreateLogsRequest{{}}, nil) - return &chronicleExporter{ - cfg: &cfg, - set: componenttest.NewNopTelemetrySettings(), - grpcClient: mockClient, - marshaler: marshaller, - } - }, - setupMocks: func(mockClient *mocks.MockIngestionServiceV2Client) { - // Simulate an error returned from the Chronicle service - err := status.Error(codes.Unavailable, "service unavailable") - mockClient.EXPECT().BatchCreateLogs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, err) - }, - expectedErr: "service unavailable", - }, - { - desc: "upload to Chronicle fails (permanent)", - setupExporter: func() *chronicleExporter { - mockClient := mocks.NewMockIngestionServiceV2Client(gomock.NewController(t)) - marshaller := NewMockMarshaler(t) - marshaller.On("MarshalRawLogs", mock.Anything, mock.Anything).Return([]*api.BatchCreateLogsRequest{{}}, nil) - return &chronicleExporter{ - cfg: &cfg, - set: componenttest.NewNopTelemetrySettings(), - grpcClient: mockClient, - marshaler: marshaller, - } - }, - setupMocks: func(mockClient *mocks.MockIngestionServiceV2Client) { - err := status.Error(codes.InvalidArgument, "Invalid argument detected.") - mockClient.EXPECT().BatchCreateLogs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, err) - }, - expectedErr: "Invalid argument detected.", - permanentErr: true, - }, - { - desc: "marshaler error", - setupExporter: func() *chronicleExporter { - mockClient := mocks.NewMockIngestionServiceV2Client(gomock.NewController(t)) - marshaller := NewMockMarshaler(t) - // Simulate an error during log marshaling - marshaller.On("MarshalRawLogs", mock.Anything, mock.Anything).Return(nil, errors.New("marshal error")) - return &chronicleExporter{ - cfg: &cfg, - set: componenttest.NewNopTelemetrySettings(), - grpcClient: mockClient, - marshaler: marshaller, - } - }, - setupMocks: func(_ *mocks.MockIngestionServiceV2Client) { - // No need to setup mocks for the client as the error occurs before the client is used - }, - expectedErr: "marshal error", - }, - { - desc: "empty log records", - setupExporter: func() *chronicleExporter { - mockClient := mocks.NewMockIngestionServiceV2Client(gomock.NewController(t)) - marshaller := NewMockMarshaler(t) - // Return an empty slice to simulate no logs to push - marshaller.On("MarshalRawLogs", mock.Anything, mock.Anything).Return([]*api.BatchCreateLogsRequest{}, nil) - return &chronicleExporter{ - cfg: &cfg, - set: componenttest.NewNopTelemetrySettings(), - grpcClient: mockClient, - marshaler: marshaller, - } - }, - setupMocks: func(_ *mocks.MockIngestionServiceV2Client) { - // Expect no calls to BatchCreateLogs since there are no logs to push - }, - expectedErr: "", - }, - } - - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - exporter := tc.setupExporter() - tc.setupMocks(exporter.grpcClient.(*mocks.MockIngestionServiceV2Client)) - - // Create a dummy plog.Logs to pass to logsDataPusher - logs := mockLogs(mockLogRecord("Test body", map[string]any{"key1": "value1"})) - - err := exporter.logsDataPusher(ctx, logs) - - if tc.expectedErr == "" { - require.NoError(t, err) - } else { - require.ErrorContains(t, err, tc.expectedErr) - if tc.permanentErr { - require.True(t, consumererror.IsPermanent(err), "Expected error to be permanent") - } else { - require.False(t, consumererror.IsPermanent(err), "Expected error to be transient") - } - } - }) - } -} diff --git a/exporter/chronicleexporter/factory.go b/exporter/chronicleexporter/factory.go index 61a9a140e..45a914aa9 100644 --- a/exporter/chronicleexporter/factory.go +++ b/exporter/chronicleexporter/factory.go @@ -16,7 +16,6 @@ package chronicleexporter import ( "context" - "errors" "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/metadata" "go.opentelemetry.io/collector/component" @@ -64,30 +63,28 @@ func createLogsExporter( ctx context.Context, params exporter.Settings, cfg component.Config, -) (exporter.Logs, error) { - chronicleCfg, ok := cfg.(*Config) - if !ok { - return nil, errors.New("invalid config type") - } - - exp, err := newExporter(chronicleCfg, params, params.ID.String()) - if err != nil { - return nil, err - } - - pusher := exp.logsDataPusher - if chronicleCfg.Protocol == protocolHTTPS { - pusher = exp.logsHTTPDataPusher +) (exp exporter.Logs, err error) { + c := cfg.(*Config) + if c.Protocol == protocolHTTPS { + exp, err = newHTTPExporter(c, params) + if err != nil { + return nil, err + } + } else { + exp, err = newGRPCExporter(c, params) + if err != nil { + return nil, err + } } return exporterhelper.NewLogs( ctx, params, - chronicleCfg, - pusher, + c, + exp.ConsumeLogs, exporterhelper.WithCapabilities(exp.Capabilities()), - exporterhelper.WithTimeout(chronicleCfg.TimeoutConfig), - exporterhelper.WithQueue(chronicleCfg.QueueConfig), - exporterhelper.WithRetry(chronicleCfg.BackOffConfig), + exporterhelper.WithTimeout(c.TimeoutConfig), + exporterhelper.WithQueue(c.QueueConfig), + exporterhelper.WithRetry(c.BackOffConfig), exporterhelper.WithStart(exp.Start), exporterhelper.WithShutdown(exp.Shutdown), ) diff --git a/exporter/chronicleexporter/factory_test.go b/exporter/chronicleexporter/factory_test.go index a542324b2..aa77d905b 100644 --- a/exporter/chronicleexporter/factory_test.go +++ b/exporter/chronicleexporter/factory_test.go @@ -38,6 +38,6 @@ func Test_createDefaultConfig(t *testing.T) { BatchRequestSizeLimitHTTP: defaultBatchRequestSizeLimitHTTP, } - actual := createDefaultConfig() + actual := NewFactory().CreateDefaultConfig() require.Equal(t, expectedCfg, actual) } diff --git a/exporter/chronicleexporter/grpc_exporter.go b/exporter/chronicleexporter/grpc_exporter.go new file mode 100644 index 000000000..20d4185ea --- /dev/null +++ b/exporter/chronicleexporter/grpc_exporter.go @@ -0,0 +1,164 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chronicleexporter + +import ( + "context" + "fmt" + "net/http" + + "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumererror" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/pdata/plog" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" + grpcgzip "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/status" +) + +const grpcScope = "https://www.googleapis.com/auth/malachite-ingestion" + +type grpcExporter struct { + cfg *Config + set component.TelemetrySettings + exporterID string + marshaler *protoMarshaler + + client api.IngestionServiceV2Client + conn *grpc.ClientConn + metrics *hostMetricsReporter +} + +func newGRPCExporter(cfg *Config, params exporter.Settings) (*grpcExporter, error) { + customerID, err := uuid.Parse(cfg.CustomerID) + if err != nil { + return nil, fmt.Errorf("parse customer ID: %w", err) + } + marshaller, err := newProtoMarshaler(*cfg, params.TelemetrySettings, customerID[:]) + if err != nil { + return nil, fmt.Errorf("create proto marshaller: %w", err) + } + return &grpcExporter{ + cfg: cfg, + set: params.TelemetrySettings, + exporterID: params.ID.String(), + marshaler: marshaller, + }, nil +} + +func (exp *grpcExporter) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: false} +} + +func (exp *grpcExporter) Start(ctx context.Context, _ component.Host) error { + conn, err := getGRPCClient(ctx, exp.cfg) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + exp.conn = conn + exp.client = api.NewIngestionServiceV2Client(conn) + + if exp.cfg.CollectAgentMetrics { + f := func(ctx context.Context, request *api.BatchCreateEventsRequest) error { + _, err := exp.client.BatchCreateEvents(ctx, request) + return err + } + metrics, err := newHostMetricsReporter(exp.cfg, exp.set, exp.exporterID, f) + if err != nil { + return fmt.Errorf("create metrics reporter: %w", err) + } + exp.metrics = metrics + exp.metrics.start() + } + + return nil +} + +func (exp *grpcExporter) Shutdown(context.Context) error { + defer http.DefaultTransport.(*http.Transport).CloseIdleConnections() + if exp.metrics != nil { + exp.metrics.shutdown() + } + if exp.conn != nil { + if err := exp.conn.Close(); err != nil { + return fmt.Errorf("connection close: %s", err) + } + } + return nil +} + +func (exp *grpcExporter) ConsumeLogs(ctx context.Context, ld plog.Logs) error { + payloads, err := exp.marshaler.MarshalRawLogs(ctx, ld) + if err != nil { + return fmt.Errorf("marshal logs: %w", err) + } + for _, payload := range payloads { + if err := exp.uploadToChronicle(ctx, payload); err != nil { + return err + } + } + return nil +} + +func (exp *grpcExporter) uploadToChronicle(ctx context.Context, request *api.BatchCreateLogsRequest) error { + if exp.metrics != nil { + totalLogs := int64(len(request.GetBatch().GetEntries())) + defer exp.metrics.recordSent(totalLogs) + } + _, err := exp.client.BatchCreateLogs(ctx, request, exp.buildOptions()...) + if err != nil { + errCode := status.Code(err) + switch errCode { + // These errors are potentially transient + // TODO interpret with https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/internal/coreinternal/errorutil/grpc.go + case codes.Canceled, + codes.Unavailable, + codes.DeadlineExceeded, + codes.ResourceExhausted, + codes.Aborted: + return fmt.Errorf("upload logs to chronicle: %w", err) + default: + return consumererror.NewPermanent(fmt.Errorf("upload logs to chronicle: %w", err)) + } + } + return nil +} + +func (exp *grpcExporter) buildOptions() []grpc.CallOption { + opts := make([]grpc.CallOption, 0) + if exp.cfg.Compression == grpcgzip.Name { + opts = append(opts, grpc.UseCompressor(grpcgzip.Name)) + } + return opts +} + +var getGRPCClient func(context.Context, *Config) (*grpc.ClientConn, error) = buildGRPCClient + +func buildGRPCClient(ctx context.Context, cfg *Config) (*grpc.ClientConn, error) { + creds, err := googleCredentials(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("load Google credentials: %w", err) + } + return grpc.NewClient(cfg.Endpoint+":443", + grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: creds.TokenSource}), + grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), + ) +} diff --git a/exporter/chronicleexporter/hostmetrics.go b/exporter/chronicleexporter/hostmetrics.go index 99b7a7422..bfc6e2976 100644 --- a/exporter/chronicleexporter/hostmetrics.go +++ b/exporter/chronicleexporter/hostmetrics.go @@ -68,9 +68,9 @@ func newHostMetricsReporter(cfg *Config, set component.TelemetrySettings, export return &hostMetricsReporter{ set: set, send: send, + startTime: now, agentID: agentID[:], exporterID: exporterID, - startTime: now, customerID: customerID[:], namespace: cfg.Namespace, stats: &api.AgentStatsEvent{ @@ -108,6 +108,13 @@ func (hmr *hostMetricsReporter) start() { }() } +func (hmr *hostMetricsReporter) shutdown() { + if hmr.cancel != nil { + hmr.cancel() + hmr.wg.Wait() + } +} + func (hmr *hostMetricsReporter) getAndReset() *api.BatchCreateEventsRequest { hmr.mutex.Lock() defer hmr.mutex.Unlock() @@ -143,13 +150,6 @@ func (hmr *hostMetricsReporter) getAndReset() *api.BatchCreateEventsRequest { return request } -func (hmr *hostMetricsReporter) shutdown() { - if hmr.cancel != nil { - hmr.cancel() - hmr.wg.Wait() - } -} - func (hmr *hostMetricsReporter) resetStats() { hmr.stats = &api.AgentStatsEvent{ ExporterStats: []*api.ExporterStats{ diff --git a/exporter/chronicleexporter/http_exporter.go b/exporter/chronicleexporter/http_exporter.go new file mode 100644 index 000000000..34a3ef7f7 --- /dev/null +++ b/exporter/chronicleexporter/http_exporter.go @@ -0,0 +1,178 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chronicleexporter + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/http" + + "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumererror" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + "golang.org/x/oauth2" + grpcgzip "google.golang.org/grpc/encoding/gzip" + "google.golang.org/protobuf/encoding/protojson" +) + +const httpScope = "https://www.googleapis.com/auth/cloud-platform" + +type httpExporter struct { + cfg *Config + set component.TelemetrySettings + marshaler *protoMarshaler + client *http.Client +} + +func newHTTPExporter(cfg *Config, params exporter.Settings) (*httpExporter, error) { + customerID, err := uuid.Parse(cfg.CustomerID) + if err != nil { + return nil, fmt.Errorf("parse customer ID: %w", err) + } + marshaller, err := newProtoMarshaler(*cfg, params.TelemetrySettings, customerID[:]) + if err != nil { + return nil, fmt.Errorf("create proto marshaller: %w", err) + } + return &httpExporter{ + cfg: cfg, + set: params.TelemetrySettings, + marshaler: marshaller, + }, nil +} + +func (exp *httpExporter) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: false} +} + +func (exp *httpExporter) Start(ctx context.Context, _ component.Host) error { + ts, err := getTokenSource(ctx, exp.cfg) + if err != nil { + return fmt.Errorf("load Google credentials: %w", err) + } + exp.client = oauth2.NewClient(context.Background(), ts) + return nil +} + +func (exp *httpExporter) Shutdown(context.Context) error { + defer http.DefaultTransport.(*http.Transport).CloseIdleConnections() + t := exp.client.Transport.(*oauth2.Transport) + if t.Base != nil { + t.Base.(*http.Transport).CloseIdleConnections() + } + return nil +} + +func (exp *httpExporter) ConsumeLogs(ctx context.Context, ld plog.Logs) error { + payloads, err := exp.marshaler.MarshalRawLogsForHTTP(ctx, ld) + if err != nil { + return fmt.Errorf("marshal logs: %w", err) + } + for logType, logTypePayloads := range payloads { + for _, payload := range logTypePayloads { + if err := exp.uploadToChronicleHTTP(ctx, payload, logType); err != nil { + return fmt.Errorf("upload to chronicle: %w", err) + } + } + } + return nil +} + +func (exp *httpExporter) uploadToChronicleHTTP(ctx context.Context, logs *api.ImportLogsRequest, logType string) error { + data, err := protojson.Marshal(logs) + if err != nil { + return fmt.Errorf("marshal protobuf logs to JSON: %w", err) + } + + var body io.Reader + if exp.cfg.Compression == grpcgzip.Name { + var b bytes.Buffer + gz := gzip.NewWriter(&b) + if _, err := gz.Write(data); err != nil { + return fmt.Errorf("gzip write: %w", err) + } + if err := gz.Close(); err != nil { + return fmt.Errorf("gzip close: %w", err) + } + body = &b + } else { + body = bytes.NewBuffer(data) + } + + request, err := http.NewRequestWithContext(ctx, "POST", getHTTPEndpoint(exp.cfg, logType), body) + if err != nil { + return fmt.Errorf("create request: %w", err) + } + + if exp.cfg.Compression == grpcgzip.Name { + request.Header.Set("Content-Encoding", "gzip") + } + + request.Header.Set("Content-Type", "application/json") + + resp, err := exp.client.Do(request) + if err != nil { + return fmt.Errorf("send request to Chronicle: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err == nil && resp.StatusCode == http.StatusOK { + return nil + } + + if err != nil { + exp.set.Logger.Warn("Failed to read response body", zap.Error(err)) + } else { + exp.set.Logger.Warn("Received non-OK response from Chronicle", zap.String("status", resp.Status), zap.ByteString("response", respBody)) + } + + // TODO interpret with https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/internal/coreinternal/errorutil/http.go + statusErr := errors.New(resp.Status) + switch resp.StatusCode { + case http.StatusInternalServerError, http.StatusServiceUnavailable: // potentially transient + return statusErr + default: + return consumererror.NewPermanent(statusErr) + } +} + +// Override for testing +var getHTTPEndpoint = buildHTTPEndpoint + +// This uses the DataPlane URL for the request +// URL for the request: https://{region}-chronicle.googleapis.com/{version}/projects/{project}/location/{region}/instances/{customerID} +func buildHTTPEndpoint(cfg *Config, logType string) string { + formatString := "https://%s-%s/v1alpha/projects/%s/locations/%s/instances/%s/logTypes/%s/logs:import" + return fmt.Sprintf(formatString, cfg.Location, cfg.Endpoint, cfg.Project, cfg.Location, cfg.CustomerID, logType) +} + +var getTokenSource func(context.Context, *Config) (oauth2.TokenSource, error) = googleTokenSource + +func googleTokenSource(ctx context.Context, cfg *Config) (oauth2.TokenSource, error) { + creds, err := googleCredentials(ctx, cfg) + if err != nil { + return nil, err + } + return creds.TokenSource, nil +} diff --git a/exporter/chronicleexporter/marshal.go b/exporter/chronicleexporter/marshal.go index 832f769b3..aaea72a86 100644 --- a/exporter/chronicleexporter/marshal.go +++ b/exporter/chronicleexporter/marshal.go @@ -48,26 +48,21 @@ var supportedLogTypes = map[string]string{ "sql_server": "MICROSOFT_SQL", } -//go:generate mockery --name logMarshaler --filename mock_log_marshaler.go --structname MockMarshaler --inpackage -type logMarshaler interface { - MarshalRawLogs(ctx context.Context, ld plog.Logs) ([]*api.BatchCreateLogsRequest, error) - MarshalRawLogsForHTTP(ctx context.Context, ld plog.Logs) (map[string][]*api.ImportLogsRequest, error) -} type protoMarshaler struct { - cfg Config - teleSettings component.TelemetrySettings - startTime time.Time - customerID []byte - collectorID []byte + cfg Config + set component.TelemetrySettings + startTime time.Time + customerID []byte + collectorID []byte } -func newProtoMarshaler(cfg Config, teleSettings component.TelemetrySettings, customerID []byte) (*protoMarshaler, error) { +func newProtoMarshaler(cfg Config, set component.TelemetrySettings, customerID []byte) (*protoMarshaler, error) { return &protoMarshaler{ - startTime: time.Now(), - cfg: cfg, - teleSettings: teleSettings, - customerID: customerID[:], - collectorID: chronicleCollectorID[:], + startTime: time.Now(), + cfg: cfg, + set: set, + customerID: customerID[:], + collectorID: chronicleCollectorID[:], }, nil } @@ -93,7 +88,7 @@ func (m *protoMarshaler) extractRawLogs(ctx context.Context, ld plog.Logs) (map[ rawLog, logType, namespace, ingestionLabels, err := m.processLogRecord(ctx, logRecord, scopeLog, resourceLog) if err != nil { - m.teleSettings.Logger.Error("Error processing log record", zap.Error(err)) + m.set.Logger.Error("Error processing log record", zap.Error(err)) continue } @@ -323,7 +318,7 @@ func (m *protoMarshaler) getRawField(ctx context.Context, field string, logRecor return "", nil } - lrExpr, err := expr.NewOTTLLogRecordExpression(field, m.teleSettings) + lrExpr, err := expr.NewOTTLLogRecordExpression(field, m.set) if err != nil { return "", fmt.Errorf("raw_log_field is invalid: %s", err) } @@ -431,7 +426,7 @@ func (m *protoMarshaler) enforceMaximumsGRPCRequest(request *api.BatchCreateLogs } if len(entries) < 2 { - m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + m.set.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) return []*api.BatchCreateLogsRequest{} } @@ -484,7 +479,7 @@ func (m *protoMarshaler) extractRawHTTPLogs(ctx context.Context, ld plog.Logs) ( logRecord := scopeLog.LogRecords().At(k) rawLog, logType, namespace, ingestionLabels, err := m.processHTTPLogRecord(ctx, logRecord, scopeLog, resourceLog) if err != nil { - m.teleSettings.Logger.Error("Error processing log record", zap.Error(err)) + m.set.Logger.Error("Error processing log record", zap.Error(err)) continue } @@ -542,7 +537,7 @@ func (m *protoMarshaler) enforceMaximumsHTTPRequest(request *api.ImportLogsReque } if len(logs) < 2 { - m.teleSettings.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + m.set.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) return []*api.ImportLogsRequest{} } diff --git a/exporter/chronicleexporter/marshal_test.go b/exporter/chronicleexporter/marshal_test.go index 4a5dfb59c..0540c1125 100644 --- a/exporter/chronicleexporter/marshal_test.go +++ b/exporter/chronicleexporter/marshal_test.go @@ -1673,7 +1673,7 @@ func Test_getRawField(t *testing.T) { for _, tc := range getRawFieldCases { t.Run(tc.name, func(t *testing.T) { m := &protoMarshaler{} - m.teleSettings.Logger = zap.NewNop() + m.set.Logger = zap.NewNop() ctx := context.Background() @@ -1691,7 +1691,7 @@ func Test_getRawField(t *testing.T) { func Benchmark_getRawField(b *testing.B) { m := &protoMarshaler{} - m.teleSettings.Logger = zap.NewNop() + m.set.Logger = zap.NewNop() ctx := context.Background() From 7a3a1bb30826aa515ee12334458c73d71ba2b8dc Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Thu, 12 Dec 2024 09:35:33 -0500 Subject: [PATCH 7/8] Minimize replaced code in grpc tests --- exporter/chronicleexporter/grpc_exporter.go | 23 +++++++++++---------- exporter/chronicleexporter/http_exporter.go | 20 ++++-------------- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/exporter/chronicleexporter/grpc_exporter.go b/exporter/chronicleexporter/grpc_exporter.go index 20d4185ea..8f57ae6f9 100644 --- a/exporter/chronicleexporter/grpc_exporter.go +++ b/exporter/chronicleexporter/grpc_exporter.go @@ -26,6 +26,7 @@ import ( "go.opentelemetry.io/collector/consumer/consumererror" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/plog" + "golang.org/x/oauth2" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -69,7 +70,12 @@ func (exp *grpcExporter) Capabilities() consumer.Capabilities { } func (exp *grpcExporter) Start(ctx context.Context, _ component.Host) error { - conn, err := getGRPCClient(ctx, exp.cfg) + ts, err := tokenSource(ctx, exp.cfg) + if err != nil { + return fmt.Errorf("load Google credentials: %w", err) + } + endpoint, dialOpts := grpcClientParams(exp.cfg.Endpoint, ts) + conn, err := grpc.NewClient(endpoint, dialOpts...) if err != nil { return fmt.Errorf("dial: %w", err) } @@ -150,15 +156,10 @@ func (exp *grpcExporter) buildOptions() []grpc.CallOption { return opts } -var getGRPCClient func(context.Context, *Config) (*grpc.ClientConn, error) = buildGRPCClient - -func buildGRPCClient(ctx context.Context, cfg *Config) (*grpc.ClientConn, error) { - creds, err := googleCredentials(ctx, cfg) - if err != nil { - return nil, fmt.Errorf("load Google credentials: %w", err) - } - return grpc.NewClient(cfg.Endpoint+":443", - grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: creds.TokenSource}), +// Override for testing +var grpcClientParams = func(cfgEndpoint string, ts oauth2.TokenSource) (string, []grpc.DialOption) { + return cfgEndpoint + ":443", []grpc.DialOption{ + grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: ts}), grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), - ) + } } diff --git a/exporter/chronicleexporter/http_exporter.go b/exporter/chronicleexporter/http_exporter.go index 34a3ef7f7..ac433281d 100644 --- a/exporter/chronicleexporter/http_exporter.go +++ b/exporter/chronicleexporter/http_exporter.go @@ -66,7 +66,7 @@ func (exp *httpExporter) Capabilities() consumer.Capabilities { } func (exp *httpExporter) Start(ctx context.Context, _ component.Host) error { - ts, err := getTokenSource(ctx, exp.cfg) + ts, err := tokenSource(ctx, exp.cfg) if err != nil { return fmt.Errorf("load Google credentials: %w", err) } @@ -119,7 +119,7 @@ func (exp *httpExporter) uploadToChronicleHTTP(ctx context.Context, logs *api.Im body = bytes.NewBuffer(data) } - request, err := http.NewRequestWithContext(ctx, "POST", getHTTPEndpoint(exp.cfg, logType), body) + request, err := http.NewRequestWithContext(ctx, "POST", httpEndpoint(exp.cfg, logType), body) if err != nil { return fmt.Errorf("create request: %w", err) } @@ -157,22 +157,10 @@ func (exp *httpExporter) uploadToChronicleHTTP(ctx context.Context, logs *api.Im } } -// Override for testing -var getHTTPEndpoint = buildHTTPEndpoint - // This uses the DataPlane URL for the request // URL for the request: https://{region}-chronicle.googleapis.com/{version}/projects/{project}/location/{region}/instances/{customerID} -func buildHTTPEndpoint(cfg *Config, logType string) string { +// Override for testing +var httpEndpoint = func(cfg *Config, logType string) string { formatString := "https://%s-%s/v1alpha/projects/%s/locations/%s/instances/%s/logTypes/%s/logs:import" return fmt.Sprintf(formatString, cfg.Location, cfg.Endpoint, cfg.Project, cfg.Location, cfg.CustomerID, logType) } - -var getTokenSource func(context.Context, *Config) (oauth2.TokenSource, error) = googleTokenSource - -func googleTokenSource(ctx context.Context, cfg *Config) (oauth2.TokenSource, error) { - creds, err := googleCredentials(ctx, cfg) - if err != nil { - return nil, err - } - return creds.TokenSource, nil -} From 160c0d7ea03ad1ca8677dca57c754c14acee8f9b Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Thu, 12 Dec 2024 10:57:51 -0500 Subject: [PATCH 8/8] Separate and internalize marshalers, and test exported surface only. --- exporter/chronicleexporter/grpc_exporter.go | 25 +- exporter/chronicleexporter/hostmetrics.go | 3 +- exporter/chronicleexporter/http_exporter.go | 30 +- .../chronicleexporter/http_exporter_test.go | 4 +- .../chronicleexporter/internal/ccid/ccid.go | 23 + .../internal/marshal/grpc.go | 247 +++ .../internal/marshal/grpc_test.go | 785 ++++++++ .../internal/marshal/http.go | 244 +++ .../internal/marshal/http_test.go | 769 ++++++++ .../internal/marshal/marshal.go | 206 ++ .../internal/marshal/marshal_test.go | 225 +++ exporter/chronicleexporter/marshal.go | 573 ------ exporter/chronicleexporter/marshal_test.go | 1706 ----------------- 13 files changed, 2538 insertions(+), 2302 deletions(-) create mode 100644 exporter/chronicleexporter/internal/ccid/ccid.go create mode 100644 exporter/chronicleexporter/internal/marshal/grpc.go create mode 100644 exporter/chronicleexporter/internal/marshal/grpc_test.go create mode 100644 exporter/chronicleexporter/internal/marshal/http.go create mode 100644 exporter/chronicleexporter/internal/marshal/http_test.go create mode 100644 exporter/chronicleexporter/internal/marshal/marshal.go create mode 100644 exporter/chronicleexporter/internal/marshal/marshal_test.go delete mode 100644 exporter/chronicleexporter/marshal.go delete mode 100644 exporter/chronicleexporter/marshal_test.go diff --git a/exporter/chronicleexporter/grpc_exporter.go b/exporter/chronicleexporter/grpc_exporter.go index 8f57ae6f9..7a450d158 100644 --- a/exporter/chronicleexporter/grpc_exporter.go +++ b/exporter/chronicleexporter/grpc_exporter.go @@ -19,7 +19,7 @@ import ( "fmt" "net/http" - "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/marshal" "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" @@ -41,7 +41,7 @@ type grpcExporter struct { cfg *Config set component.TelemetrySettings exporterID string - marshaler *protoMarshaler + marshaler *marshal.GRPC client api.IngestionServiceV2Client conn *grpc.ClientConn @@ -49,19 +49,24 @@ type grpcExporter struct { } func newGRPCExporter(cfg *Config, params exporter.Settings) (*grpcExporter, error) { - customerID, err := uuid.Parse(cfg.CustomerID) + marshaler, err := marshal.NewGRPC(marshal.Config{ + CustomerID: cfg.CustomerID, + Namespace: cfg.Namespace, + LogType: cfg.LogType, + RawLogField: cfg.RawLogField, + OverrideLogType: cfg.OverrideLogType, + IngestionLabels: cfg.IngestionLabels, + BatchRequestSizeLimit: cfg.BatchRequestSizeLimitGRPC, + BatchLogCountLimit: cfg.BatchLogCountLimitGRPC, + }, params.TelemetrySettings) if err != nil { - return nil, fmt.Errorf("parse customer ID: %w", err) - } - marshaller, err := newProtoMarshaler(*cfg, params.TelemetrySettings, customerID[:]) - if err != nil { - return nil, fmt.Errorf("create proto marshaller: %w", err) + return nil, fmt.Errorf("create proto marshaler: %w", err) } return &grpcExporter{ cfg: cfg, set: params.TelemetrySettings, exporterID: params.ID.String(), - marshaler: marshaller, + marshaler: marshaler, }, nil } @@ -112,7 +117,7 @@ func (exp *grpcExporter) Shutdown(context.Context) error { } func (exp *grpcExporter) ConsumeLogs(ctx context.Context, ld plog.Logs) error { - payloads, err := exp.marshaler.MarshalRawLogs(ctx, ld) + payloads, err := exp.marshaler.MarshalLogs(ctx, ld) if err != nil { return fmt.Errorf("marshal logs: %w", err) } diff --git a/exporter/chronicleexporter/hostmetrics.go b/exporter/chronicleexporter/hostmetrics.go index bfc6e2976..7feadb67f 100644 --- a/exporter/chronicleexporter/hostmetrics.go +++ b/exporter/chronicleexporter/hostmetrics.go @@ -22,6 +22,7 @@ import ( "time" "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/ccid" "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" "github.com/shirou/gopsutil/v3/process" "go.opentelemetry.io/collector/component" @@ -122,7 +123,7 @@ func (hmr *hostMetricsReporter) getAndReset() *api.BatchCreateEventsRequest { now := timestamppb.Now() batchID := uuid.New() source := &api.EventSource{ - CollectorId: chronicleCollectorID[:], + CollectorId: ccid.ChronicleCollectorID[:], Namespace: hmr.namespace, CustomerId: hmr.customerID, } diff --git a/exporter/chronicleexporter/http_exporter.go b/exporter/chronicleexporter/http_exporter.go index ac433281d..6fbc8a9fa 100644 --- a/exporter/chronicleexporter/http_exporter.go +++ b/exporter/chronicleexporter/http_exporter.go @@ -23,7 +23,7 @@ import ( "io" "net/http" - "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/marshal" "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" @@ -41,23 +41,33 @@ const httpScope = "https://www.googleapis.com/auth/cloud-platform" type httpExporter struct { cfg *Config set component.TelemetrySettings - marshaler *protoMarshaler + marshaler *marshal.HTTP client *http.Client } func newHTTPExporter(cfg *Config, params exporter.Settings) (*httpExporter, error) { - customerID, err := uuid.Parse(cfg.CustomerID) + marshaler, err := marshal.NewHTTP(marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: cfg.CustomerID, + Namespace: cfg.Namespace, + LogType: cfg.LogType, + RawLogField: cfg.RawLogField, + OverrideLogType: cfg.OverrideLogType, + IngestionLabels: cfg.IngestionLabels, + BatchRequestSizeLimit: cfg.BatchRequestSizeLimitHTTP, + BatchLogCountLimit: cfg.BatchLogCountLimitHTTP, + }, + Project: cfg.Project, + Location: cfg.Location, + Forwarder: cfg.Forwarder, + }, params.TelemetrySettings) if err != nil { - return nil, fmt.Errorf("parse customer ID: %w", err) - } - marshaller, err := newProtoMarshaler(*cfg, params.TelemetrySettings, customerID[:]) - if err != nil { - return nil, fmt.Errorf("create proto marshaller: %w", err) + return nil, fmt.Errorf("create proto marshaler: %w", err) } return &httpExporter{ cfg: cfg, set: params.TelemetrySettings, - marshaler: marshaller, + marshaler: marshaler, }, nil } @@ -84,7 +94,7 @@ func (exp *httpExporter) Shutdown(context.Context) error { } func (exp *httpExporter) ConsumeLogs(ctx context.Context, ld plog.Logs) error { - payloads, err := exp.marshaler.MarshalRawLogsForHTTP(ctx, ld) + payloads, err := exp.marshaler.MarshalLogs(ctx, ld) if err != nil { return fmt.Errorf("marshal logs: %w", err) } diff --git a/exporter/chronicleexporter/http_exporter_test.go b/exporter/chronicleexporter/http_exporter_test.go index f20b27445..491c10d8e 100644 --- a/exporter/chronicleexporter/http_exporter_test.go +++ b/exporter/chronicleexporter/http_exporter_test.go @@ -125,7 +125,7 @@ func TestHTTPExporter(t *testing.T) { return logs }(), expectedRequests: 1, - expectedErr: "upload logs to chronicle: 503 Service Unavailable", + expectedErr: "upload to chronicle: 503 Service Unavailable", permanentErr: false, }, { @@ -144,7 +144,7 @@ func TestHTTPExporter(t *testing.T) { return logs }(), expectedRequests: 1, - expectedErr: "Permanent error: upload logs to chronicle: 401 Unauthorized", + expectedErr: "upload to chronicle: Permanent error: 401 Unauthorized", permanentErr: true, }, } diff --git a/exporter/chronicleexporter/internal/ccid/ccid.go b/exporter/chronicleexporter/internal/ccid/ccid.go new file mode 100644 index 000000000..63541aa4d --- /dev/null +++ b/exporter/chronicleexporter/internal/ccid/ccid.go @@ -0,0 +1,23 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ccid exposes a hardcoded UUID that is used to identify bindplane agents in Chronicle. +package ccid + +import ( + "github.com/google/uuid" +) + +// ChronicleCollectorID is a specific collector ID for Chronicle. It's used to identify bindplane agents in Chronicle. +var ChronicleCollectorID = uuid.MustParse("aaaa1111-aaaa-1111-aaaa-1111aaaa1111") diff --git a/exporter/chronicleexporter/internal/marshal/grpc.go b/exporter/chronicleexporter/internal/marshal/grpc.go new file mode 100644 index 000000000..2818457e3 --- /dev/null +++ b/exporter/chronicleexporter/internal/marshal/grpc.go @@ -0,0 +1,247 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package marshal contains marshalers for grpc and http +package marshal + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// GRPC is a marshaler for gRPC protos +type GRPC struct { + protoMarshaler +} + +// NewGRPC creates a new GRPC marshaler +func NewGRPC(cfg Config, set component.TelemetrySettings) (*GRPC, error) { + m, err := newProtoMarshaler(cfg, set) + if err != nil { + return nil, err + } + return &GRPC{protoMarshaler: *m}, nil +} + +// MarshalLogs marshals logs into gRPC requests +func (m *GRPC) MarshalLogs(ctx context.Context, ld plog.Logs) ([]*api.BatchCreateLogsRequest, error) { + rawLogs, namespace, ingestionLabels, err := m.extractRawLogs(ctx, ld) + if err != nil { + return nil, fmt.Errorf("extract raw logs: %w", err) + } + return m.constructPayloads(rawLogs, namespace, ingestionLabels), nil +} + +func (m *GRPC) extractRawLogs(ctx context.Context, ld plog.Logs) (map[string][]*api.LogEntry, map[string]string, map[string][]*api.Label, error) { + entries := make(map[string][]*api.LogEntry) + namespaceMap := make(map[string]string) + ingestionLabelsMap := make(map[string][]*api.Label) + + for i := 0; i < ld.ResourceLogs().Len(); i++ { + resourceLog := ld.ResourceLogs().At(i) + for j := 0; j < resourceLog.ScopeLogs().Len(); j++ { + scopeLog := resourceLog.ScopeLogs().At(j) + for k := 0; k < scopeLog.LogRecords().Len(); k++ { + logRecord := scopeLog.LogRecords().At(k) + rawLog, logType, namespace, ingestionLabels, err := m.processLogRecord(ctx, logRecord, scopeLog, resourceLog) + + if err != nil { + m.set.Logger.Error("Error processing log record", zap.Error(err)) + continue + } + + if rawLog == "" { + continue + } + + var timestamp time.Time + + if logRecord.Timestamp() != 0 { + timestamp = logRecord.Timestamp().AsTime() + } else { + timestamp = logRecord.ObservedTimestamp().AsTime() + } + + entry := &api.LogEntry{ + Timestamp: timestamppb.New(timestamp), + CollectionTime: timestamppb.New(logRecord.ObservedTimestamp().AsTime()), + Data: []byte(rawLog), + } + entries[logType] = append(entries[logType], entry) + // each logType maps to exactly 1 namespace value + if namespace != "" { + if _, ok := namespaceMap[logType]; !ok { + namespaceMap[logType] = namespace + } + } + if len(ingestionLabels) > 0 { + // each logType maps to a list of ingestion labels + if _, exists := ingestionLabelsMap[logType]; !exists { + ingestionLabelsMap[logType] = make([]*api.Label, 0) + } + existingLabels := make(map[string]struct{}) + for _, label := range ingestionLabelsMap[logType] { + existingLabels[label.Key] = struct{}{} + } + for _, label := range ingestionLabels { + // only add to ingestionLabelsMap if the label is unique + if _, ok := existingLabels[label.Key]; !ok { + ingestionLabelsMap[logType] = append(ingestionLabelsMap[logType], label) + existingLabels[label.Key] = struct{}{} + } + } + } + } + } + } + return entries, namespaceMap, ingestionLabelsMap, nil +} + +func (m *GRPC) processLogRecord(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, string, string, []*api.Label, error) { + rawLog, err := m.getRawLog(ctx, logRecord, scope, resource) + if err != nil { + return "", "", "", nil, err + } + logType, err := m.getLogType(ctx, logRecord, scope, resource) + if err != nil { + return "", "", "", nil, err + } + namespace, err := m.getNamespace(ctx, logRecord, scope, resource) + if err != nil { + return "", "", "", nil, err + } + ingestionLabels, err := m.getIngestionLabels(logRecord) + if err != nil { + return "", "", "", nil, err + } + return rawLog, logType, namespace, ingestionLabels, nil +} + +func (m *GRPC) getIngestionLabels(logRecord plog.LogRecord) ([]*api.Label, error) { + // check for labels in attributes["chronicle_ingestion_labels"] + ingestionLabels, err := m.getRawNestedFields(chronicleIngestionLabelsPrefix, logRecord) + if err != nil { + return []*api.Label{}, fmt.Errorf("get chronicle ingestion labels: %w", err) + } + + if len(ingestionLabels) != 0 { + return ingestionLabels, nil + } + // use labels defined in config if needed + configLabels := make([]*api.Label, 0) + for key, value := range m.cfg.IngestionLabels { + configLabels = append(configLabels, &api.Label{ + Key: key, + Value: value, + }) + } + return configLabels, nil +} + +func (m *GRPC) getRawNestedFields(field string, logRecord plog.LogRecord) ([]*api.Label, error) { + var nestedFields []*api.Label + logRecord.Attributes().Range(func(key string, value pcommon.Value) bool { + if !strings.HasPrefix(key, field) { + return true + } + // Extract the key name from the nested field + cleanKey := strings.Trim(key[len(field):], `[]"`) + var jsonMap map[string]string + + // If needs to be parsed as JSON + if err := json.Unmarshal([]byte(value.AsString()), &jsonMap); err == nil { + for k, v := range jsonMap { + nestedFields = append(nestedFields, &api.Label{Key: k, Value: v}) + } + } else { + nestedFields = append(nestedFields, &api.Label{Key: cleanKey, Value: value.AsString()}) + } + return true + }) + return nestedFields, nil +} + +func (m *GRPC) constructPayloads(rawLogs map[string][]*api.LogEntry, namespaceMap map[string]string, ingestionLabelsMap map[string][]*api.Label) []*api.BatchCreateLogsRequest { + payloads := make([]*api.BatchCreateLogsRequest, 0, len(rawLogs)) + for logType, entries := range rawLogs { + if len(entries) > 0 { + namespace, ok := namespaceMap[logType] + if !ok { + namespace = m.cfg.Namespace + } + ingestionLabels := ingestionLabelsMap[logType] + + request := m.buildGRPCRequest(entries, logType, namespace, ingestionLabels) + + payloads = append(payloads, m.enforceMaximumsGRPCRequest(request)...) + } + } + return payloads +} + +func (m *GRPC) enforceMaximumsGRPCRequest(request *api.BatchCreateLogsRequest) []*api.BatchCreateLogsRequest { + size := proto.Size(request) + entries := request.Batch.Entries + if size <= m.cfg.BatchRequestSizeLimit && len(entries) <= m.cfg.BatchLogCountLimit { + return []*api.BatchCreateLogsRequest{ + request, + } + } + + if len(entries) < 2 { + m.set.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + return []*api.BatchCreateLogsRequest{} + } + + // split request into two + mid := len(entries) / 2 + leftHalf := entries[:mid] + rightHalf := entries[mid:] + + request.Batch.Entries = leftHalf + otherHalfRequest := m.buildGRPCRequest(rightHalf, request.Batch.LogType, request.Batch.Source.Namespace, request.Batch.Source.Labels) + + // re-enforce max size restriction on each half + enforcedRequest := m.enforceMaximumsGRPCRequest(request) + enforcedOtherHalfRequest := m.enforceMaximumsGRPCRequest(otherHalfRequest) + + return append(enforcedRequest, enforcedOtherHalfRequest...) +} + +func (m *GRPC) buildGRPCRequest(entries []*api.LogEntry, logType, namespace string, ingestionLabels []*api.Label) *api.BatchCreateLogsRequest { + return &api.BatchCreateLogsRequest{ + Batch: &api.LogEntryBatch{ + StartTime: timestamppb.New(m.startTime), + Entries: entries, + LogType: logType, + Source: &api.EventSource{ + CollectorId: m.collectorID, + CustomerId: m.customerID, + Labels: ingestionLabels, + Namespace: namespace, + }, + }, + } +} diff --git a/exporter/chronicleexporter/internal/marshal/grpc_test.go b/exporter/chronicleexporter/internal/marshal/grpc_test.go new file mode 100644 index 000000000..b9ebc6145 --- /dev/null +++ b/exporter/chronicleexporter/internal/marshal/grpc_test.go @@ -0,0 +1,785 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshal_test + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/marshal" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestGRPC(t *testing.T) { + logger := zap.NewNop() + + tests := []struct { + name string + cfg marshal.Config + logRecords func() plog.Logs + expectations func(t *testing.T, requests []*api.BatchCreateLogsRequest, startTime time.Time) + }{ + { + name: "Single log record with expected data", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Test log message", map[string]any{"log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["env"]`: "prod"})) + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, startTime time.Time) { + require.Len(t, requests, 1) + batch := requests[0].Batch + require.Equal(t, "WINEVTLOG", batch.LogType) + require.Len(t, batch.Entries, 1) + + // Convert Data (byte slice) to string for comparison + logDataAsString := string(batch.Entries[0].Data) + expectedLogData := `Test log message` + require.Equal(t, expectedLogData, logDataAsString) + + require.NotNil(t, batch.StartTime) + require.True(t, timestamppb.New(startTime).AsTime().Equal(batch.StartTime.AsTime()), "Start time should be set correctly") + }, + }, + { + name: "Single log record with expected data, no log_type, namespace, or ingestion labels", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: true, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Test log message", nil)) + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, startTime time.Time) { + require.Len(t, requests, 1) + batch := requests[0].Batch + require.Equal(t, "WINEVTLOG", batch.LogType) + require.Equal(t, "", batch.Source.Namespace) + require.Equal(t, 0, len(batch.Source.Labels)) + require.Len(t, batch.Entries, 1) + + // Convert Data (byte slice) to string for comparison + logDataAsString := string(batch.Entries[0].Data) + expectedLogData := `Test log message` + require.Equal(t, expectedLogData, logDataAsString) + + require.NotNil(t, batch.StartTime) + require.True(t, timestamppb.New(startTime).AsTime().Equal(batch.StartTime.AsTime()), "Start time should be set correctly") + }, + }, + { + name: "Multiple log records", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + return logs + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + require.Len(t, requests, 1, "Expected a single batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") + // Verifying the first log entry data + require.Equal(t, "First log message", string(batch.Entries[0].Data)) + // Verifying the second log entry data + require.Equal(t, "Second log message", string(batch.Entries[1].Data)) + }, + }, + { + name: "Log record with attributes", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "attributes", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("", map[string]any{"key1": "value1", "log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"})) + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + require.Len(t, requests, 1) + batch := requests[0].Batch + require.Len(t, batch.Entries, 1) + + // Assuming the attributes are marshaled into the Data field as a JSON string + expectedData := `{"key1":"value1", "log_type":"WINEVTLOG", "namespace":"test", "chronicle_ingestion_label[\"key1\"]": "value1", "chronicle_ingestion_label[\"key2\"]": "value2"}` + actualData := string(batch.Entries[0].Data) + require.JSONEq(t, expectedData, actualData, "Log attributes should match expected") + }, + }, + { + name: "No log records", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "DEFAULT", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return plog.NewLogs() // No log records added + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + require.Len(t, requests, 0, "Expected no requests due to no log records") + }, + }, + { + name: "No log type set in config or attributes", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + RawLogField: "body", + OverrideLogType: true, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Log without logType", map[string]any{"namespace": "test", `ingestion_label["realkey1"]`: "realvalue1", `ingestion_label["realkey2"]`: "realvalue2"})) + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + require.Len(t, requests, 1) + batch := requests[0].Batch + require.Equal(t, "", batch.LogType, "Expected log type to be empty") + }, + }, + { + name: "Multiple log records with duplicate data, no log type in attributes", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + return logs + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify one request for log type in config + require.Len(t, requests, 1, "Expected a single batch request") + batch := requests[0].Batch + // verify batch source labels + require.Len(t, batch.Source.Labels, 2) + require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") + // Verifying the first log entry data + require.Equal(t, "First log message", string(batch.Entries[0].Data)) + // Verifying the second log entry data + require.Equal(t, "Second log message", string(batch.Entries[1].Data)) + }, + }, + { + name: "Multiple log records with different data, no log type in attributes", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) + return logs + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify one request for one log type + require.Len(t, requests, 1, "Expected a single batch request") + batch := requests[0].Batch + require.Equal(t, "WINEVTLOG", batch.LogType) + require.Equal(t, "", batch.Source.Namespace) + // verify batch source labels + require.Len(t, batch.Source.Labels, 4) + require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") + // Verifying the first log entry data + require.Equal(t, "First log message", string(batch.Entries[0].Data)) + // Verifying the second log entry data + require.Equal(t, "Second log message", string(batch.Entries[1].Data)) + }, + }, + { + name: "Override log type with attribute", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "DEFAULT", // This should be overridden by the log_type attribute + RawLogField: "body", + OverrideLogType: true, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"log_type": "windows_event.application", "namespace": "test", `ingestion_label["realkey1"]`: "realvalue1", `ingestion_label["realkey2"]`: "realvalue2"})) + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + require.Len(t, requests, 1) + batch := requests[0].Batch + require.Equal(t, "WINEVTLOG", batch.LogType, "Expected log type to be overridden by attribute") + }, + }, + { + name: "Override log type with chronicle attribute", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "DEFAULT", // This should be overridden by the chronicle_log_type attribute + RawLogField: "body", + OverrideLogType: true, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"chronicle_log_type": "ASOC_ALERT", "chronicle_namespace": "test", `chronicle_ingestion_label["realkey1"]`: "realvalue1", `chronicle_ingestion_label["realkey2"]`: "realvalue2"})) + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + require.Len(t, requests, 1) + batch := requests[0].Batch + require.Equal(t, "ASOC_ALERT", batch.LogType, "Expected log type to be overridden by attribute") + require.Equal(t, "test", batch.Source.Namespace, "Expected namespace to be overridden by attribute") + expectedLabels := map[string]string{ + "realkey1": "realvalue1", + "realkey2": "realvalue2", + } + for _, label := range batch.Source.Labels { + require.Equal(t, expectedLabels[label.Key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Multiple log records with duplicate data, log type in attributes", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + return logs + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 1 request, 2 batches for same log type + require.Len(t, requests, 1, "Expected a single batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") + // verify batch for first log + require.Equal(t, "WINEVTLOGS", batch.LogType) + require.Equal(t, "test1", batch.Source.Namespace) + require.Len(t, batch.Source.Labels, 2) + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + for _, label := range batch.Source.Labels { + require.Equal(t, expectedLabels[label.Key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Multiple log records with different data, log type in attributes", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS2", "chronicle_namespace": "test2", `chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 2 requests, with 1 batch for different log types + require.Len(t, requests, 2, "Expected a two batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 1, "Expected one log entries in the batch") + // verify batch for first log + require.Contains(t, batch.LogType, "WINEVTLOGS") + require.Contains(t, batch.Source.Namespace, "test") + require.Len(t, batch.Source.Labels, 2) + + batch2 := requests[1].Batch + require.Len(t, batch2.Entries, 1, "Expected one log entries in the batch") + // verify batch for second log + require.Contains(t, batch2.LogType, "WINEVTLOGS") + require.Contains(t, batch2.Source.Namespace, "test") + require.Len(t, batch2.Source.Labels, 2) + // verify ingestion labels + for _, req := range requests { + for _, label := range req.Batch.Source.Labels { + require.Contains(t, []string{ + "key1", + "key2", + "key3", + "key4", + }, label.Key) + require.Contains(t, []string{ + "value1", + "value2", + "value3", + "value4", + }, label.Value) + } + } + }, + }, + { + name: "Many logs, all one batch", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + for i := 0; i < 1000; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr("Log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 1 request, with 1 batch + require.Len(t, requests, 1, "Expected a one-batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 1000, "Expected 1000 log entries in the batch") + // verify batch for first log + require.Contains(t, batch.LogType, "WINEVTLOGS") + require.Contains(t, batch.Source.Namespace, "test") + require.Len(t, batch.Source.Labels, 2) + + // verify ingestion labels + for _, req := range requests { + for _, label := range req.Batch.Source.Labels { + require.Contains(t, []string{ + "key1", + "key2", + "key3", + "key4", + }, label.Key) + require.Contains(t, []string{ + "value1", + "value2", + "value3", + "value4", + }, label.Value) + } + } + }, + }, + { + name: "Single batch split into multiple because more than 1000 logs", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + for i := 0; i < 1001; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr("Log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 1 request, with 1 batch + require.Len(t, requests, 2, "Expected a two-batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 500, "Expected 500 log entries in the first batch") + // verify batch for first log + require.Contains(t, batch.LogType, "WINEVTLOGS") + require.Contains(t, batch.Source.Namespace, "test") + require.Len(t, batch.Source.Labels, 2) + + batch2 := requests[1].Batch + require.Len(t, batch2.Entries, 501, "Expected 501 log entries in the second batch") + // verify batch for first log + require.Contains(t, batch2.LogType, "WINEVTLOGS") + require.Contains(t, batch2.Source.Namespace, "test") + require.Len(t, batch2.Source.Labels, 2) + + // verify ingestion labels + for _, req := range requests { + for _, label := range req.Batch.Source.Labels { + require.Contains(t, []string{ + "key1", + "key2", + "key3", + "key4", + }, label.Key) + require.Contains(t, []string{ + "value1", + "value2", + "value3", + "value4", + }, label.Value) + } + } + }, + }, + { + name: "Recursively split batch, exceeds 1000 entries multiple times", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + for i := 0; i < 2002; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr("Log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 1 request, with 1 batch + require.Len(t, requests, 4, "Expected a four-batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 500, "Expected 500 log entries in the first batch") + // verify batch for first log + require.Contains(t, batch.LogType, "WINEVTLOGS") + require.Contains(t, batch.Source.Namespace, "test") + require.Len(t, batch.Source.Labels, 2) + + batch2 := requests[1].Batch + require.Len(t, batch2.Entries, 501, "Expected 501 log entries in the second batch") + // verify batch for first log + require.Contains(t, batch2.LogType, "WINEVTLOGS") + require.Contains(t, batch2.Source.Namespace, "test") + require.Len(t, batch2.Source.Labels, 2) + + batch3 := requests[2].Batch + require.Len(t, batch3.Entries, 500, "Expected 500 log entries in the third batch") + // verify batch for first log + require.Contains(t, batch3.LogType, "WINEVTLOGS") + require.Contains(t, batch3.Source.Namespace, "test") + require.Len(t, batch3.Source.Labels, 2) + + batch4 := requests[3].Batch + require.Len(t, batch4.Entries, 501, "Expected 501 log entries in the fourth batch") + // verify batch for first log + require.Contains(t, batch4.LogType, "WINEVTLOGS") + require.Contains(t, batch4.Source.Namespace, "test") + require.Len(t, batch4.Source.Labels, 2) + + // verify ingestion labels + for _, req := range requests { + for _, label := range req.Batch.Source.Labels { + require.Contains(t, []string{ + "key1", + "key2", + "key3", + "key4", + }, label.Key) + require.Contains(t, []string{ + "value1", + "value2", + "value3", + "value4", + }, label.Value) + } + } + }, + }, + { + name: "Single batch split into multiple because request size too large", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + // create 640 logs with size 8192 bytes each - totalling 5242880 bytes. non-body fields put us over limit + for i := 0; i < 640; i++ { + record1 := logRecords.AppendEmpty() + body := tokenWithLength(8192) + record1.Body().SetStr(string(body)) + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify request, with 1 batch + require.Len(t, requests, 2, "Expected a two-batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 320, "Expected 320 log entries in the first batch") + // verify batch for first log + require.Contains(t, batch.LogType, "WINEVTLOGS") + require.Contains(t, batch.Source.Namespace, "test") + require.Len(t, batch.Source.Labels, 2) + + batch2 := requests[1].Batch + require.Len(t, batch2.Entries, 320, "Expected 320 log entries in the second batch") + // verify batch for first log + require.Contains(t, batch2.LogType, "WINEVTLOGS") + require.Contains(t, batch2.Source.Namespace, "test") + require.Len(t, batch2.Source.Labels, 2) + + // verify ingestion labels + for _, req := range requests { + for _, label := range req.Batch.Source.Labels { + require.Contains(t, []string{ + "key1", + "key2", + "key3", + "key4", + }, label.Key) + require.Contains(t, []string{ + "value1", + "value2", + "value3", + "value4", + }, label.Value) + } + } + }, + }, + { + name: "Recursively split batch into multiple because request size too large", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + // create 1280 logs with size 8192 bytes each - totalling 5242880 * 2 bytes. non-body fields put us over twice the limit + for i := 0; i < 1280; i++ { + record1 := logRecords.AppendEmpty() + body := tokenWithLength(8192) + record1.Body().SetStr(string(body)) + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 1 request, with 1 batch + require.Len(t, requests, 4, "Expected a four-batch request") + batch := requests[0].Batch + require.Len(t, batch.Entries, 320, "Expected 320 log entries in the first batch") + // verify batch for first log + require.Contains(t, batch.LogType, "WINEVTLOGS") + require.Contains(t, batch.Source.Namespace, "test") + require.Len(t, batch.Source.Labels, 2) + + batch2 := requests[1].Batch + require.Len(t, batch2.Entries, 320, "Expected 320 log entries in the second batch") + // verify batch for first log + require.Contains(t, batch2.LogType, "WINEVTLOGS") + require.Contains(t, batch2.Source.Namespace, "test") + require.Len(t, batch2.Source.Labels, 2) + + batch3 := requests[2].Batch + require.Len(t, batch3.Entries, 320, "Expected 320 log entries in the third batch") + // verify batch for first log + require.Contains(t, batch3.LogType, "WINEVTLOGS") + require.Contains(t, batch3.Source.Namespace, "test") + require.Len(t, batch3.Source.Labels, 2) + + batch4 := requests[3].Batch + require.Len(t, batch4.Entries, 320, "Expected 320 log entries in the fourth batch") + // verify batch for first log + require.Contains(t, batch4.LogType, "WINEVTLOGS") + require.Contains(t, batch4.Source.Namespace, "test") + require.Len(t, batch4.Source.Labels, 2) + + // verify ingestion labels + for _, req := range requests { + for _, label := range req.Batch.Source.Labels { + require.Contains(t, []string{ + "key1", + "key2", + "key3", + "key4", + }, label.Key) + require.Contains(t, []string{ + "value1", + "value2", + "value3", + "value4", + }, label.Value) + } + } + }, + }, + { + name: "Unsplittable batch, single log exceeds max request size", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + body := tokenWithLength(5242881) + record1.Body().SetStr(string(body)) + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + return logs + }, + + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // verify 1 request, with 1 batch + require.Len(t, requests, 0, "Expected a zero requests") + }, + }, + { + name: "Multiple valid log records + unsplittable log entries", + cfg: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + tooLargeBody := string(tokenWithLength(5242881)) + // first normal log, then impossible to split log + logRecords1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + record1 := logRecords1.AppendEmpty() + record1.Body().SetStr("First log message") + tooLargeRecord1 := logRecords1.AppendEmpty() + tooLargeRecord1.Body().SetStr(tooLargeBody) + // first impossible to split log, then normal log + logRecords2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + tooLargeRecord2 := logRecords2.AppendEmpty() + tooLargeRecord2.Body().SetStr(tooLargeBody) + record2 := logRecords2.AppendEmpty() + record2.Body().SetStr("Second log message") + return logs + }, + expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest, _ time.Time) { + // this is a kind of weird edge case, the overly large logs makes the final requests quite inefficient, but it's going to be so rare that the inefficiency isn't a real concern + require.Len(t, requests, 2, "Expected two batch requests") + batch1 := requests[0].Batch + require.Len(t, batch1.Entries, 1, "Expected one log entry in the first batch") + // Verifying the first log entry data + require.Equal(t, "First log message", string(batch1.Entries[0].Data)) + + batch2 := requests[1].Batch + require.Len(t, batch2.Entries, 1, "Expected one log entry in the second batch") + // Verifying the second log entry data + require.Equal(t, "Second log message", string(batch2.Entries[0].Data)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + marshaler, err := marshal.NewGRPC(tt.cfg, component.TelemetrySettings{Logger: logger}) + require.NoError(t, err) + + logs := tt.logRecords() + requests, err := marshaler.MarshalLogs(context.Background(), logs) + require.NoError(t, err) + + tt.expectations(t, requests, marshaler.StartTime()) + }) + } +} diff --git a/exporter/chronicleexporter/internal/marshal/http.go b/exporter/chronicleexporter/internal/marshal/http.go new file mode 100644 index 000000000..f0bc6e67f --- /dev/null +++ b/exporter/chronicleexporter/internal/marshal/http.go @@ -0,0 +1,244 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshal + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// HTTPConfig is the configuration for the HTTP marshaler +type HTTPConfig struct { + Config + Project string + Location string + Forwarder string +} + +// HTTP is a marshaler for HTTP protos +type HTTP struct { + protoMarshaler + project string + location string + forwarder string +} + +// NewHTTP creates a new HTTP marshaler +func NewHTTP(cfg HTTPConfig, set component.TelemetrySettings) (*HTTP, error) { + m, err := newProtoMarshaler(cfg.Config, set) + if err != nil { + return nil, err + } + return &HTTP{ + protoMarshaler: *m, + project: cfg.Project, + location: cfg.Location, + forwarder: cfg.Forwarder, + }, nil +} + +// MarshalLogs marshals logs into HTTP payloads +func (m *HTTP) MarshalLogs(ctx context.Context, ld plog.Logs) (map[string][]*api.ImportLogsRequest, error) { + rawLogs, err := m.extractRawHTTPLogs(ctx, ld) + if err != nil { + return nil, fmt.Errorf("extract raw logs: %w", err) + } + return m.constructHTTPPayloads(rawLogs), nil +} + +func (m *HTTP) extractRawHTTPLogs(ctx context.Context, ld plog.Logs) (map[string][]*api.Log, error) { + entries := make(map[string][]*api.Log) + for i := 0; i < ld.ResourceLogs().Len(); i++ { + resourceLog := ld.ResourceLogs().At(i) + for j := 0; j < resourceLog.ScopeLogs().Len(); j++ { + scopeLog := resourceLog.ScopeLogs().At(j) + for k := 0; k < scopeLog.LogRecords().Len(); k++ { + logRecord := scopeLog.LogRecords().At(k) + rawLog, logType, namespace, ingestionLabels, err := m.processHTTPLogRecord(ctx, logRecord, scopeLog, resourceLog) + if err != nil { + m.set.Logger.Error("Error processing log record", zap.Error(err)) + continue + } + + if rawLog == "" { + continue + } + + var timestamp time.Time + if logRecord.Timestamp() != 0 { + timestamp = logRecord.Timestamp().AsTime() + } else { + timestamp = logRecord.ObservedTimestamp().AsTime() + } + + entry := &api.Log{ + LogEntryTime: timestamppb.New(timestamp), + CollectionTime: timestamppb.New(logRecord.ObservedTimestamp().AsTime()), + Data: []byte(rawLog), + EnvironmentNamespace: namespace, + Labels: ingestionLabels, + } + entries[logType] = append(entries[logType], entry) + } + } + } + + return entries, nil +} + +func (m *HTTP) processHTTPLogRecord(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, string, string, map[string]*api.Log_LogLabel, error) { + rawLog, err := m.getRawLog(ctx, logRecord, scope, resource) + if err != nil { + return "", "", "", nil, err + } + + logType, err := m.getLogType(ctx, logRecord, scope, resource) + if err != nil { + return "", "", "", nil, err + } + namespace, err := m.getNamespace(ctx, logRecord, scope, resource) + if err != nil { + return "", "", "", nil, err + } + ingestionLabels, err := m.getHTTPIngestionLabels(logRecord) + if err != nil { + return "", "", "", nil, err + } + + return rawLog, logType, namespace, ingestionLabels, nil +} + +func (m *HTTP) getHTTPIngestionLabels(logRecord plog.LogRecord) (map[string]*api.Log_LogLabel, error) { + // Check for labels in attributes["chronicle_ingestion_labels"] + ingestionLabels, err := m.getHTTPRawNestedFields(chronicleIngestionLabelsPrefix, logRecord) + if err != nil { + return nil, fmt.Errorf("get chronicle ingestion labels: %w", err) + } + + if len(ingestionLabels) != 0 { + return ingestionLabels, nil + } + + // use labels defined in the config if needed + configLabels := make(map[string]*api.Log_LogLabel) + for key, value := range m.cfg.IngestionLabels { + configLabels[key] = &api.Log_LogLabel{ + Value: value, + } + } + return configLabels, nil +} + +func (m *HTTP) getHTTPRawNestedFields(field string, logRecord plog.LogRecord) (map[string]*api.Log_LogLabel, error) { + nestedFields := make(map[string]*api.Log_LogLabel) // Map with key as string and value as Log_LogLabel + logRecord.Attributes().Range(func(key string, value pcommon.Value) bool { + if !strings.HasPrefix(key, field) { + return true + } + // Extract the key name from the nested field + cleanKey := strings.Trim(key[len(field):], `[]"`) + var jsonMap map[string]string + + // If needs to be parsed as JSON + if err := json.Unmarshal([]byte(value.AsString()), &jsonMap); err == nil { + for k, v := range jsonMap { + nestedFields[k] = &api.Log_LogLabel{ + Value: v, + } + } + } else { + nestedFields[cleanKey] = &api.Log_LogLabel{ + Value: value.AsString(), + } + } + return true + }) + + return nestedFields, nil +} + +func (m *HTTP) buildForwarderString() string { + format := "projects/%s/locations/%s/instances/%s/forwarders/%s" + return fmt.Sprintf(format, m.project, m.location, m.customerID, m.forwarder) +} + +func (m *HTTP) constructHTTPPayloads(rawLogs map[string][]*api.Log) map[string][]*api.ImportLogsRequest { + payloads := make(map[string][]*api.ImportLogsRequest, len(rawLogs)) + + for logType, entries := range rawLogs { + if len(entries) > 0 { + request := m.buildHTTPRequest(entries) + + payloads[logType] = m.enforceMaximumsHTTPRequest(request) + } + } + return payloads +} + +func (m *HTTP) enforceMaximumsHTTPRequest(request *api.ImportLogsRequest) []*api.ImportLogsRequest { + size := proto.Size(request) + logs := request.GetInlineSource().Logs + if size <= m.cfg.BatchRequestSizeLimit && len(logs) <= m.cfg.BatchLogCountLimit { + return []*api.ImportLogsRequest{ + request, + } + } + + if len(logs) < 2 { + m.set.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) + return []*api.ImportLogsRequest{} + } + + // split request into two + mid := len(logs) / 2 + leftHalf := logs[:mid] + rightHalf := logs[mid:] + + request.GetInlineSource().Logs = leftHalf + otherHalfRequest := m.buildHTTPRequest(rightHalf) + + // re-enforce max size restriction on each half + enforcedRequest := m.enforceMaximumsHTTPRequest(request) + enforcedOtherHalfRequest := m.enforceMaximumsHTTPRequest(otherHalfRequest) + + return append(enforcedRequest, enforcedOtherHalfRequest...) +} + +func (m *HTTP) buildHTTPRequest(entries []*api.Log) *api.ImportLogsRequest { + return &api.ImportLogsRequest{ + // TODO: Add parent and hint + // We don't yet have solid guidance on what these should be + Parent: "", + Hint: "", + + Source: &api.ImportLogsRequest_InlineSource{ + InlineSource: &api.ImportLogsRequest_LogsInlineSource{ + Forwarder: m.buildForwarderString(), + Logs: entries, + }, + }, + } +} diff --git a/exporter/chronicleexporter/internal/marshal/http_test.go b/exporter/chronicleexporter/internal/marshal/http_test.go new file mode 100644 index 000000000..121a34c18 --- /dev/null +++ b/exporter/chronicleexporter/internal/marshal/http_test.go @@ -0,0 +1,769 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshal_test + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/marshal" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" +) + +func TestHTTP(t *testing.T) { + logger := zap.NewNop() + + tests := []struct { + name string + cfg marshal.HTTPConfig + labels []*api.Label + logRecords func() plog.Logs + expectations func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) + }{ + { + name: "Single log record with expected data", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + Project: "test-project", + Location: "us", + Forwarder: uuid.New().String(), + }, + labels: []*api.Label{ + {Key: "env", Value: "prod"}, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Test log message", map[string]any{"log_type": "WINEVTLOG", "namespace": "test"})) + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, _ time.Time) { + require.Len(t, requests, 1) + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + require.Len(t, logs, 1) + // Convert Data (byte slice) to string for comparison + logDataAsString := string(logs[0].Data) + expectedLogData := `Test log message` + require.Equal(t, expectedLogData, logDataAsString) + }, + }, + { + name: "Multiple log records", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + labels: []*api.Label{ + {Key: "env", Value: "staging"}, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + return logs + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1, "Expected a single batch request") + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + require.Len(t, logs, 2, "Expected two log entries in the batch") + // Verifying the first log entry data + require.Equal(t, "First log message", string(logs[0].Data)) + // Verifying the second log entry data + require.Equal(t, "Second log message", string(logs[1].Data)) + }, + }, + { + name: "Log record with attributes", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "attributes", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + labels: []*api.Label{}, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("", map[string]any{"key1": "value1", "log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"})) + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1) + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + // Assuming the attributes are marshaled into the Data field as a JSON string + expectedData := `{"key1":"value1", "log_type":"WINEVTLOG", "namespace":"test", "chronicle_ingestion_label[\"key1\"]": "value1", "chronicle_ingestion_label[\"key2\"]": "value2"}` + actualData := string(logs[0].Data) + require.JSONEq(t, expectedData, actualData, "Log attributes should match expected") + }, + }, + { + name: "No log records", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "DEFAULT", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + labels: []*api.Label{}, + logRecords: func() plog.Logs { + return plog.NewLogs() // No log records added + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 0, "Expected no requests due to no log records") + }, + }, + { + name: "No log type set in config or attributes", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "attributes", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + labels: []*api.Label{}, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("", map[string]any{"key1": "value1", "log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"})) + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1) + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + // Assuming the attributes are marshaled into the Data field as a JSON string + expectedData := `{"key1":"value1", "log_type":"WINEVTLOG", "namespace":"test", "chronicle_ingestion_label[\"key1\"]": "value1", "chronicle_ingestion_label[\"key2\"]": "value2"}` + actualData := string(logs[0].Data) + require.JSONEq(t, expectedData, actualData, "Log attributes should match expected") + }, + }, + { + name: "Multiple log records with duplicate data, no log type in attributes", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + return logs + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + // verify one request for log type in config + require.Len(t, requests, 1, "Expected a single batch request") + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + // verify batch source labels + require.Len(t, logs[0].Labels, 2) + require.Len(t, logs, 2, "Expected two log entries in the batch") + // Verifying the first log entry data + require.Equal(t, "First log message", string(logs[0].Data)) + // Verifying the second log entry data + require.Equal(t, "Second log message", string(logs[1].Data)) + }, + }, + { + name: "Multiple log records with different data, no log type in attributes", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) + return logs + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + // verify one request for one log type + require.Len(t, requests, 1, "Expected a single batch request") + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + require.Len(t, logs, 2, "Expected two log entries in the batch") + require.Equal(t, "", logs[0].EnvironmentNamespace) + // verify batch source labels + require.Len(t, logs[0].Labels, 2) + require.Len(t, logs[1].Labels, 2) + // Verifying the first log entry data + require.Equal(t, "First log message", string(logs[0].Data)) + // Verifying the second log entry data + require.Equal(t, "Second log message", string(logs[1].Data)) + }, + }, + { + name: "Override log type with attribute", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "DEFAULT", // This should be overridden by the log_type attribute + RawLogField: "body", + OverrideLogType: true, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"log_type": "windows_event.application", "namespace": "test", `ingestion_label["realkey1"]`: "realvalue1", `ingestion_label["realkey2"]`: "realvalue2"})) + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1) + logs := requests["WINEVTLOG"][0].GetInlineSource().Logs + require.NotEqual(t, len(logs), 0) + }, + }, + { + name: "Override log type with chronicle attribute", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "DEFAULT", // This should be overridden by the chronicle_log_type attribute + RawLogField: "body", + OverrideLogType: true, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"chronicle_log_type": "ASOC_ALERT", "chronicle_namespace": "test", `chronicle_ingestion_label["realkey1"]`: "realvalue1", `chronicle_ingestion_label["realkey2"]`: "realvalue2"})) + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1) + logs := requests["ASOC_ALERT"][0].GetInlineSource().Logs + require.Equal(t, "test", logs[0].EnvironmentNamespace, "Expected namespace to be overridden by attribute") + expectedLabels := map[string]string{ + "realkey1": "realvalue1", + "realkey2": "realvalue2", + } + for key, label := range logs[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Multiple log records with duplicate data, log type in attributes", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + return logs + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + // verify 1 request, 2 batches for same log type + require.Len(t, requests, 1, "Expected a single batch request") + logs := requests["WINEVTLOGS"][0].GetInlineSource().Logs + require.Len(t, logs, 2, "Expected two log entries in the batch") + // verify variables + require.Equal(t, "test1", logs[0].EnvironmentNamespace) + require.Len(t, logs[0].Labels, 2) + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + for key, label := range logs[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Multiple log records with different data, log type in attributes", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + + record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record2.Body().SetStr("Second log message") + record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS2", "chronicle_namespace": "test2", `chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) + return logs + }, + + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value3", + "key4": "value4", + } + // verify 2 requests, with 1 batch for different log types + require.Len(t, requests, 2, "Expected a two batch request") + + logs1 := requests["WINEVTLOGS1"][0].GetInlineSource().Logs + require.Len(t, logs1, 1, "Expected one log entries in the batch") + // verify variables for first log + require.Equal(t, logs1[0].EnvironmentNamespace, "test1") + require.Len(t, logs1[0].Labels, 2) + for key, label := range logs1[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs2 := requests["WINEVTLOGS2"][0].GetInlineSource().Logs + require.Len(t, logs2, 1, "Expected one log entries in the batch") + // verify variables for second log + require.Equal(t, logs2[0].EnvironmentNamespace, "test2") + require.Len(t, logs2[0].Labels, 2) + for key, label := range logs2[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Many log records all one batch", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + for i := 0; i < 1000; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + + return logs + }, + + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + // verify 1 requests + require.Len(t, requests, 1, "Expected a one batch request") + + logs1 := requests["WINEVTLOGS1"][0].GetInlineSource().Logs + require.Len(t, logs1, 1000, "Expected one thousand log entries in the batch") + // verify variables for first log + require.Equal(t, logs1[0].EnvironmentNamespace, "test1") + require.Len(t, logs1[0].Labels, 2) + for key, label := range logs1[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Many log records split into two batches", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + for i := 0; i < 1001; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + + return logs + }, + + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + // verify 1 request log type + require.Len(t, requests, 1, "Expected one log type for the requests") + winEvtLogRequests := requests["WINEVTLOGS1"] + require.Len(t, winEvtLogRequests, 2, "Expected two batches") + + logs1 := winEvtLogRequests[0].GetInlineSource().Logs + require.Len(t, logs1, 500, "Expected 500 log entries in the first batch") + // verify variables for first log + require.Equal(t, logs1[0].EnvironmentNamespace, "test1") + require.Len(t, logs1[0].Labels, 2) + for key, label := range logs1[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs2 := winEvtLogRequests[1].GetInlineSource().Logs + require.Len(t, logs2, 501, "Expected 501 log entries in the second batch") + // verify variables for first log + require.Equal(t, logs2[0].EnvironmentNamespace, "test1") + require.Len(t, logs2[0].Labels, 2) + for key, label := range logs2[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Recursively split batch multiple times because too many logs", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + for i := 0; i < 2002; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr("First log message") + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + + return logs + }, + + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + // verify 1 request log type + require.Len(t, requests, 1, "Expected one log type for the requests") + winEvtLogRequests := requests["WINEVTLOGS1"] + require.Len(t, winEvtLogRequests, 4, "Expected four batches") + + logs1 := winEvtLogRequests[0].GetInlineSource().Logs + require.Len(t, logs1, 500, "Expected 500 log entries in the first batch") + // verify variables for first log + require.Equal(t, logs1[0].EnvironmentNamespace, "test1") + require.Len(t, logs1[0].Labels, 2) + for key, label := range logs1[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs2 := winEvtLogRequests[1].GetInlineSource().Logs + require.Len(t, logs2, 501, "Expected 501 log entries in the second batch") + // verify variables for first log + require.Equal(t, logs2[0].EnvironmentNamespace, "test1") + require.Len(t, logs2[0].Labels, 2) + for key, label := range logs2[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs3 := winEvtLogRequests[2].GetInlineSource().Logs + require.Len(t, logs3, 500, "Expected 500 log entries in the third batch") + // verify variables for first log + require.Equal(t, logs3[0].EnvironmentNamespace, "test1") + require.Len(t, logs3[0].Labels, 2) + for key, label := range logs3[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs4 := winEvtLogRequests[3].GetInlineSource().Logs + require.Len(t, logs4, 501, "Expected 501 log entries in the fourth batch") + // verify variables for first log + require.Equal(t, logs4[0].EnvironmentNamespace, "test1") + require.Len(t, logs4[0].Labels, 2) + for key, label := range logs4[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Many log records split into two batches because request size too large", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + // 8192 * 640 = 5242880 + body := tokenWithLength(8192) + for i := 0; i < 640; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr(string(body)) + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + + return logs + }, + + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + // verify 1 request log type + require.Len(t, requests, 1, "Expected one log type for the requests") + winEvtLogRequests := requests["WINEVTLOGS1"] + require.Len(t, winEvtLogRequests, 2, "Expected two batches") + + logs1 := winEvtLogRequests[0].GetInlineSource().Logs + require.Len(t, logs1, 320, "Expected 320 log entries in the first batch") + // verify variables for first log + require.Equal(t, logs1[0].EnvironmentNamespace, "test1") + require.Len(t, logs1[0].Labels, 2) + for key, label := range logs1[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs2 := winEvtLogRequests[1].GetInlineSource().Logs + require.Len(t, logs2, 320, "Expected 320 log entries in the second batch") + // verify variables for first log + require.Equal(t, logs2[0].EnvironmentNamespace, "test1") + require.Len(t, logs2[0].Labels, 2) + for key, label := range logs2[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Recursively split into batches because request size too large", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 2000, + BatchRequestSizeLimit: 5242880, + }, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + // 8192 * 1280 = 5242880 * 2 + body := tokenWithLength(8192) + for i := 0; i < 1280; i++ { + record1 := logRecords.AppendEmpty() + record1.Body().SetStr(string(body)) + record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) + } + + return logs + }, + + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + expectedLabels := map[string]string{ + "key1": "value1", + "key2": "value2", + } + // verify 1 request log type + require.Len(t, requests, 1, "Expected one log type for the requests") + winEvtLogRequests := requests["WINEVTLOGS1"] + require.Len(t, winEvtLogRequests, 4, "Expected four batches") + + logs1 := winEvtLogRequests[0].GetInlineSource().Logs + require.Len(t, logs1, 320, "Expected 320 log entries in the first batch") + // verify variables for first log + require.Equal(t, logs1[0].EnvironmentNamespace, "test1") + require.Len(t, logs1[0].Labels, 2) + for key, label := range logs1[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs2 := winEvtLogRequests[1].GetInlineSource().Logs + require.Len(t, logs2, 320, "Expected 320 log entries in the second batch") + // verify variables for first log + require.Equal(t, logs2[0].EnvironmentNamespace, "test1") + require.Len(t, logs2[0].Labels, 2) + for key, label := range logs2[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs3 := winEvtLogRequests[2].GetInlineSource().Logs + require.Len(t, logs3, 320, "Expected 320 log entries in the third batch") + // verify variables for first log + require.Equal(t, logs3[0].EnvironmentNamespace, "test1") + require.Len(t, logs3[0].Labels, 2) + for key, label := range logs3[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + + logs4 := winEvtLogRequests[3].GetInlineSource().Logs + require.Len(t, logs4, 320, "Expected 320 log entries in the fourth batch") + // verify variables for first log + require.Equal(t, logs4[0].EnvironmentNamespace, "test1") + require.Len(t, logs4[0].Labels, 2) + for key, label := range logs4[0].Labels { + require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") + } + }, + }, + { + name: "Unsplittable log record, single log exceeds request size limit", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 100000, + }, + }, + labels: []*api.Label{ + {Key: "env", Value: "staging"}, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + record1.Body().SetStr(string(tokenWithLength(100000))) + return logs + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1, "Expected one log type") + require.Len(t, requests["WINEVTLOG"], 0, "Expected WINEVTLOG log type to have zero requests") + }, + }, + { + name: "Unsplittable log record, single log exceeds request size limit, mixed with okay logs", + cfg: marshal.HTTPConfig{ + Config: marshal.Config{ + CustomerID: uuid.New().String(), + LogType: "WINEVTLOG", + RawLogField: "body", + OverrideLogType: false, + BatchLogCountLimit: 1000, + BatchRequestSizeLimit: 100000, + }, + }, + labels: []*api.Label{ + {Key: "env", Value: "staging"}, + }, + logRecords: func() plog.Logs { + logs := plog.NewLogs() + tooLargeBody := string(tokenWithLength(100001)) + // first normal log, then impossible to split log + logRecords1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + record1 := logRecords1.AppendEmpty() + record1.Body().SetStr("First log message") + tooLargeRecord1 := logRecords1.AppendEmpty() + tooLargeRecord1.Body().SetStr(tooLargeBody) + // first impossible to split log, then normal log + logRecords2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() + tooLargeRecord2 := logRecords2.AppendEmpty() + tooLargeRecord2.Body().SetStr(tooLargeBody) + record2 := logRecords2.AppendEmpty() + record2.Body().SetStr("Second log message") + return logs + }, + expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest, startTime time.Time) { + require.Len(t, requests, 1, "Expected one log type") + winEvtLogRequests := requests["WINEVTLOG"] + require.Len(t, winEvtLogRequests, 2, "Expected WINEVTLOG log type to have zero requests") + + logs1 := winEvtLogRequests[0].GetInlineSource().Logs + require.Len(t, logs1, 1, "Expected 1 log entry in the first batch") + require.Equal(t, string(logs1[0].Data), "First log message") + + logs2 := winEvtLogRequests[1].GetInlineSource().Logs + require.Len(t, logs2, 1, "Expected 1 log entry in the second batch") + require.Equal(t, string(logs2[0].Data), "Second log message") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + marshaler, err := marshal.NewHTTP(tt.cfg, component.TelemetrySettings{Logger: logger}) + require.NoError(t, err) + + logs := tt.logRecords() + requests, err := marshaler.MarshalLogs(context.Background(), logs) + require.NoError(t, err) + + tt.expectations(t, requests, marshaler.StartTime()) + }) + } +} diff --git a/exporter/chronicleexporter/internal/marshal/marshal.go b/exporter/chronicleexporter/internal/marshal/marshal.go new file mode 100644 index 000000000..99ce132f3 --- /dev/null +++ b/exporter/chronicleexporter/internal/marshal/marshal.go @@ -0,0 +1,206 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshal + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/ccid" + "github.com/observiq/bindplane-otel-collector/expr" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottllog" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" +) + +const logTypeField = `attributes["log_type"]` +const chronicleLogTypeField = `attributes["chronicle_log_type"]` +const chronicleNamespaceField = `attributes["chronicle_namespace"]` +const chronicleIngestionLabelsPrefix = `chronicle_ingestion_label` + +var supportedLogTypes = map[string]string{ + "windows_event.security": "WINEVTLOG", + "windows_event.application": "WINEVTLOG", + "windows_event.system": "WINEVTLOG", + "sql_server": "MICROSOFT_SQL", +} + +// Config is a subset of the HTTPConfig but if we ever identify a need for GRPC-specific config fields, +// then we should make it a shared unexported struct and embed it in both HTTPConfig and Config. +type Config struct { + CustomerID string + Namespace string + LogType string + RawLogField string + OverrideLogType bool + IngestionLabels map[string]string + BatchRequestSizeLimit int + BatchLogCountLimit int +} + +type protoMarshaler struct { + cfg Config + set component.TelemetrySettings + startTime time.Time + customerID []byte + collectorID []byte +} + +func newProtoMarshaler(cfg Config, set component.TelemetrySettings) (*protoMarshaler, error) { + customerID, err := uuid.Parse(cfg.CustomerID) + if err != nil { + return nil, fmt.Errorf("parse customer ID: %w", err) + } + return &protoMarshaler{ + startTime: time.Now(), + cfg: cfg, + set: set, + customerID: customerID[:], + collectorID: ccid.ChronicleCollectorID[:], + }, nil +} + +func (m *protoMarshaler) StartTime() time.Time { + return m.startTime +} + +func (m *protoMarshaler) getRawLog(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { + if m.cfg.RawLogField == "" { + entireLogRecord := map[string]any{ + "body": logRecord.Body().Str(), + "attributes": logRecord.Attributes().AsRaw(), + "resource_attributes": resource.Resource().Attributes().AsRaw(), + } + + bytesLogRecord, err := json.Marshal(entireLogRecord) + if err != nil { + return "", fmt.Errorf("marshal log record: %w", err) + } + + return string(bytesLogRecord), nil + } + return GetRawField(ctx, m.set, m.cfg.RawLogField, logRecord, scope, resource) +} + +func (m *protoMarshaler) getLogType(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { + // check for attributes in attributes["chronicle_log_type"] + logType, err := GetRawField(ctx, m.set, chronicleLogTypeField, logRecord, scope, resource) + if err != nil { + return "", fmt.Errorf("get chronicle log type: %w", err) + } + if logType != "" { + return logType, nil + } + + if m.cfg.OverrideLogType { + logType, err := GetRawField(ctx, m.set, logTypeField, logRecord, scope, resource) + + if err != nil { + return "", fmt.Errorf("get log type: %w", err) + } + if logType != "" { + if chronicleLogType, ok := supportedLogTypes[logType]; ok { + return chronicleLogType, nil + } + } + } + + return m.cfg.LogType, nil +} + +func (m *protoMarshaler) getNamespace(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { + // check for attributes in attributes["chronicle_namespace"] + namespace, err := GetRawField(ctx, m.set, chronicleNamespaceField, logRecord, scope, resource) + if err != nil { + return "", fmt.Errorf("get chronicle log type: %w", err) + } + if namespace != "" { + return namespace, nil + } + return m.cfg.Namespace, nil +} + +// GetRawField is a helper function to extract a field from a log record using an OTTL expression. +func GetRawField(ctx context.Context, set component.TelemetrySettings, field string, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { + switch field { + case "body": + switch logRecord.Body().Type() { + case pcommon.ValueTypeStr: + return logRecord.Body().Str(), nil + case pcommon.ValueTypeMap: + bytes, err := json.Marshal(logRecord.Body().AsRaw()) + if err != nil { + return "", fmt.Errorf("marshal log body: %w", err) + } + return string(bytes), nil + } + case logTypeField: + attributes := logRecord.Attributes().AsRaw() + if logType, ok := attributes["log_type"]; ok { + if v, ok := logType.(string); ok { + return v, nil + } + } + return "", nil + case chronicleLogTypeField: + attributes := logRecord.Attributes().AsRaw() + if logType, ok := attributes["chronicle_log_type"]; ok { + if v, ok := logType.(string); ok { + return v, nil + } + } + return "", nil + case chronicleNamespaceField: + attributes := logRecord.Attributes().AsRaw() + if namespace, ok := attributes["chronicle_namespace"]; ok { + if v, ok := namespace.(string); ok { + return v, nil + } + } + return "", nil + } + + lrExpr, err := expr.NewOTTLLogRecordExpression(field, set) + if err != nil { + return "", fmt.Errorf("raw_log_field is invalid: %s", err) + } + tCtx := ottllog.NewTransformContext(logRecord, scope.Scope(), resource.Resource(), scope, resource) + + lrExprResult, err := lrExpr.Execute(ctx, tCtx) + if err != nil { + return "", fmt.Errorf("execute log record expression: %w", err) + } + + if lrExprResult == nil { + return "", nil + } + + switch result := lrExprResult.(type) { + case string: + return result, nil + case pcommon.Map: + bytes, err := json.Marshal(result.AsRaw()) + if err != nil { + return "", fmt.Errorf("marshal log record expression result: %w", err) + } + return string(bytes), nil + default: + return "", fmt.Errorf("unsupported log record expression result type: %T", lrExprResult) + } +} diff --git a/exporter/chronicleexporter/internal/marshal/marshal_test.go b/exporter/chronicleexporter/internal/marshal/marshal_test.go new file mode 100644 index 000000000..10a6b40a2 --- /dev/null +++ b/exporter/chronicleexporter/internal/marshal/marshal_test.go @@ -0,0 +1,225 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshal_test + +import ( + "context" + "testing" + + "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/internal/marshal" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/pdata/plog" + "golang.org/x/exp/rand" +) + +func Test_GetRawField(t *testing.T) { + for _, tc := range getRawFieldCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + set := componenttest.NewNopTelemetrySettings() + rawField, err := marshal.GetRawField(ctx, set, tc.field, tc.logRecord, tc.scope, tc.resource) + if tc.expectErrStr != "" { + require.Contains(t, err.Error(), tc.expectErrStr) + return + } + require.NoError(t, err) + require.Equal(t, tc.expect, rawField) + }) + } +} + +func Benchmark_GetRawField(b *testing.B) { + ctx := context.Background() + set := componenttest.NewNopTelemetrySettings() + for _, tc := range getRawFieldCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = marshal.GetRawField(ctx, set, tc.field, tc.logRecord, tc.scope, tc.resource) + } + }) + } +} + +func tokenWithLength(length int) []byte { + charset := "abcdefghijklmnopqrstuvwxyz" + b := make([]byte, length) + for i := range b { + b[i] = charset[rand.Intn(len(charset))] + } + return b +} + +func mockLogRecord(body string, attributes map[string]any) plog.LogRecord { + lr := plog.NewLogRecord() + lr.Body().SetStr(body) + for k, v := range attributes { + switch val := v.(type) { + case string: + lr.Attributes().PutStr(k, val) + default: + } + } + return lr +} + +func mockLogs(record plog.LogRecord) plog.Logs { + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + sl := rl.ScopeLogs().AppendEmpty() + record.CopyTo(sl.LogRecords().AppendEmpty()) + return logs +} + +type getRawFieldCase struct { + name string + field string + logRecord plog.LogRecord + scope plog.ScopeLogs + resource plog.ResourceLogs + expect string + expectErrStr string +} + +// Used by tests and benchmarks +var getRawFieldCases = []getRawFieldCase{ + { + name: "String body", + field: "body", + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Body().SetStr("703604000x80800000000000003562SystemWIN-L6PC55MPB98Print Spoolerstopped530070006F006F006C00650072002F0031000000") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "703604000x80800000000000003562SystemWIN-L6PC55MPB98Print Spoolerstopped530070006F006F006C00650072002F0031000000", + }, + { + name: "Empty body", + field: "body", + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Body().SetStr("") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "", + }, + { + name: "Map body", + field: "body", + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Body().SetEmptyMap() + lr.Body().Map().PutStr("param1", "Print Spooler") + lr.Body().Map().PutStr("param2", "stopped") + lr.Body().Map().PutStr("binary", "530070006F006F006C00650072002F0031000000") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: `{"binary":"530070006F006F006C00650072002F0031000000","param1":"Print Spooler","param2":"stopped"}`, + }, + { + name: "Map body field", + field: "body[\"param1\"]", + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Body().SetEmptyMap() + lr.Body().Map().PutStr("param1", "Print Spooler") + lr.Body().Map().PutStr("param2", "stopped") + lr.Body().Map().PutStr("binary", "530070006F006F006C00650072002F0031000000") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "Print Spooler", + }, + { + name: "Map body field missing", + field: "body[\"missing\"]", + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Body().SetEmptyMap() + lr.Body().Map().PutStr("param1", "Print Spooler") + lr.Body().Map().PutStr("param2", "stopped") + lr.Body().Map().PutStr("binary", "530070006F006F006C00650072002F0031000000") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "", + }, + { + name: "Attribute log_type", + field: `attributes["log_type"]`, + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Attributes().PutStr("status", "200") + lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") + lr.Attributes().PutStr("log_type", "WINEVTLOG") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "WINEVTLOG", + }, + { + name: "Attribute log_type missing", + field: `attributes["log_type"]`, + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Attributes().PutStr("status", "200") + lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "", + }, + { + name: "Attribute chronicle_log_type", + field: `attributes["chronicle_log_type"]`, + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Attributes().PutStr("status", "200") + lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") + lr.Attributes().PutStr("chronicle_log_type", "MICROSOFT_SQL") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "MICROSOFT_SQL", + }, + { + name: "Attribute chronicle_namespace", + field: `attributes["chronicle_namespace"]`, + logRecord: func() plog.LogRecord { + lr := plog.NewLogRecord() + lr.Attributes().PutStr("status", "200") + lr.Attributes().PutStr("log_type", "k8s-container") + lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") + lr.Attributes().PutStr("chronicle_log_type", "MICROSOFT_SQL") + lr.Attributes().PutStr("chronicle_namespace", "test") + return lr + }(), + scope: plog.NewScopeLogs(), + resource: plog.NewResourceLogs(), + expect: "test", + }, +} diff --git a/exporter/chronicleexporter/marshal.go b/exporter/chronicleexporter/marshal.go deleted file mode 100644 index aaea72a86..000000000 --- a/exporter/chronicleexporter/marshal.go +++ /dev/null @@ -1,573 +0,0 @@ -// Copyright observIQ, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chronicleexporter - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "time" - - "github.com/google/uuid" - "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" - "github.com/observiq/bindplane-otel-collector/expr" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottllog" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/plog" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" -) - -const logTypeField = `attributes["log_type"]` -const chronicleLogTypeField = `attributes["chronicle_log_type"]` -const chronicleNamespaceField = `attributes["chronicle_namespace"]` -const chronicleIngestionLabelsPrefix = `chronicle_ingestion_label` - -// This is a specific collector ID for Chronicle. It's used to identify bindplane agents in Chronicle. -var chronicleCollectorID = uuid.MustParse("aaaa1111-aaaa-1111-aaaa-1111aaaa1111") - -var supportedLogTypes = map[string]string{ - "windows_event.security": "WINEVTLOG", - "windows_event.application": "WINEVTLOG", - "windows_event.system": "WINEVTLOG", - "sql_server": "MICROSOFT_SQL", -} - -type protoMarshaler struct { - cfg Config - set component.TelemetrySettings - startTime time.Time - customerID []byte - collectorID []byte -} - -func newProtoMarshaler(cfg Config, set component.TelemetrySettings, customerID []byte) (*protoMarshaler, error) { - return &protoMarshaler{ - startTime: time.Now(), - cfg: cfg, - set: set, - customerID: customerID[:], - collectorID: chronicleCollectorID[:], - }, nil -} - -func (m *protoMarshaler) MarshalRawLogs(ctx context.Context, ld plog.Logs) ([]*api.BatchCreateLogsRequest, error) { - rawLogs, namespace, ingestionLabels, err := m.extractRawLogs(ctx, ld) - if err != nil { - return nil, fmt.Errorf("extract raw logs: %w", err) - } - return m.constructPayloads(rawLogs, namespace, ingestionLabels), nil -} - -func (m *protoMarshaler) extractRawLogs(ctx context.Context, ld plog.Logs) (map[string][]*api.LogEntry, map[string]string, map[string][]*api.Label, error) { - entries := make(map[string][]*api.LogEntry) - namespaceMap := make(map[string]string) - ingestionLabelsMap := make(map[string][]*api.Label) - - for i := 0; i < ld.ResourceLogs().Len(); i++ { - resourceLog := ld.ResourceLogs().At(i) - for j := 0; j < resourceLog.ScopeLogs().Len(); j++ { - scopeLog := resourceLog.ScopeLogs().At(j) - for k := 0; k < scopeLog.LogRecords().Len(); k++ { - logRecord := scopeLog.LogRecords().At(k) - rawLog, logType, namespace, ingestionLabels, err := m.processLogRecord(ctx, logRecord, scopeLog, resourceLog) - - if err != nil { - m.set.Logger.Error("Error processing log record", zap.Error(err)) - continue - } - - if rawLog == "" { - continue - } - - var timestamp time.Time - - if logRecord.Timestamp() != 0 { - timestamp = logRecord.Timestamp().AsTime() - } else { - timestamp = logRecord.ObservedTimestamp().AsTime() - } - - entry := &api.LogEntry{ - Timestamp: timestamppb.New(timestamp), - CollectionTime: timestamppb.New(logRecord.ObservedTimestamp().AsTime()), - Data: []byte(rawLog), - } - entries[logType] = append(entries[logType], entry) - // each logType maps to exactly 1 namespace value - if namespace != "" { - if _, ok := namespaceMap[logType]; !ok { - namespaceMap[logType] = namespace - } - } - if len(ingestionLabels) > 0 { - // each logType maps to a list of ingestion labels - if _, exists := ingestionLabelsMap[logType]; !exists { - ingestionLabelsMap[logType] = make([]*api.Label, 0) - } - existingLabels := make(map[string]struct{}) - for _, label := range ingestionLabelsMap[logType] { - existingLabels[label.Key] = struct{}{} - } - for _, label := range ingestionLabels { - // only add to ingestionLabelsMap if the label is unique - if _, ok := existingLabels[label.Key]; !ok { - ingestionLabelsMap[logType] = append(ingestionLabelsMap[logType], label) - existingLabels[label.Key] = struct{}{} - } - } - } - } - } - } - return entries, namespaceMap, ingestionLabelsMap, nil -} - -func (m *protoMarshaler) processLogRecord(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, string, string, []*api.Label, error) { - rawLog, err := m.getRawLog(ctx, logRecord, scope, resource) - if err != nil { - return "", "", "", nil, err - } - logType, err := m.getLogType(ctx, logRecord, scope, resource) - if err != nil { - return "", "", "", nil, err - } - namespace, err := m.getNamespace(ctx, logRecord, scope, resource) - if err != nil { - return "", "", "", nil, err - } - ingestionLabels, err := m.getIngestionLabels(logRecord) - if err != nil { - return "", "", "", nil, err - } - return rawLog, logType, namespace, ingestionLabels, nil -} - -func (m *protoMarshaler) processHTTPLogRecord(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, string, string, map[string]*api.Log_LogLabel, error) { - rawLog, err := m.getRawLog(ctx, logRecord, scope, resource) - if err != nil { - return "", "", "", nil, err - } - - logType, err := m.getLogType(ctx, logRecord, scope, resource) - if err != nil { - return "", "", "", nil, err - } - namespace, err := m.getNamespace(ctx, logRecord, scope, resource) - if err != nil { - return "", "", "", nil, err - } - ingestionLabels, err := m.getHTTPIngestionLabels(logRecord) - if err != nil { - return "", "", "", nil, err - } - - return rawLog, logType, namespace, ingestionLabels, nil -} - -func (m *protoMarshaler) getRawLog(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { - if m.cfg.RawLogField == "" { - entireLogRecord := map[string]any{ - "body": logRecord.Body().Str(), - "attributes": logRecord.Attributes().AsRaw(), - "resource_attributes": resource.Resource().Attributes().AsRaw(), - } - - bytesLogRecord, err := json.Marshal(entireLogRecord) - if err != nil { - return "", fmt.Errorf("marshal log record: %w", err) - } - - return string(bytesLogRecord), nil - } - return m.getRawField(ctx, m.cfg.RawLogField, logRecord, scope, resource) -} - -func (m *protoMarshaler) getLogType(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { - // check for attributes in attributes["chronicle_log_type"] - logType, err := m.getRawField(ctx, chronicleLogTypeField, logRecord, scope, resource) - if err != nil { - return "", fmt.Errorf("get chronicle log type: %w", err) - } - if logType != "" { - return logType, nil - } - - if m.cfg.OverrideLogType { - logType, err := m.getRawField(ctx, logTypeField, logRecord, scope, resource) - - if err != nil { - return "", fmt.Errorf("get log type: %w", err) - } - if logType != "" { - if chronicleLogType, ok := supportedLogTypes[logType]; ok { - return chronicleLogType, nil - } - } - } - - return m.cfg.LogType, nil -} - -func (m *protoMarshaler) getNamespace(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { - // check for attributes in attributes["chronicle_namespace"] - namespace, err := m.getRawField(ctx, chronicleNamespaceField, logRecord, scope, resource) - if err != nil { - return "", fmt.Errorf("get chronicle log type: %w", err) - } - if namespace != "" { - return namespace, nil - } - return m.cfg.Namespace, nil -} - -func (m *protoMarshaler) getIngestionLabels(logRecord plog.LogRecord) ([]*api.Label, error) { - // check for labels in attributes["chronicle_ingestion_labels"] - ingestionLabels, err := m.getRawNestedFields(chronicleIngestionLabelsPrefix, logRecord) - if err != nil { - return []*api.Label{}, fmt.Errorf("get chronicle ingestion labels: %w", err) - } - - if len(ingestionLabels) != 0 { - return ingestionLabels, nil - } - // use labels defined in config if needed - configLabels := make([]*api.Label, 0) - for key, value := range m.cfg.IngestionLabels { - configLabels = append(configLabels, &api.Label{ - Key: key, - Value: value, - }) - } - return configLabels, nil -} - -func (m *protoMarshaler) getHTTPIngestionLabels(logRecord plog.LogRecord) (map[string]*api.Log_LogLabel, error) { - // Check for labels in attributes["chronicle_ingestion_labels"] - ingestionLabels, err := m.getHTTPRawNestedFields(chronicleIngestionLabelsPrefix, logRecord) - if err != nil { - return nil, fmt.Errorf("get chronicle ingestion labels: %w", err) - } - - if len(ingestionLabels) != 0 { - return ingestionLabels, nil - } - - // use labels defined in the config if needed - configLabels := make(map[string]*api.Log_LogLabel) - for key, value := range m.cfg.IngestionLabels { - configLabels[key] = &api.Log_LogLabel{ - Value: value, - } - } - return configLabels, nil -} - -func (m *protoMarshaler) getRawField(ctx context.Context, field string, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) { - switch field { - case "body": - switch logRecord.Body().Type() { - case pcommon.ValueTypeStr: - return logRecord.Body().Str(), nil - case pcommon.ValueTypeMap: - bytes, err := json.Marshal(logRecord.Body().AsRaw()) - if err != nil { - return "", fmt.Errorf("marshal log body: %w", err) - } - return string(bytes), nil - } - case logTypeField: - attributes := logRecord.Attributes().AsRaw() - if logType, ok := attributes["log_type"]; ok { - if v, ok := logType.(string); ok { - return v, nil - } - } - return "", nil - case chronicleLogTypeField: - attributes := logRecord.Attributes().AsRaw() - if logType, ok := attributes["chronicle_log_type"]; ok { - if v, ok := logType.(string); ok { - return v, nil - } - } - return "", nil - case chronicleNamespaceField: - attributes := logRecord.Attributes().AsRaw() - if namespace, ok := attributes["chronicle_namespace"]; ok { - if v, ok := namespace.(string); ok { - return v, nil - } - } - return "", nil - } - - lrExpr, err := expr.NewOTTLLogRecordExpression(field, m.set) - if err != nil { - return "", fmt.Errorf("raw_log_field is invalid: %s", err) - } - tCtx := ottllog.NewTransformContext(logRecord, scope.Scope(), resource.Resource(), scope, resource) - - lrExprResult, err := lrExpr.Execute(ctx, tCtx) - if err != nil { - return "", fmt.Errorf("execute log record expression: %w", err) - } - - if lrExprResult == nil { - return "", nil - } - - switch result := lrExprResult.(type) { - case string: - return result, nil - case pcommon.Map: - bytes, err := json.Marshal(result.AsRaw()) - if err != nil { - return "", fmt.Errorf("marshal log record expression result: %w", err) - } - return string(bytes), nil - default: - return "", fmt.Errorf("unsupported log record expression result type: %T", lrExprResult) - } -} - -func (m *protoMarshaler) getRawNestedFields(field string, logRecord plog.LogRecord) ([]*api.Label, error) { - var nestedFields []*api.Label - logRecord.Attributes().Range(func(key string, value pcommon.Value) bool { - if !strings.HasPrefix(key, field) { - return true - } - // Extract the key name from the nested field - cleanKey := strings.Trim(key[len(field):], `[]"`) - var jsonMap map[string]string - - // If needs to be parsed as JSON - if err := json.Unmarshal([]byte(value.AsString()), &jsonMap); err == nil { - for k, v := range jsonMap { - nestedFields = append(nestedFields, &api.Label{Key: k, Value: v}) - } - } else { - nestedFields = append(nestedFields, &api.Label{Key: cleanKey, Value: value.AsString()}) - } - return true - }) - return nestedFields, nil -} - -func (m *protoMarshaler) getHTTPRawNestedFields(field string, logRecord plog.LogRecord) (map[string]*api.Log_LogLabel, error) { - nestedFields := make(map[string]*api.Log_LogLabel) // Map with key as string and value as Log_LogLabel - logRecord.Attributes().Range(func(key string, value pcommon.Value) bool { - if !strings.HasPrefix(key, field) { - return true - } - // Extract the key name from the nested field - cleanKey := strings.Trim(key[len(field):], `[]"`) - var jsonMap map[string]string - - // If needs to be parsed as JSON - if err := json.Unmarshal([]byte(value.AsString()), &jsonMap); err == nil { - for k, v := range jsonMap { - nestedFields[k] = &api.Log_LogLabel{ - Value: v, - } - } - } else { - nestedFields[cleanKey] = &api.Log_LogLabel{ - Value: value.AsString(), - } - } - return true - }) - - return nestedFields, nil -} - -func (m *protoMarshaler) constructPayloads(rawLogs map[string][]*api.LogEntry, namespaceMap map[string]string, ingestionLabelsMap map[string][]*api.Label) []*api.BatchCreateLogsRequest { - payloads := make([]*api.BatchCreateLogsRequest, 0, len(rawLogs)) - for logType, entries := range rawLogs { - if len(entries) > 0 { - namespace, ok := namespaceMap[logType] - if !ok { - namespace = m.cfg.Namespace - } - ingestionLabels := ingestionLabelsMap[logType] - - request := m.buildGRPCRequest(entries, logType, namespace, ingestionLabels) - - payloads = append(payloads, m.enforceMaximumsGRPCRequest(request)...) - } - } - return payloads -} - -func (m *protoMarshaler) enforceMaximumsGRPCRequest(request *api.BatchCreateLogsRequest) []*api.BatchCreateLogsRequest { - size := proto.Size(request) - entries := request.Batch.Entries - if size <= m.cfg.BatchRequestSizeLimitGRPC && len(entries) <= m.cfg.BatchLogCountLimitGRPC { - return []*api.BatchCreateLogsRequest{ - request, - } - } - - if len(entries) < 2 { - m.set.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) - return []*api.BatchCreateLogsRequest{} - } - - // split request into two - mid := len(entries) / 2 - leftHalf := entries[:mid] - rightHalf := entries[mid:] - - request.Batch.Entries = leftHalf - otherHalfRequest := m.buildGRPCRequest(rightHalf, request.Batch.LogType, request.Batch.Source.Namespace, request.Batch.Source.Labels) - - // re-enforce max size restriction on each half - enforcedRequest := m.enforceMaximumsGRPCRequest(request) - enforcedOtherHalfRequest := m.enforceMaximumsGRPCRequest(otherHalfRequest) - - return append(enforcedRequest, enforcedOtherHalfRequest...) -} - -func (m *protoMarshaler) buildGRPCRequest(entries []*api.LogEntry, logType, namespace string, ingestionLabels []*api.Label) *api.BatchCreateLogsRequest { - return &api.BatchCreateLogsRequest{ - Batch: &api.LogEntryBatch{ - StartTime: timestamppb.New(m.startTime), - Entries: entries, - LogType: logType, - Source: &api.EventSource{ - CollectorId: m.collectorID, - CustomerId: m.customerID, - Labels: ingestionLabels, - Namespace: namespace, - }, - }, - } -} - -func (m *protoMarshaler) MarshalRawLogsForHTTP(ctx context.Context, ld plog.Logs) (map[string][]*api.ImportLogsRequest, error) { - rawLogs, err := m.extractRawHTTPLogs(ctx, ld) - if err != nil { - return nil, fmt.Errorf("extract raw logs: %w", err) - } - return m.constructHTTPPayloads(rawLogs), nil -} - -func (m *protoMarshaler) extractRawHTTPLogs(ctx context.Context, ld plog.Logs) (map[string][]*api.Log, error) { - entries := make(map[string][]*api.Log) - for i := 0; i < ld.ResourceLogs().Len(); i++ { - resourceLog := ld.ResourceLogs().At(i) - for j := 0; j < resourceLog.ScopeLogs().Len(); j++ { - scopeLog := resourceLog.ScopeLogs().At(j) - for k := 0; k < scopeLog.LogRecords().Len(); k++ { - logRecord := scopeLog.LogRecords().At(k) - rawLog, logType, namespace, ingestionLabels, err := m.processHTTPLogRecord(ctx, logRecord, scopeLog, resourceLog) - if err != nil { - m.set.Logger.Error("Error processing log record", zap.Error(err)) - continue - } - - if rawLog == "" { - continue - } - - var timestamp time.Time - if logRecord.Timestamp() != 0 { - timestamp = logRecord.Timestamp().AsTime() - } else { - timestamp = logRecord.ObservedTimestamp().AsTime() - } - - entry := &api.Log{ - LogEntryTime: timestamppb.New(timestamp), - CollectionTime: timestamppb.New(logRecord.ObservedTimestamp().AsTime()), - Data: []byte(rawLog), - EnvironmentNamespace: namespace, - Labels: ingestionLabels, - } - entries[logType] = append(entries[logType], entry) - } - } - } - - return entries, nil -} - -func buildForwarderString(cfg Config) string { - format := "projects/%s/locations/%s/instances/%s/forwarders/%s" - return fmt.Sprintf(format, cfg.Project, cfg.Location, cfg.CustomerID, cfg.Forwarder) -} - -func (m *protoMarshaler) constructHTTPPayloads(rawLogs map[string][]*api.Log) map[string][]*api.ImportLogsRequest { - payloads := make(map[string][]*api.ImportLogsRequest, len(rawLogs)) - - for logType, entries := range rawLogs { - if len(entries) > 0 { - request := m.buildHTTPRequest(entries) - - payloads[logType] = m.enforceMaximumsHTTPRequest(request) - } - } - return payloads -} - -func (m *protoMarshaler) enforceMaximumsHTTPRequest(request *api.ImportLogsRequest) []*api.ImportLogsRequest { - size := proto.Size(request) - logs := request.GetInlineSource().Logs - if size <= m.cfg.BatchRequestSizeLimitHTTP && len(logs) <= m.cfg.BatchLogCountLimitHTTP { - return []*api.ImportLogsRequest{ - request, - } - } - - if len(logs) < 2 { - m.set.Logger.Error("Single entry exceeds max request size. Dropping entry", zap.Int("size", size)) - return []*api.ImportLogsRequest{} - } - - // split request into two - mid := len(logs) / 2 - leftHalf := logs[:mid] - rightHalf := logs[mid:] - - request.GetInlineSource().Logs = leftHalf - otherHalfRequest := m.buildHTTPRequest(rightHalf) - - // re-enforce max size restriction on each half - enforcedRequest := m.enforceMaximumsHTTPRequest(request) - enforcedOtherHalfRequest := m.enforceMaximumsHTTPRequest(otherHalfRequest) - - return append(enforcedRequest, enforcedOtherHalfRequest...) -} - -func (m *protoMarshaler) buildHTTPRequest(entries []*api.Log) *api.ImportLogsRequest { - return &api.ImportLogsRequest{ - // TODO: Add parent and hint - // We don't yet have solid guidance on what these should be - Parent: "", - Hint: "", - - Source: &api.ImportLogsRequest_InlineSource{ - InlineSource: &api.ImportLogsRequest_LogsInlineSource{ - Forwarder: buildForwarderString(m.cfg), - Logs: entries, - }, - }, - } -} diff --git a/exporter/chronicleexporter/marshal_test.go b/exporter/chronicleexporter/marshal_test.go deleted file mode 100644 index 0540c1125..000000000 --- a/exporter/chronicleexporter/marshal_test.go +++ /dev/null @@ -1,1706 +0,0 @@ -// Copyright observIQ, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chronicleexporter - -import ( - "context" - "testing" - "time" - - "github.com/google/uuid" - "github.com/observiq/bindplane-otel-collector/exporter/chronicleexporter/protos/api" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/pdata/plog" - "go.uber.org/zap" - "golang.org/x/exp/rand" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func TestProtoMarshaler_MarshalRawLogs(t *testing.T) { - logger := zap.NewNop() - startTime := time.Now() - - tests := []struct { - name string - cfg Config - logRecords func() plog.Logs - expectations func(t *testing.T, requests []*api.BatchCreateLogsRequest) - }{ - { - name: "Single log record with expected data", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Test log message", map[string]any{"log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["env"]`: "prod"})) - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1) - batch := requests[0].Batch - require.Equal(t, "WINEVTLOG", batch.LogType) - require.Len(t, batch.Entries, 1) - - // Convert Data (byte slice) to string for comparison - logDataAsString := string(batch.Entries[0].Data) - expectedLogData := `Test log message` - require.Equal(t, expectedLogData, logDataAsString) - - require.NotNil(t, batch.StartTime) - require.True(t, timestamppb.New(startTime).AsTime().Equal(batch.StartTime.AsTime()), "Start time should be set correctly") - }, - }, - { - name: "Single log record with expected data, no log_type, namespace, or ingestion labels", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: true, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Test log message", nil)) - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1) - batch := requests[0].Batch - require.Equal(t, "WINEVTLOG", batch.LogType) - require.Equal(t, "", batch.Source.Namespace) - require.Equal(t, 0, len(batch.Source.Labels)) - require.Len(t, batch.Entries, 1) - - // Convert Data (byte slice) to string for comparison - logDataAsString := string(batch.Entries[0].Data) - expectedLogData := `Test log message` - require.Equal(t, expectedLogData, logDataAsString) - - require.NotNil(t, batch.StartTime) - require.True(t, timestamppb.New(startTime).AsTime().Equal(batch.StartTime.AsTime()), "Start time should be set correctly") - }, - }, - { - name: "Multiple log records", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - return logs - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1, "Expected a single batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") - // Verifying the first log entry data - require.Equal(t, "First log message", string(batch.Entries[0].Data)) - // Verifying the second log entry data - require.Equal(t, "Second log message", string(batch.Entries[1].Data)) - }, - }, - { - name: "Log record with attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "attributes", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("", map[string]any{"key1": "value1", "log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"})) - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1) - batch := requests[0].Batch - require.Len(t, batch.Entries, 1) - - // Assuming the attributes are marshaled into the Data field as a JSON string - expectedData := `{"key1":"value1", "log_type":"WINEVTLOG", "namespace":"test", "chronicle_ingestion_label[\"key1\"]": "value1", "chronicle_ingestion_label[\"key2\"]": "value2"}` - actualData := string(batch.Entries[0].Data) - require.JSONEq(t, expectedData, actualData, "Log attributes should match expected") - }, - }, - { - name: "No log records", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "DEFAULT", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return plog.NewLogs() // No log records added - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 0, "Expected no requests due to no log records") - }, - }, - { - name: "No log type set in config or attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - RawLogField: "body", - OverrideLogType: true, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Log without logType", map[string]any{"namespace": "test", `ingestion_label["realkey1"]`: "realvalue1", `ingestion_label["realkey2"]`: "realvalue2"})) - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1) - batch := requests[0].Batch - require.Equal(t, "", batch.LogType, "Expected log type to be empty") - }, - }, - { - name: "Multiple log records with duplicate data, no log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - return logs - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify one request for log type in config - require.Len(t, requests, 1, "Expected a single batch request") - batch := requests[0].Batch - // verify batch source labels - require.Len(t, batch.Source.Labels, 2) - require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") - // Verifying the first log entry data - require.Equal(t, "First log message", string(batch.Entries[0].Data)) - // Verifying the second log entry data - require.Equal(t, "Second log message", string(batch.Entries[1].Data)) - }, - }, - { - name: "Multiple log records with different data, no log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) - return logs - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify one request for one log type - require.Len(t, requests, 1, "Expected a single batch request") - batch := requests[0].Batch - require.Equal(t, "WINEVTLOG", batch.LogType) - require.Equal(t, "", batch.Source.Namespace) - // verify batch source labels - require.Len(t, batch.Source.Labels, 4) - require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") - // Verifying the first log entry data - require.Equal(t, "First log message", string(batch.Entries[0].Data)) - // Verifying the second log entry data - require.Equal(t, "Second log message", string(batch.Entries[1].Data)) - }, - }, - { - name: "Override log type with attribute", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "DEFAULT", // This should be overridden by the log_type attribute - RawLogField: "body", - OverrideLogType: true, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"log_type": "windows_event.application", "namespace": "test", `ingestion_label["realkey1"]`: "realvalue1", `ingestion_label["realkey2"]`: "realvalue2"})) - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1) - batch := requests[0].Batch - require.Equal(t, "WINEVTLOG", batch.LogType, "Expected log type to be overridden by attribute") - }, - }, - { - name: "Override log type with chronicle attribute", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "DEFAULT", // This should be overridden by the chronicle_log_type attribute - RawLogField: "body", - OverrideLogType: true, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"chronicle_log_type": "ASOC_ALERT", "chronicle_namespace": "test", `chronicle_ingestion_label["realkey1"]`: "realvalue1", `chronicle_ingestion_label["realkey2"]`: "realvalue2"})) - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - require.Len(t, requests, 1) - batch := requests[0].Batch - require.Equal(t, "ASOC_ALERT", batch.LogType, "Expected log type to be overridden by attribute") - require.Equal(t, "test", batch.Source.Namespace, "Expected namespace to be overridden by attribute") - expectedLabels := map[string]string{ - "realkey1": "realvalue1", - "realkey2": "realvalue2", - } - for _, label := range batch.Source.Labels { - require.Equal(t, expectedLabels[label.Key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Multiple log records with duplicate data, log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - return logs - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 1 request, 2 batches for same log type - require.Len(t, requests, 1, "Expected a single batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 2, "Expected two log entries in the batch") - // verify batch for first log - require.Equal(t, "WINEVTLOGS", batch.LogType) - require.Equal(t, "test1", batch.Source.Namespace) - require.Len(t, batch.Source.Labels, 2) - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - for _, label := range batch.Source.Labels { - require.Equal(t, expectedLabels[label.Key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Multiple log records with different data, log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS2", "chronicle_namespace": "test2", `chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 2 requests, with 1 batch for different log types - require.Len(t, requests, 2, "Expected a two batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 1, "Expected one log entries in the batch") - // verify batch for first log - require.Contains(t, batch.LogType, "WINEVTLOGS") - require.Contains(t, batch.Source.Namespace, "test") - require.Len(t, batch.Source.Labels, 2) - - batch2 := requests[1].Batch - require.Len(t, batch2.Entries, 1, "Expected one log entries in the batch") - // verify batch for second log - require.Contains(t, batch2.LogType, "WINEVTLOGS") - require.Contains(t, batch2.Source.Namespace, "test") - require.Len(t, batch2.Source.Labels, 2) - // verify ingestion labels - for _, req := range requests { - for _, label := range req.Batch.Source.Labels { - require.Contains(t, []string{ - "key1", - "key2", - "key3", - "key4", - }, label.Key) - require.Contains(t, []string{ - "value1", - "value2", - "value3", - "value4", - }, label.Value) - } - } - }, - }, - { - name: "Many logs, all one batch", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - for i := 0; i < 1000; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr("Log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 1 request, with 1 batch - require.Len(t, requests, 1, "Expected a one-batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 1000, "Expected 1000 log entries in the batch") - // verify batch for first log - require.Contains(t, batch.LogType, "WINEVTLOGS") - require.Contains(t, batch.Source.Namespace, "test") - require.Len(t, batch.Source.Labels, 2) - - // verify ingestion labels - for _, req := range requests { - for _, label := range req.Batch.Source.Labels { - require.Contains(t, []string{ - "key1", - "key2", - "key3", - "key4", - }, label.Key) - require.Contains(t, []string{ - "value1", - "value2", - "value3", - "value4", - }, label.Value) - } - } - }, - }, - { - name: "Single batch split into multiple because more than 1000 logs", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - for i := 0; i < 1001; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr("Log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 1 request, with 1 batch - require.Len(t, requests, 2, "Expected a two-batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 500, "Expected 500 log entries in the first batch") - // verify batch for first log - require.Contains(t, batch.LogType, "WINEVTLOGS") - require.Contains(t, batch.Source.Namespace, "test") - require.Len(t, batch.Source.Labels, 2) - - batch2 := requests[1].Batch - require.Len(t, batch2.Entries, 501, "Expected 501 log entries in the second batch") - // verify batch for first log - require.Contains(t, batch2.LogType, "WINEVTLOGS") - require.Contains(t, batch2.Source.Namespace, "test") - require.Len(t, batch2.Source.Labels, 2) - - // verify ingestion labels - for _, req := range requests { - for _, label := range req.Batch.Source.Labels { - require.Contains(t, []string{ - "key1", - "key2", - "key3", - "key4", - }, label.Key) - require.Contains(t, []string{ - "value1", - "value2", - "value3", - "value4", - }, label.Value) - } - } - }, - }, - { - name: "Recursively split batch, exceeds 1000 entries multiple times", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - for i := 0; i < 2002; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr("Log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 1 request, with 1 batch - require.Len(t, requests, 4, "Expected a four-batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 500, "Expected 500 log entries in the first batch") - // verify batch for first log - require.Contains(t, batch.LogType, "WINEVTLOGS") - require.Contains(t, batch.Source.Namespace, "test") - require.Len(t, batch.Source.Labels, 2) - - batch2 := requests[1].Batch - require.Len(t, batch2.Entries, 501, "Expected 501 log entries in the second batch") - // verify batch for first log - require.Contains(t, batch2.LogType, "WINEVTLOGS") - require.Contains(t, batch2.Source.Namespace, "test") - require.Len(t, batch2.Source.Labels, 2) - - batch3 := requests[2].Batch - require.Len(t, batch3.Entries, 500, "Expected 500 log entries in the third batch") - // verify batch for first log - require.Contains(t, batch3.LogType, "WINEVTLOGS") - require.Contains(t, batch3.Source.Namespace, "test") - require.Len(t, batch3.Source.Labels, 2) - - batch4 := requests[3].Batch - require.Len(t, batch4.Entries, 501, "Expected 501 log entries in the fourth batch") - // verify batch for first log - require.Contains(t, batch4.LogType, "WINEVTLOGS") - require.Contains(t, batch4.Source.Namespace, "test") - require.Len(t, batch4.Source.Labels, 2) - - // verify ingestion labels - for _, req := range requests { - for _, label := range req.Batch.Source.Labels { - require.Contains(t, []string{ - "key1", - "key2", - "key3", - "key4", - }, label.Key) - require.Contains(t, []string{ - "value1", - "value2", - "value3", - "value4", - }, label.Value) - } - } - }, - }, - { - name: "Single batch split into multiple because request size too large", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - // create 640 logs with size 8192 bytes each - totalling 5242880 bytes. non-body fields put us over limit - for i := 0; i < 640; i++ { - record1 := logRecords.AppendEmpty() - body := tokenWithLength(8192) - record1.Body().SetStr(string(body)) - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify request, with 1 batch - require.Len(t, requests, 2, "Expected a two-batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 320, "Expected 320 log entries in the first batch") - // verify batch for first log - require.Contains(t, batch.LogType, "WINEVTLOGS") - require.Contains(t, batch.Source.Namespace, "test") - require.Len(t, batch.Source.Labels, 2) - - batch2 := requests[1].Batch - require.Len(t, batch2.Entries, 320, "Expected 320 log entries in the second batch") - // verify batch for first log - require.Contains(t, batch2.LogType, "WINEVTLOGS") - require.Contains(t, batch2.Source.Namespace, "test") - require.Len(t, batch2.Source.Labels, 2) - - // verify ingestion labels - for _, req := range requests { - for _, label := range req.Batch.Source.Labels { - require.Contains(t, []string{ - "key1", - "key2", - "key3", - "key4", - }, label.Key) - require.Contains(t, []string{ - "value1", - "value2", - "value3", - "value4", - }, label.Value) - } - } - }, - }, - { - name: "Recursively split batch into multiple because request size too large", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - // create 1280 logs with size 8192 bytes each - totalling 5242880 * 2 bytes. non-body fields put us over twice the limit - for i := 0; i < 1280; i++ { - record1 := logRecords.AppendEmpty() - body := tokenWithLength(8192) - record1.Body().SetStr(string(body)) - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 1 request, with 1 batch - require.Len(t, requests, 4, "Expected a four-batch request") - batch := requests[0].Batch - require.Len(t, batch.Entries, 320, "Expected 320 log entries in the first batch") - // verify batch for first log - require.Contains(t, batch.LogType, "WINEVTLOGS") - require.Contains(t, batch.Source.Namespace, "test") - require.Len(t, batch.Source.Labels, 2) - - batch2 := requests[1].Batch - require.Len(t, batch2.Entries, 320, "Expected 320 log entries in the second batch") - // verify batch for first log - require.Contains(t, batch2.LogType, "WINEVTLOGS") - require.Contains(t, batch2.Source.Namespace, "test") - require.Len(t, batch2.Source.Labels, 2) - - batch3 := requests[2].Batch - require.Len(t, batch3.Entries, 320, "Expected 320 log entries in the third batch") - // verify batch for first log - require.Contains(t, batch3.LogType, "WINEVTLOGS") - require.Contains(t, batch3.Source.Namespace, "test") - require.Len(t, batch3.Source.Labels, 2) - - batch4 := requests[3].Batch - require.Len(t, batch4.Entries, 320, "Expected 320 log entries in the fourth batch") - // verify batch for first log - require.Contains(t, batch4.LogType, "WINEVTLOGS") - require.Contains(t, batch4.Source.Namespace, "test") - require.Len(t, batch4.Source.Labels, 2) - - // verify ingestion labels - for _, req := range requests { - for _, label := range req.Batch.Source.Labels { - require.Contains(t, []string{ - "key1", - "key2", - "key3", - "key4", - }, label.Key) - require.Contains(t, []string{ - "value1", - "value2", - "value3", - "value4", - }, label.Value) - } - } - }, - }, - { - name: "Unsplittable batch, single log exceeds max request size", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - body := tokenWithLength(5242881) - record1.Body().SetStr(string(body)) - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - return logs - }, - - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // verify 1 request, with 1 batch - require.Len(t, requests, 0, "Expected a zero requests") - }, - }, - { - name: "Multiple valid log records + unsplittable log entries", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitGRPC: 1000, - BatchRequestSizeLimitGRPC: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - tooLargeBody := string(tokenWithLength(5242881)) - // first normal log, then impossible to split log - logRecords1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - record1 := logRecords1.AppendEmpty() - record1.Body().SetStr("First log message") - tooLargeRecord1 := logRecords1.AppendEmpty() - tooLargeRecord1.Body().SetStr(tooLargeBody) - // first impossible to split log, then normal log - logRecords2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - tooLargeRecord2 := logRecords2.AppendEmpty() - tooLargeRecord2.Body().SetStr(tooLargeBody) - record2 := logRecords2.AppendEmpty() - record2.Body().SetStr("Second log message") - return logs - }, - expectations: func(t *testing.T, requests []*api.BatchCreateLogsRequest) { - // this is a kind of weird edge case, the overly large logs makes the final requests quite inefficient, but it's going to be so rare that the inefficiency isn't a real concern - require.Len(t, requests, 2, "Expected two batch requests") - batch1 := requests[0].Batch - require.Len(t, batch1.Entries, 1, "Expected one log entry in the first batch") - // Verifying the first log entry data - require.Equal(t, "First log message", string(batch1.Entries[0].Data)) - - batch2 := requests[1].Batch - require.Len(t, batch2.Entries, 1, "Expected one log entry in the second batch") - // Verifying the second log entry data - require.Equal(t, "Second log message", string(batch2.Entries[0].Data)) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - customerID, err := uuid.Parse(tt.cfg.CustomerID) - require.NoError(t, err) - - marshaler, err := newProtoMarshaler(tt.cfg, component.TelemetrySettings{Logger: logger}, customerID[:]) - marshaler.startTime = startTime - require.NoError(t, err) - - logs := tt.logRecords() - requests, err := marshaler.MarshalRawLogs(context.Background(), logs) - require.NoError(t, err) - - tt.expectations(t, requests) - }) - } -} - -func TestProtoMarshaler_MarshalRawLogsForHTTP(t *testing.T) { - logger := zap.NewNop() - startTime := time.Now() - - tests := []struct { - name string - cfg Config - labels []*api.Label - logRecords func() plog.Logs - expectations func(t *testing.T, requests map[string][]*api.ImportLogsRequest) - }{ - { - name: "Single log record with expected data", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - Protocol: protocolHTTPS, - Project: "test-project", - Location: "us", - Forwarder: uuid.New().String(), - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - labels: []*api.Label{ - {Key: "env", Value: "prod"}, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Test log message", map[string]any{"log_type": "WINEVTLOG", "namespace": "test"})) - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1) - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - require.Len(t, logs, 1) - // Convert Data (byte slice) to string for comparison - logDataAsString := string(logs[0].Data) - expectedLogData := `Test log message` - require.Equal(t, expectedLogData, logDataAsString) - }, - }, - { - name: "Multiple log records", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - labels: []*api.Label{ - {Key: "env", Value: "staging"}, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - return logs - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1, "Expected a single batch request") - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - require.Len(t, logs, 2, "Expected two log entries in the batch") - // Verifying the first log entry data - require.Equal(t, "First log message", string(logs[0].Data)) - // Verifying the second log entry data - require.Equal(t, "Second log message", string(logs[1].Data)) - }, - }, - { - name: "Log record with attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "attributes", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - labels: []*api.Label{}, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("", map[string]any{"key1": "value1", "log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"})) - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1) - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - // Assuming the attributes are marshaled into the Data field as a JSON string - expectedData := `{"key1":"value1", "log_type":"WINEVTLOG", "namespace":"test", "chronicle_ingestion_label[\"key1\"]": "value1", "chronicle_ingestion_label[\"key2\"]": "value2"}` - actualData := string(logs[0].Data) - require.JSONEq(t, expectedData, actualData, "Log attributes should match expected") - }, - }, - { - name: "No log records", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "DEFAULT", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - labels: []*api.Label{}, - logRecords: func() plog.Logs { - return plog.NewLogs() // No log records added - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 0, "Expected no requests due to no log records") - }, - }, - { - name: "No log type set in config or attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "attributes", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - labels: []*api.Label{}, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("", map[string]any{"key1": "value1", "log_type": "WINEVTLOG", "namespace": "test", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"})) - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1) - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - // Assuming the attributes are marshaled into the Data field as a JSON string - expectedData := `{"key1":"value1", "log_type":"WINEVTLOG", "namespace":"test", "chronicle_ingestion_label[\"key1\"]": "value1", "chronicle_ingestion_label[\"key2\"]": "value2"}` - actualData := string(logs[0].Data) - require.JSONEq(t, expectedData, actualData, "Log attributes should match expected") - }, - }, - { - name: "Multiple log records with duplicate data, no log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{"chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - return logs - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - // verify one request for log type in config - require.Len(t, requests, 1, "Expected a single batch request") - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - // verify batch source labels - require.Len(t, logs[0].Labels, 2) - require.Len(t, logs, 2, "Expected two log entries in the batch") - // Verifying the first log entry data - require.Equal(t, "First log message", string(logs[0].Data)) - // Verifying the second log entry data - require.Equal(t, "Second log message", string(logs[1].Data)) - }, - }, - { - name: "Multiple log records with different data, no log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{`chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) - return logs - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - // verify one request for one log type - require.Len(t, requests, 1, "Expected a single batch request") - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - require.Len(t, logs, 2, "Expected two log entries in the batch") - require.Equal(t, "", logs[0].EnvironmentNamespace) - // verify batch source labels - require.Len(t, logs[0].Labels, 2) - require.Len(t, logs[1].Labels, 2) - // Verifying the first log entry data - require.Equal(t, "First log message", string(logs[0].Data)) - // Verifying the second log entry data - require.Equal(t, "Second log message", string(logs[1].Data)) - }, - }, - { - name: "Override log type with attribute", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "DEFAULT", // This should be overridden by the log_type attribute - RawLogField: "body", - OverrideLogType: true, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"log_type": "windows_event.application", "namespace": "test", `ingestion_label["realkey1"]`: "realvalue1", `ingestion_label["realkey2"]`: "realvalue2"})) - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1) - logs := requests["WINEVTLOG"][0].GetInlineSource().Logs - require.NotEqual(t, len(logs), 0) - }, - }, - { - name: "Override log type with chronicle attribute", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "DEFAULT", // This should be overridden by the chronicle_log_type attribute - RawLogField: "body", - OverrideLogType: true, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - return mockLogs(mockLogRecord("Log with overridden type", map[string]any{"chronicle_log_type": "ASOC_ALERT", "chronicle_namespace": "test", `chronicle_ingestion_label["realkey1"]`: "realvalue1", `chronicle_ingestion_label["realkey2"]`: "realvalue2"})) - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1) - logs := requests["ASOC_ALERT"][0].GetInlineSource().Logs - require.Equal(t, "test", logs[0].EnvironmentNamespace, "Expected namespace to be overridden by attribute") - expectedLabels := map[string]string{ - "realkey1": "realvalue1", - "realkey2": "realvalue2", - } - for key, label := range logs[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Multiple log records with duplicate data, log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - return logs - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - // verify 1 request, 2 batches for same log type - require.Len(t, requests, 1, "Expected a single batch request") - logs := requests["WINEVTLOGS"][0].GetInlineSource().Logs - require.Len(t, logs, 2, "Expected two log entries in the batch") - // verify variables - require.Equal(t, "test1", logs[0].EnvironmentNamespace) - require.Len(t, logs[0].Labels, 2) - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - for key, label := range logs[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Multiple log records with different data, log type in attributes", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - - record2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record2.Body().SetStr("Second log message") - record2.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS2", "chronicle_namespace": "test2", `chronicle_ingestion_label["key3"]`: "value3", `chronicle_ingestion_label["key4"]`: "value4"}) - return logs - }, - - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - "key3": "value3", - "key4": "value4", - } - // verify 2 requests, with 1 batch for different log types - require.Len(t, requests, 2, "Expected a two batch request") - - logs1 := requests["WINEVTLOGS1"][0].GetInlineSource().Logs - require.Len(t, logs1, 1, "Expected one log entries in the batch") - // verify variables for first log - require.Equal(t, logs1[0].EnvironmentNamespace, "test1") - require.Len(t, logs1[0].Labels, 2) - for key, label := range logs1[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs2 := requests["WINEVTLOGS2"][0].GetInlineSource().Logs - require.Len(t, logs2, 1, "Expected one log entries in the batch") - // verify variables for second log - require.Equal(t, logs2[0].EnvironmentNamespace, "test2") - require.Len(t, logs2[0].Labels, 2) - for key, label := range logs2[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Many log records all one batch", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - for i := 0; i < 1000; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - - return logs - }, - - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - // verify 1 requests - require.Len(t, requests, 1, "Expected a one batch request") - - logs1 := requests["WINEVTLOGS1"][0].GetInlineSource().Logs - require.Len(t, logs1, 1000, "Expected one thousand log entries in the batch") - // verify variables for first log - require.Equal(t, logs1[0].EnvironmentNamespace, "test1") - require.Len(t, logs1[0].Labels, 2) - for key, label := range logs1[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Many log records split into two batches", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - for i := 0; i < 1001; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - - return logs - }, - - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - // verify 1 request log type - require.Len(t, requests, 1, "Expected one log type for the requests") - winEvtLogRequests := requests["WINEVTLOGS1"] - require.Len(t, winEvtLogRequests, 2, "Expected two batches") - - logs1 := winEvtLogRequests[0].GetInlineSource().Logs - require.Len(t, logs1, 500, "Expected 500 log entries in the first batch") - // verify variables for first log - require.Equal(t, logs1[0].EnvironmentNamespace, "test1") - require.Len(t, logs1[0].Labels, 2) - for key, label := range logs1[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs2 := winEvtLogRequests[1].GetInlineSource().Logs - require.Len(t, logs2, 501, "Expected 501 log entries in the second batch") - // verify variables for first log - require.Equal(t, logs2[0].EnvironmentNamespace, "test1") - require.Len(t, logs2[0].Labels, 2) - for key, label := range logs2[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Recursively split batch multiple times because too many logs", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - for i := 0; i < 2002; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr("First log message") - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - - return logs - }, - - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - // verify 1 request log type - require.Len(t, requests, 1, "Expected one log type for the requests") - winEvtLogRequests := requests["WINEVTLOGS1"] - require.Len(t, winEvtLogRequests, 4, "Expected four batches") - - logs1 := winEvtLogRequests[0].GetInlineSource().Logs - require.Len(t, logs1, 500, "Expected 500 log entries in the first batch") - // verify variables for first log - require.Equal(t, logs1[0].EnvironmentNamespace, "test1") - require.Len(t, logs1[0].Labels, 2) - for key, label := range logs1[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs2 := winEvtLogRequests[1].GetInlineSource().Logs - require.Len(t, logs2, 501, "Expected 501 log entries in the second batch") - // verify variables for first log - require.Equal(t, logs2[0].EnvironmentNamespace, "test1") - require.Len(t, logs2[0].Labels, 2) - for key, label := range logs2[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs3 := winEvtLogRequests[2].GetInlineSource().Logs - require.Len(t, logs3, 500, "Expected 500 log entries in the third batch") - // verify variables for first log - require.Equal(t, logs3[0].EnvironmentNamespace, "test1") - require.Len(t, logs3[0].Labels, 2) - for key, label := range logs3[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs4 := winEvtLogRequests[3].GetInlineSource().Logs - require.Len(t, logs4, 501, "Expected 501 log entries in the fourth batch") - // verify variables for first log - require.Equal(t, logs4[0].EnvironmentNamespace, "test1") - require.Len(t, logs4[0].Labels, 2) - for key, label := range logs4[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Many log records split into two batches because request size too large", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - // 8192 * 640 = 5242880 - body := tokenWithLength(8192) - for i := 0; i < 640; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr(string(body)) - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - - return logs - }, - - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - // verify 1 request log type - require.Len(t, requests, 1, "Expected one log type for the requests") - winEvtLogRequests := requests["WINEVTLOGS1"] - require.Len(t, winEvtLogRequests, 2, "Expected two batches") - - logs1 := winEvtLogRequests[0].GetInlineSource().Logs - require.Len(t, logs1, 320, "Expected 320 log entries in the first batch") - // verify variables for first log - require.Equal(t, logs1[0].EnvironmentNamespace, "test1") - require.Len(t, logs1[0].Labels, 2) - for key, label := range logs1[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs2 := winEvtLogRequests[1].GetInlineSource().Logs - require.Len(t, logs2, 320, "Expected 320 log entries in the second batch") - // verify variables for first log - require.Equal(t, logs2[0].EnvironmentNamespace, "test1") - require.Len(t, logs2[0].Labels, 2) - for key, label := range logs2[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Recursively split into batches because request size too large", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 2000, - BatchRequestSizeLimitHTTP: 5242880, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - // 8192 * 1280 = 5242880 * 2 - body := tokenWithLength(8192) - for i := 0; i < 1280; i++ { - record1 := logRecords.AppendEmpty() - record1.Body().SetStr(string(body)) - record1.Attributes().FromRaw(map[string]any{"chronicle_log_type": "WINEVTLOGS1", "chronicle_namespace": "test1", `chronicle_ingestion_label["key1"]`: "value1", `chronicle_ingestion_label["key2"]`: "value2"}) - } - - return logs - }, - - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - expectedLabels := map[string]string{ - "key1": "value1", - "key2": "value2", - } - // verify 1 request log type - require.Len(t, requests, 1, "Expected one log type for the requests") - winEvtLogRequests := requests["WINEVTLOGS1"] - require.Len(t, winEvtLogRequests, 4, "Expected four batches") - - logs1 := winEvtLogRequests[0].GetInlineSource().Logs - require.Len(t, logs1, 320, "Expected 320 log entries in the first batch") - // verify variables for first log - require.Equal(t, logs1[0].EnvironmentNamespace, "test1") - require.Len(t, logs1[0].Labels, 2) - for key, label := range logs1[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs2 := winEvtLogRequests[1].GetInlineSource().Logs - require.Len(t, logs2, 320, "Expected 320 log entries in the second batch") - // verify variables for first log - require.Equal(t, logs2[0].EnvironmentNamespace, "test1") - require.Len(t, logs2[0].Labels, 2) - for key, label := range logs2[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs3 := winEvtLogRequests[2].GetInlineSource().Logs - require.Len(t, logs3, 320, "Expected 320 log entries in the third batch") - // verify variables for first log - require.Equal(t, logs3[0].EnvironmentNamespace, "test1") - require.Len(t, logs3[0].Labels, 2) - for key, label := range logs3[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - - logs4 := winEvtLogRequests[3].GetInlineSource().Logs - require.Len(t, logs4, 320, "Expected 320 log entries in the fourth batch") - // verify variables for first log - require.Equal(t, logs4[0].EnvironmentNamespace, "test1") - require.Len(t, logs4[0].Labels, 2) - for key, label := range logs4[0].Labels { - require.Equal(t, expectedLabels[key], label.Value, "Expected ingestion label to be overridden by attribute") - } - }, - }, - { - name: "Unsplittable log record, single log exceeds request size limit", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 100000, - }, - labels: []*api.Label{ - {Key: "env", Value: "staging"}, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - record1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - record1.Body().SetStr(string(tokenWithLength(100000))) - return logs - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1, "Expected one log type") - require.Len(t, requests["WINEVTLOG"], 0, "Expected WINEVTLOG log type to have zero requests") - }, - }, - { - name: "Unsplittable log record, single log exceeds request size limit, mixed with okay logs", - cfg: Config{ - CustomerID: uuid.New().String(), - LogType: "WINEVTLOG", - RawLogField: "body", - OverrideLogType: false, - BatchLogCountLimitHTTP: 1000, - BatchRequestSizeLimitHTTP: 100000, - }, - labels: []*api.Label{ - {Key: "env", Value: "staging"}, - }, - logRecords: func() plog.Logs { - logs := plog.NewLogs() - tooLargeBody := string(tokenWithLength(100001)) - // first normal log, then impossible to split log - logRecords1 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - record1 := logRecords1.AppendEmpty() - record1.Body().SetStr("First log message") - tooLargeRecord1 := logRecords1.AppendEmpty() - tooLargeRecord1.Body().SetStr(tooLargeBody) - // first impossible to split log, then normal log - logRecords2 := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() - tooLargeRecord2 := logRecords2.AppendEmpty() - tooLargeRecord2.Body().SetStr(tooLargeBody) - record2 := logRecords2.AppendEmpty() - record2.Body().SetStr("Second log message") - return logs - }, - expectations: func(t *testing.T, requests map[string][]*api.ImportLogsRequest) { - require.Len(t, requests, 1, "Expected one log type") - winEvtLogRequests := requests["WINEVTLOG"] - require.Len(t, winEvtLogRequests, 2, "Expected WINEVTLOG log type to have zero requests") - - logs1 := winEvtLogRequests[0].GetInlineSource().Logs - require.Len(t, logs1, 1, "Expected 1 log entry in the first batch") - require.Equal(t, string(logs1[0].Data), "First log message") - - logs2 := winEvtLogRequests[1].GetInlineSource().Logs - require.Len(t, logs2, 1, "Expected 1 log entry in the second batch") - require.Equal(t, string(logs2[0].Data), "Second log message") - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - customerID, err := uuid.Parse(tt.cfg.CustomerID) - require.NoError(t, err) - - marshaler, err := newProtoMarshaler(tt.cfg, component.TelemetrySettings{Logger: logger}, customerID[:]) - marshaler.startTime = startTime - require.NoError(t, err) - - logs := tt.logRecords() - requests, err := marshaler.MarshalRawLogsForHTTP(context.Background(), logs) - require.NoError(t, err) - - tt.expectations(t, requests) - }) - } -} - -func tokenWithLength(length int) []byte { - charset := "abcdefghijklmnopqrstuvwxyz" - b := make([]byte, length) - for i := range b { - b[i] = charset[rand.Intn(len(charset))] - } - return b -} - -func mockLogRecord(body string, attributes map[string]any) plog.LogRecord { - lr := plog.NewLogRecord() - lr.Body().SetStr(body) - for k, v := range attributes { - switch val := v.(type) { - case string: - lr.Attributes().PutStr(k, val) - default: - } - } - return lr -} - -func mockLogs(record plog.LogRecord) plog.Logs { - logs := plog.NewLogs() - rl := logs.ResourceLogs().AppendEmpty() - sl := rl.ScopeLogs().AppendEmpty() - record.CopyTo(sl.LogRecords().AppendEmpty()) - return logs -} - -type getRawFieldCase struct { - name string - field string - logRecord plog.LogRecord - scope plog.ScopeLogs - resource plog.ResourceLogs - expect string - expectErrStr string -} - -// Used by tests and benchmarks -var getRawFieldCases = []getRawFieldCase{ - { - name: "String body", - field: "body", - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Body().SetStr("703604000x80800000000000003562SystemWIN-L6PC55MPB98Print Spoolerstopped530070006F006F006C00650072002F0031000000") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "703604000x80800000000000003562SystemWIN-L6PC55MPB98Print Spoolerstopped530070006F006F006C00650072002F0031000000", - }, - { - name: "Empty body", - field: "body", - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Body().SetStr("") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "", - }, - { - name: "Map body", - field: "body", - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Body().SetEmptyMap() - lr.Body().Map().PutStr("param1", "Print Spooler") - lr.Body().Map().PutStr("param2", "stopped") - lr.Body().Map().PutStr("binary", "530070006F006F006C00650072002F0031000000") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: `{"binary":"530070006F006F006C00650072002F0031000000","param1":"Print Spooler","param2":"stopped"}`, - }, - { - name: "Map body field", - field: "body[\"param1\"]", - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Body().SetEmptyMap() - lr.Body().Map().PutStr("param1", "Print Spooler") - lr.Body().Map().PutStr("param2", "stopped") - lr.Body().Map().PutStr("binary", "530070006F006F006C00650072002F0031000000") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "Print Spooler", - }, - { - name: "Map body field missing", - field: "body[\"missing\"]", - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Body().SetEmptyMap() - lr.Body().Map().PutStr("param1", "Print Spooler") - lr.Body().Map().PutStr("param2", "stopped") - lr.Body().Map().PutStr("binary", "530070006F006F006C00650072002F0031000000") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "", - }, - { - name: "Attribute log_type", - field: `attributes["log_type"]`, - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Attributes().PutStr("status", "200") - lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") - lr.Attributes().PutStr("log_type", "WINEVTLOG") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "WINEVTLOG", - }, - { - name: "Attribute log_type missing", - field: `attributes["log_type"]`, - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Attributes().PutStr("status", "200") - lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "", - }, - { - name: "Attribute chronicle_log_type", - field: `attributes["chronicle_log_type"]`, - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Attributes().PutStr("status", "200") - lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") - lr.Attributes().PutStr("chronicle_log_type", "MICROSOFT_SQL") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "MICROSOFT_SQL", - }, - { - name: "Attribute chronicle_namespace", - field: `attributes["chronicle_namespace"]`, - logRecord: func() plog.LogRecord { - lr := plog.NewLogRecord() - lr.Attributes().PutStr("status", "200") - lr.Attributes().PutStr("log_type", "k8s-container") - lr.Attributes().PutStr("log.file.name", "/var/log/containers/agent_agent_ns.log") - lr.Attributes().PutStr("chronicle_log_type", "MICROSOFT_SQL") - lr.Attributes().PutStr("chronicle_namespace", "test") - return lr - }(), - scope: plog.NewScopeLogs(), - resource: plog.NewResourceLogs(), - expect: "test", - }, -} - -func Test_getRawField(t *testing.T) { - for _, tc := range getRawFieldCases { - t.Run(tc.name, func(t *testing.T) { - m := &protoMarshaler{} - m.set.Logger = zap.NewNop() - - ctx := context.Background() - - rawField, err := m.getRawField(ctx, tc.field, tc.logRecord, tc.scope, tc.resource) - if tc.expectErrStr != "" { - require.Contains(t, err.Error(), tc.expectErrStr) - return - } - - require.NoError(t, err) - require.Equal(t, tc.expect, rawField) - }) - } -} - -func Benchmark_getRawField(b *testing.B) { - m := &protoMarshaler{} - m.set.Logger = zap.NewNop() - - ctx := context.Background() - - for _, tc := range getRawFieldCases { - b.ResetTimer() - b.Run(tc.name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = m.getRawField(ctx, tc.field, tc.logRecord, tc.scope, tc.resource) - } - }) - } -}