diff --git a/.changelog/1452.breaking.txt b/.changelog/1452.breaking.txt new file mode 100644 index 0000000000..bd70709a8d --- /dev/null +++ b/.changelog/1452.breaking.txt @@ -0,0 +1 @@ +feat(sumologicexporter)!: remove deprecated json_logs \ No newline at end of file diff --git a/docs/upgrading.md b/docs/upgrading.md index 34c5a4ab69..de9516b11f 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -1,5 +1,7 @@ # Upgrading +- [Upgrading to v0.95.0-sumo-0](#upgrading-to-v0950-sumo-0) + - [`sumologic` exporter: remove `json_logs`](#sumologic-exporter-remove-json_logs) - [Upgrading to v0.94.0-sumo-0](#upgrading-to-v0940-sumo-0) - [`servicegraph` processor: removed in favor of `servicegraph` connector](#servicegraph-processor-removed-in-favor-of-servicegraph-connector) - [Upgrading to v0.92.0-sumo-0](#upgrading-to-v0920-sumo-0) @@ -50,6 +52,14 @@ - [Removing unnecessary metadata using the resourceprocessor](#removing-unnecessary-metadata-using-the-resourceprocessor) - [Moving record-level attributes used for metadata to the resource level](#moving-record-level-attributes-used-for-metadata-to-the-resource-level) +## Upgrading to v0.95.0-sumo-0 + +### `sumologic` exporter: remove `json_logs` + +`json_logs` has been removed in favor of `transform` processor. + +Please follow [the migration process](#sumologic-exporter-deprecate-json_logs) + ## Upgrading to v0.94.0-sumo-0 ### `servicegraph` processor: removed in favor of `servicegraph` connector diff --git a/pkg/exporter/sumologicexporter/README.md b/pkg/exporter/sumologicexporter/README.md index 4bd585658a..9c514cc35e 100644 --- a/pkg/exporter/sumologicexporter/README.md +++ b/pkg/exporter/sumologicexporter/README.md @@ -61,25 +61,6 @@ exporters: # default = true clear_logs_timestamp: {true, false} - json_logs: - # defines which key will be used to attach the log body at. - # This option affects JSON log format only. - # By default this is "log". - log_key: - # defines whether to include a timestamp field when sending - # JSON logs, which would contain UNIX epoch timestamp in milliseconds. - # This option affects JSON log format only. - # default = true. - add_timestamp: {true, false} - # when add_timestamp is set to true then this key defines what is the name - # of the timestamp key. - # default = "timestamp". - timestamp_key: - # When flatten_body is set to true and log is a map, - # log's body is going to be flattened and `log_key` won't be used - # default = false - flatten_body: {true, false} - # instructs sumologicexporter to use an edpoint automatically generated by # sumologicextension; # to use direct endpoint, set it `auth` to `null` and set the endpoint configuration diff --git a/pkg/exporter/sumologicexporter/config.go b/pkg/exporter/sumologicexporter/config.go index a31ab347b8..eb8f958046 100644 --- a/pkg/exporter/sumologicexporter/config.go +++ b/pkg/exporter/sumologicexporter/config.go @@ -70,33 +70,11 @@ type Config struct { // By default this is true. ClearLogsTimestamp bool `mapstructure:"clear_logs_timestamp"` - JSONLogs `mapstructure:"json_logs"` - // StickySessionEnabled defines if sticky session support is enable. // By default this is false. StickySessionEnabled bool `mapstructure:"sticky_session_enabled"` } -type JSONLogs struct { - // LogKey defines which key will be used to attach the log body at. - // This option affects JSON log format only. - // By default this is "log". - LogKey string `mapstructure:"log_key"` - // AddTimestamp defines whether to include a timestamp field when sending - // JSON logs, which would contain UNIX epoch timestamp in milliseconds. - // This option affects JSON log format only. - // By default this is true. - AddTimestamp bool `mapstructure:"add_timestamp"` - // When add_timestamp is set to true then this key defines what is the name - // of the timestamp key. - // By default this is "timestamp". - TimestampKey string `mapstructure:"timestamp_key"` - // When flatten_body is set to true and log is a map, - // log's body is going to be flattened and `log_key` won't be used - // By default this is false. - FlattenBody bool `mapstructure:"flatten_body"` -} - // CreateDefaultHTTPClientSettings returns default http client settings func CreateDefaultHTTPClientSettings() confighttp.HTTPClientSettings { return confighttp.HTTPClientSettings{ @@ -230,12 +208,6 @@ const ( DefaultClearLogsTimestamp bool = true // DefaultLogKey defines default LogKey value DefaultLogKey string = "log" - // DefaultAddTimestamp defines default AddTimestamp value - DefaultAddTimestamp bool = true - // DefaultTimestampKey defines default TimestampKey value - DefaultTimestampKey string = "timestamp" - // DefaultFlattenBody defines default FlattenBody value - DefaultFlattenBody bool = false // DefaultDropRoutingAttribute defines default DropRoutingAttribute DefaultDropRoutingAttribute string = "" // DefaultStickySessionEnabled defines default StickySessionEnabled value diff --git a/pkg/exporter/sumologicexporter/factory.go b/pkg/exporter/sumologicexporter/factory.go index da7da6c386..6b98604db2 100644 --- a/pkg/exporter/sumologicexporter/factory.go +++ b/pkg/exporter/sumologicexporter/factory.go @@ -51,13 +51,7 @@ func createDefaultConfig() component.Config { MetricFormat: DefaultMetricFormat, Client: DefaultClient, ClearLogsTimestamp: DefaultClearLogsTimestamp, - JSONLogs: JSONLogs{ - LogKey: DefaultLogKey, - AddTimestamp: DefaultAddTimestamp, - TimestampKey: DefaultTimestampKey, - FlattenBody: DefaultFlattenBody, - }, - TraceFormat: OTLPTraceFormat, + TraceFormat: OTLPTraceFormat, HTTPClientSettings: CreateDefaultHTTPClientSettings(), BackOffConfig: configretry.NewDefaultBackOffConfig(), diff --git a/pkg/exporter/sumologicexporter/factory_test.go b/pkg/exporter/sumologicexporter/factory_test.go index 0ec6da3a86..a803d1beb3 100644 --- a/pkg/exporter/sumologicexporter/factory_test.go +++ b/pkg/exporter/sumologicexporter/factory_test.go @@ -44,12 +44,7 @@ func TestCreateDefaultConfig(t *testing.T) { MetricFormat: "otlp", Client: "otelcol", ClearLogsTimestamp: true, - JSONLogs: JSONLogs{ - LogKey: "log", - AddTimestamp: true, - TimestampKey: "timestamp", - }, - TraceFormat: "otlp", + TraceFormat: "otlp", HTTPClientSettings: confighttp.HTTPClientSettings{ Timeout: 30 * time.Second, diff --git a/pkg/exporter/sumologicexporter/sender.go b/pkg/exporter/sumologicexporter/sender.go index be551142e5..31976ae106 100644 --- a/pkg/exporter/sumologicexporter/sender.go +++ b/pkg/exporter/sumologicexporter/sender.go @@ -127,7 +127,6 @@ type sender struct { config *Config client *http.Client prometheusFormatter prometheusFormatter - jsonLogsConfig JSONLogs dataUrlMetrics string dataUrlLogs string dataUrlTraces string @@ -177,7 +176,6 @@ func newSender( config: cfg, client: cl, prometheusFormatter: pf, - jsonLogsConfig: cfg.JSONLogs, dataUrlMetrics: metricsUrl, dataUrlLogs: logsUrl, dataUrlTraces: tracesUrl, @@ -358,25 +356,10 @@ func (s *sender) logToText(record plog.LogRecord) string { func (s *sender) logToJSON(record plog.LogRecord) (string, error) { recordCopy := plog.NewLogRecord() record.CopyTo(recordCopy) - if s.jsonLogsConfig.AddTimestamp { - addJSONTimestamp(recordCopy.Attributes(), s.jsonLogsConfig.TimestampKey, recordCopy.Timestamp()) - } // Only append the body when it's not empty to prevent sending 'null' log. if body := recordCopy.Body(); !isEmptyAttributeValue(body) { - if s.jsonLogsConfig.FlattenBody && body.Type() == pcommon.ValueTypeMap { - // Cannot use CopyTo, as it overrides data.orig's values - body.Map().Range(func(k string, v pcommon.Value) bool { - _, ok := recordCopy.Attributes().Get(k) - - if !ok { - v.CopyTo(recordCopy.Attributes().PutEmpty(k)) - } - return true - }) - } else { - body.CopyTo(recordCopy.Attributes().PutEmpty(s.jsonLogsConfig.LogKey)) - } + body.CopyTo(recordCopy.Attributes().PutEmpty(DefaultLogKey)) } nextLine := new(bytes.Buffer) @@ -391,21 +374,6 @@ func (s *sender) logToJSON(record plog.LogRecord) (string, error) { return strings.TrimSuffix(nextLine.String(), "\n"), nil } -var timeZeroUTC = time.Unix(0, 0).UTC() - -// addJSONTimestamp adds a timestamp field to record attribtues before sending -// out the logs as JSON. -// If the attached timestamp is equal to 0 (millisecond based UNIX timestamp) -// then send out current time formatted as milliseconds since January 1, 1970. -func addJSONTimestamp(attrs pcommon.Map, timestampKey string, pt pcommon.Timestamp) { - t := pt.AsTime() - if t == timeZeroUTC { - attrs.PutInt(timestampKey, time.Now().UnixMilli()) - } else { - attrs.PutInt(timestampKey, t.UnixMilli()) - } -} - func isEmptyAttributeValue(att pcommon.Value) bool { switch att.Type() { case pcommon.ValueTypeEmpty: @@ -428,7 +396,7 @@ func isEmptyAttributeValue(att pcommon.Value) bool { // returns array of records which has not been sent correctly and error func (s *sender) sendNonOTLPLogs(ctx context.Context, rl plog.ResourceLogs, flds fields) ([]plog.LogRecord, error) { if s.config.LogFormat == OTLPLogFormat { - return nil, fmt.Errorf("Attempting to send OTLP logs as non-OTLP data") + return nil, fmt.Errorf("attempting to send OTLP logs as non-OTLP data") } var ( diff --git a/pkg/exporter/sumologicexporter/sender_test.go b/pkg/exporter/sumologicexporter/sender_test.go index 0a2903a113..2f1c5559f5 100644 --- a/pkg/exporter/sumologicexporter/sender_test.go +++ b/pkg/exporter/sumologicexporter/sender_test.go @@ -471,34 +471,14 @@ func TestSendLogsJsonConfig(t *testing.T) { }{ { name: "default config", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: DefaultLogKey, - AddTimestamp: DefaultAddTimestamp, - TimestampKey: DefaultTimestampKey, - FlattenBody: DefaultFlattenBody, - } - }, - }, - bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + + bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log"}` + `\n` + - `{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}`, + `{"key1":"value1","key2":"value2","log":"Another example log"}`, logsFunc: twoLogsFunc, }, { - name: "empty body", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: DefaultLogKey, - AddTimestamp: DefaultAddTimestamp, - TimestampKey: DefaultTimestampKey, - FlattenBody: DefaultFlattenBody, - } - }, - }, - bodyRegex: `{"key1":"value1","key2":"value2","timestamp":\d{13}}`, + name: "empty body", + bodyRegex: `{"key1":"value1","key2":"value2"}`, logsFunc: func() plog.ResourceLogs { rls := plog.NewResourceLogs() @@ -511,84 +491,10 @@ func TestSendLogsJsonConfig(t *testing.T) { return rls }, }, - { - name: "disabled add timestamp", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: DefaultLogKey, - AddTimestamp: false, - } - }, - }, - bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log"}` + - `\n` + - `{"key1":"value1","key2":"value2","log":"Another example log"}`, - logsFunc: twoLogsFunc, - }, - { - name: "enabled add timestamp with custom timestamp key", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: DefaultLogKey, - AddTimestamp: true, - TimestampKey: "xxyy_zz", - } - }, - }, - bodyRegex: `{"key1":"value1","key2":"value2","log":"Example log","xxyy_zz":\d{13}}` + - `\n` + - `{"key1":"value1","key2":"value2","log":"Another example log","xxyy_zz":\d{13}}`, - logsFunc: twoLogsFunc, - }, - { - name: "custom log key", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: "log_vendor_key", - AddTimestamp: DefaultAddTimestamp, - TimestampKey: DefaultTimestampKey, - FlattenBody: DefaultFlattenBody, - } - }, - }, - bodyRegex: `{"key1":"value1","key2":"value2","log_vendor_key":"Example log","timestamp":\d{13}}` + - `\n` + - `{"key1":"value1","key2":"value2","log_vendor_key":"Another example log","timestamp":\d{13}}`, - logsFunc: twoLogsFunc, - }, - { - name: "flatten body", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: "log_vendor_key", - AddTimestamp: DefaultAddTimestamp, - TimestampKey: DefaultTimestampKey, - FlattenBody: true, - } - }, - }, - bodyRegex: `{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` + - `"g":{"h":"i","j":false,"k":12,"l":11.1},"m":"n","timestamp":\d{13}}`, - logsFunc: twoComplexBodyLogsFunc, - }, { name: "complex body", - configOpts: []func(*Config){ - func(c *Config) { - c.JSONLogs = JSONLogs{ - LogKey: "log_vendor_key", - AddTimestamp: DefaultAddTimestamp, - TimestampKey: DefaultTimestampKey, - FlattenBody: DefaultFlattenBody, - } - }, - }, - bodyRegex: `{"log_vendor_key":{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` + - `"g":{"h":"i","j":false,"k":12,"l":11.1}},"m":"n","timestamp":\d{13}}`, + bodyRegex: `{"log":{"a":"b","c":false,"d":20,"e":20.5,"f":\["p",true,13,19.3\],` + + `"g":{"h":"i","j":false,"k":12,"l":11.1}},"m":"n"}`, logsFunc: twoComplexBodyLogsFunc, }, } @@ -620,9 +526,9 @@ func TestSendLogsJson(t *testing.T) { func(w http.ResponseWriter, req *http.Request) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Example log"}` regex += `\n` - regex += `{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Another example log"}` assert.Regexp(t, regex, body) assert.Equal(t, "key=value", req.Header.Get("X-Sumo-Fields")) @@ -659,9 +565,9 @@ func TestSendLogsJsonHTLM(t *testing.T) { func(w http.ResponseWriter, req *http.Request) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Example log"}` regex += `\n` - regex += `{"key1":"value1","key2":"value2","log":"

Another example log

","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"

Another example log

"}` assert.Regexp(t, regex, body) assert.Equal(t, "key=value", req.Header.Get("X-Sumo-Fields")) @@ -698,9 +604,9 @@ func TestSendLogsJsonMultitype(t *testing.T) { func(w http.ResponseWriter, req *http.Request) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":{"lk1":"lv1","lk2":13},"timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":{"lk1":"lv1","lk2":13}}` regex += `\n` - regex += `{"key1":"value1","key2":"value2","log":\["lv2",13\],"timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":\["lv2",13\]}` assert.Regexp(t, regex, body) assert.Equal(t, "key=value", req.Header.Get("X-Sumo-Fields")) @@ -752,13 +658,13 @@ func TestSendLogsJsonSplit(t *testing.T) { func(w http.ResponseWriter, req *http.Request) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Example log"}` assert.Regexp(t, regex, body) }, func(w http.ResponseWriter, req *http.Request) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Another example log"}` assert.Regexp(t, regex, body) }, }) @@ -795,14 +701,14 @@ func TestSendLogsJsonSplitFailedOne(t *testing.T) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Example log"}` assert.Regexp(t, regex, body) }, func(w http.ResponseWriter, req *http.Request) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Another example log"}` assert.Regexp(t, regex, body) }, }) @@ -840,7 +746,7 @@ func TestSendLogsJsonSplitFailedAll(t *testing.T) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Example log"}` assert.Regexp(t, regex, body) }, func(w http.ResponseWriter, req *http.Request) { @@ -849,7 +755,7 @@ func TestSendLogsJsonSplitFailedAll(t *testing.T) { body := extractBody(t, req) var regex string - regex += `{"key1":"value1","key2":"value2","log":"Another example log","timestamp":\d{13}}` + regex += `{"key1":"value1","key2":"value2","log":"Another example log"}` assert.Regexp(t, regex, body) }, })