chore: Merge release/v1.66.0 to main for release prep (#2027)

cmd/plugindocgen/go.mod

@@ -3,7 +3,7 @@ module github.com/observiq/bindplane-agent/plugindocgen
 go 1.22.7
 
 require (
-	github.com/observiq/bindplane-agent/receiver/pluginreceiver v1.65.0
+	github.com/observiq/bindplane-agent/receiver/pluginreceiver v1.66.0
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -45,7 +45,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/shirou/gopsutil/v4 v4.24.10 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect

cmd/plugindocgen/go.sum

@@ -615,8 +615,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=

counter/go.mod

@@ -2,7 +2,7 @@ module github.com/observiq/bindplane-agent/counter
 
 go 1.22.7
 
-require github.com/stretchr/testify v1.9.0
+require github.com/stretchr/testify v1.10.0
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect

counter/go.sum

@@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

docs/development.md

@@ -33,9 +33,9 @@ The steps are as follows:
     ./scripts/update-otel.sh {COLLECTOR_VERSION} {CONTRIB_VERSION} {PDATA_VERSION}
     ```
     Grab the different versions from OTEL's GitHub by checking the latest release versions of [collector-contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib) and the [collector](https://github.com/open-telemetry/opentelemetry-collector). They should be the same.
-    The pdata version can be found by checking which version of it is imported in the collector-contrib [go.mod file](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/go.mod). The specific line with the pdata version should look similar to
-    ```
-    go.opentelemetry.io/collector/pdata v1.0.0 // indirect
+    The pdata version can be found by checking the latest release notes of the collector in a header with {PDATA_VERSION}/{COLLECTOR_VERSION}. All three version arguments should include the v - an example run of the script would look like this:
+    ```sh
+    ./scripts/update-otel.sh v0.114.0 v0.114.0 v1.20.0
     ```
 
 2. Run `make tidy`
@@ -46,25 +46,15 @@ The steps are as follows:
     ```
     These should be the same versions used in the first step.
 
-4. Update the `mdatagen` package in the `install-tools` command in the Makefile to now point to the same version as collector-contrib.
-    
-    Specifically, this line is the one to update:
-    ```
-    .PHONY: install-tools
-    install-tools:
-        ...
-        go install github.com/open-telemetry/opentelemetry-collector-contrib/cmd/[email protected]
-    ``` 
-
-5. Run `make install-tools`
+4. Run `make install-tools`
 
-6. Run `make generate`
+5. Run `make generate`
 
-7. Run `make tidy`
+6. Run `make tidy`
 
-8. Run `make ci-checks`
+7. Run `make ci-checks`
 
-If all was successful, the repo has had it's OTEL dependencies updated to the latest version. 
+If all was successful, the repo has had its OTEL dependencies updated to the latest version. 
 
 There is potential for tests to fail, deprecation issues, code changes, or a variety of other problems to arise, but once the above steps are successful the repo can be updated.
 

exporter/azureblobexporter/go.mod

@@ -4,7 +4,7 @@ go 1.22.7
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	go.opentelemetry.io/collector/component v0.114.0
 	go.opentelemetry.io/collector/component/componenttest v0.114.0
 	go.opentelemetry.io/collector/config/configretry v1.20.0

exporter/azureblobexporter/go.sum

@@ -66,8 +66,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opentelemetry.io/collector/component v0.114.0 h1:SVGbm5LvHGSTEDv7p92oPuBgK5tuiWR82I9+LL4TtBE=

exporter/chronicleexporter/go.mod

@@ -5,10 +5,10 @@ go 1.22.7
 require (
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.6.0
-	github.com/observiq/bindplane-agent/expr v1.65.0
+	github.com/observiq/bindplane-agent/expr v1.66.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.114.0
 	github.com/shirou/gopsutil/v3 v3.24.5
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	go.opentelemetry.io/collector/component v0.114.0
 	go.opentelemetry.io/collector/component/componenttest v0.114.0
 	go.opentelemetry.io/collector/config/configretry v1.20.0

exporter/chronicleexporter/go.sum

@@ -101,8 +101,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=

exporter/chronicleexporter/marshal.go

@@ -163,18 +163,26 @@ func (m *protoMarshaler) processLogRecord(ctx context.Context, logRecord plog.Lo
 	return rawLog, logType, namespace, ingestionLabels, nil
 }
 
-func (m *protoMarshaler) processHTTPLogRecord(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, string, error) {
+func (m *protoMarshaler) processHTTPLogRecord(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, string, string, map[string]*api.Log_LogLabel, error) {
 	rawLog, err := m.getRawLog(ctx, logRecord, scope, resource)
 	if err != nil {
-		return "", "", err
+		return "", "", "", nil, err
 	}
 
 	logType, err := m.getLogType(ctx, logRecord, scope, resource)
 	if err != nil {
-		return "", "", err
+		return "", "", "", nil, err
+	}
+	namespace, err := m.getNamespace(ctx, logRecord, scope, resource)
+	if err != nil {
+		return "", "", "", nil, err
+	}
+	ingestionLabels, err := m.getHTTPIngestionLabels(logRecord)
+	if err != nil {
+		return "", "", "", nil, err
 	}
 
-	return rawLog, logType, nil
+	return rawLog, logType, namespace, ingestionLabels, nil
 }
 
 func (m *protoMarshaler) getRawLog(ctx context.Context, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) {
@@ -239,6 +247,7 @@ func (m *protoMarshaler) getIngestionLabels(logRecord plog.LogRecord) ([]*api.La
 	if err != nil {
 		return []*api.Label{}, fmt.Errorf("get chronicle ingestion labels: %w", err)
 	}
+
 	if len(ingestionLabels) != 0 {
 		return ingestionLabels, nil
 	}
@@ -253,6 +262,27 @@ func (m *protoMarshaler) getIngestionLabels(logRecord plog.LogRecord) ([]*api.La
 	return configLabels, nil
 }
 
+func (m *protoMarshaler) getHTTPIngestionLabels(logRecord plog.LogRecord) (map[string]*api.Log_LogLabel, error) {
+	// Check for labels in attributes["chronicle_ingestion_labels"]
+	ingestionLabels, err := m.getHTTPRawNestedFields(chronicleIngestionLabelsPrefix, logRecord)
+	if err != nil {
+		return nil, fmt.Errorf("get chronicle ingestion labels: %w", err)
+	}
+
+	if len(ingestionLabels) != 0 {
+		return ingestionLabels, nil
+	}
+
+	// use labels defined in the config if needed
+	configLabels := make(map[string]*api.Log_LogLabel)
+	for key, value := range m.cfg.IngestionLabels {
+		configLabels[key] = &api.Log_LogLabel{
+			Value: value,
+		}
+	}
+	return configLabels, nil
+}
+
 func (m *protoMarshaler) getRawField(ctx context.Context, field string, logRecord plog.LogRecord, scope plog.ScopeLogs, resource plog.ResourceLogs) (string, error) {
 	switch field {
 	case "body":
@@ -344,6 +374,34 @@ func (m *protoMarshaler) getRawNestedFields(field string, logRecord plog.LogReco
 	return nestedFields, nil
 }
 
+func (m *protoMarshaler) getHTTPRawNestedFields(field string, logRecord plog.LogRecord) (map[string]*api.Log_LogLabel, error) {
+	nestedFields := make(map[string]*api.Log_LogLabel) // Map with key as string and value as Log_LogLabel
+	logRecord.Attributes().Range(func(key string, value pcommon.Value) bool {
+		if !strings.HasPrefix(key, field) {
+			return true
+		}
+		// Extract the key name from the nested field
+		cleanKey := strings.Trim(key[len(field):], `[]"`)
+		var jsonMap map[string]string
+
+		// If needs to be parsed as JSON
+		if err := json.Unmarshal([]byte(value.AsString()), &jsonMap); err == nil {
+			for k, v := range jsonMap {
+				nestedFields[k] = &api.Log_LogLabel{
+					Value: v,
+				}
+			}
+		} else {
+			nestedFields[cleanKey] = &api.Log_LogLabel{
+				Value: value.AsString(),
+			}
+		}
+		return true
+	})
+
+	return nestedFields, nil
+}
+
 func (m *protoMarshaler) constructPayloads(rawLogs map[string][]*api.LogEntry, namespaceMap map[string]string, ingestionLabelsMap map[string][]*api.Label) []*api.BatchCreateLogsRequest {
 	payloads := make([]*api.BatchCreateLogsRequest, 0, len(rawLogs))
 	for logType, entries := range rawLogs {
@@ -387,7 +445,7 @@ func (m *protoMarshaler) extractRawHTTPLogs(ctx context.Context, ld plog.Logs) (
 			scopeLog := resourceLog.ScopeLogs().At(j)
 			for k := 0; k < scopeLog.LogRecords().Len(); k++ {
 				logRecord := scopeLog.LogRecords().At(k)
-				rawLog, logType, err := m.processHTTPLogRecord(ctx, logRecord, scopeLog, resourceLog)
+				rawLog, logType, namespace, ingestionLabels, err := m.processHTTPLogRecord(ctx, logRecord, scopeLog, resourceLog)
 				if err != nil {
 					m.teleSettings.Logger.Error("Error processing log record", zap.Error(err))
 					continue
@@ -405,9 +463,11 @@ func (m *protoMarshaler) extractRawHTTPLogs(ctx context.Context, ld plog.Logs) (
 				}
 
 				entry := &api.Log{
-					LogEntryTime:   timestamppb.New(timestamp),
-					CollectionTime: timestamppb.New(logRecord.ObservedTimestamp().AsTime()),
-					Data:           []byte(rawLog),
+					LogEntryTime:         timestamppb.New(timestamp),
+					CollectionTime:       timestamppb.New(logRecord.ObservedTimestamp().AsTime()),
+					Data:                 []byte(rawLog),
+					EnvironmentNamespace: namespace,
+					Labels:               ingestionLabels,
 				}
 				entries[logType] = append(entries[logType], entry)
 			}