[receiver/googlepubsub] Add Google Cloud Logging encoding

This commit addresses the need to accept logs from Google Cloud Platform
into an OpenTelemetry pipeline. The goal is to introduce the
`cloud_logging` encoding option for the receiver, allowing users to
seamlessly integrate Cloud Logging logs into their OpenTelemetry
pipeline.

issue #23184

Co-authored-by: Alex Van Boxel <[email protected]>

.chloggen/pubsubreceiver-add-cloudlogging-support.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: googlepubsubreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for GoogleCloud logging encoding
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [29299]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/googlecloudpubsubreceiver/README.md

@@ -23,8 +23,8 @@ The following configuration options are supported:
 * `subscription` (Required): The subscription name to receive OTLP data from. The subscription name  should be a 
   fully qualified resource name (eg: `projects/otel-project/subscriptions/otlp`).
 * `encoding` (Optional): The encoding that will be used to received data from the subscription. This can either be
-  `otlp_proto_trace`, `otlp_proto_metric`, `otlp_proto_log`, or `raw_text` (see `encoding`).  This will only be used as 
-  a fallback, when no `content-type` attribute is present.
+  `otlp_proto_trace`, `otlp_proto_metric`, `otlp_proto_log`, `cloud_logging`, or `raw_text` (see `encoding`).  This will
+  only be used as a fallback, when no `content-type` attribute is present.
 * `compression` (Optional): The compression that will be used on received data from the subscription. When set it can 
   only be `gzip`. This will only be used as a fallback, when no `content-encoding` attribute is present.
 * `endpoint` (Optional): Override the default Pubsub Endpoint, useful when connecting to the PubSub emulator instance
@@ -46,20 +46,29 @@ You should not need to set the encoding of the subscription as the receiver will
 by looking at the `ce-type` and `content-type` attributes of the message. Only when those attributes are not set 
 must the `encoding` field in the configuration be set. 
 
-| ce-type] | ce-datacontenttype | encoding | description |
-| --- | --- | --- | --- |
-| org.opentelemetry.otlp.traces.v1 | application/protobuf |  | Decode OTLP trace message |
-| org.opentelemetry.otlp.metrics.v1 | application/protobuf |  | Decode OTLP metric message |
-| org.opentelemetry.otlp.logs.v1 | application/json |  | Decode OTLP log message |
-| - | - | otlp_proto_trace | Decode OTLP trace message |
-| - | - | otlp_proto_metric | Decode OTLP trace message |
-| - | - | otlp_proto_log | Decode OTLP trace message |
-| - | - | raw_text | Wrap in an OTLP log message |
+| ce-type                           | ce-datacontenttype   | encoding          | description                                    |
+|-----------------------------------|----------------------|-------------------|------------------------------------------------|
+| org.opentelemetry.otlp.traces.v1  | application/protobuf |                   | Decode OTLP trace message                      |
+| org.opentelemetry.otlp.metrics.v1 | application/protobuf |                   | Decode OTLP metric message                     |
+| org.opentelemetry.otlp.logs.v1    | application/json     |                   | Decode OTLP log message                        |
+| -                                 | -                    | otlp_proto_trace  | Decode OTLP trace message                      |
+| -                                 | -                    | otlp_proto_metric | Decode OTLP trace message                      |
+| -                                 | -                    | otlp_proto_log    | Decode OTLP trace message                      |
+| -                                 | -                    | cloud_logging     | Decode [Cloud Logging] [LogEntry] message type |
+| -                                 | -                    | raw_text          | Wrap in an OTLP log message                    |
 
 When the `encoding` configuration is set, the attributes on the message are ignored.
 
-The receiver can be used for ingesting arbitrary text message on a Pubsub subscription and wrap them in OTLP Log
-message, making it a convenient way to ingest log lines from Pubsub.
+With `cloud_logging`, the receiver can be used to bring Cloud Logging messages into an OpenTelemetry pipeline. You'll
+first need to [set up a logging sink][sink-docs] with a Pub/Sub topic as its destination. Note that the `cloud_logging`
+integration is considered **alpha** as the semantic convention on some of the conversion are not stabilized yet.
+
+With `raw_text`, the receiver can be used for ingesting arbitrary text message on a Pubsub subscription, wrapping them
+in OTLP Log messages, making it a convenient way to ingest raw log lines from Pubsub.
+
+[Cloud Logging]: https://cloud.google.com/logging
+[LogEntry]: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry
+[sink-docs]: https://cloud.google.com/logging/docs/export/configure_export_v2#creating_sink
 
 ## Pubsub subscription
 

receiver/googlecloudpubsubreceiver/config.go

@@ -46,8 +46,9 @@ func (config *Config) validateForLog() error {
 	case "otlp_proto_log":
 	case "raw_text":
 	case "raw_json":
+	case "cloud_logging":
 	default:
-		return fmt.Errorf("log encoding %v is not supported.  supported encoding formats include [otlp_proto_log,raw_text,raw_json]", config.Encoding)
+		return fmt.Errorf("log encoding %v is not supported.  supported encoding formats include [otlp_proto_log,raw_text,raw_json,cloud_logging]", config.Encoding)
 	}
 	return nil
 }

receiver/googlecloudpubsubreceiver/go.mod

@@ -3,7 +3,11 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/google
 go 1.20
 
 require (
+	cloud.google.com/go/logging v1.9.0
 	cloud.google.com/go/pubsub v1.36.1
+	github.com/google/go-cmp v0.6.0
+	github.com/iancoleman/strcase v0.3.0
+	github.com/json-iterator/go v1.1.12
 	github.com/stretchr/testify v1.8.4
 	go.opentelemetry.io/collector/component v0.93.1-0.20240130182548-89388addcc7f
 	go.opentelemetry.io/collector/confmap v0.93.1-0.20240130182548-89388addcc7f
@@ -13,16 +17,21 @@ require (
 	go.opentelemetry.io/collector/receiver v0.93.1-0.20240130182548-89388addcc7f
 	go.opentelemetry.io/otel/metric v1.22.0
 	go.opentelemetry.io/otel/trace v1.22.0
+	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.26.0
 	google.golang.org/api v0.160.0
+	google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac
+	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe
 	google.golang.org/grpc v1.61.0
+	google.golang.org/protobuf v1.32.0
 )
 
 require (
 	cloud.google.com/go v0.112.0 // indirect
 	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.5 // indirect
+	cloud.google.com/go/longrunning v0.5.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -33,11 +42,9 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.0.1 // indirect
@@ -63,7 +70,6 @@ require (
 	go.opentelemetry.io/otel/exporters/prometheus v0.45.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.22.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
@@ -72,10 +78,7 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

receiver/googlecloudpubsubreceiver/internal/common_protos.go

@@ -0,0 +1,8 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/googlecloudpubsubreceiver/internal"
+
+import (
+	_ "google.golang.org/genproto/googleapis/cloud/audit" // support decoding Cloud Audit logs
+)