From 6642b136b9de56bbe63ad2d47719501b7511e8ed Mon Sep 17 00:00:00 2001 From: joreetz-otto Date: Thu, 14 Dec 2023 16:12:46 +0100 Subject: [PATCH 1/3] Add option to fetch metadata and add it to resources Metadata will be fetched from Google Metadata Service and Environment --- pkg/gcp/services.go | 64 +++++++++++++++++++++++++++++----- pkg/gke/metadata.go | 85 +++++++++++++++++++++++++++++++++++++++++++++ pkg/gke/resource.go | 35 +++++++++++++++++++ 3 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 pkg/gke/metadata.go diff --git a/pkg/gcp/services.go b/pkg/gcp/services.go index 16607be..1caeb5c 100644 --- a/pkg/gcp/services.go +++ b/pkg/gcp/services.go @@ -9,6 +9,7 @@ import ( "cloud.google.com/go/logging" texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" "github.com/otto-de/sherlock-microservice/pkg/gke" + "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" "google.golang.org/genproto/googleapis/api/monitoredres" ) @@ -38,10 +39,11 @@ func NewLoggingClient(project string) (*logging.Client, error) { } type discoveryOption struct { - clusterName string - namespace string - pod string - containerName string + clusterName string + containerName string + namespace string + pod string + gkeAutoDiscoverMetaData bool } func WithKubernetes(clusterName, namespace, pod, containerName string) discoveryOption { @@ -53,9 +55,36 @@ func WithKubernetes(clusterName, namespace, pod, containerName string) discovery } } +func WithGKEAutoDiscoverMetaData() discoveryOption { + /* + This option will try to auto discover available metadata from the Google Cloud metadata service and environment variables. + For the container name it will use the environment variable CONTAINER_NAME. + For the pod name it will use the environment variable POD_NAME. + For the namespace it will use the environment variable NAMESPACE. + These variables cant be fetched from the metadata service. + Set the environment variables in the deployment manifest. + + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: test-container + + If the container name is equal to the pod name you might use the same field for both. + + */ + return discoveryOption{ + gkeAutoDiscoverMetaData: true, + } +} + // DiscoverServices builds clients for all Services that we use. func DiscoverServices(project, serviceName string, tracerProviderOptions []sdktrace.TracerProviderOption, opts ...discoveryOption) (*Services, error) { - loggingClient, err := NewLoggingClient(project) if err != nil { return nil, err @@ -83,19 +112,36 @@ func DiscoverServices(project, serviceName string, tracerProviderOptions []sdktr panic(err) } - tp := sdktrace.NewTracerProvider(append(tracerProviderOptions, sdktrace.WithBatcher(exporter))...) - s := &Services{ Logging: loggingClient, ErrorReporting: errorClient, - TracerProvider: tp, } + var traceResource *resource.Resource for _, opt := range opts { - if opt.pod != "" { + if opt.gkeAutoDiscoverMetaData { + metadata, err := gke.GetMetaData() + if err != nil { + defer logger.Flush() + + logger.Log(logging.Entry{ + Severity: logging.Info, + Payload: fmt.Sprintf("Error getting MetaData: %s", err), + }) + continue + } + + s.MonitoredResource = gke.MonitoredResourceFromMetaData(metadata) + traceResource = gke.TraceResourceFromMetaData(serviceName, metadata) + } else if opt.pod != "" { s.MonitoredResource = gke.MonitoredResource(s.Logging, project, opt.clusterName, opt.namespace, opt.pod, opt.containerName) } } + if traceResource != nil { + tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithResource(traceResource)) + } + + s.TracerProvider = sdktrace.NewTracerProvider(append(tracerProviderOptions, sdktrace.WithBatcher(exporter))...) return s, nil } diff --git a/pkg/gke/metadata.go b/pkg/gke/metadata.go new file mode 100644 index 0000000..3eca68f --- /dev/null +++ b/pkg/gke/metadata.go @@ -0,0 +1,85 @@ +package gke + +import ( + "encoding/json" + "os" + "strings" + + "cloud.google.com/go/compute/metadata" +) + +type GKEMetaData struct { + ClusterLocation string + ClusterName string + ContainerName string + Namespace string + InstanceName string + InstanceID int64 + NumericProjectID int64 + PodName string + ProjectID string + Zone string +} + +type instance struct { + Instance struct { + Attributes struct { + ClusterLocation string `json:"cluster-location"` + ClusterName string `json:"cluster-name"` + ClusterUID string `json:"cluster-uid"` + } `json:"attributes"` + Hostname string `json:"hostname"` + ID int64 `json:"id"` + Name string `json:"name"` + NetworkInterfaces map[string]struct { + Ipv6s string `json:"ipv6s"` + } `json:"networkInterfaces"` + ServiceAccounts map[string]struct { + Aliases []string `json:"aliases"` + Email string `json:"email"` + Scopes []string `json:"scopes"` + } `json:"serviceAccounts"` + Zone string `json:"zone"` + } `json:"instance"` + Project struct { + NumericProjectID int64 `json:"numericProjectId"` + ProjectID string `json:"projectId"` + } `json:"project"` +} + +func parseInstanceJSON(jsonData []byte) (*instance, error) { + var instance instance + err := json.Unmarshal(jsonData, &instance) + if err != nil { + return nil, err + } + return &instance, nil +} + +func GetMetaData() (*GKEMetaData, error) { + s, err := metadata.Get("/?recursive=true") + if err != nil { + return &GKEMetaData{}, err + } + + i, err := parseInstanceJSON([]byte(s)) + if err != nil { + return &GKEMetaData{}, err + } + + zp := strings.Split(i.Instance.Zone, "/") + zoneName := zp[len(zp)-1] + + return &GKEMetaData{ + ClusterLocation: i.Instance.Attributes.ClusterLocation, + ClusterName: i.Instance.Attributes.ClusterName, + ContainerName: os.Getenv("CONTAINER_NAME"), + Namespace: os.Getenv("POD_NAMESPACE"), + InstanceName: i.Instance.Name, + InstanceID: i.Instance.ID, + NumericProjectID: i.Project.NumericProjectID, + PodName: os.Getenv("POD_NAME"), + ProjectID: i.Project.ProjectID, + Zone: zoneName, + }, nil +} diff --git a/pkg/gke/resource.go b/pkg/gke/resource.go index 4ea2be9..bee2d07 100644 --- a/pkg/gke/resource.go +++ b/pkg/gke/resource.go @@ -5,6 +5,9 @@ import ( "cloud.google.com/go/compute/metadata" "cloud.google.com/go/logging" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" "google.golang.org/genproto/googleapis/api/monitoredres" ) @@ -53,3 +56,35 @@ func MonitoredResource(l *logging.Client, project, clusterName, namespace, pod, return res } + +func MonitoredResourceFromMetaData(metadata *GKEMetaData) *monitoredres.MonitoredResource { + return &monitoredres.MonitoredResource{ + Type: "gke_container", + Labels: map[string]string{ + "project_id": metadata.ProjectID, + "cluster_name": metadata.ClusterName, + "namespace_id": metadata.Namespace, + "instance_id": string(metadata.InstanceID), + "pod_id": metadata.PodName, + "container_name": metadata.ContainerName, + "zone": metadata.Zone, + }, + } +} + +func TraceResourceFromMetaData(serviceName string, metadata *GKEMetaData) *resource.Resource { + return resource.NewWithAttributes( + "", + semconv.CloudProviderGCP, + semconv.CloudPlatformGCPKubernetesEngine, + semconv.ServiceNameKey.String(serviceName), + attribute.String("g.co/r/k8s_container/project_id", metadata.ProjectID), + attribute.String("g.co/r/k8s_container/cluster_name", metadata.ClusterName), + attribute.String("g.co/r/k8s_container/namespace", metadata.Namespace), + attribute.String("g.co/r/k8s_container/location", metadata.ClusterLocation), + attribute.String("g.co/r/k8s_container/node_name", metadata.InstanceName), + attribute.String("g.co/r/k8s_container/pod_name", metadata.PodName), + attribute.String("g.co/r/k8s_container/container_name", metadata.ContainerName), + attribute.String("g.co/r/k8s_container/zone", metadata.Zone), + ) +} From 0473239380408cd571bafca4420fa424b5efeedf Mon Sep 17 00:00:00 2001 From: joreetz-otto Date: Fri, 15 Dec 2023 09:00:21 +0100 Subject: [PATCH 2/3] Update WithGKEAutoDiscoverMetaData doc --- pkg/gcp/services.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/gcp/services.go b/pkg/gcp/services.go index 1caeb5c..ea54bb6 100644 --- a/pkg/gcp/services.go +++ b/pkg/gcp/services.go @@ -58,6 +58,7 @@ func WithKubernetes(clusterName, namespace, pod, containerName string) discovery func WithGKEAutoDiscoverMetaData() discoveryOption { /* This option will try to auto discover available metadata from the Google Cloud metadata service and environment variables. + The metadata will be used to create the monitored resource and trace resource. For the container name it will use the environment variable CONTAINER_NAME. For the pod name it will use the environment variable POD_NAME. For the namespace it will use the environment variable NAMESPACE. @@ -66,17 +67,14 @@ func WithGKEAutoDiscoverMetaData() discoveryOption { - name: POD_NAME valueFrom: - fieldRef: - fieldPath: metadata.name + fieldRef: + fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: - fieldRef: - fieldPath: metadata.namespace + fieldRef: + fieldPath: metadata.namespace - name: CONTAINER_NAME value: test-container - - If the container name is equal to the pod name you might use the same field for both. - */ return discoveryOption{ gkeAutoDiscoverMetaData: true, From 2d639aab6f62e954b2ce5237228c8195b7033e5c Mon Sep 17 00:00:00 2001 From: joreetz-otto Date: Fri, 15 Dec 2023 10:02:02 +0100 Subject: [PATCH 3/3] Dont force flush logs --- pkg/gcp/services.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/gcp/services.go b/pkg/gcp/services.go index ea54bb6..b54b1f1 100644 --- a/pkg/gcp/services.go +++ b/pkg/gcp/services.go @@ -120,8 +120,6 @@ func DiscoverServices(project, serviceName string, tracerProviderOptions []sdktr if opt.gkeAutoDiscoverMetaData { metadata, err := gke.GetMetaData() if err != nil { - defer logger.Flush() - logger.Log(logging.Entry{ Severity: logging.Info, Payload: fmt.Sprintf("Error getting MetaData: %s", err),