Skip to content

Commit

Permalink
Add opentelemetry endpoint tracer
Browse files Browse the repository at this point in the history
  • Loading branch information
sagikazarmark committed May 8, 2020
1 parent 0712076 commit 6f65481
Show file tree
Hide file tree
Showing 6 changed files with 426 additions and 0 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ require (
github.com/stretchr/testify v1.5.1
github.com/vektah/gqlparser/v2 v2.0.1
go.opencensus.io v0.22.3
go.opentelemetry.io/otel v0.4.3
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d
google.golang.org/grpc v1.28.1
Expand Down
11 changes: 11 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7 h1:qELHH0AWCvf98Yf+CNIJx9vOZOfHFDDzgDRYsnNk/vs=
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
Expand Down Expand Up @@ -51,6 +53,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/benbjohnson/clock v1.0.0 h1:78Jk/r6m4wCi6sndMpty7A//t4dw/RW5fV4ZgDVfX1w=
github.com/benbjohnson/clock v1.0.0/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
Expand Down Expand Up @@ -172,6 +176,9 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
Expand Down Expand Up @@ -334,6 +341,7 @@ github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
Expand Down Expand Up @@ -494,6 +502,8 @@ go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/otel v0.4.3 h1:CroUX/0O1ZDcF0iWOO8gwYFWb5EbdSF0/C1yosO+Vhs=
go.opentelemetry.io/otel v0.4.3/go.mod h1:jzBIgIzK43Iu1BpDAXwqOd6UPsSAk+ewVZ5ofSXw4Ek=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down Expand Up @@ -654,6 +664,7 @@ google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0W
google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d h1:I7Vuu5Ejagca+VcgfBINHke3xwjCTYnIG4Q57fv0wYY=
google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
Expand Down
4 changes: 4 additions & 0 deletions internal/platform/kitx/tracing/opentelemetry/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// Package opentelemetry provides Go kit integration to the OpenTelemetry project.
//
// OpenTelemetry makes robust, portable telemetry a built-in feature of cloud-native software.
package opentelemetry
105 changes: 105 additions & 0 deletions internal/platform/kitx/tracing/opentelemetry/endpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package opentelemetry

import (
"context"
"strconv"

"go.opentelemetry.io/otel/api/core"
"go.opentelemetry.io/otel/api/global"
"go.opentelemetry.io/otel/api/key"
"go.opentelemetry.io/otel/api/trace"
"google.golang.org/grpc/codes"

"github.com/go-kit/kit/endpoint"
"github.com/go-kit/kit/sd/lb"
)

// TraceEndpointDefaultName is the default endpoint span name to use.
const TraceEndpointDefaultName = "gokit/endpoint"

// TraceEndpoint returns an Endpoint middleware, tracing a Go kit endpoint.
// This endpoint tracer should be used in combination with a Go kit Transport
// tracing middleware, generic OpenTelemetry transport middleware or custom before
// and after transport functions as service propagation of SpanContext is not
// provided in this middleware.
func TraceEndpoint(options ...EndpointOption) endpoint.Middleware {
cfg := &EndpointOptions{}

global.Tracer("")

for _, o := range options {
o(cfg)
}

if cfg.Tracer == nil {
cfg.Tracer = global.Tracer("")
}

if cfg.DefaultName == "" {
cfg.DefaultName = TraceEndpointDefaultName
}

return func(next endpoint.Endpoint) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (response interface{}, err error) {
name := cfg.DefaultName

if cfg.GetName != nil {
if newName := cfg.GetName(ctx, name); newName != "" {
name = newName
}
}

ctx, span := cfg.Tracer.Start(
ctx, name,
trace.WithAttributes(cfg.Attributes...),
trace.WithAttributes(cfg.getAttributes(ctx)...),
)
defer span.End()

defer func() {
if err != nil {
if lberr, ok := err.(lb.RetryError); ok {
// handle errors originating from lb.Retry
attrs := make([]core.KeyValue, 0, len(lberr.RawErrors))
for idx, rawErr := range lberr.RawErrors {
attrs = append(attrs, key.String("gokit.retry.error."+strconv.Itoa(idx+1), rawErr.Error()))
}

span.SetAttributes(attrs...)
span.SetStatus(codes.Unknown, lberr.Final.Error())

return
}

// generic error
span.SetStatus(codes.Unknown, err.Error())

return
}

// test for business error
if res, ok := response.(endpoint.Failer); ok && res.Failed() != nil {
span.SetAttributes(key.String("gokit.business.error", res.Failed().Error()))

if cfg.IgnoreBusinessError {
// status ok

return
}

// treating business error as real error in span.
span.SetStatus(codes.Unknown, res.Failed().Error())

return
}

// no errors identified
// status ok
}()

response, err = next(ctx, request)

return
}
}
}
102 changes: 102 additions & 0 deletions internal/platform/kitx/tracing/opentelemetry/endpoint_options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package opentelemetry

import (
"context"

"go.opentelemetry.io/otel/api/core"
"go.opentelemetry.io/otel/api/trace"
)

// EndpointOptions holds the options for tracing an endpoint
type EndpointOptions struct {
// Tracer (if specified) is used for starting new spans.
// Falls back to a global tracer with an empty name.
//
// See https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/api.md#obtaining-a-tracer
Tracer trace.Tracer

// DefaultName is used as a fallback if GetName is not specified.
DefaultName string

// IgnoreBusinessError if set to true will not treat a business error
// identified through the endpoint.Failer interface as a span error.
IgnoreBusinessError bool

// Attributes holds the default attributes which will be set on span
// creation by our Endpoint middleware.
Attributes []core.KeyValue

// GetName is an optional function that can set the span name based on the existing name
// for the endpoint and information in the context.
//
// If the function is nil, or the returned name is empty, the existing name for the endpoint is used.
GetName func(ctx context.Context, name string) string

// GetAttributes is an optional function that can extract trace attributes
// from the context and add them to the span.
GetAttributes func(ctx context.Context) []core.KeyValue
}

func (o EndpointOptions) getAttributes(ctx context.Context) []core.KeyValue {
if o.GetAttributes == nil {
return nil
}

return o.GetAttributes(ctx)
}

// EndpointOption allows for functional options to our OpenTelemetry endpoint
// tracing middleware.
type EndpointOption func(*EndpointOptions)

// WithEndpointConfig sets all configuration options at once by use of the
// EndpointOptions struct.
func WithEndpointConfig(options EndpointOptions) EndpointOption {
return func(o *EndpointOptions) {
*o = options
}
}

// WithTracer sets the tracer.
func WithTracer(tracer trace.Tracer) EndpointOption {
return func(o *EndpointOptions) {
o.Tracer = tracer
}
}

// WithDefaultName sets the default name.
func WithDefaultName(defaultName string) EndpointOption {
return func(o *EndpointOptions) {
o.DefaultName = defaultName
}
}

// WithEndpointAttributes sets the default attributes for the spans created by
// the Endpoint tracer.
func WithEndpointAttributes(attrs ...core.KeyValue) EndpointOption {
return func(o *EndpointOptions) {
o.Attributes = attrs
}
}

// WithIgnoreBusinessError if set to true will not treat a business error
// identified through the endpoint.Failer interface as a span error.
func WithIgnoreBusinessError(val bool) EndpointOption {
return func(o *EndpointOptions) {
o.IgnoreBusinessError = val
}
}

// WithSpanName extracts additional attributes from the request context.
func WithSpanName(fn func(ctx context.Context, name string) string) EndpointOption {
return func(o *EndpointOptions) {
o.GetName = fn
}
}

// WithSpanAttributes extracts additional attributes from the request context.
func WithSpanAttributes(fn func(ctx context.Context) []core.KeyValue) EndpointOption {
return func(o *EndpointOptions) {
o.GetAttributes = fn
}
}
Loading

0 comments on commit 6f65481

Please sign in to comment.