Skip to content

Commit

Permalink
feat(autologs): init
Browse files Browse the repository at this point in the history
  • Loading branch information
ernado committed Dec 10, 2023
1 parent bfc7514 commit f409aa4
Show file tree
Hide file tree
Showing 8 changed files with 132 additions and 80 deletions.
49 changes: 3 additions & 46 deletions cmd/oteldb/oteldb.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,13 @@ package main

import (
"context"
"os"
"strings"

"github.com/go-faster/errors"
"github.com/go-faster/sdk/app"
"github.com/go-faster/sdk/zctx"
"go.opentelemetry.io/collector/pdata/plog/plogotlp"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"github.com/go-faster/oteldb/internal/autologs"
"github.com/go-faster/oteldb/internal/autozpages"
"github.com/go-faster/oteldb/internal/zapotel"
)

func main() {
Expand All @@ -28,44 +20,9 @@ func main() {
defer func() {
_ = shutdown(context.Background())
}()
if os.Getenv("OTEL_LOGS_EXPORTER") == "otlp" {
// Setting zap -> otel.
otelOptions := []otelgrpc.Option{
otelgrpc.WithTracerProvider(m.TracerProvider()),
otelgrpc.WithMeterProvider(m.MeterProvider()),
}
// Only PoC, should be replaced with real initialization
// and moved to go-faster/sdk.
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
if endpoint == "" {
endpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
}
if endpoint == "" {
endpoint = "localhost:4317"
}
endpoint = strings.TrimPrefix(endpoint, "http://")
conn, err := grpc.DialContext(ctx, endpoint,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelOptions...)),
)
if err != nil {
return errors.Wrap(err, "dial logs endpoint")
}
res, err := app.Resource(ctx)
if err != nil {
return errors.Wrap(err, "get resource")
}
otelCore := zapotel.New(lg.Level(), res, plogotlp.NewGRPCClient(conn))
// Update logger down the stack.
lg.Info("Setting up OTLP log exporter")
lg = lg.WithOptions(
zap.WrapCore(func(core zapcore.Core) zapcore.Core {
return zapcore.NewTee(core, otelCore)
}),
)
ctx = zctx.Base(ctx, lg)
if ctx, err = autologs.Setup(ctx, m); err != nil {
return errors.Wrap(err, "setup logs")
}

root, err := newApp(ctx, m)
if err != nil {
return errors.Wrap(err, "setup")
Expand Down
10 changes: 10 additions & 0 deletions cmd/otelfaker/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,21 @@ import (
"net/http"
"time"

"github.com/go-faster/errors"
"github.com/go-faster/sdk/app"
"github.com/go-faster/sdk/zctx"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.uber.org/zap"

"github.com/go-faster/oteldb/internal/autologs"
)

func client(ctx context.Context, lg *zap.Logger, m *app.Metrics) error {
ctx, err := autologs.Setup(ctx, m)
if err != nil {
return errors.Wrap(err, "setup logs")
}

httpTransport := otelhttp.NewTransport(http.DefaultTransport,
otelhttp.WithTracerProvider(m.TracerProvider()),
otelhttp.WithMeterProvider(m.MeterProvider()),
Expand All @@ -28,6 +36,7 @@ func client(ctx context.Context, lg *zap.Logger, m *app.Metrics) error {
ctx, span := tracer.Start(ctx, "sendRequest")
defer span.End()

time.Sleep(time.Millisecond * 40)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://server:8080/api/hello", http.NoBody)
if err != nil {
lg.Error("create request", zap.Error(err))
Expand All @@ -44,6 +53,7 @@ func client(ctx context.Context, lg *zap.Logger, m *app.Metrics) error {
zap.Int("status", resp.StatusCode),
zap.String("url", req.URL.String()),
)
time.Sleep(time.Millisecond * 40)
}
sendRequest(ctx)
ticker := time.NewTicker(time.Second)
Expand Down
29 changes: 24 additions & 5 deletions cmd/otelfaker/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,26 @@ import (
"net/http"
"time"

"github.com/go-faster/errors"
"github.com/go-faster/sdk/app"
"github.com/go-faster/sdk/zctx"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"

"github.com/go-faster/oteldb/internal/autologs"
)

func server(ctx context.Context, lg *zap.Logger, m *app.Metrics) error {
ctx, err := autologs.Setup(ctx, m)
if err != nil {
return errors.Wrap(err, "setup logs")
}

g, ctx := errgroup.WithContext(ctx)
mux := http.NewServeMux()

Expand All @@ -34,22 +44,31 @@ func server(ctx context.Context, lg *zap.Logger, m *app.Metrics) error {
EnableOpenMetrics: true,
}))
tracer := m.TracerProvider().Tracer("server")
doProcessing := func(ctx context.Context) {
ctx, span := tracer.Start(ctx, "doProcessing")
doProcessing := func(ctx context.Context, step string) {
ctx, span := tracer.Start(ctx, "doProcessing",
trace.WithAttributes(
attribute.String("step", step),
),
)
defer span.End()
zctx.From(ctx).Info("processing")
// Sleep some random time.
duration := time.Duration(100+rand.Intn(100)) * time.Millisecond // #nosec G404
time.Sleep(duration)
requestDurations.(prometheus.ExemplarObserver).ObserveWithExemplar(duration.Seconds(), prometheus.Labels{
"trace_id": span.SpanContext().TraceID().String(),
"span_id": span.SpanContext().SpanID().String(),
})
zctx.From(ctx).Info("got request", zap.Duration("duration", duration))
}
mux.HandleFunc("/api/hello", func(w http.ResponseWriter, r *http.Request) {
ctx, span := tracer.Start(r.Context(), "hello")
ctx, span := tracer.Start(r.Context(), "handleHello")
defer span.End()
doProcessing(ctx)
zctx.From(ctx).Info("got request")
time.Sleep(time.Millisecond * 40)
doProcessing(ctx, "alpha")
doProcessing(ctx, "beta")
zctx.From(ctx).Info("processed")
time.Sleep(time.Millisecond * 40)
w.WriteHeader(http.StatusOK)
})
srv := &http.Server{
Expand Down
35 changes: 22 additions & 13 deletions dev/local/ch-demo/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ services:
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_INSECURE=true
- OTEL_METRICS_EXPORTER=none
- OTEL_TRACES_EXPORTER=none
- OTEL_RESOURCE_ATTRIBUTES=service.name=chotel
depends_on:
- clickhouse
Expand All @@ -39,10 +40,13 @@ services:
dockerfile: otelfaker.Dockerfile
command: ["client"]
environment:
- OTEL_METRICS_EXPORTER=none
- OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317
- OTEL_EXPORTER_PROMETHEUS_HOST=0.0.0.0
- OTEL_LOG_LEVEL=debug
- OTEL_METRICS_EXPORTER=otlp
- OTEL_LOGS_EXPORTER=otlp
- OTEL_TRACES_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_INSECURE=true
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317
- OTEL_RESOURCE_ATTRIBUTES=service.name=client
server:
restart: always
Expand All @@ -51,10 +55,13 @@ services:
context: ../../../
dockerfile: otelfaker.Dockerfile
environment:
- OTEL_METRICS_EXPORTER=none
- OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317
- OTEL_EXPORTER_PROMETHEUS_HOST=0.0.0.0
- OTEL_LOG_LEVEL=debug
- OTEL_METRICS_EXPORTER=otlp
- OTEL_LOGS_EXPORTER=otlp
- OTEL_TRACES_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_INSECURE=true
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317
- OTEL_RESOURCE_ATTRIBUTES=service.name=server

oteldb:
Expand All @@ -64,12 +71,14 @@ services:
environment:
- OTELDB_STORAGE=ch
- CH_DSN=clickhouse://clickhouse:9000
- OTEL_LOG_LEVEL=debug
- METRICS_ADDR=:3201
- OTEL_METRICS_EXPORTER=none
- OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317
- OTEL_LOGS_EXPORTER=none
- OTEL_LOG_LEVEL=info
- OTEL_METRICS_EXPORTER=otlp
- OTEL_LOGS_EXPORTER=otlp
- OTEL_TRACES_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_INSECURE=true
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317
- OTEL_RESOURCE_ATTRIBUTES=service.name=oteldb
ports:
- "9090:9090" # prometheus API
Expand Down
10 changes: 4 additions & 6 deletions dev/local/ch-full/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,11 @@ services:
- OTELDB_STORAGE=ch
- CH_DSN=clickhouse://clickhouse:9000
- OTEL_LOG_LEVEL=debug
- METRICS_ADDR=:3201
- OTEL_METRICS_EXPORTER=prometheus
- OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317
- OTEL_METRICS_EXPORTER=otlp
- OTEL_LOGS_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://otelcol:4317
- OTEL_TRACES_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317
- OTEL_RESOURCE_ATTRIBUTES=service.name=go-faster.oteldb
ports:
- "9090:9090" # prometheus API
Expand Down
8 changes: 3 additions & 5 deletions dev/local/ch-remote-write/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,11 @@ services:
- OTELDB_STORAGE=ch
- CH_DSN=clickhouse://clickhouse:9000
- OTEL_LOG_LEVEL=debug
- METRICS_ADDR=:3201
- OTEL_METRICS_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317
- OTEL_LOGS_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://otelcol:4317
- OTEL_TRACES_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317
- OTEL_RESOURCE_ATTRIBUTES=service.name=go-faster.oteldb
ports:
- "9090:9090" # prometheus API
Expand Down
9 changes: 4 additions & 5 deletions dev/local/ch/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,12 @@ services:
- OTELDB_STORAGE=ch
- CH_DSN=clickhouse://clickhouse:9000
- OTEL_LOG_LEVEL=debug
- METRICS_ADDR=:3201
- OTEL_METRICS_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317
- OTEL_LOGS_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://otelcol:4317
- OTEL_TRACES_EXPORTER=otlp
- OTEL_EXPORTER_OTLP_PROTOCOL=grpc
- OTEL_EXPORTER_OTLP_INSECURE=true
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317
- OTEL_RESOURCE_ATTRIBUTES=service.name=go-faster.oteldb
ports:
- "9090:9090" # prometheus API
Expand Down
62 changes: 62 additions & 0 deletions internal/autologs/autologs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package autologs

import (
"context"
"os"
"strings"

"github.com/go-faster/errors"
"github.com/go-faster/sdk/app"
"github.com/go-faster/sdk/zctx"
"go.opentelemetry.io/collector/pdata/plog/plogotlp"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"github.com/go-faster/oteldb/internal/zapotel"
)

// Setup OTLP log exporter if configured.
func Setup(ctx context.Context, m *app.Metrics) (context.Context, error) {
if os.Getenv("OTEL_LOGS_EXPORTER") != "otlp" {
return ctx, nil
}
// Setting zap -> otel.
otelOptions := []otelgrpc.Option{
otelgrpc.WithTracerProvider(m.TracerProvider()),
otelgrpc.WithMeterProvider(m.MeterProvider()),
}
// Only PoC, should be replaced with real initialization
// and moved to go-faster/sdk.
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
if endpoint == "" {
endpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
}
if endpoint == "" {
endpoint = "localhost:4317"
}
endpoint = strings.TrimPrefix(endpoint, "http://")
conn, err := grpc.DialContext(ctx, endpoint,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelOptions...)),
)
if err != nil {
return ctx, errors.Wrap(err, "dial logs endpoint")
}
res, err := app.Resource(ctx)
if err != nil {
return ctx, errors.Wrap(err, "get resource")
}
lg := zctx.From(ctx)
otelCore := zapotel.New(lg.Level(), res, plogotlp.NewGRPCClient(conn))
// Update logger down the stack.
lg.Info("Setting up OTLP log exporter")
lg = lg.WithOptions(
zap.WrapCore(func(core zapcore.Core) zapcore.Core {
return zapcore.NewTee(core, otelCore)
}),
)
return zctx.Base(ctx, lg), nil
}

0 comments on commit f409aa4

Please sign in to comment.