Add observability stack for SOLDR (#40)

* Add observability stack for SOLDR * fix: naming of services, containers, volumes and hostnames * fix: add mapping device-id from host OS to the vx-containers to pinning generated service ID * fix: reorder services in the docker compose file * fix: default option of ES Java opts * fix: bug with delivery ES metrics from exporter * feat: add loki service support in compose * fix: pinning observability components version * fix: docker compose restart policy to unless-stopped * feat: add collecting logs and sending it to loki via otel collector * Update README.md Co-authored-by: Dmitry Ng <[email protected]> Co-authored-by: Denis Tarasov <[email protected]> Co-authored-by: Mikhail Kochegarov <[email protected]>
vxcontrol · Dec 30, 2022 · 0f887a6 · 0f887a6
1 parent 0e04087
commit 0f887a6
Show file tree

Hide file tree

Showing 14 changed files with 702 additions and 65 deletions.
diff --git a/.env.template b/.env.template
@@ -49,3 +49,12 @@ MINIO_ENDPOINT=http://127.0.0.1:9000
 
 # internal services
 INTERNAL_NET_HOST=127.0.0.1
+
+# observability
+OTEL_ADDR=otel.local:8148
+GRAFANA_SERVER_HOST=0.0.0.0
+GRAFANA_SERVER_PORT=3000
+MASTER_PASSWORD=P@ssw0rd
+ELK_VERSION=7.14.1
+ES_JAVA_OPTS="-Xmx2g -Xms2g"
+SPAN_STORAGE_TYPE=elasticsearch
diff --git a/README.md b/README.md
@@ -79,6 +79,12 @@ docker compose pull
 docker compose up -d
 ```
 
+#### Run observability stack components
+
+```bash
+docker compose --profile obs up -d
+```
+
 #### Stop services
 
 ```bash
@@ -219,6 +225,23 @@ Launch on of the available debug tasks:
 - launch vxagent
 - launch web ui
 
+### Observability stack
+
+Observability stack collect metrics, traces, logs from SOLDR components. Stack consist of:
+- `Grafana` - querying and visualizing observability data
+- `VictoriaMetrics` - datastore for server and SOLDR components metrics
+- `node-exporter` and `elasticsearch-exporter` - scraping metrics
+- `Jaeger` - storing and querying traces
+- `Elasticsearch` - datastore for `Jaeger`
+- `OpenTelemetry collector` - single entry point to receive, process and export all observability data 
+
+> For more information about collector, visit [`https://opentelemetry.io/docs/collector`](https://opentelemetry.io/docs/collector).
+
+Run observability stack components and then open `Grafana` in a browser [`https://localhost:3000`](https://localhost:3000). Default credentials: `admin/admin`. After default password for `admin` user changed, you can check provisioned SOLDR dashboards by click `Dashboards` icon on menu bar. On this SOLDR dashboards you can view server, agents and modules resource utilization, events statistics etc. 
+For checking traces, you need to click `Explore` icon on menu bar, then choose `Jaeger` data source from dropdown in the top left. Now you can query and filter traces by `Service Name`, `Operation Name`, `Tags`, `duration` and `Time ranges`.
+
+> Full observability stack requires more resources compare to clean SOLDR.
+
 ### Clean up the project
 
 Remove all build files and other security keys:

diff --git a/cmd/agent/main.go b/cmd/agent/main.go
@@ -292,7 +292,13 @@ func configureLogging(ctx context.Context, c *config.Config) (func(), error) {
 		}
 	}
 	tracerClient := observability.NewHookTracerClient(c.TracerConfigClient)
-	tracerProvider, err := observability.NewTracerProvider(ctx, tracerClient, serviceName, c.Version, attr)
+	tracerProvider, err := observability.NewTracerProvider(
+		ctx,
+		tracerClient,
+		serviceName,
+		c.Version,
+		attr,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize a tracer provider for logging: %w", err)
 	}
@@ -305,12 +311,27 @@ func configureLogging(ctx context.Context, c *config.Config) (func(), error) {
 		}
 	}
 	meterClient := observability.NewHookMeterClient(c.MeterConfigClient)
-	meterProvider, err := observability.NewMeterProvider(ctx, meterClient, serviceName, c.Version, attr)
+	meterProvider, err := observability.NewMeterProvider(
+		ctx,
+		meterClient,
+		serviceName,
+		c.Version,
+		attr,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialized a metrics provider for logging")
 	}
 
-	observability.InitObserver(ctx, tracerProvider, meterProvider, tracerClient, meterClient, serviceName, logLevels)
+	observability.InitObserver(
+		ctx,
+		tracerProvider,
+		meterProvider,
+		tracerClient,
+		meterClient,
+		serviceName,
+		c.Version,
+		logLevels,
+	)
 	return func() {
 		observability.Observer.Close()
 	}, nil

diff --git a/cmd/api/main.go b/cmd/api/main.go
@@ -31,6 +31,8 @@ import (
 	"soldr/internal/version"
 )
 
+const serviceName = "vxapi"
+
 type Config struct {
 	Debug            bool `config:"debug"`
 	Develop          bool `config:"is_develop"`
@@ -191,15 +193,21 @@ func main() {
 	serviceS3ConnectionStorage := mem.NewServiceS3ConnectionStorage()
 
 	tracerClient := observability.NewProxyTracerClient(
-		observability.NewOtlpTracerClient(cfg.Tracing.Addr),
+		observability.NewOtlpTracerAndLoggerClient(cfg.Tracing.Addr),
 		observability.NewHookTracerClient(&observability.HookClientConfig{
 			ResendTimeout:   observability.DefaultResendTimeout,
 			QueueSizeLimit:  observability.DefaultQueueSizeLimit,
 			PacketSizeLimit: observability.DefaultPacketSizeLimit,
 		}),
 	)
 	attr := attribute.String("api_server_id", system.MakeAgentID())
-	tracerProvider, err := observability.NewTracerProvider(ctx, tracerClient, "vxapi", version.GetBinaryVersion(), attr)
+	tracerProvider, err := observability.NewTracerProvider(
+		ctx,
+		tracerClient,
+		serviceName,
+		version.GetBinaryVersion(),
+		attr,
+	)
 	if err != nil {
 		logger.WithError(err).Error("could not create tracer provider")
 		return
@@ -216,7 +224,13 @@ func main() {
 		logger.WithError(err).Error("could not create meter client")
 		return
 	}
-	meterProvider, err := observability.NewMeterProvider(ctx, meterClient, "vxapi", version.GetBinaryVersion(), attr)
+	meterProvider, err := observability.NewMeterProvider(
+		ctx,
+		meterClient,
+		serviceName,
+		version.GetBinaryVersion(),
+		attr,
+	)
 	if err != nil {
 		logger.WithError(err).Error("could not create meter provider")
 		return
@@ -232,7 +246,16 @@ func main() {
 	if cfg.Debug {
 		logLevels = append(logLevels, logrus.DebugLevel)
 	}
-	observability.InitObserver(ctx, tracerProvider, meterProvider, tracerClient, meterClient, "vxapi", logLevels)
+	observability.InitObserver(
+		ctx,
+		tracerProvider,
+		meterProvider,
+		tracerClient,
+		meterClient,
+		serviceName,
+		version.GetBinaryVersion(),
+		logLevels,
+	)
 
 	gormMeter := meterProvider.Meter("vxapi-meter")
 	if err = meter.InitGormMetrics(gormMeter); err != nil {
@@ -241,8 +264,8 @@ func main() {
 	}
 
 	// initialize system metric collection in current observer instance
-	observability.Observer.StartProcessMetricCollect("vxapi", version.GetBinaryVersion(), attr)
-	observability.Observer.StartGoRuntimeMetricCollect("vxapi", version.GetBinaryVersion(), attr)
+	observability.Observer.StartProcessMetricCollect(serviceName, version.GetBinaryVersion(), attr)
+	observability.Observer.StartGoRuntimeMetricCollect(serviceName, version.GetBinaryVersion(), attr)
 	defer observability.Observer.Close()
 
 	exchanger := srvevents.NewExchanger()

diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -485,7 +485,7 @@ func getLogDir(configLogDir string) (string, error) {
 
 func initObserver(server *Server, _ *logrus.Entry) (func(), error) {
 	server.tracerClient = observability.NewProxyTracerClient(
-		observability.NewOtlpTracerClient(server.config.OtelAddr),
+		observability.NewOtlpTracerAndLoggerClient(server.config.OtelAddr),
 		observability.NewHookTracerClient(&observability.HookClientConfig{
 			ResendTimeout:   100 * time.Millisecond,
 			QueueSizeLimit:  100 * 1024 * 1024, // 100 MB
@@ -502,11 +502,23 @@ func initObserver(server *Server, _ *logrus.Entry) (func(), error) {
 	)
 	attr := attribute.String("server_id", system.MakeAgentID())
 	ctx := context.Background()
-	tracerProvider, err := observability.NewTracerProvider(ctx, server.tracerClient, serviceName, server.version, attr)
+	tracerProvider, err := observability.NewTracerProvider(
+		ctx,
+		server.tracerClient,
+		serviceName,
+		server.version,
+		attr,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize a tracer provider: %w", err)
 	}
-	meterProvider, err := observability.NewMeterProvider(ctx, server.metricsClient, serviceName, server.version, attr)
+	meterProvider, err := observability.NewMeterProvider(
+		ctx,
+		server.metricsClient,
+		serviceName,
+		server.version,
+		attr,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize a metrics provider: %w", err)
 	}
@@ -524,7 +536,16 @@ func initObserver(server *Server, _ *logrus.Entry) (func(), error) {
 	} else {
 		logrus.SetLevel(logrus.InfoLevel)
 	}
-	observability.InitObserver(ctx, tracerProvider, meterProvider, server.tracerClient, server.metricsClient, serviceName, logLevels)
+	observability.InitObserver(
+		ctx,
+		tracerProvider,
+		meterProvider,
+		server.tracerClient,
+		server.metricsClient,
+		serviceName,
+		server.version,
+		logLevels,
+	)
 	return func() {
 		observability.Observer.Close()
 	}, nil