Prometheus-style exporter

AccumulateNetwork · Jun 26, 2024 · a550157 · a550157
1 parent 8b8d45a
commit a550157
Show file tree

Hide file tree

Showing 8 changed files with 111 additions and 16 deletions.
diff --git a/cmd/accumulated/run/config.yml b/cmd/accumulated/run/config.yml
@@ -97,10 +97,17 @@ Telemetry:
     - name: Enabled
       type: bool
       pointer: true
+    - name: Stdout
+      type: bool
+      pointer: true
     - name: Oltp
       type: OltpConfig
       pointer: true
       marshal-as: reference
+    - name: Export
+      type: HttpListener
+      pointer: true
+      marshal-as: reference
 
 OltpConfig:
   non-binary: true

diff --git a/cmd/accumulated/run/instance.go b/cmd/accumulated/run/instance.go
@@ -18,9 +18,15 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/exp/ioc"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/api/v3/p2p"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"golang.org/x/exp/slices"
 )
 
+var meter = otel.Meter("gitlab.com/accumulatenetwork/accumulate/cmd/accumulated/run")
+var serviceUp = must(meter.Int64Counter("accumulated_service_up"))
+
 type Instance struct {
 	config  *Config
 	rootDir string
@@ -189,6 +195,15 @@ func (inst *Instance) StartFiltered(predicate func(Service) bool) (err error) {
 			if err != nil {
 				return errors.UnknownError.WithFormat("start service %v: %w", svc.Type(), err)
 			}
+
+			serviceUp.Add(inst.context, 1, metric.WithAttributes(
+				attribute.String("type", svc.Type().String())))
+
+			inst.cleanup(func(ctx context.Context) error {
+				serviceUp.Add(inst.context, -1, metric.WithAttributes(
+					attribute.String("type", svc.Type().String())))
+				return nil
+			})
 		}
 	}
 

diff --git a/cmd/accumulated/run/telemetry.go b/cmd/accumulated/run/telemetry.go
@@ -15,11 +15,14 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"gitlab.com/accumulatenetwork/accumulate"
 	"gitlab.com/accumulatenetwork/accumulate/exp/telemetry"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
+	promex "go.opentelemetry.io/otel/exporters/prometheus"
 	"go.opentelemetry.io/otel/exporters/stdout/stdoutlog"
+	"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
 	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
 	"go.opentelemetry.io/otel/log/global"
 	"go.opentelemetry.io/otel/propagation"
@@ -96,7 +99,41 @@ func (t *Telemetry) setupTraceProvider(res *resource.Resource, inst *Instance) e
 }
 
 func (t *Telemetry) setupMeterProvider(res *resource.Resource, inst *Instance) error {
+	otelProm := &telemetry.OtelPromProducer{Gatherer: prometheus.DefaultGatherer}
+	options := []metric.Option{
+		metric.WithResource(res),
+	}
+
+	if t.Export != nil {
+		registerer := prometheus.NewRegistry()
+		exporter, err := promex.New(
+			promex.WithoutScopeInfo(), // The HTTP handler breaks without this
+			promex.WithRegisterer(registerer),
+			promex.WithProducer(otelProm))
+		if err != nil {
+			return err
+		}
+
+		options = append(options,
+			metric.WithReader(exporter))
+
+		t.Export.applyHttpDefaults()
+		_, err = t.Export.startHTTP(inst, promhttp.InstrumentMetricHandler(registerer,
+			promhttp.HandlerFor(registerer,
+				promhttp.HandlerOpts{MaxRequestsInFlight: int(*t.Export.ConnectionLimit)})))
+		if err != nil {
+			return err
+		}
+	}
+
 	var exporters []metric.Exporter
+	if setDefaultPtr(&t.Stdout, false) {
+		exporter, err := stdoutmetric.New()
+		if err != nil {
+			return err
+		}
+		exporters = append(exporters, exporter)
+	}
 
 	if t.Oltp != nil {
 		auth := base64.StdEncoding.EncodeToString([]byte(t.Oltp.Username + ":" + t.Oltp.Password))
@@ -111,17 +148,12 @@ func (t *Telemetry) setupMeterProvider(res *resource.Resource, inst *Instance) e
 		exporters = append(exporters, exporter)
 	}
 
-	options := []metric.Option{
-		metric.WithResource(res),
-	}
-
-	otelProm := &telemetry.OtelPromProducer{Gatherer: prometheus.DefaultGatherer}
 	for _, exporter := range exporters {
 		options = append(options,
 			metric.WithReader(
 				metric.NewPeriodicReader(&filterExporter{Exporter: exporter},
 					metric.WithProducer(otelProm),
-					metric.WithInterval(1*time.Minute))))
+					metric.WithInterval(10*time.Second))))
 	}
 
 	provider := metric.NewMeterProvider(options...)

diff --git a/cmd/accumulated/run/types_gen.go b/cmd/accumulated/run/types_gen.go
@@ -280,8 +280,10 @@ type SubnodeService struct {
 }
 
 type Telemetry struct {
-	Enabled *bool       `json:"enabled,omitempty" form:"enabled" query:"enabled" validate:"required"`
-	Oltp    *OltpConfig `json:"oltp,omitempty" form:"oltp" query:"oltp" validate:"required"`
+	Enabled *bool         `json:"enabled,omitempty" form:"enabled" query:"enabled" validate:"required"`
+	Stdout  *bool         `json:"stdout,omitempty" form:"stdout" query:"stdout" validate:"required"`
+	Oltp    *OltpConfig   `json:"oltp,omitempty" form:"oltp" query:"oltp" validate:"required"`
+	Export  *HttpListener `json:"export,omitempty" form:"export" query:"export" validate:"required"`
 }
 
 type TransientPrivateKey struct {
@@ -1001,9 +1003,16 @@ func (v *Telemetry) Copy() *Telemetry {
 		u.Enabled = new(bool)
 		*u.Enabled = *v.Enabled
 	}
+	if v.Stdout != nil {
+		u.Stdout = new(bool)
+		*u.Stdout = *v.Stdout
+	}
 	if v.Oltp != nil {
 		u.Oltp = (v.Oltp).Copy()
 	}
+	if v.Export != nil {
+		u.Export = (v.Export).Copy()
+	}
 
 	return u
 }
@@ -1824,13 +1833,29 @@ func (v *Telemetry) Equal(u *Telemetry) bool {
 		return false
 	}
 	switch {
+	case v.Stdout == u.Stdout:
+		// equal
+	case v.Stdout == nil || u.Stdout == nil:
+		return false
+	case !(*v.Stdout == *u.Stdout):
+		return false
+	}
+	switch {
 	case v.Oltp == u.Oltp:
 		// equal
 	case v.Oltp == nil || u.Oltp == nil:
 		return false
 	case !((v.Oltp).Equal(u.Oltp)):
 		return false
 	}
+	switch {
+	case v.Export == u.Export:
+		// equal
+	case v.Export == nil || u.Export == nil:
+		return false
+	case !((v.Export).Equal(u.Export)):
+		return false
+	}
 
 	return true
 }

diff --git a/cmd/accumulated/run/utils.go b/cmd/accumulated/run/utils.go
@@ -36,6 +36,13 @@ var (
 	portAccP2P  = portOffset(config.PortOffsetAccumulateP2P)
 )
 
+func must[V any](v V, err error) V {
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
 func Ptr[T any](v T) *T { return &v }
 
 func setDefaultPtr[V any](ptr **V, def V) V {

diff --git a/exp/telemetry/otel_prom.go b/exp/telemetry/otel_prom.go
@@ -90,7 +90,10 @@ func (p *OtelPromProducer) Produce(ctx context.Context) ([]metricdata.ScopeMetri
 
 			fOtel.Metrics = append(fOtel.Metrics, mOtel)
 		}
-		otel = append(otel, fOtel)
+
+		if len(fOtel.Metrics) > 0 {
+			otel = append(otel, fOtel)
+		}
 	}
 	return otel, nil
 }

diff --git a/go.mod b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/go-playground/validator/v10 v10.9.0
 	github.com/golangci/golangci-lint v1.56.3-0.20240311192827-d18acc5b51ed
 	github.com/pelletier/go-toml v1.9.5
-	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/common v0.53.0
 	github.com/rinchsan/gosimports v0.1.5
 	github.com/rs/zerolog v1.29.0
@@ -79,6 +79,7 @@ require (
 	go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.3.0
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0
 	go.opentelemetry.io/otel/log v0.3.0
+	go.opentelemetry.io/otel/metric v1.27.0
 	go.opentelemetry.io/otel/sdk v1.27.0
 	go.opentelemetry.io/otel/sdk/log v0.3.0
 	go.opentelemetry.io/otel/sdk/metric v1.27.0
@@ -236,7 +237,6 @@ require (
 	go.opentelemetry.io/collector/pdata v1.10.0 // indirect
 	go.opentelemetry.io/collector/semconv v0.98.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
-	go.opentelemetry.io/otel/metric v1.27.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/automaxprocs v1.5.3 // indirect
@@ -367,7 +367,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/polyfloyd/go-errorlint v1.4.8 // indirect
 	github.com/prometheus/client_model v0.6.1
-	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/procfs v0.15.0 // indirect
 	github.com/quasilyte/go-ruleguard v0.4.2 // indirect
 	github.com/quasilyte/gogrep v0.5.0 // indirect
 	github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
@@ -406,6 +406,8 @@ require (
 	gitlab.com/bosi/decorder v0.4.1 // indirect
 	go.etcd.io/bbolt v1.3.6
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.49.0
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.23.0

diff --git a/go.sum b/go.sum
@@ -1054,8 +1054,8 @@ github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
-github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -1080,8 +1080,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek=
+github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk=
 github.com/prometheus/prometheus v0.52.1 h1:BrQ29YG+mzdGh8DgHPirHbeMGNqtL+INe0rqg7ttBJ4=
 github.com/prometheus/prometheus v0.52.1/go.mod h1:3z74cVsmVH0iXOR5QBjB7Pa6A0KJeEAK5A6UsmAFb1g=
 github.com/quasilyte/go-ruleguard v0.4.2 h1:htXcXDK6/rO12kiTHKfHuqR4kr3Y4M0J0rOL6CH/BYs=
@@ -1335,8 +1335,12 @@ go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg=
 go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc=
+go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ=
+go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM=
 go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.3.0 h1:6aGq6rMOdOx9B385JpF1OpeL18+6Ho8bTFdxy10oEGY=
 go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.3.0/go.mod h1:fdZI+pB2Y6Dpl3Uf+1ZPrkX6cnwsUAhjK1f9yCAlJIM=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 h1:/jlt1Y8gXWiHG9FBx6cJaIC5hYx5Fe64nC8w5Cylt/0=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0/go.mod h1:bmToOGOBZ4hA9ghphIc1PAf66VA8KOtsuy3+ScStG20=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o=
 go.opentelemetry.io/otel/log v0.3.0 h1:kJRFkpUFYtny37NQzL386WbznUByZx186DpEMKhEGZs=