From 032531ee5dd8379fd10952d7816fa9d930128b67 Mon Sep 17 00:00:00 2001 From: IAvecilla Date: Thu, 10 Oct 2024 18:39:08 -0300 Subject: [PATCH 1/2] feat: add new aggregator metric for created tasks --- aggregator/internal/pkg/aggregator.go | 1 + metrics/metrics.go | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 5cd6cbc29..f17591edc 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -372,6 +372,7 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.logger.Fatalf("BLS aggregation service error when initializing new task: %s", err) } + agg.metrics.IncAggregatorReceivedTasks() agg.taskMutex.Unlock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Adding new task") agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) diff --git a/metrics/metrics.go b/metrics/metrics.go index 819f6d4a8..11c0829c4 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -12,10 +12,11 @@ import ( ) type Metrics struct { - ipPortAddress string - logger logging.Logger - numAggregatedResponses prometheus.Counter - numOperatorTaskResponses prometheus.Counter + ipPortAddress string + logger logging.Logger + numAggregatedResponses prometheus.Counter + numAggregatorReceivedTasks prometheus.Counter + numOperatorTaskResponses prometheus.Counter } const alignedNamespace = "aligned" @@ -34,6 +35,11 @@ func NewMetrics(ipPortAddress string, reg prometheus.Registerer, logger logging. Name: "operator_responses", Help: "Number of proof verified by the operator and sent to the Aligned Service Manager", }), + numAggregatorReceivedTasks: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Namespace: alignedNamespace, + Name: "aggregator_received_tasks", + Help: "Number of tasks received by the Service Manager", + }), } } @@ -57,6 +63,10 @@ func (m *Metrics) Start(ctx context.Context, reg prometheus.Gatherer) <-chan err return errC } +func (m *Metrics) IncAggregatorReceivedTasks() { + m.numAggregatorReceivedTasks.Inc() +} + func (m *Metrics) IncAggregatedResponses() { m.numAggregatedResponses.Inc() } From 64b7e05e2f9b7e28962c62d16d67dc8723ee3d35 Mon Sep 17 00:00:00 2001 From: IAvecilla Date: Thu, 10 Oct 2024 18:39:35 -0300 Subject: [PATCH 2/2] feat: update grafana dashboard --- .../dashboards/aligned/aggregator.json | 375 ++++++++++-------- 1 file changed, 218 insertions(+), 157 deletions(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator.json b/grafana/provisioning/dashboards/aligned/aggregator.json index d94b8a5ca..503b1c073 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator.json +++ b/grafana/provisioning/dashboards/aligned/aggregator.json @@ -22,6 +22,32 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Batcher", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 10, + "panels": [], + "title": "Aggregator", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -37,8 +63,20 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "dark-red", "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 2 } ] } @@ -47,11 +85,11 @@ }, "gridPos": { "h": 7, - "w": 5, + "w": 10, "x": 0, - "y": 0 + "y": 2 }, - "id": 2, + "id": 9, "options": { "colorMode": "value", "graphMode": "none", @@ -64,9 +102,7 @@ "fields": "", "values": false }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "textMode": "auto" }, "pluginVersion": "10.1.10", "targets": [ @@ -78,69 +114,17 @@ "disableTextWrap": false, "editorMode": "builder", "exemplar": false, - "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[10y]))", - "format": "table", + "expr": "floor(increase(aligned_aggregator_received_tasks{job=\"aligned-aggregator\"}[$__range]))", "fullMetaSearch": false, + "hide": true, "includeNullMetadata": true, "instant": false, "interval": "", - "legendFormat": "__auto", + "legendFormat": "{{label_name}}", "range": true, "refId": "A", "useBackend": false - } - ], - "title": "Total Responses", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 5, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.1.10", - "targets": [ { "datasource": { "type": "prometheus", @@ -148,18 +132,50 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[$__range]))", "fullMetaSearch": false, + "hide": true, "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, - "refId": "A", + "refId": "B", "useBackend": false + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A - $B", + "hide": false, + "reducer": "last", + "refId": "C", + "type": "math" + } + ], + "title": "Tasks Not Verified", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "lower", + "options": { + "value": 0 + } + }, + "fieldName": "C {bot=\"aggregator\", instance=\"host.docker.internal:9091\", job=\"aligned-aggregator\"}" + } + ], + "match": "any", + "type": "exclude" + } } ], - "title": "New Responses", "type": "stat" }, { @@ -221,10 +237,10 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 13, + "h": 7, + "w": 10, "x": 10, - "y": 0 + "y": 2 }, "id": 1, "options": { @@ -258,7 +274,7 @@ "useBackend": false } ], - "title": "Aggregator", + "title": "Verified Tasks", "type": "timeseries" }, { @@ -266,49 +282,45 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], - "max": 30, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 25 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 7, "w": 5, "x": 0, - "y": 7 + "y": 9 }, - "id": 3, + "id": 8, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "showThresholdLabels": true, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "10.1.10", "targets": [ @@ -318,8 +330,8 @@ "uid": "prometheus" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(aligned_aggregated_responses{bot=\"aggregator\"}[1m]) * 60", + "editorMode": "builder", + "expr": "floor(increase(aligned_aggregator_received_tasks{bot=\"aggregator\"}[10y]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -329,43 +341,48 @@ "useBackend": false } ], - "title": "Responses Rate per Min", - "type": "gauge" + "title": "Total Tasks Received", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], - "max": 10, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 7, "w": 5, "x": 5, - "y": 7 + "y": 9 }, - "id": 6, + "id": 7, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -374,8 +391,7 @@ "fields": "", "values": false }, - "showThresholdLabels": true, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "10.1.10", "targets": [ @@ -384,16 +400,20 @@ "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "rate(aligned_aggregated_responses{bot=\"aggregator\"}[1m])", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "floor(increase(aligned_aggregator_received_tasks{bot=\"aggregator\"}[$__range]))", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Responses Rate per Sec", - "type": "gauge" + "title": "Tasks Received", + "type": "stat" }, { "datasource": { @@ -403,38 +423,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -443,10 +432,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -455,24 +440,99 @@ }, "gridPos": { "h": 7, - "w": 13, - "x": 10, - "y": 9 + "w": 5, + "x": 0, + "y": 16 }, - "id": 4, + "id": 2, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[10y]))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false } + ], + "title": "Total Tasks Verified", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 16 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -480,8 +540,9 @@ "uid": "prometheus" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(aligned_aggregated_responses{bot=\"aggregator\"}[1m]) * 60", + "editorMode": "builder", + "exemplar": false, + "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[$__range]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -491,11 +552,11 @@ "useBackend": false } ], - "title": "Responses Rate per Min", - "type": "timeseries" + "title": "Tasks Verified", + "type": "stat" } ], - "refresh": "auto", + "refresh": "5s", "schemaVersion": 38, "style": "dark", "tags": [], @@ -510,6 +571,6 @@ "timezone": "browser", "title": "Aggregator Data", "uid": "aggregator", - "version": 3, + "version": 19, "weekStart": "" -} \ No newline at end of file +}