From 8e8da218878b007b0bbb441d29c2aef19baedf42 Mon Sep 17 00:00:00 2001 From: Julian Ventura Date: Mon, 9 Dec 2024 15:49:43 -0300 Subject: [PATCH 1/4] Add queue len and queue size in bytes metrics --- batcher/aligned-batcher/src/lib.rs | 23 +++++++++++++++++++---- batcher/aligned-batcher/src/metrics.rs | 11 +++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index b760d9c82..dd4389380 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -16,6 +16,7 @@ use tokio::time::timeout; use types::batch_state::BatchState; use types::user_state::UserState; +use batch_queue::calculate_batch_size; use std::collections::HashMap; use std::env; use std::net::SocketAddr; @@ -1042,11 +1043,15 @@ impl Batcher { ), BatchQueueEntryPriority::new(max_fee, nonce), ); + + // Update metrics + let queue_len = batch_state_lock.batch_queue.len(); + let queue_size_bytes = calculate_batch_size(&batch_state_lock.batch_queue)?; - info!( - "Current batch queue length: {}", - batch_state_lock.batch_queue.len() - ); + self.metrics.queue_len.set(queue_len as i64); + self.metrics.queue_size_bytes.set(queue_size_bytes as i64); + + info!("Current batch queue length: {}", queue_len); let mut proof_submitter_addr = proof_submitter_addr; @@ -1226,6 +1231,13 @@ impl Batcher { ))?; } + // Update metrics + let queue_len = batch_state_lock.batch_queue.len(); + let queue_size_bytes = calculate_batch_size(&batch_state_lock.batch_queue)?; + + self.metrics.queue_len.set(queue_len as i64); + self.metrics.queue_size_bytes.set(queue_size_bytes as i64); + Ok(()) } @@ -1373,6 +1385,9 @@ impl Batcher { batch_state_lock .user_states .insert(nonpaying_replacement_addr, nonpaying_user_state); + + self.metrics.queue_len.set(0); + self.metrics.queue_size_bytes.set(0); } /// Receives new block numbers, checks if conditions are met for submission and diff --git a/batcher/aligned-batcher/src/metrics.rs b/batcher/aligned-batcher/src/metrics.rs index a7c6f26e3..5299d604e 100644 --- a/batcher/aligned-batcher/src/metrics.rs +++ b/batcher/aligned-batcher/src/metrics.rs @@ -19,6 +19,8 @@ pub struct BatcherMetrics { pub batcher_started: IntCounter, pub gas_price_used_on_latest_batch: IntGauge, pub broken_ws_connections: IntCounter, + pub queue_len: IntGauge, + pub queue_size_bytes: IntGauge, } impl BatcherMetrics { @@ -46,6 +48,11 @@ impl BatcherMetrics { "broken_ws_connections_count", "Broken websocket connections" ))?; + let queue_len = register_int_gauge!(opts!("queue_len", "Amount of proofs in the queue"))?; + let queue_size_bytes = register_int_gauge!(opts!( + "queue_size_bytes", + "Accumulated size in bytes of all proofs in the queue" + ))?; registry.register(Box::new(open_connections.clone()))?; registry.register(Box::new(received_proofs.clone()))?; @@ -56,6 +63,8 @@ impl BatcherMetrics { registry.register(Box::new(gas_price_used_on_latest_batch.clone()))?; registry.register(Box::new(batcher_started.clone()))?; registry.register(Box::new(broken_ws_connections.clone()))?; + registry.register(Box::new(queue_len.clone()))?; + registry.register(Box::new(queue_size_bytes.clone()))?; let metrics_route = warp::path!("metrics") .and(warp::any().map(move || registry.clone())) @@ -77,6 +86,8 @@ impl BatcherMetrics { batcher_started, gas_price_used_on_latest_batch, broken_ws_connections, + queue_len, + queue_size_bytes, }) } From 2f421cdea879fbba9ff2abc1b4f5415880d40a51 Mon Sep 17 00:00:00 2001 From: Julian Ventura Date: Mon, 9 Dec 2024 17:54:07 -0300 Subject: [PATCH 2/4] Fix formatting --- batcher/aligned-batcher/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index dd4389380..b509d3bf6 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1043,7 +1043,7 @@ impl Batcher { ), BatchQueueEntryPriority::new(max_fee, nonce), ); - + // Update metrics let queue_len = batch_state_lock.batch_queue.len(); let queue_size_bytes = calculate_batch_size(&batch_state_lock.batch_queue)?; From 36baa0a4f51f88c9d8cdabc7208d7ca8afea0076 Mon Sep 17 00:00:00 2001 From: Julian Ventura Date: Tue, 10 Dec 2024 16:42:40 -0300 Subject: [PATCH 3/4] Add method to update both queue metrics at the same time --- batcher/aligned-batcher/src/lib.rs | 12 +++++------- batcher/aligned-batcher/src/metrics.rs | 5 +++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index b509d3bf6..e97e5e1b2 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -1047,9 +1047,8 @@ impl Batcher { // Update metrics let queue_len = batch_state_lock.batch_queue.len(); let queue_size_bytes = calculate_batch_size(&batch_state_lock.batch_queue)?; - - self.metrics.queue_len.set(queue_len as i64); - self.metrics.queue_size_bytes.set(queue_size_bytes as i64); + self.metrics + .update_queue_metrics(queue_len as i64, queue_size_bytes as i64); info!("Current batch queue length: {}", queue_len); @@ -1235,8 +1234,8 @@ impl Batcher { let queue_len = batch_state_lock.batch_queue.len(); let queue_size_bytes = calculate_batch_size(&batch_state_lock.batch_queue)?; - self.metrics.queue_len.set(queue_len as i64); - self.metrics.queue_size_bytes.set(queue_size_bytes as i64); + self.metrics + .update_queue_metrics(queue_len as i64, queue_size_bytes as i64); Ok(()) } @@ -1386,8 +1385,7 @@ impl Batcher { .user_states .insert(nonpaying_replacement_addr, nonpaying_user_state); - self.metrics.queue_len.set(0); - self.metrics.queue_size_bytes.set(0); + self.metrics.update_queue_metrics(0, 0); } /// Receives new block numbers, checks if conditions are met for submission and diff --git a/batcher/aligned-batcher/src/metrics.rs b/batcher/aligned-batcher/src/metrics.rs index 5299d604e..3e50dc8ee 100644 --- a/batcher/aligned-batcher/src/metrics.rs +++ b/batcher/aligned-batcher/src/metrics.rs @@ -117,4 +117,9 @@ impl BatcherMetrics { pub fn user_error(&self, label_values: &[&str]) { self.user_errors.with_label_values(label_values).inc(); } + + pub fn update_queue_metrics(&self, queue_len: i64, queue_size: i64) { + self.queue_len.set(queue_len); + self.queue_size_bytes.set(queue_size); + } } From 1bd4ad8cf64448d0e476352e1cdaff9a0f3610ea Mon Sep 17 00:00:00 2001 From: Julian Ventura Date: Tue, 10 Dec 2024 17:12:35 -0300 Subject: [PATCH 4/4] Add grafana dashboard --- .../aligned/aggregator_batcher.json | 340 +++++++++++++++++- 1 file changed, 337 insertions(+), 3 deletions(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index 929ff29ea..d21d97e56 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": 2, "links": [], "liveNow": false, "panels": [ @@ -2625,6 +2625,340 @@ } ], "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of proofs in the queue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 69 + }, + "id": 43, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "queue_len", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Number of proofs in the queue", + "range": true, + "refId": "Queue Length", + "useBackend": false + } + ], + "title": "Queue Length", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of proofs in the queue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 3, + "y": 69 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "queue_len", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Queue Length", + "range": true, + "refId": "Queue Length", + "useBackend": false + } + ], + "title": "Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum of all proof sizes in the queue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 76 + }, + "id": 45, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "queue_size_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Size in bytes of the queue", + "range": true, + "refId": "Queue Size", + "useBackend": false + } + ], + "title": "Queue Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum of all proof sizes in the queue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 3, + "y": 76 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "queue_size_bytes", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Queue Size", + "range": true, + "refId": "Queue Size", + "useBackend": false + } + ], + "title": "Queue Size", + "type": "timeseries" } ], "refresh": "5s", @@ -2642,6 +2976,6 @@ "timezone": "browser", "title": "System Data", "uid": "aggregator", - "version": 9, + "version": 45, "weekStart": "" -} \ No newline at end of file +}