Prometheus/Grafana setup (#413)

* Basic Prometheus/Grafana setup

* fix metrics endpoint

* Add full setup instructions

* docker volume, improved instructions

crates/relayer-handlers/src/routes/metric.rs

@@ -11,13 +11,6 @@ use webb_relayer_context::RelayerContext;
 use webb_relayer_utils::metric::Metrics;
 use webb_relayer_utils::HandlerError;
 
-/// Response with metrics message
-#[derive(Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RelayerMetricResponse {
-    metrics: String,
-}
-
 /// Response with resource metrics data
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -33,14 +26,11 @@ pub struct ResourceMetricResponse {
 /// Handles relayer metric requests
 ///
 /// Returns a Result with the `MetricResponse` on success
-pub async fn handle_metric_info(
-) -> Result<Json<RelayerMetricResponse>, HandlerError> {
+pub async fn handle_metric_info() -> Result<String, HandlerError> {
     let metric_gathered = Metrics::gather_metrics().map_err(|e| {
         HandlerError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())
     })?;
-    Ok(Json(RelayerMetricResponse {
-        metrics: metric_gathered,
-    }))
+    Ok(metric_gathered)
 }
 
 /// Handles relayer metric requests for evm based resource

docker/grafana/README.md

@@ -0,0 +1,15 @@
+Grafana Monitoring for relayer
+===
+
+Use the following steps for monitoring:
+
+- Execute relayer on localhost as usual
+- Alternatively, change the target value from `localhost:9955` to the address of a remote relayer
+- Run `docker-compose up` in this directory
+- Open Prometheus on `http://localhost:9090`. You can directly query for data
+- Open Grafana on `http://localhost:3000`. Use the following steps for initial setup:
+    - Login with `admin` / `admin`
+    - Click the cogwheel icon on the bottom left and go to "Data sources"
+    - Click "Add new data source", add Prometheus with URL `http://localhost:9090/` and click "Save & test"
+    - Click "Dashboard" in the top left menu and select "Import"
+    - Upload `grafana-dashboard.json` from this folder

docker/grafana/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+  grafana:
+    image: grafana/grafana:9.4.7
+    network_mode: "host"
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+    volumes:
+      - grafana-data:/var/lib/grafana
+
+  prometheus:
+    image: prom/prometheus:v2.43.0
+    network_mode: "host"
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+
+volumes:
+  grafana-data:

docker/grafana/grafana-dashboard.json

@@ -0,0 +1,142 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "target": {
+            "limit": 100,
+            "matchAny": false,
+            "tags": [],
+            "type": "dashboard"
+          },
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 1,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "BtDYMgE4z"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 0
+        },
+        "id": 2,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "BtDYMgE4z"
+            },
+            "editorMode": "builder",
+            "expr": "bridge_watcher_back_off",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Panel Title",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "revision": 1,
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now-6h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Relayer",
+    "uid": "c3fyMgPVz",
+    "version": 2,
+    "weekStart": ""
+  }

docker/grafana/prometheus.yml

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 30s
+
+scrape_configs:
+  - job_name: 'webb-relayer'
+    metrics_path: '/api/v1/metrics'
+
+    static_configs:
+      - targets: ['localhost:9955']

docker/docker-compose.yml → docker/prod/docker-compose.yml

@@ -7,8 +7,8 @@ services:
     env_file:
       - ./nginx-certbot.env
     ports:
-      - 80:80
-      - 443:443
+      - "80:80"
+      - "443:443"
     volumes:
       - nginx_secrets:/etc/letsencrypt
       - ./user_conf.d:/etc/nginx/user_conf.d
@@ -17,8 +17,18 @@ services:
     restart: unless-stopped
     depends_on: [nginx]
     ports:
-      - 9955:9955
+      - "9955:9955"
     volumes:
       - ./config:/config
+  prometheus:
+    image: prom/prometheus:v2.43.0
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+  grafana:
+    image: grafana/grafana:9.4.7
+    ports:
+      - "3000:3000"
 volumes:
   nginx_secrets: