From 771d1899e337c433ace1f62318fcf0cde86c95e2 Mon Sep 17 00:00:00 2001 From: Max Kurapov Date: Mon, 15 Jul 2024 21:02:04 +0200 Subject: [PATCH] feat: add telemetry stack to local playground (#2804) * feat: add telemetry stack to localenv under command * chore: remove tempo * chore(localenv): update prometheus scrape interval * chore: explicitly set otel collector endpoint * chore: change prometheus scrape interval to 15s * chore: update dashboard queries * chore: add readme * chore: set auto-refresh within grafana dashboard * chore: add psql command * chore: formatting --- localenv/telemetry/README.md | 27 ++ localenv/telemetry/docker-compose.yml | 47 ++++ .../provisioning/dashboards/default.yaml | 7 + .../provisioning/dashboards/example.json | 248 ++++++++++++++++++ .../provisioning/datasources/datasources.yaml | 11 + localenv/telemetry/otel-collector-config.yaml | 21 ++ localenv/telemetry/prometheus.yaml | 5 + package.json | 2 + 8 files changed, 368 insertions(+) create mode 100644 localenv/telemetry/README.md create mode 100644 localenv/telemetry/docker-compose.yml create mode 100644 localenv/telemetry/grafana/provisioning/dashboards/default.yaml create mode 100644 localenv/telemetry/grafana/provisioning/dashboards/example.json create mode 100644 localenv/telemetry/grafana/provisioning/datasources/datasources.yaml create mode 100644 localenv/telemetry/otel-collector-config.yaml create mode 100644 localenv/telemetry/prometheus.yaml diff --git a/localenv/telemetry/README.md b/localenv/telemetry/README.md new file mode 100644 index 0000000000..6638b9eaf0 --- /dev/null +++ b/localenv/telemetry/README.md @@ -0,0 +1,27 @@ +# Local Playground Telemetry + +This folder contains telemetry components that can be run as part of the local playground. + +## Overview + +The telemetry components include: + +- **OpenTelemetry Collector**: Collects and processes telemetry data from `cloud-nine-backend` and `happy-life-backend` services. +- **Prometheus**: Scrapes metrics from the OpenTelemetry collector, and stores them. +- **Grafana**: Visualizes metrics from Prometheus. + +## Usage + +From the root of the repository, run: + +``` +pnpm localenv:compose:telemetry up +``` + +If wanting to use Postgres instead of TigerBeetle for the accounting database: + +``` +pnpm localenv:compose:psql:telemetry up +``` + +Once the components are running, you can access Grafana at http://localhost:4500. Grafana comes with an example dashboard of some of our basic metrics. diff --git a/localenv/telemetry/docker-compose.yml b/localenv/telemetry/docker-compose.yml new file mode 100644 index 0000000000..89fd3771ae --- /dev/null +++ b/localenv/telemetry/docker-compose.yml @@ -0,0 +1,47 @@ +services: + cloud-nine-backend: + environment: + ENABLE_TELEMETRY: true + LIVENET: false + OPEN_TELEMETRY_COLLECTOR_URLS: http://otel-collector:4317 + + happy-life-backend: + environment: + ENABLE_TELEMETRY: true + LIVENET: false + OPEN_TELEMETRY_COLLECTOR_URLS: http://otel-collector:4317 + + otel-collector: + hostname: otel-collector + image: otel/opentelemetry-collector:latest + command: "--config=/etc/otel-collector-config.yaml" + networks: + - rafiki + volumes: + - ../telemetry/otel-collector-config.yaml:/etc/otel-collector-config.yaml + + prometheus: + image: prom/prometheus:latest + command: "--config.file=/etc/prometheus/prometheus.yaml --log.level=debug" + networks: + - rafiki + volumes: + - ../telemetry/prometheus.yaml:/etc/prometheus/prometheus.yaml + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:latest + networks: + - rafiki + ports: + - '4500:3000' + volumes: + - grafana_storage:/var/lib/grafana + - ../telemetry/grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + +volumes: + grafana_storage: diff --git a/localenv/telemetry/grafana/provisioning/dashboards/default.yaml b/localenv/telemetry/grafana/provisioning/dashboards/default.yaml new file mode 100644 index 0000000000..190f281c6c --- /dev/null +++ b/localenv/telemetry/grafana/provisioning/dashboards/default.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 + +providers: + - name: Example + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/localenv/telemetry/grafana/provisioning/dashboards/example.json b/localenv/telemetry/grafana/provisioning/dashboards/example.json new file mode 100644 index 0000000000..8826b6ecf8 --- /dev/null +++ b/localenv/telemetry/grafana/provisioning/dashboards/example.json @@ -0,0 +1,248 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(increase(transactions_amount_total[30s]) * 0.5 * 0.0001)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction Amount", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (source) (round(increase(transactions_total[30s]) * 0.5))", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction Count", + "type": "timeseries" + } + ], + "refresh": "15s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "2m", + "5m", + "10m", + "15m", + "30m", + "1h" + ] + }, + "timezone": "browser", + "title": "Example Dashboard", + "uid": "fdr58stwkr6yof", + "version": 1, + "weekStart": "" +} diff --git a/localenv/telemetry/grafana/provisioning/datasources/datasources.yaml b/localenv/telemetry/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000000..b5ba6389b3 --- /dev/null +++ b/localenv/telemetry/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + isDefault: true + version: 1 + editable: true diff --git a/localenv/telemetry/otel-collector-config.yaml b/localenv/telemetry/otel-collector-config.yaml new file mode 100644 index 0000000000..d99ca845bf --- /dev/null +++ b/localenv/telemetry/otel-collector-config.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: + +exporters: + debug: + verbosity: detailed + prometheus: + endpoint: 0.0.0.0:8491 + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus, debug] diff --git a/localenv/telemetry/prometheus.yaml b/localenv/telemetry/prometheus.yaml new file mode 100644 index 0000000000..c61d0f4c7b --- /dev/null +++ b/localenv/telemetry/prometheus.yaml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: prometheus + scrape_interval: 15s + static_configs: + - targets: ['otel-collector:8491'] diff --git a/package.json b/package.json index 5e29966d59..ba8fcc7282 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "build": "tsc --build", "localenv:compose:psql": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml", "localenv:compose": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml -f ./localenv/tigerbeetle/docker-compose.yml --env-file ./localenv/tigerbeetle/.env.tigerbeetle", + "localenv:compose:psql:telemetry": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml -f ./localenv/telemetry/docker-compose.yml", + "localenv:compose:telemetry": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml -f ./localenv/tigerbeetle/docker-compose.yml -f ./localenv/telemetry/docker-compose.yml --env-file ./localenv/tigerbeetle/.env.tigerbeetle", "localenv:seed:auth": "pnpm -C ./packages/auth knex seed:run --env=development && pnpm -C ./packages/auth knex seed:run --env=peerdevelopment", "sanity": "pnpm -r build && pnpm -r test", "localenv:compose:autopeer": "run-p tunnel:start wait-tunnel:localenv:compose",