From bae452a331b16ccd4b36b2b9bbefecc854276a5e Mon Sep 17 00:00:00 2001 From: "Dima K." Date: Wed, 30 Oct 2024 21:18:55 -0700 Subject: [PATCH] [Observability] Foundation for load testing telemetry (#832) ## Summary Refactor the foundation for E2E tokenomics observability w/ lots of new data points. Key changes include: - `x/tokenomics` telemetry - Begin/End blockers execution time management - Custom `poktroll` telemetry config in `app.toml` ## Issue - #762 --------- Co-authored-by: Daniel Olshansky --- Tiltfile | 29 +- api/poktroll/application/types.pulsar.go | 14 +- api/poktroll/tokenomics/event.pulsar.go | 2 +- app/app.go | 5 + cmd/poktrolld/cmd/config.go | 67 +- config.yml | 26 +- go.mod | 2 +- .../cosmos_sdk_insights.json | 1318 +++++++++++++++++ .../permissionless_demand_observability.json | 1314 ++++++++++++++++ .../grafana-dashboards/tokenomics_relays.json | 523 +++++++ .../observability-prometheus-stack.yaml | 23 +- localnet/poktrolld/config/app.toml | 7 +- ...appgate_server_config_localnet_vscode.yaml | 2 +- .../relayminer_config_localnet_vscode.yaml | 12 +- pkg/appgateserver/cmd/cmd.go | 5 +- pkg/relayer/cmd/cmd.go | 5 +- pkg/relayer/proxy/metrics.go | 4 +- pkg/relayer/proxy/synchronous.go | 7 +- telemetry/block.go | 8 + telemetry/common.go | 54 + telemetry/event_counters.go | 156 +- telemetry/telemetry.go | 31 + telemetry/tokens.go | 61 + x/application/module/abci.go | 5 + x/application/types/types.pb.go | 14 +- x/proof/keeper/msg_server_create_claim.go | 31 +- x/proof/keeper/msg_server_submit_proof.go | 45 +- x/proof/keeper/proof_validation.go | 5 + x/session/keeper/query_get_session.go | 2 +- x/session/keeper/session_hydrator.go | 13 +- x/supplier/module/abci.go | 5 + x/tokenomics/keeper/settle_pending_claims.go | 47 +- x/tokenomics/keeper/token_logic_modules.go | 26 +- x/tokenomics/module/abci.go | 42 +- x/tokenomics/types/event.pb.go | 2 +- 35 files changed, 3744 insertions(+), 168 deletions(-) create mode 100644 localnet/grafana-dashboards/cosmos_sdk_insights.json create mode 100644 localnet/grafana-dashboards/permissionless_demand_observability.json create mode 100644 localnet/grafana-dashboards/tokenomics_relays.json create mode 100644 telemetry/common.go create mode 100644 telemetry/telemetry.go create mode 100644 telemetry/tokens.go diff --git a/Tiltfile b/Tiltfile index cd9b4a177..bc0d78b92 100644 --- a/Tiltfile +++ b/Tiltfile @@ -6,7 +6,7 @@ load("ext://deployment", "deployment_create") load("ext://execute_in_pod", "execute_in_pod") # A list of directories where changes trigger a hot-reload of the validator -hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"] +hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg", "telemetry"] def merge_dicts(base, updates): @@ -38,14 +38,26 @@ localnet_config_defaults = { "enabled": True, "grafana": {"defaultDashboardsEnabled": False}, }, - "relayminers": {"count": 1, "delve": {"enabled": False}}, + "relayminers": { + "count": 1, + "delve": {"enabled": False}, + "logs": { + "level": "debug", + }, + }, "gateways": { "count": 1, "delve": {"enabled": False}, + "logs": { + "level": "debug", + }, }, "appgateservers": { "count": 1, "delve": {"enabled": False}, + "logs": { + "level": "debug", + }, }, "ollama": { "enabled": False, @@ -100,8 +112,10 @@ if localnet_config["observability"]["enabled"]: helm_repo("prometheus-community", "https://prometheus-community.github.io/helm-charts") helm_repo("grafana-helm-repo", "https://grafana.github.io/helm-charts") - # Increase timeout for building the image - update_settings(k8s_upsert_timeout_secs=60) + # Timeout is increased to 120 seconds (default is 30) because a slow internet connection + # could timeout pulling the image. + # container images. + update_settings(k8s_upsert_timeout_secs=120) helm_resource( "observability", @@ -226,6 +240,7 @@ helm_resource( "--set=logs.format=" + str(localnet_config["validator"]["logs"]["format"]), "--set=serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), "--set=development.delve.enabled=" + str(localnet_config["validator"]["delve"]["enabled"]), + "--set=image.repository=poktrolld", ], image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], @@ -244,6 +259,8 @@ for x in range(localnet_config["relayminers"]["count"]): "--values=./localnet/kubernetes/values-relayminer-" + str(actor_number) + ".yaml", "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), "--set=development.delve.enabled=" + str(localnet_config["relayminers"]["delve"]["enabled"]), + "--set=logLevel=" + str(localnet_config["relayminers"]["logs"]["level"]), + "--set=image.repository=poktrolld", ], image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], @@ -284,6 +301,8 @@ for x in range(localnet_config["appgateservers"]["count"]): "--set=config.signing_key=app" + str(actor_number), "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), "--set=development.delve.enabled=" + str(localnet_config["appgateservers"]["delve"]["enabled"]), + "--set=logLevel=" + str(localnet_config["appgateservers"]["logs"]["level"]), + "--set=image.repository=poktrolld", ], image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], @@ -325,6 +344,8 @@ for x in range(localnet_config["gateways"]["count"]): "--set=config.signing_key=gateway" + str(actor_number), "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), "--set=development.delve.enabled=" + str(localnet_config["gateways"]["delve"]["enabled"]), + "--set=logLevel=" + str(localnet_config["gateways"]["logs"]["level"]), + "--set=image.repository=poktrolld", ], image_deps=["poktrolld"], image_keys=[("image.repository", "image.tag")], diff --git a/api/poktroll/application/types.pulsar.go b/api/poktroll/application/types.pulsar.go index 4a7751438..d1f8f492f 100644 --- a/api/poktroll/application/types.pulsar.go +++ b/api/poktroll/application/types.pulsar.go @@ -2190,14 +2190,12 @@ type Application struct { Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` // The Bech32 address of the application. Stake *v1beta1.Coin `protobuf:"bytes,2,opt,name=stake,proto3" json:"stake,omitempty"` // The total amount of uPOKT the application has staked - // As per this discussion: - // https://github.com/pokt-network/poktroll/pull/750#discussion_r1735025033 - // The number of service_configs is limited to 1 per service. - // This is to ensure that an application could not over service by making multiple - // claims settelments compete to burn the same stake. - // A slice of service_configs is still maintained to allow for future multi-service - // capabilities to be added, such as off-chain application stake tracking by suppliers: - // https://www.notion.so/buildwithgrove/Off-chain-Application-Stake-Tracking-6a8bebb107db4f7f9dc62cbe7ba555f7 + // CRITICAL_DEV_NOTE: The number of service_configs must be EXACTLY ONE. + // This prevents applications from over-servicing. + // The field is kept repeated (a list) for both legacy and future logic reaosns. + // References: + // - https://github.com/pokt-network/poktroll/pull/750#discussion_r1735025033 + // - https://www.notion.so/buildwithgrove/Off-chain-Application-Stake-Tracking-6a8bebb107db4f7f9dc62cbe7ba555f7 ServiceConfigs []*shared.ApplicationServiceConfig `protobuf:"bytes,3,rep,name=service_configs,json=serviceConfigs,proto3" json:"service_configs,omitempty"` // The list of services this appliccation is configured to request service for // TODO_BETA: Rename `delegatee_gateway_addresses` to `gateway_addresses_delegated_to`. // Ensure to rename all relevant configs, comments, variables, function names, etc as well. diff --git a/api/poktroll/tokenomics/event.pulsar.go b/api/poktroll/tokenomics/event.pulsar.go index cf18db79d..4a8a3c0e5 100644 --- a/api/poktroll/tokenomics/event.pulsar.go +++ b/api/poktroll/tokenomics/event.pulsar.go @@ -3730,7 +3730,7 @@ func (x *EventSupplierSlashed) GetSlashingAmount() *v1beta1.Coin { } // EventApplicationReimbursementRequest is emitted when an application requests -// a reimbursement +// a reimbursement. type EventApplicationReimbursementRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/app/app.go b/app/app.go index 8384a82f4..b98b86b42 100644 --- a/app/app.go +++ b/app/app.go @@ -302,6 +302,11 @@ func New( return nil, err } + // Set up poktroll telemetry using `app.toml` configuration options (in addition to cosmos-sdk telemetry config). + if err := telemetry.New(appOpts); err != nil { + return nil, err + } + return app, nil } diff --git a/cmd/poktrolld/cmd/config.go b/cmd/poktrolld/cmd/config.go index 003c6a690..61a441fa7 100644 --- a/cmd/poktrolld/cmd/config.go +++ b/cmd/poktrolld/cmd/config.go @@ -9,10 +9,27 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" "github.com/pokt-network/poktroll/app" + "github.com/pokt-network/poktroll/telemetry" ) var once sync.Once +// PoktrollAppConfig represents a poktroll-specific part of `app.toml` file. +// Checkout `customAppConfigTemplate()` for additional information about each setting. +type PoktrollAppConfig struct { + Telemetry telemetry.PoktrollTelemetryConfig `mapstructure:"telemetry"` +} + +// poktrollAppConfigDefaults sets default values to render in `app.toml`. +// Checkout `customAppConfigTemplate()` for additional information about each setting. +func poktrollAppConfigDefaults() PoktrollAppConfig { + return PoktrollAppConfig{ + Telemetry: telemetry.PoktrollTelemetryConfig{ + CardinalityLevel: "medium", + }, + } +} + func InitSDKConfig() { once.Do(func() { checkOrInitSDKConfig() @@ -90,6 +107,7 @@ func initAppConfig() (string, interface{}) { // The following code snippet is just for reference. type CustomAppConfig struct { serverconfig.Config `mapstructure:",squash"` + Poktroll PoktrollAppConfig `mapstructure:"poktroll"` } // Optionally allow the chain developer to overwrite the SDK's default @@ -113,7 +131,9 @@ func initAppConfig() (string, interface{}) { srvCfg.MinGasPrices = "0.000000001upokt" // Also adjust ignite's `config.yml`. srvCfg.Mempool.MaxTxs = 10000 srvCfg.Telemetry.Enabled = true - srvCfg.Telemetry.PrometheusRetentionTime = 60 // in seconds. This turns on Prometheus support. + // Positive non-zero value turns on Prometheus support. + // Prometheus metrics are removed from the exporter when retention time is reached. + srvCfg.Telemetry.PrometheusRetentionTime = 60 * 60 * 24 // in seconds. srvCfg.Telemetry.MetricsSink = "mem" srvCfg.Pruning = "nothing" // archiving node by default srvCfg.API.Enable = true @@ -121,19 +141,38 @@ func initAppConfig() (string, interface{}) { srvCfg.GRPCWeb.Enable = true customAppConfig := CustomAppConfig{ - Config: *srvCfg, + Config: *srvCfg, + Poktroll: poktrollAppConfigDefaults(), } - customAppTemplate := serverconfig.DefaultConfigTemplate - // Edit the default template file - // - // customAppTemplate := serverconfig.DefaultConfigTemplate + ` - // [wasm] - // # This is the maximum sdk gas (wasm and storage) that we allow for any x/wasm "smart" queries - // query_gas_limit = 300000 - // # This is the number of wasm vm instances we keep cached in memory for speed-up - // # Warning: this is currently unstable and may lead to crashes, best to keep for 0 unless testing locally - // lru_size = 0` - - return customAppTemplate, customAppConfig + return customPoktrollAppConfigTemplate(), customAppConfig +} + +// customPoktrollAppConfigTemplate extends the default configuration `app.toml` file with our own configs. +// They are going to be used by validators and full-nodes. +// These configs are rendered using default values from `poktrollAppConfigDefaults()`. +func customPoktrollAppConfigTemplate() string { + return serverconfig.DefaultConfigTemplate + ` + ############################################################################### + ### Poktroll ### + ############################################################################### + + # Poktroll-specific app configuration for Full Nodes and Validators. + [poktroll] + + # Telemetry configuration in addition to the [telemetry] settings. + [poktroll.telemetry] + + # Cardinality level for telemetry metrics collection + # This controls the level of detail (number of unique labels) in metrics. + # Options: + # - "low": Collects basic metrics with low cardinality. + # Suitable for production environments with tight performance constraints. + # - "medium": Collects a moderate number of labels, balancing detail and performance. + # Suitable for moderate workloads or staging environments. + # - "high": WARNING: WILL CAUSE STRESS TO YOUR MONITORING ENVIRONMENT! Collects detailed metrics with high + # cardinality, including labels with many unique values (e.g., application_id, session_id). + # Recommended for debugging or testing environments. + cardinality-level = "{{ .Poktroll.Telemetry.CardinalityLevel }}" + ` } diff --git a/config.yml b/config.yml index 73ea3a945..e2cabd9a8 100644 --- a/config.yml +++ b/config.yml @@ -91,7 +91,11 @@ validators: # minimum-gas-prices: 0.000000001upokt telemetry: enabled: true - prometheus-retention-time: "600" # seconds + poktroll: + telemetry: + # "high" produces a lot of timeseries. + # ONLY suitable for small networks such as LocalNet. + cardinality-level: high config: moniker: "validator1" consensus: @@ -139,28 +143,28 @@ genesis: denom: upokt bank: supply: - - amount: "1003000204" + - amount: "1102000204" denom: upokt balances: # Application module - address: pokt1rl3gjgzexmplmds3tq3r3yk84zlwdl6djzgsvm coins: - - amount: "1000068" # Equals to the total of all app stakes below + - amount: "100000068" # MUST BE equal to the total of all app stakes below denom: upokt # Supplier module - address: pokt1j40dzzmn6cn9kxku7a5tjnud6hv37vesr5ccaa coins: - - amount: "1000068" # Equals to the total of all supplier stakes below + - amount: "1000068" # MUST BE equal to the total of all supplier stakes below denom: upokt # Gateway module - address: pokt1f6j7u6875p2cvyrgjr0d2uecyzah0kget9vlpl coins: - - amount: "1000068" # Equals to the total of all gateway stakes below + - amount: "1000068" # MUST BE equal to the total of all gateway stakes below denom: upokt # Service module - address: pokt1nhmtqf4gcmpxu0p6e53hpgtwj0llmsqpxtumcf coins: - - amount: "1000000000" # Equals to one add_service_fee below + - amount: "1000000000" # MUST BE equal to one add_service_fee below denom: upokt application: params: @@ -171,9 +175,8 @@ genesis: denom: upokt applicationList: - address: pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4 - delegatee_gateway_addresses: [ - pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 - ] + delegatee_gateway_addresses: + [pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4] service_configs: - service_id: anvil stake: @@ -182,9 +185,8 @@ genesis: amount: "100000068" # ~100 POKT denom: upokt - address: pokt184zvylazwu4queyzpl0gyz9yf5yxm2kdhh9hpm - delegatee_gateway_addresses: [ - pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 - ] + delegatee_gateway_addresses: + [pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4] service_configs: - service_id: rest stake: diff --git a/go.mod b/go.mod index 91de15f1d..024364e1b 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( require ( cosmossdk.io/x/tx v0.13.4 github.com/jhump/protoreflect v1.16.0 + github.com/mitchellh/mapstructure v1.5.0 ) require ( @@ -224,7 +225,6 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect diff --git a/localnet/grafana-dashboards/cosmos_sdk_insights.json b/localnet/grafana-dashboards/cosmos_sdk_insights.json new file mode 100644 index 000000000..157314cdc --- /dev/null +++ b/localnet/grafana-dashboards/cosmos_sdk_insights.json @@ -0,0 +1,1318 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "title": "Block processing times", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*: avg$" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(begin_blocker{job=\"$job\"}) by (module, quantile)", + "instant": false, + "legendFormat": "{{module}}: q{{quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(begin_blocker_sum{job=\"$job\"}) by (module) / sum(begin_blocker_count{job=\"$job\"}) by (module)", + "hide": false, + "instant": false, + "legendFormat": "{{module}}: avg", + "range": true, + "refId": "B" + } + ], + "title": "Begin Blocker Time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*: avg$" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(end_blocker{job=\"$job\"}) by (module, quantile)", + "instant": false, + "legendFormat": "{{module}}: q{{quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(end_blocker_sum{job=\"$job\"}) by (module) / sum(end_blocker_count{job=\"$job\"}) by (module)", + "hide": false, + "instant": false, + "legendFormat": "{{module}}: avg", + "range": true, + "refId": "B" + } + ], + "title": "End Blocker Time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 13, + "options": { + "calculate": true, + "calculation": { + "yBuckets": { + "scale": { + "type": "linear" + } + } + }, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-green", + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Greens", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (increase(cometbft_consensus_block_interval_seconds_bucket[$__rate_interval]))", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CometBFT Consensus Block Interval", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*: avg$" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(poktroll_proof_validation{job=\"$job\"}) by (quantile)", + "instant": false, + "legendFormat": " q{{quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(poktroll_proof_validation_sum{job=\"$job\"}) / sum(poktroll_proof_validation_count{job=\"$job\"})", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "B" + } + ], + "title": "Proof Validation Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 9, + "panels": [], + "title": "Tokenomics", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(poktroll_relay_ema{job=\"$job\",service_id=~\"$service_id\"}) by (service_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "EMA by ServiceID", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(poktroll_relay_mining_difficulty{job=\"$job\",service_id=~\"$service_id\"}) by (service_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RelayMining difficulty by ServiceID", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_relays{job=\"$job\",service_id=~\"$service_id\"}[1m])) by (service_id, claim_proof_stage)", + "instant": false, + "legendFormat": "{{service_id}}-{{claim_proof_stage}}", + "range": true, + "refId": "A" + } + ], + "title": "On-Chain Relays", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_compute_units{job=\"$job\",service_id=~\"$service_id\"}[1m])) by (service_id, claim_proof_stage)", + "instant": false, + "legendFormat": "{{service_id}}-{{claim_proof_stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Compute Units", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_proof_requirements{job=\"$job\",service_id=~\"$service_id\"}[1m])) by (service_id, reason)", + "instant": false, + "legendFormat": "{{service_id}}-{{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Proof requirements", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 3 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_minted_tokens{job=\"$job\"}[1m])) by (module)", + "instant": false, + "legendFormat": "minted by {{module}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_burned_tokens{job=\"$job\"}[1m])) by (module)", + "hide": false, + "instant": false, + "legendFormat": "burned by {{module}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_slashed_tokens{job=\"$job\"}[1m])) by (module)", + "hide": false, + "instant": false, + "legendFormat": "slashed by {{module}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_minted_tokens{job=\"$job\"}[1m])) - sum(increase(poktroll_burned_tokens{job=\"$job\"}[1m]))", + "hide": false, + "instant": false, + "legendFormat": "Minted total (minus burned)", + "range": true, + "refId": "D" + } + ], + "title": "Minted / Burned / Slashed Tokens", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(poktroll_claims{job=\"$job\",service_id=~\"$service_id\"}[1m])) by (service_id, claim_proof_stage)", + "instant": false, + "legendFormat": "{{service_id}}-{{claim_proof_stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Claims", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "validator-poktroll-validator", + "value": "validator-poktroll-validator" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(cometbft_consensus_height,job)", + "hide": 0, + "includeAll": false, + "label": "Full Node", + "multi": false, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(cometbft_consensus_height,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(poktroll_proof_requirements,service_id)", + "hide": 0, + "includeAll": true, + "label": "Service ID", + "multi": true, + "name": "service_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(poktroll_proof_requirements,service_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Protocol / Insights and On-Chain data", + "uid": "adzickiu028lcb", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/localnet/grafana-dashboards/permissionless_demand_observability.json b/localnet/grafana-dashboards/permissionless_demand_observability.json new file mode 100644 index 000000000..9fffee6e9 --- /dev/null +++ b/localnet/grafana-dashboards/permissionless_demand_observability.json @@ -0,0 +1,1314 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cometbft_consensus_block_size_bytes{})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Block size", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n b.timestamp AS time,\n COUNT(m.id) AS num_proofs\nFROM\n localnet.messages m\nJOIN\n localnet.transactions t ON t.id = m.transaction_id\nJOIN\n localnet.blocks b ON b.id = t.block_id\nWHERE\n m.type_url = '/poktroll.proof.MsgSubmitProof'\n AND b.timestamp BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n b.timestamp\nORDER BY\n b.timestamp;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Number of proofs", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "increase(appgateserver_relay_requests_total[1m])", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Actual relays", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(b.\"timestamp\", '1m') AS time,\n SUM(trim(both '\"' from ea_num_relays.value)::bigint) AS total_num_relays,\n (ea_claim.value::json ->> 'supplier_operator_address') AS supplier_operator_address,\n (ea_claim.value::json -> 'session_header' ->> 'service_id') AS service_id\nFROM\n localnet.events e\nJOIN\n localnet.event_attributes ea_num_relays ON ea_num_relays.event_id = e.id\n AND ea_num_relays.key = 'num_relays'\nJOIN\n localnet.event_attributes ea_claim ON ea_claim.event_id = e.id\n AND ea_claim.key = 'claim'\nJOIN\n localnet.blocks b ON b.id = e.block_id\nWHERE\n e.type = 'poktroll.proof.EventClaimCreated'\n AND b.\"timestamp\" BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n time, supplier_operator_address, service_id\nORDER BY\n time;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "On chain relays from claims", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(poktroll_relay_ema{service_id=~\"$service_id\"}) by (service_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "EMA by service", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(poktroll_relay_mining_difficulty{service_id=~\"$service_id\"}) by (service_id)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RM difficulty by service", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 8, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "-- Average number of proofs per application\n\nWITH proofs_per_block AS (\n SELECT\n b.height AS block_height,\n m.json::jsonb -> 'sessionHeader' ->> 'applicationAddress' AS application_address,\n COUNT(m.id) AS num_proofs\n FROM\n localnet.messages m\n JOIN\n localnet.transactions t ON t.id = m.transaction_id\n JOIN\n localnet.blocks b ON b.id = t.block_id\n WHERE\n m.type_url = '/poktroll.proof.MsgSubmitProof'\n AND b.timestamp BETWEEN $__timeFrom() AND $__timeTo()\n GROUP BY\n b.height, application_address\n)\n\nSELECT\n application_address AS application,\n AVG(num_proofs) AS avg_num_proofs\nFROM\n proofs_per_block\nGROUP BY\n application\nORDER BY\n avg_num_proofs DESC;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Average Number of Proofs per Application", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 9, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "-- Average number of proofs per service\n\nWITH proofs_per_block AS (\n SELECT\n b.height AS block_height,\n m.json::jsonb -> 'sessionHeader' ->> 'serviceId' AS service_id,\n COUNT(m.id) AS num_proofs\n FROM\n localnet.messages m\n JOIN\n localnet.transactions t ON t.id = m.transaction_id\n JOIN\n localnet.blocks b ON b.id = t.block_id\n WHERE\n m.type_url = '/poktroll.proof.MsgSubmitProof'\n AND b.timestamp BETWEEN $__timeFrom() AND $__timeTo()\n GROUP BY\n b.height, service_id\n)\n\nSELECT\n service_id AS service,\n AVG(num_proofs) AS avg_num_proofs\nFROM\n proofs_per_block\nGROUP BY\n service\nORDER BY\n avg_num_proofs DESC;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Average Number of Proofs per Service", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 32 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n b.timestamp AS time,\n COUNT(m.id) AS num_proofs,\n m.json::jsonb -> 'sessionHeader' ->> 'applicationAddress' AS application_address\nFROM\n localnet.messages m\nJOIN\n localnet.transactions t ON t.id = m.transaction_id\nJOIN\n localnet.blocks b ON b.id = t.block_id\nWHERE\n m.type_url = '/poktroll.proof.MsgSubmitProof'\n AND b.timestamp BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n b.timestamp, application_address\nORDER BY\n b.timestamp;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Proof per application", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 32 + }, + "id": 6, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n b.timestamp AS time,\n COUNT(m.id) AS num_proofs,\n m.json::jsonb -> 'sessionHeader' ->> 'serviceId' AS service_id\nFROM\n localnet.messages m\nJOIN\n localnet.transactions t ON t.id = m.transaction_id\nJOIN\n localnet.blocks b ON b.id = t.block_id\nWHERE\n m.type_url = '/poktroll.proof.MsgSubmitProof'\n AND b.timestamp BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n b.timestamp, service_id\nORDER BY\n b.timestamp;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Proof per service", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 32 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n b.timestamp AS time,\n COUNT(m.id) AS num_proofs,\n m.json::jsonb ->> 'supplierOperatorAddress' AS supplier_operator_address\nFROM\n localnet.messages m\nJOIN\n localnet.transactions t ON t.id = m.transaction_id\nJOIN\n localnet.blocks b ON b.id = t.block_id\nWHERE\n m.type_url = '/poktroll.proof.MsgSubmitProof'\n AND b.timestamp BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n b.timestamp, supplier_operator_address\nORDER BY\n b.timestamp;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Proof per supplier operator", + "type": "barchart" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "-- Serviced relays (from claims)\nWITH serviced_relays AS (\n SELECT\n (ea_claim.value::json ->> 'supplier_operator_address') AS supplier_operator_address,\n SUM(trim(both '\"' from ea_num_relays.value)::bigint) AS total_serviced_relays\n FROM\n localnet.events e\n JOIN\n localnet.event_attributes ea_num_relays ON ea_num_relays.event_id = e.id\n AND ea_num_relays.key = 'num_relays'\n JOIN\n localnet.event_attributes ea_claim ON ea_claim.event_id = e.id\n AND ea_claim.key = 'claim'\n JOIN\n localnet.blocks b ON b.id = e.block_id\n WHERE\n e.type = 'poktroll.proof.EventClaimCreated'\n AND b.\"timestamp\" BETWEEN $__timeFrom() AND $__timeTo()\n GROUP BY\n supplier_operator_address\n),\n\n-- Estimated relays (from proofs)\nestimated_relays AS (\n SELECT\n (ea_proof.value::json ->> 'supplier_operator_address') AS supplier_operator_address,\n SUM(trim(both '\"' from ea_num_relays.value)::bigint) AS total_estimated_relays\n FROM\n localnet.events e\n JOIN\n localnet.event_attributes ea_num_relays ON ea_num_relays.event_id = e.id\n AND ea_num_relays.key = 'num_relays'\n JOIN\n localnet.event_attributes ea_proof ON ea_proof.event_id = e.id\n AND ea_proof.key = 'proof'\n JOIN\n localnet.blocks b ON b.id = e.block_id\n WHERE\n e.type = 'poktroll.proof.EventProofSubmitted'\n AND b.\"timestamp\" BETWEEN $__timeFrom() AND $__timeTo()\n GROUP BY\n supplier_operator_address\n)\n\n-- Combine the totals per supplier\nSELECT\n COALESCE(s.supplier_operator_address, e.supplier_operator_address) AS supplier_operator_address,\n COALESCE(s.total_serviced_relays, 0) AS total_serviced_relays,\n COALESCE(e.total_estimated_relays, 0) AS total_estimated_relays\nFROM\n serviced_relays s\nFULL OUTER JOIN\n estimated_relays e ON s.supplier_operator_address = e.supplier_operator_address\nORDER BY\n supplier_operator_address;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "cumulative number of relays claims vs proofs per supplier", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Permissionless demand observability", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/localnet/grafana-dashboards/tokenomics_relays.json b/localnet/grafana-dashboards/tokenomics_relays.json new file mode 100644 index 000000000..f6228f499 --- /dev/null +++ b/localnet/grafana-dashboards/tokenomics_relays.json @@ -0,0 +1,523 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 5, + "y": 0 + }, + "id": 4, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## What does this dashboard show?\n\nAs relays flow through the network\nAppGate -> RelayMiner and RelayMiner creates proofs and claims, we can capture the whole relay cycle.\n\n1. shows the actual amount of relays on AppGate;\n2. shows the actual amount of relays processed by RelayMiner;\n3. the amount of relays from the on-chain information using `EventClaimCreated`;\n4. relays from `EventProofSubmitted`;", + "mode": "markdown" + }, + "pluginVersion": "11.2.2+security-01", + "title": "Wht is this dashboard?", + "type": "text" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "increase(appgateserver_relay_requests_total[$group_by])", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Actual AppGate Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "increase(relayminer_requests_total[$group_by])", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Actual RelayMIner Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(b.\"timestamp\", '$group_by') AS time,\n SUM(trim(both '\"' from ea_num_relays.value)::bigint) AS total_num_relays,\n (ea_claim.value::json ->> 'supplier_operator_address') AS supplier_operator_address,\n (ea_claim.value::json -> 'session_header' ->> 'service_id') AS service_id\nFROM\n localnet.events e\nJOIN\n localnet.event_attributes ea_num_relays ON ea_num_relays.event_id = e.id\n AND ea_num_relays.key = 'num_relays'\nJOIN\n localnet.event_attributes ea_claim ON ea_claim.event_id = e.id\n AND ea_claim.key = 'claim'\nJOIN\n localnet.blocks b ON b.id = e.block_id\nWHERE\n e.type = 'poktroll.proof.EventClaimCreated'\n AND b.\"timestamp\" BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n time, supplier_operator_address, service_id\nORDER BY\n time;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Relays (from on-chain claims)", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PCC52D03280B7034C" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(b.\"timestamp\", '$group_by') AS time,\n SUM(trim(both '\"' from ea_num_relays.value)::bigint) AS total_num_relays,\n (ea_proof.value::json ->> 'supplier_operator_address') AS supplier_operator_address,\n (ea_proof.value::json -> 'session_header' ->> 'service_id') AS service_id\nFROM\n localnet.events e\nJOIN\n localnet.event_attributes ea_num_relays ON ea_num_relays.event_id = e.id\n AND ea_num_relays.key = 'num_relays'\nJOIN\n localnet.event_attributes ea_proof ON ea_proof.event_id = e.id\n AND ea_proof.key = 'proof'\nJOIN\n localnet.blocks b ON b.id = e.block_id\nWHERE\n e.type = 'poktroll.proof.EventProofSubmitted'\n AND b.\"timestamp\" BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY\n time, supplier_operator_address, service_id\nORDER BY\n time;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Relays (from on-chain proofs)", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "1m", + "value": "1m" + }, + "hide": 0, + "includeAll": false, + "label": "Group by time / Window", + "multi": false, + "name": "group_by", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + } + ], + "query": "1m,5m,10m,15m", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Protocol / Tokenomics / Relays life-cycle", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/localnet/kubernetes/observability-prometheus-stack.yaml b/localnet/kubernetes/observability-prometheus-stack.yaml index c6a29b67b..566ac33fd 100644 --- a/localnet/kubernetes/observability-prometheus-stack.yaml +++ b/localnet/kubernetes/observability-prometheus-stack.yaml @@ -17,7 +17,28 @@ grafana: viewers_can_edit: true security: allow_embedding: true + additionalDataSources: + - name: PostgreSQL + type: postgres + access: proxy + url: postgres-service:5432 + database: postgres + user: postgres + secureJsonData: + password: postgres + jsonData: + sslmode: disable + postgresVersion: 1604 # Adjust to match your PostgreSQL version + # timescaledb: false # Set to true if you are using TimescaleDB + schema: localnet # Specify your postgres schema here prometheus: prometheusSpec: - serviceMonitorSelectorNilUsesHelmValues: false + scrapeConfigSelector: + matchLabels: null + serviceMonitorSelector: + matchLabels: null + ruleSelector: + matchLabels: null + podMonitorSelector: + matchLabels: null diff --git a/localnet/poktrolld/config/app.toml b/localnet/poktrolld/config/app.toml index c80f545ae..d5135735e 100644 --- a/localnet/poktrolld/config/app.toml +++ b/localnet/poktrolld/config/app.toml @@ -34,6 +34,11 @@ query-gas-limit = "0" [mempool] max-txs = 10000 +[poktroll] + + [poktroll.telemetry] + cardinality-level = "high" + [rpc] cors_allowed_origins = ["*"] @@ -56,6 +61,6 @@ query-gas-limit = "0" enabled = true global-labels = [] metrics-sink = "mem" - prometheus-retention-time = "600" + prometheus-retention-time = 86400 service-name = "" statsd-addr = "" diff --git a/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml b/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml index 7c7e475da..2dab42371 100644 --- a/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml +++ b/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml @@ -1,5 +1,5 @@ query_node_rpc_url: tcp://localhost:26657 -query_node_grpc_url: tcp://localhost:9090 +query_node_grpc_url: tcp://localhost:36658 signing_key: app1 self_signing: true listening_endpoint: http://0.0.0.0:42069 diff --git a/localnet/poktrolld/config/relayminer_config_localnet_vscode.yaml b/localnet/poktrolld/config/relayminer_config_localnet_vscode.yaml index 74ba44dde..451914f9f 100644 --- a/localnet/poktrolld/config/relayminer_config_localnet_vscode.yaml +++ b/localnet/poktrolld/config/relayminer_config_localnet_vscode.yaml @@ -22,9 +22,9 @@ metrics: enabled: true addr: :9070 pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36657 + query_node_rpc_url: tcp://localhost:26657 + query_node_grpc_url: tcp://localhost:36658 + tx_node_rpc_url: tcp://localhost:26657 suppliers: - service_id: anvil listen_url: http://0.0.0.0:8545 @@ -38,6 +38,12 @@ suppliers: backend_url: http://localhost:11434/ publicly_exposed_endpoints: - relayminer1 + - service_id: rest + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://rest:10000/ + publicly_exposed_endpoints: + - relayminer1 pprof: enabled: false addr: localhost:6070 diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go index 740104182..124d8ceb5 100644 --- a/pkg/appgateserver/cmd/cmd.go +++ b/pkg/appgateserver/cmd/cmd.go @@ -31,6 +31,8 @@ var ( flagNodeRPCURL string // flagNodeGRPCURL is the variable containing the Cosmos node GRPC URL flag value. flagNodeGRPCURL string + // flagLogLevel is the variable to set a log level (used by cosmos and polylog). + flagLogLevel string ) // AppGateServerCmd returns the Cobra command for running the AppGate server. @@ -74,6 +76,7 @@ provided that: cmd.Flags().StringVar(&flagNodeRPCURL, cosmosflags.FlagNode, omittedDefaultFlagValue, "Register the default Cosmos node flag, which is needed to initialize the Cosmos query context correctly. It can be used to override the `QueryNodeUrl` field in the config file if specified.") cmd.Flags().StringVar(&flagNodeGRPCURL, cosmosflags.FlagGRPC, omittedDefaultFlagValue, "Register the default Cosmos node grpc flag, which is needed to initialize the Cosmos query context with grpc correctly. It can be used to override the `QueryNodeGRPCUrl` field in the config file if specified.") cmd.Flags().Bool(cosmosflags.FlagGRPCInsecure, true, "Used to initialize the Cosmos query context with grpc security options. It can be used to override the `QueryNodeGRPCInsecure` field in the config file if specified.") + cmd.Flags().StringVar(&flagLogLevel, cosmosflags.FlagLogLevel, "debug", "The logging level (debug|info|warn|error)") return cmd } @@ -99,7 +102,7 @@ func runAppGateServer(cmd *cobra.Command, _ []string) error { // TODO_TECHDEBT: populate logger from the config (ideally, from viper). loggerOpts := []polylog.LoggerOption{ - polyzero.WithLevel(polyzero.DebugLevel), + polyzero.WithLevel(polyzero.ParseLevel(flagLogLevel)), polyzero.WithOutput(os.Stderr), } diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index c8f0df94e..98df7e375 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -39,6 +39,8 @@ var ( flagNodeRPCURL string // flagNodeGRPCURL is the variable containing the Cosmos node GRPC URL flag value. flagNodeGRPCURL string + // flagLogLevel is the variable to set a log level (used by cosmos and polylog). + flagLogLevel string ) // RelayerCmd returns the Cobra command for running the relay miner. @@ -72,6 +74,7 @@ for such operations.`, cmd.Flags().StringVar(&flagNodeGRPCURL, cosmosflags.FlagGRPC, omittedDefaultFlagValue, "Register the default Cosmos node grpc flag, which is needed to initialize the Cosmos query context with grpc correctly. It can be used to override the `QueryNodeGRPCURL` field in the config file if specified.") cmd.Flags().Bool(cosmosflags.FlagGRPCInsecure, true, "Used to initialize the Cosmos query context with grpc security options. It can be used to override the `QueryNodeGRPCInsecure` field in the config file if specified.") cmd.Flags().String(cosmosflags.FlagChainID, "poktroll", "The network chain ID") + cmd.Flags().StringVar(&flagLogLevel, cosmosflags.FlagLogLevel, "debug", "The logging level (debug|info|warn|error)") return cmd } @@ -97,7 +100,7 @@ func runRelayer(cmd *cobra.Command, _ []string) error { // TODO_TECHDEBT: populate logger from the config (ideally, from viper). loggerOpts := []polylog.LoggerOption{ - polyzero.WithLevel(polyzero.DebugLevel), + polyzero.WithLevel(polyzero.ParseLevel(flagLogLevel)), polyzero.WithOutput(os.Stderr), } diff --git a/pkg/relayer/proxy/metrics.go b/pkg/relayer/proxy/metrics.go index d42f09c4b..4fdea226e 100644 --- a/pkg/relayer/proxy/metrics.go +++ b/pkg/relayer/proxy/metrics.go @@ -29,7 +29,7 @@ var ( Subsystem: relayMinerProcess, Name: requestsTotal, Help: "Total number of requests processed, labeled by service ID.", - }, []string{"service_id"}) + }, []string{"service_id", "supplier_operator_address"}) // relaysErrorsTotal is a Counter for total error events in the relay miner. // It increments with each error, labeled by 'service_id', @@ -45,7 +45,7 @@ var ( }, []string{"service_id"}) // relaysSuccessTotal is a Counter metric for successful requests in the relay miner. - // It increments with each successful request, labeled by ''service_id'. + // It increments with each successful request, labeled by 'service_id'. relaysSuccessTotal = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Subsystem: relayMinerProcess, Name: requestsSuccessTotal, diff --git a/pkg/relayer/proxy/synchronous.go b/pkg/relayer/proxy/synchronous.go index e3767eef6..df3bb8e35 100644 --- a/pkg/relayer/proxy/synchronous.go +++ b/pkg/relayer/proxy/synchronous.go @@ -174,7 +174,10 @@ func (sync *synchronousRPCServer) ServeHTTP(writer http.ResponseWriter, request } // Increment the relays counter. - relaysTotal.With("service_id", supplierServiceId).Add(1) + relaysTotal.With( + "service_id", supplierServiceId, + "supplier_operator_address", relayRequest.Meta.SupplierOperatorAddress, + ).Add(1) defer func() { duration := time.Since(startTime).Seconds() @@ -203,7 +206,7 @@ func (sync *synchronousRPCServer) ServeHTTP(writer http.ResponseWriter, request return } - sync.logger.Info().Fields(map[string]any{ + sync.logger.Debug().Fields(map[string]any{ "application_address": relay.Res.Meta.SessionHeader.ApplicationAddress, "service_id": relay.Res.Meta.SessionHeader.ServiceId, "session_start_height": relay.Res.Meta.SessionHeader.SessionStartBlockHeight, diff --git a/telemetry/block.go b/telemetry/block.go index b012a7d4a..59074f783 100644 --- a/telemetry/block.go +++ b/telemetry/block.go @@ -77,6 +77,10 @@ func (mal metricsABCIListener) ListenFinalizeBlock( req abci.RequestFinalizeBlock, res abci.ResponseFinalizeBlock, ) error { + if !isTelemetyEnabled() { + return nil + } + telemetry.SetGauge( float32(res.Size()), blockResultSizeBytesMetric, @@ -94,5 +98,9 @@ func (mal metricsABCIListener) ListenCommit( res abci.ResponseCommit, changeSet []*storetypes.StoreKVPair, ) error { + if !isTelemetyEnabled() { + return nil + } + return nil } diff --git a/telemetry/common.go b/telemetry/common.go new file mode 100644 index 000000000..28f5dc59b --- /dev/null +++ b/telemetry/common.go @@ -0,0 +1,54 @@ +package telemetry + +import ( + cosmostelemetry "github.com/cosmos/cosmos-sdk/telemetry" + "github.com/hashicorp/go-metrics" +) + +// MetricNameKeys prefixes metrics with `poktroll` for easy identification. +// E.g., `("hodlers", "regret_level")` yields `poktroll_hodlers_regret_level` — great for tracking FOMO as hodlers rethink choices. +// Returns a slice of strings as `go-metric`, the underlying metrics library, expects. +func MetricNameKeys(metrics ...string) []string { + result := make([]string, 0, len(metrics)+1) + result = append(result, metricNamePrefix) + result = append(result, metrics...) + return result +} + +// isTelemetyEnabled returns whether is telemetry turned on in the config file `app.toml` - cosmos-sdk's telemetry section. +func isTelemetyEnabled() bool { + return cosmostelemetry.IsTelemetryEnabled() +} + +// appendMediumCardinalityLabels only creates the label if cardinality if set to "medium" or higher. +// A good example for a "medium" cardinality use-case is `service_id`: +// - This is a network wide parameter +// - It is dependenon the permissionless nature of the network and can grow unbounded +// - We're keeping an option to turn off such labels to avoid metric bloat +// +// Configuration option is exposed in app.toml under the `poktroll.telemetry` section. +func appendMediumCardinalityLabels(labels []metrics.Label, labelPairs ...metrics.Label) []metrics.Label { + if globalTelemetryConfig.CardinalityLevel == "medium" || globalTelemetryConfig.CardinalityLevel == "high" { + return append(labels, labelPairs...) + } + return labels +} + +// appendHighCardinalityLabels only creates the label if cardinality is set to "high". +// A good example of high cardinality labels is `application_address` or `supplier_address`: +// - This setting, on a large network, will slow down both the full node and the metric scraping system. +// - These labels need to be exposed for local development, debugging and performance troubleshooting. +// +// Additional references on cardinality: https://www.robustperception.io/cardinality-is-key/ +// Configuration option is exposed in app.toml under the `poktroll.telemetry` section. +func appendHighCardinalityLabels(labels []metrics.Label, labelPairs ...metrics.Label) []metrics.Label { + if globalTelemetryConfig.CardinalityLevel == "high" { + return append(labels, labelPairs...) + } + return labels +} + +// toMetricLabel takes simple key and value of the label to return metrics.Label. +func toMetricLabel(key, value string) metrics.Label { + return cosmostelemetry.NewLabel(key, value) +} diff --git a/telemetry/event_counters.go b/telemetry/event_counters.go index 054e67ddb..2a36580be 100644 --- a/telemetry/event_counters.go +++ b/telemetry/event_counters.go @@ -10,16 +10,15 @@ import ( "github.com/cosmos/cosmos-sdk/telemetry" "github.com/hashicorp/go-metrics" - - prooftypes "github.com/pokt-network/poktroll/x/proof/types" ) const ( - // TODO_DECIDE: Decide if we want to continue using these generic metrics keys - // or opt for specific keys for each event_type. - // See: https://github.com/pokt-network/poktroll/pull/631#discussion_r1653760820 - eventTypeMetricKey = "event_type" - eventTypeMetricKeyGauge = "event_type_gauge" + // Prefix all metric names with "poktroll" for easier search + metricNamePrefix = "poktroll" + + // Label Names + applicationAddressLabelName = "app_addr" + supplierOperatorAddressLabelName = "sup_op_addr" ) // EventSuccessCounter increments a counter with the given data type and success status. @@ -28,15 +27,25 @@ func EventSuccessCounter( getValue func() float32, isSuccessful func() bool, ) { - successResult := strconv.FormatBool(isSuccessful()) + if !isTelemetyEnabled() { + return + } + value := getValue() + var metricName []string + + if isSuccessful() { + metricName = MetricNameKeys("successful", "events") + } else { + metricName = MetricNameKeys("failed", "events") + } + telemetry.IncrCounterWithLabels( - []string{eventTypeMetricKey}, + metricName, value, []metrics.Label{ {Name: "type", Value: eventType}, - {Name: "is_successful", Value: successResult}, }, ) } @@ -46,23 +55,34 @@ func EventSuccessCounter( // probabilistic selection, above compute unit threshold). // If err is not nil, the counter is not incremented but Prometheus will ingest this event. func ProofRequirementCounter( - reason prooftypes.ProofRequirementReason, + reason string, + serviceId string, + applicationAddress string, + supplierOperatorAddress string, err error, ) { + if !isTelemetyEnabled() { + return + } + incrementAmount := 1 - isRequired := strconv.FormatBool(reason != prooftypes.ProofRequirementReason_NOT_REQUIRED) labels := []metrics.Label{ - {Name: "proof_required_reason", Value: reason.String()}, - {Name: "is_required", Value: isRequired}, + {Name: "reason", Value: reason}, } + labels = appendMediumCardinalityLabels(labels, toMetricLabel("service_id", serviceId)) + labels = appendHighCardinalityLabels( + labels, + toMetricLabel(applicationAddressLabelName, applicationAddress), + toMetricLabel(supplierOperatorAddressLabelName, supplierOperatorAddress), + ) // Ensure the counter is not incremented if there was an error. if err != nil { - incrementAmount = 0 + return } telemetry.IncrCounterWithLabels( - []string{eventTypeMetricKey}, + MetricNameKeys("proof", "requirements"), float32(incrementAmount), labels, ) @@ -72,15 +92,27 @@ func ProofRequirementCounter( // which are represented by on-chain claims at the given ClaimProofStage. // If err is not nil, the counter is not incremented but Prometheus will ingest this event. func ClaimComputeUnitsCounter( - claimProofStage prooftypes.ClaimProofStage, + claimProofStage string, numComputeUnits uint64, + serviceId string, + applicationAddress string, + supplierOperatorAddress string, err error, ) { + if !isTelemetyEnabled() { + return + } + incrementAmount := numComputeUnits labels := []metrics.Label{ - {Name: "unit", Value: "compute_units"}, - {Name: "claim_proof_stage", Value: claimProofStage.String()}, + {Name: "claim_proof_stage", Value: claimProofStage}, } + labels = appendMediumCardinalityLabels(labels, toMetricLabel("service_id", serviceId)) + labels = appendHighCardinalityLabels( + labels, + toMetricLabel(applicationAddressLabelName, applicationAddress), + toMetricLabel(supplierOperatorAddressLabelName, supplierOperatorAddress), + ) // Ensure the counter is not incremented if there was an error. if err != nil { @@ -88,7 +120,7 @@ func ClaimComputeUnitsCounter( } telemetry.IncrCounterWithLabels( - []string{eventTypeMetricKey}, + MetricNameKeys("compute_units"), float32(incrementAmount), labels, ) @@ -99,15 +131,27 @@ func ClaimComputeUnitsCounter( // If err is not nil, the counter is not incremented and an "error" label is added // with the error's message. I.e., Prometheus will ingest this event. func ClaimRelaysCounter( - claimProofStage prooftypes.ClaimProofStage, + claimProofStage string, numRelays uint64, + serviceId string, + applicationAddress string, + supplierOperatorAddress string, err error, ) { + if !isTelemetyEnabled() { + return + } + incrementAmount := numRelays labels := []metrics.Label{ - {Name: "unit", Value: "relays"}, - {Name: "claim_proof_stage", Value: claimProofStage.String()}, + {Name: "claim_proof_stage", Value: claimProofStage}, } + labels = appendMediumCardinalityLabels(labels, toMetricLabel("service_id", serviceId)) + labels = appendHighCardinalityLabels( + labels, + toMetricLabel(applicationAddressLabelName, applicationAddress), + toMetricLabel(supplierOperatorAddressLabelName, supplierOperatorAddress), + ) // Ensure the counter is not incremented if there was an error. if err != nil { @@ -115,7 +159,7 @@ func ClaimRelaysCounter( } telemetry.IncrCounterWithLabels( - []string{eventTypeMetricKey}, + MetricNameKeys("relays"), float32(incrementAmount), labels, ) @@ -125,23 +169,36 @@ func ClaimRelaysCounter( // ClaimProofStage. // If err is not nil, the counter is not incremented but Prometheus will ingest this event. func ClaimCounter( - claimProofStage prooftypes.ClaimProofStage, + claimProofStage string, numClaims uint64, + serviceId string, + applicationAddress string, + supplierOperatorAddress string, err error, ) { + if !isTelemetyEnabled() { + return + } + incrementAmount := numClaims labels := []metrics.Label{ - {Name: "unit", Value: "claims"}, - {Name: "claim_proof_stage", Value: claimProofStage.String()}, + {Name: "claim_proof_stage", Value: claimProofStage}, } + labels = appendMediumCardinalityLabels(labels, toMetricLabel("service_id", serviceId)) + labels = appendHighCardinalityLabels( + labels, + toMetricLabel(applicationAddressLabelName, applicationAddress), + toMetricLabel(supplierOperatorAddressLabelName, supplierOperatorAddress), + ) + // Ensure the counter is not incremented if there was an error. if err != nil { incrementAmount = 0 } telemetry.IncrCounterWithLabels( - []string{eventTypeMetricKey}, + MetricNameKeys("claims"), float32(incrementAmount), labels, ) @@ -151,13 +208,15 @@ func ClaimCounter( // of the relay mining difficulty. The serviceId is used as a label to be able to // track the difficulty for each service. func RelayMiningDifficultyGauge(difficulty float32, serviceId string) { - labels := []metrics.Label{ - {Name: "type", Value: "relay_mining_difficulty"}, - {Name: "service_id", Value: serviceId}, + if !isTelemetyEnabled() { + return } + labels := []metrics.Label{} + labels = appendMediumCardinalityLabels(labels, toMetricLabel("service_id", serviceId)) + telemetry.SetGaugeWithLabels( - []string{eventTypeMetricKeyGauge}, + MetricNameKeys("relay_mining", "difficulty"), difficulty, labels, ) @@ -166,14 +225,39 @@ func RelayMiningDifficultyGauge(difficulty float32, serviceId string) { // RelayEMAGauge sets a gauge which tracks the relay EMA for a service. // The serviceId is used as a label to be able to track the EMA for each service. func RelayEMAGauge(relayEMA uint64, serviceId string) { - labels := []metrics.Label{ - {Name: "type", Value: "relay_ema"}, - {Name: "service_id", Value: serviceId}, + if !isTelemetyEnabled() { + return } + labels := []metrics.Label{} + labels = appendMediumCardinalityLabels(labels, toMetricLabel("service_id", serviceId)) + telemetry.SetGaugeWithLabels( - []string{eventTypeMetricKeyGauge}, + MetricNameKeys("relay", "ema"), float32(relayEMA), labels, ) } + +// SessionSuppliersGauge sets a gauge which tracks the number of candidates available +// for session suppliers at the given maxPerSession value. +// The serviceId is used as a label to be able to track this information for each service. +func SessionSuppliersGauge(numCandidates int, maxPerSession int, serviceId string) { + if !isTelemetyEnabled() { + return + } + + maxPerSessionStr := strconv.Itoa(maxPerSession) + labels := []metrics.Label{} + labels = appendMediumCardinalityLabels( + labels, + toMetricLabel("service_id", serviceId), + toMetricLabel("max_per_session", maxPerSessionStr), + ) + + telemetry.SetGaugeWithLabels( + MetricNameKeys("session", "suppliers"), + float32(numCandidates), + labels, + ) +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go new file mode 100644 index 000000000..348b73898 --- /dev/null +++ b/telemetry/telemetry.go @@ -0,0 +1,31 @@ +package telemetry + +import ( + "fmt" + + servertypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/mitchellh/mapstructure" +) + +// globalTelemetryConfig stores poktroll specific telemetry configurations. +// Set once on initialization and remains constant during runtime. +var globalTelemetryConfig PoktrollTelemetryConfig + +// PoktrollTelemetryConfig represents the telemetry protion of the custom poktroll config section in `app.toml`. +type PoktrollTelemetryConfig struct { + CardinalityLevel string `mapstructure:"cardinality-level"` +} + +// New sets the globalTelemetryConfig for telemetry package. +func New(appOpts servertypes.AppOptions) error { + // Extract the map from appOpts. + // `poktroll.telemetry` comes from `app.toml` which is parsed into a map. + telemetryMap := appOpts.Get("poktroll.telemetry").(map[string]interface{}) + + // Use mapstructure to decode the map into the struct + if err := mapstructure.Decode(telemetryMap, &globalTelemetryConfig); err != nil { + return fmt.Errorf("error decoding poktroll.telemetry config: %v", err) + } + + return nil +} diff --git a/telemetry/tokens.go b/telemetry/tokens.go new file mode 100644 index 000000000..a62b3355f --- /dev/null +++ b/telemetry/tokens.go @@ -0,0 +1,61 @@ +package telemetry + +import ( + cosmostelemetry "github.com/cosmos/cosmos-sdk/telemetry" + "github.com/hashicorp/go-metrics" +) + +// TODO_MAINNET(@bryanchriswhite): Revisit how telemetry is managed under `x/tokenomics` to ensure that it +// complies with the new hardened settlement approach. + +// TODO_MAINNET(@red-0ne, #897): Minted, burnt and slashd tokens values might not be completely accurate. +// While we're keeping this metric for now consider removing in favor of utilizing the `cosmos-exporter` which uses on-chain data. +// Context: https://github.com/cosmos/cosmos-sdk/issues/21614, https://github.com/pokt-network/poktroll/pull/832 + +// MintedTokensFromModule is a function to track token minting from a specific module. +// The metric used is an increment counter, and the label includes the module name for context. +func MintedTokensFromModule(module string, amount float32) { + if !isTelemetyEnabled() { + return + } + + cosmostelemetry.IncrCounterWithLabels( + MetricNameKeys("minted", "tokens"), + amount, + []metrics.Label{ + cosmostelemetry.NewLabel("module", module), + }, + ) +} + +// BurnedTokensFromModule is a function to track token burning from a specific module. +// The metric used is an increment counter, and the label includes the module name for context. +func BurnedTokensFromModule(module string, amount float32) { + if !isTelemetyEnabled() { + return + } + + cosmostelemetry.IncrCounterWithLabels( + MetricNameKeys("burned", "tokens"), + amount, + []metrics.Label{ + cosmostelemetry.NewLabel("module", module), + }, + ) +} + +// SlashedTokensFromModule is a function to track token slashing from a specific module. +// The metric used is an increment counter, and the label includes the module name for context. +func SlashedTokensFromModule(module string, amount float32) { + if !isTelemetyEnabled() { + return + } + + cosmostelemetry.IncrCounterWithLabels( + MetricNameKeys("slashed", "tokens"), + amount, + []metrics.Label{ + cosmostelemetry.NewLabel("module", module), + }, + ) +} diff --git a/x/application/module/abci.go b/x/application/module/abci.go index 4890fa062..3a1b7b337 100644 --- a/x/application/module/abci.go +++ b/x/application/module/abci.go @@ -1,13 +1,18 @@ package application import ( + cosmostelemetry "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/pokt-network/poktroll/x/application/keeper" + "github.com/pokt-network/poktroll/x/application/types" ) // EndBlocker is called every block and handles application related updates. func EndBlocker(ctx sdk.Context, k keeper.Keeper) error { + // Telemetry: measure the end-block execution time following standard cosmos-sdk practices. + defer cosmostelemetry.ModuleMeasureSince(types.ModuleName, cosmostelemetry.Now(), cosmostelemetry.MetricKeyEndBlocker) + if err := k.EndBlockerAutoUndelegateFromUnstakedGateways(ctx); err != nil { return err } diff --git a/x/application/types/types.pb.go b/x/application/types/types.pb.go index 9e5b8ff6c..32bc0cc13 100644 --- a/x/application/types/types.pb.go +++ b/x/application/types/types.pb.go @@ -31,14 +31,12 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type Application struct { Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` Stake *types.Coin `protobuf:"bytes,2,opt,name=stake,proto3" json:"stake,omitempty"` - // As per this discussion: - // https://github.com/pokt-network/poktroll/pull/750#discussion_r1735025033 - // The number of service_configs is limited to 1 per service. - // This is to ensure that an application could not over service by making multiple - // claims settelments compete to burn the same stake. - // A slice of service_configs is still maintained to allow for future multi-service - // capabilities to be added, such as off-chain application stake tracking by suppliers: - // https://www.notion.so/buildwithgrove/Off-chain-Application-Stake-Tracking-6a8bebb107db4f7f9dc62cbe7ba555f7 + // CRITICAL_DEV_NOTE: The number of service_configs must be EXACTLY ONE. + // This prevents applications from over-servicing. + // The field is kept repeated (a list) for both legacy and future logic reaosns. + // References: + // - https://github.com/pokt-network/poktroll/pull/750#discussion_r1735025033 + // - https://www.notion.so/buildwithgrove/Off-chain-Application-Stake-Tracking-6a8bebb107db4f7f9dc62cbe7ba555f7 ServiceConfigs []*types1.ApplicationServiceConfig `protobuf:"bytes,3,rep,name=service_configs,json=serviceConfigs,proto3" json:"service_configs,omitempty"` // TODO_BETA: Rename `delegatee_gateway_addresses` to `gateway_addresses_delegated_to`. // Ensure to rename all relevant configs, comments, variables, function names, etc as well. diff --git a/x/proof/keeper/msg_server_create_claim.go b/x/proof/keeper/msg_server_create_claim.go index 01a5f509a..f95e262b1 100644 --- a/x/proof/keeper/msg_server_create_claim.go +++ b/x/proof/keeper/msg_server_create_claim.go @@ -11,6 +11,7 @@ import ( "github.com/pokt-network/poktroll/telemetry" "github.com/pokt-network/poktroll/x/proof/types" + sessiontypes "github.com/pokt-network/poktroll/x/session/types" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) @@ -26,16 +27,6 @@ func (k msgServer) CreateClaim( numClaimComputeUnits uint64 ) - // Defer telemetry calls so that they reference the final values the relevant variables. - defer func() { - // Only increment these metrics counters if handling a new claim. - if !isExistingClaim { - telemetry.ClaimCounter(types.ClaimProofStage_CLAIMED, 1, err) - telemetry.ClaimRelaysCounter(types.ClaimProofStage_CLAIMED, numRelays, err) - telemetry.ClaimComputeUnitsCounter(types.ClaimProofStage_CLAIMED, numClaimComputeUnits, err) - } - }() - logger := k.Logger().With("method", "CreateClaim") sdkCtx := cosmostypes.UnwrapSDKContext(ctx) logger.Info("creating claim") @@ -52,6 +43,9 @@ func (k msgServer) CreateClaim( return nil, status.Error(codes.InvalidArgument, err.Error()) } + // Defer telemetry calls to a helper function to keep business logic clean. + defer k.finalizeCreateClaimTelemetry(session, msg, isExistingClaim, numRelays, numClaimComputeUnits, err) + // Construct and insert claim claim = types.Claim{ SupplierOperatorAddress: msg.GetSupplierOperatorAddress(), @@ -154,3 +148,20 @@ func (k msgServer) CreateClaim( Claim: &claim, }, nil } + +// finalizeCreateClaimTelemetry defers telemetry calls to be executed after business logic, +// incrementing counters based on whether a new claim was handled successfully. +// Meant to run deferred. +func (k msgServer) finalizeCreateClaimTelemetry(session *sessiontypes.Session, msg *types.MsgCreateClaim, isExistingClaim bool, numRelays, numClaimComputeUnits uint64, err error) { + // Only increment these metrics counters if handling a new claim. + if !isExistingClaim { + serviceId := session.Header.ServiceId + applicationAddress := session.Header.ApplicationAddress + supplierOperatorAddress := msg.GetSupplierOperatorAddress() + claimProofStage := types.ClaimProofStage_CLAIMED.String() + + telemetry.ClaimCounter(claimProofStage, 1, serviceId, applicationAddress, supplierOperatorAddress, err) + telemetry.ClaimRelaysCounter(claimProofStage, numRelays, serviceId, applicationAddress, supplierOperatorAddress, err) + telemetry.ClaimComputeUnitsCounter(claimProofStage, numClaimComputeUnits, serviceId, applicationAddress, supplierOperatorAddress, err) + } +} diff --git a/x/proof/keeper/msg_server_submit_proof.go b/x/proof/keeper/msg_server_submit_proof.go index 6496a3f30..c19fbd2e6 100644 --- a/x/proof/keeper/msg_server_submit_proof.go +++ b/x/proof/keeper/msg_server_submit_proof.go @@ -16,6 +16,7 @@ import ( "github.com/pokt-network/poktroll/telemetry" "github.com/pokt-network/poktroll/x/proof/types" servicekeeper "github.com/pokt-network/poktroll/x/service/keeper" + sessiontypes "github.com/pokt-network/poktroll/x/session/types" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) @@ -47,16 +48,6 @@ func (k msgServer) SubmitProof( numClaimComputeUnits uint64 ) - // Defer telemetry calls so that they reference the final values the relevant variables. - defer func() { - // Only increment these metrics counters if handling a new claim. - if !isExistingProof { - telemetry.ClaimCounter(types.ClaimProofStage_PROVEN, 1, err) - telemetry.ClaimRelaysCounter(types.ClaimProofStage_PROVEN, numRelays, err) - telemetry.ClaimComputeUnitsCounter(types.ClaimProofStage_PROVEN, numClaimComputeUnits, err) - } - }() - logger := k.Logger().With("method", "SubmitProof") sdkCtx := cosmostypes.UnwrapSDKContext(ctx) logger.Info("About to start submitting proof") @@ -73,6 +64,9 @@ func (k msgServer) SubmitProof( return nil, status.Error(codes.InvalidArgument, err.Error()) } + // Defer telemetry calls so that they reference the final values the relevant variables. + defer k.finalizeSubmitProofTelemetry(session, msg, isExistingProof, numRelays, numClaimComputeUnits, err) + if err = k.deductProofSubmissionFee(ctx, msg.GetSupplierOperatorAddress()); err != nil { logger.Error(fmt.Sprintf("failed to deduct proof submission fee: %v", err)) return nil, status.Error(codes.FailedPrecondition, err.Error()) @@ -219,9 +213,7 @@ func (k Keeper) ProofRequirementForClaim(ctx context.Context, claim *types.Claim var requirementReason = types.ProofRequirementReason_NOT_REQUIRED // Defer telemetry calls so that they reference the final values the relevant variables. - defer func() { - telemetry.ProofRequirementCounter(requirementReason, err) - }() + defer k.finalizeProofRequirementTelemetry(requirementReason, claim, err) proofParams := k.GetParams(ctx) sharedParams := k.sharedKeeper.GetParams(ctx) @@ -322,3 +314,30 @@ func (k Keeper) getProofRequirementSeedBlockHash( // the block that is before the earliest block at which a proof can be committed. return k.sessionKeeper.GetBlockHash(ctx, earliestSupplierProofCommitHeight-1), nil } + +// finalizeSubmitProofTelemetry finalizes telemetry updates for SubmitProof, incrementing counters as needed. +// Meant to run deferred. +func (k msgServer) finalizeSubmitProofTelemetry(session *sessiontypes.Session, msg *types.MsgSubmitProof, isExistingProof bool, numRelays, numClaimComputeUnits uint64, err error) { + if !isExistingProof { + serviceId := session.Header.ServiceId + applicationAddress := session.Header.ApplicationAddress + supplierOperatorAddress := msg.GetSupplierOperatorAddress() + claimProofStage := types.ClaimProofStage_PROVEN.String() + + telemetry.ClaimCounter(claimProofStage, 1, serviceId, applicationAddress, supplierOperatorAddress, err) + telemetry.ClaimRelaysCounter(claimProofStage, numRelays, serviceId, applicationAddress, supplierOperatorAddress, err) + telemetry.ClaimComputeUnitsCounter(claimProofStage, numClaimComputeUnits, serviceId, applicationAddress, supplierOperatorAddress, err) + } +} + +// finalizeProofRequirementTelemetry finalizes telemetry updates for proof requirements. +// Meant to run deferred. +func (k Keeper) finalizeProofRequirementTelemetry(requirementReason types.ProofRequirementReason, claim *types.Claim, err error) { + telemetry.ProofRequirementCounter( + requirementReason.String(), + claim.SessionHeader.ServiceId, + claim.SessionHeader.ApplicationAddress, + claim.SupplierOperatorAddress, + err, + ) +} diff --git a/x/proof/keeper/proof_validation.go b/x/proof/keeper/proof_validation.go index 614c5cf03..778d66d8c 100644 --- a/x/proof/keeper/proof_validation.go +++ b/x/proof/keeper/proof_validation.go @@ -32,9 +32,11 @@ import ( "bytes" "context" + cosmostelemetry "github.com/cosmos/cosmos-sdk/telemetry" "github.com/pokt-network/smt" "github.com/pokt-network/poktroll/pkg/crypto/protocol" + "github.com/pokt-network/poktroll/telemetry" "github.com/pokt-network/poktroll/x/proof/types" servicekeeper "github.com/pokt-network/poktroll/x/service/keeper" servicetypes "github.com/pokt-network/poktroll/x/service/types" @@ -59,6 +61,9 @@ func (k Keeper) EnsureValidProof( ctx context.Context, proof *types.Proof, ) error { + // Telemetry: measure execution time. + defer cosmostelemetry.MeasureSince(cosmostelemetry.Now(), telemetry.MetricNameKeys("proof", "validation")...) + logger := k.Logger().With("method", "ValidateProof") // Retrieve the supplier operator's public key. diff --git a/x/session/keeper/query_get_session.go b/x/session/keeper/query_get_session.go index 166c77d37..bc25fff0e 100644 --- a/x/session/keeper/query_get_session.go +++ b/x/session/keeper/query_get_session.go @@ -36,7 +36,7 @@ func (k Keeper) GetSession(ctx context.Context, req *types.QueryGetSessionReques blockHeight = req.BlockHeight } - k.Logger().Info(fmt.Sprintf("Getting session for height: %d", blockHeight)) + k.Logger().Debug(fmt.Sprintf("Getting session for height: %d", blockHeight)) sessionHydrator := NewSessionHydrator(req.ApplicationAddress, req.ServiceId, blockHeight) session, err := k.HydrateSession(ctx, sessionHydrator) diff --git a/x/session/keeper/session_hydrator.go b/x/session/keeper/session_hydrator.go index 5f259aa78..8dae1a7a5 100644 --- a/x/session/keeper/session_hydrator.go +++ b/x/session/keeper/session_hydrator.go @@ -12,6 +12,7 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" _ "golang.org/x/crypto/sha3" + "github.com/pokt-network/poktroll/telemetry" "github.com/pokt-network/poktroll/x/session/types" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) @@ -66,22 +67,22 @@ func (k Keeper) HydrateSession(ctx context.Context, sh *sessionHydrator) (*types if err := k.hydrateSessionMetadata(ctx, sh); err != nil { return nil, err } - logger.Info("Finished hydrating session metadata") + logger.Debug("Finished hydrating session metadata") if err := k.hydrateSessionID(ctx, sh); err != nil { return nil, err } - logger.Info(fmt.Sprintf("Finished hydrating session ID: %s", sh.sessionHeader.SessionId)) + logger.Debug(fmt.Sprintf("Finished hydrating session ID: %s", sh.sessionHeader.SessionId)) if err := k.hydrateSessionApplication(ctx, sh); err != nil { return nil, err } - logger.Info(fmt.Sprintf("Finished hydrating session application: %+v", sh.session.Application)) + logger.Debug(fmt.Sprintf("Finished hydrating session application: %+v", sh.session.Application)) if err := k.hydrateSessionSuppliers(ctx, sh); err != nil { return nil, err } - logger.Info("Finished hydrating session suppliers") + logger.Debug("Finished hydrating session suppliers") sh.session.Header = sh.sessionHeader sh.session.SessionId = sh.sessionHeader.SessionId @@ -196,6 +197,8 @@ func (k Keeper) hydrateSessionSuppliers(ctx context.Context, sh *sessionHydrator } } + defer telemetry.SessionSuppliersGauge(len(candidateSuppliers), NumSupplierPerSession, sh.sessionHeader.ServiceId) + if len(candidateSuppliers) == 0 { logger.Error("[ERROR] no suppliers found for session") return types.ErrSessionSuppliersNotFound.Wrapf( @@ -206,7 +209,7 @@ func (k Keeper) hydrateSessionSuppliers(ctx context.Context, sh *sessionHydrator } if len(candidateSuppliers) < NumSupplierPerSession { - logger.Info(fmt.Sprintf( + logger.Debug(fmt.Sprintf( "Number of available suppliers (%d) is less than the maximum number of possible suppliers per session (%d)", len(candidateSuppliers), NumSupplierPerSession, diff --git a/x/supplier/module/abci.go b/x/supplier/module/abci.go index 095e33501..f448fd5f0 100644 --- a/x/supplier/module/abci.go +++ b/x/supplier/module/abci.go @@ -3,11 +3,16 @@ package supplier import ( sdk "github.com/cosmos/cosmos-sdk/types" + cosmostelemetry "github.com/cosmos/cosmos-sdk/telemetry" "github.com/pokt-network/poktroll/x/supplier/keeper" + "github.com/pokt-network/poktroll/x/supplier/types" ) // EndBlocker is called every block and handles supplier related updates. func EndBlocker(ctx sdk.Context, k keeper.Keeper) error { + // Telemetry: measure the end-block execution time following standard cosmos-sdk practices. + defer cosmostelemetry.ModuleMeasureSince(types.ModuleName, cosmostelemetry.Now(), cosmostelemetry.MetricKeyEndBlocker) + // TODO_IMPROVE: Add logs and/or telemetry on the number of unbonded suppliers. if err := k.EndBlockerUnbondSuppliers(ctx); err != nil { return err diff --git a/x/tokenomics/keeper/settle_pending_claims.go b/x/tokenomics/keeper/settle_pending_claims.go index 37a48b3f2..7555563a4 100644 --- a/x/tokenomics/keeper/settle_pending_claims.go +++ b/x/tokenomics/keeper/settle_pending_claims.go @@ -9,6 +9,7 @@ import ( "github.com/cosmos/cosmos-sdk/types/query" "github.com/pokt-network/poktroll/app/volatile" + "github.com/pokt-network/poktroll/telemetry" apptypes "github.com/pokt-network/poktroll/x/application/types" prooftypes "github.com/pokt-network/poktroll/x/proof/types" servicekeeper "github.com/pokt-network/poktroll/x/service/keeper" @@ -179,6 +180,18 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) ( expiredResult.NumClaims++ expiredResult.NumRelays += numClaimRelays expiredResult.NumComputeUnits += numClaimComputeUnits + + // Telemetry - defer telemetry calls so that they reference the final values the relevant variables. + defer k.finalizeTelemetry( + prooftypes.ClaimProofStage_EXPIRED, + claim.SessionHeader.ServiceId, + claim.SessionHeader.ApplicationAddress, + claim.SupplierOperatorAddress, + numClaimRelays, + numClaimComputeUnits, + err, + ) + continue } } @@ -238,7 +251,18 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) ( settledResult.NumComputeUnits += numClaimComputeUnits settledResult.RelaysPerServiceMap[claim.SessionHeader.ServiceId] += numClaimRelays - logger.Info(fmt.Sprintf("Successfully settled claim for session ID %q at block height %d", claim.SessionHeader.SessionId, blockHeight)) + logger.Debug(fmt.Sprintf("Successfully settled claim for session ID %q at block height %d", claim.SessionHeader.SessionId, blockHeight)) + + // Telemetry - defer telemetry calls so that they reference the final values the relevant variables. + defer k.finalizeTelemetry( + prooftypes.ClaimProofStage_SETTLED, + claim.SessionHeader.ServiceId, + claim.SessionHeader.ApplicationAddress, + claim.SupplierOperatorAddress, + numClaimRelays, + numClaimComputeUnits, + err, + ) } // Slash all the suppliers that have been marked for slashing slashingCount times. @@ -358,6 +382,11 @@ func (k Keeper) slashSupplierStake( return err } + // Update telemetry information + if totalSlashingCoin.Amount.IsInt64() { + defer telemetry.SlashedTokensFromModule(suppliertypes.ModuleName, float32(totalSlashingCoin.Amount.Int64())) + } + supplierToSlash.Stake = &remainingStakeCoin logger.Info(fmt.Sprintf( @@ -447,3 +476,19 @@ func (k Keeper) getApplicationInitialStakeMap( return applicationInitialStakeMap, nil } + +// finalizeTelemetry logs telemetry metrics for a claim based on its stage (e.g., EXPIRED, SETTLED). +// Meant to run deferred. +func (k Keeper) finalizeTelemetry( + claimProofStage prooftypes.ClaimProofStage, + serviceId string, + applicationAddress string, + supplierOperatorAddress string, + numRelays uint64, + numClaimComputeUnits uint64, + err error, +) { + telemetry.ClaimCounter(claimProofStage.String(), 1, serviceId, applicationAddress, supplierOperatorAddress, err) + telemetry.ClaimRelaysCounter(claimProofStage.String(), numRelays, serviceId, applicationAddress, supplierOperatorAddress, err) + telemetry.ClaimComputeUnitsCounter(claimProofStage.String(), numClaimComputeUnits, serviceId, applicationAddress, supplierOperatorAddress, err) +} diff --git a/x/tokenomics/keeper/token_logic_modules.go b/x/tokenomics/keeper/token_logic_modules.go index 3dba9136a..f6bad19d5 100644 --- a/x/tokenomics/keeper/token_logic_modules.go +++ b/x/tokenomics/keeper/token_logic_modules.go @@ -376,7 +376,13 @@ func (k Keeper) TokenLogicModuleRelayBurnEqualsMint( err, ) } - logger.Info(fmt.Sprintf("minted (%v) coins in the supplier module", settlementCoin)) + + // Update telemetry information + if settlementCoin.Amount.IsInt64() { + defer telemetry.MintedTokensFromModule(suppliertypes.ModuleName, float32(settlementCoin.Amount.Int64())) + } + + logger.Debug(fmt.Sprintf("minted (%v) coins in the supplier module", settlementCoin)) // Distribute the rewards to the supplier's shareholders based on the rev share percentage. if err := k.distributeSupplierRewardsToShareHolders(ctx, supplier, service.Id, settlementCoin.Amount.Uint64()); err != nil { @@ -386,7 +392,7 @@ func (k Keeper) TokenLogicModuleRelayBurnEqualsMint( err, ) } - logger.Info(fmt.Sprintf("sent (%v) from the supplier module to the supplier account with address %q", settlementCoin, supplier.OperatorAddress)) + logger.Debug(fmt.Sprintf("sent (%v) from the supplier module to the supplier account with address %q", settlementCoin, supplier.OperatorAddress)) // Burn uPOKT from the application module account which was held in escrow // on behalf of the application account. @@ -395,7 +401,13 @@ func (k Keeper) TokenLogicModuleRelayBurnEqualsMint( ); err != nil { return tokenomicstypes.ErrTokenomicsApplicationModuleBurn.Wrapf("burning %s from the application module account: %v", settlementCoin, err) } - logger.Info(fmt.Sprintf("burned (%v) from the application module account", settlementCoin)) + + // Update telemetry information + if settlementCoin.Amount.IsInt64() { + defer telemetry.BurnedTokensFromModule(apptypes.ModuleName, float32(settlementCoin.Amount.Int64())) + } + + logger.Debug(fmt.Sprintf("burned (%v) from the application module account", settlementCoin)) // Update the application's on-chain stake newAppStake, err := application.Stake.SafeSub(settlementCoin) @@ -403,7 +415,7 @@ func (k Keeper) TokenLogicModuleRelayBurnEqualsMint( return tokenomicstypes.ErrTokenomicsApplicationNewStakeInvalid.Wrapf("application %q stake cannot be reduced to a negative amount %v", application.Address, newAppStake) } application.Stake = &newAppStake - logger.Info(fmt.Sprintf("updated application %q stake to %v", application.Address, newAppStake)) + logger.Debug(fmt.Sprintf("updated application %q stake to %v", application.Address, newAppStake)) return nil } @@ -436,6 +448,12 @@ func (k Keeper) TokenLogicModuleGlobalMint( return tokenomicstypes.ErrTokenomicsModuleMintFailed.Wrapf( "minting (%s) to the tokenomics module account: %v", newMintCoin, err) } + + // Update telemetry information + if newMintCoin.Amount.IsInt64() { + defer telemetry.MintedTokensFromModule(tokenomicstypes.ModuleName, float32(newMintCoin.Amount.Int64())) + } + logger.Info(fmt.Sprintf("minted (%s) to the tokenomics module account", newMintCoin)) // Send a portion of the rewards to the application diff --git a/x/tokenomics/module/abci.go b/x/tokenomics/module/abci.go index 8140263c9..fc0776c68 100644 --- a/x/tokenomics/module/abci.go +++ b/x/tokenomics/module/abci.go @@ -3,16 +3,20 @@ package tokenomics import ( "fmt" + cosmostelemetry "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/pokt-network/poktroll/pkg/crypto/protocol" "github.com/pokt-network/poktroll/telemetry" - prooftypes "github.com/pokt-network/poktroll/x/proof/types" "github.com/pokt-network/poktroll/x/tokenomics/keeper" + "github.com/pokt-network/poktroll/x/tokenomics/types" ) // EndBlocker called at every block and settles all pending claims. func EndBlocker(ctx sdk.Context, k keeper.Keeper) (err error) { + // Telemetry: measure the end-block execution time following standard cosmos-sdk practices. + defer cosmostelemetry.ModuleMeasureSince(types.ModuleName, cosmostelemetry.Now(), cosmostelemetry.MetricKeyEndBlocker) + logger := k.Logger().With("method", "EndBlocker") // NB: There are two main reasons why we settle expiring claims in the end @@ -33,42 +37,6 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper) (err error) { expiredResult.NumClaims, )) - // Telemetry - defer telemetry calls so that they reference the final values the relevant variables. - defer func() { - telemetry.ClaimCounter( - prooftypes.ClaimProofStage_SETTLED, - settledResult.NumClaims, - err, - ) - telemetry.ClaimRelaysCounter( - prooftypes.ClaimProofStage_SETTLED, - settledResult.NumRelays, - err, - ) - telemetry.ClaimComputeUnitsCounter( - prooftypes.ClaimProofStage_SETTLED, - settledResult.NumComputeUnits, - err, - ) - - telemetry.ClaimCounter( - prooftypes.ClaimProofStage_EXPIRED, - expiredResult.NumClaims, - err, - ) - telemetry.ClaimRelaysCounter( - prooftypes.ClaimProofStage_EXPIRED, - expiredResult.NumRelays, - err, - ) - telemetry.ClaimComputeUnitsCounter( - prooftypes.ClaimProofStage_EXPIRED, - expiredResult.NumComputeUnits, - err, - ) - // TODO_POST_MAINNET: Add a counter for expired compute units. - }() - // Update the relay mining difficulty for every service that settled pending // claims based on how many estimated relays were serviced for it. difficultyPerServiceMap, err := k.UpdateRelayMiningDifficulty(ctx, settledResult.RelaysPerServiceMap) diff --git a/x/tokenomics/types/event.pb.go b/x/tokenomics/types/event.pb.go index 8570fd911..5e69f719f 100644 --- a/x/tokenomics/types/event.pb.go +++ b/x/tokenomics/types/event.pb.go @@ -370,7 +370,7 @@ func (m *EventSupplierSlashed) GetSlashingAmount() *types1.Coin { } // EventApplicationReimbursementRequest is emitted when an application requests -// a reimbursement +// a reimbursement. type EventApplicationReimbursementRequest struct { ApplicationAddr string `protobuf:"bytes,1,opt,name=application_addr,json=applicationAddr,proto3" json:"application_addr,omitempty"` SupplierOperatorAddr string `protobuf:"bytes,2,opt,name=supplier_operator_addr,json=supplierOperatorAddr,proto3" json:"supplier_operator_addr,omitempty"`