[Observability] Foundation for load testing telemetry

Tiltfile

@@ -6,7 +6,7 @@ load("ext://deployment", "deployment_create")
 load("ext://execute_in_pod", "execute_in_pod")
 
 # A list of directories where changes trigger a hot-reload of the validator
-hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"]
+hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg", "telemetry"]
 
 
 def merge_dicts(base, updates):
@@ -38,14 +38,26 @@ localnet_config_defaults = {
         "enabled": True,
         "grafana": {"defaultDashboardsEnabled": False},
     },
-    "relayminers": {"count": 1, "delve": {"enabled": False}},
+    "relayminers": {
+        "count": 1,
+        "delve": {"enabled": False},
+        "logs": {
+            "level": "debug",
+        },
+    },
     "gateways": {
         "count": 1,
         "delve": {"enabled": False},
+        "logs": {
+            "level": "debug",
+        },
     },
     "appgateservers": {
         "count": 1,
         "delve": {"enabled": False},
+        "logs": {
+            "level": "debug",
+        },
     },
     "ollama": {
         "enabled": False,
@@ -100,8 +112,9 @@ if localnet_config["observability"]["enabled"]:
     helm_repo("prometheus-community", "https://prometheus-community.github.io/helm-charts")
     helm_repo("grafana-helm-repo", "https://grafana.github.io/helm-charts")
 
-    # Increase timeout for building the image
-    update_settings(k8s_upsert_timeout_secs=60)
+    # Increase timeout for building the imagedefault is 30, which can be too low for slow internet connections to pull
+    # container images.
+    update_settings(k8s_upsert_timeout_secs=120)
 
     helm_resource(
         "observability",
@@ -226,6 +239,7 @@ helm_resource(
         "--set=logs.format=" + str(localnet_config["validator"]["logs"]["format"]),
         "--set=serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
         "--set=development.delve.enabled=" + str(localnet_config["validator"]["delve"]["enabled"]),
+        "--set=image.repository=poktrolld",
     ],
     image_deps=["poktrolld"],
     image_keys=[("image.repository", "image.tag")],
@@ -244,6 +258,8 @@ for x in range(localnet_config["relayminers"]["count"]):
             "--values=./localnet/kubernetes/values-relayminer-" + str(actor_number) + ".yaml",
             "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
             "--set=development.delve.enabled=" + str(localnet_config["relayminers"]["delve"]["enabled"]),
+            "--set=logLevel=" + str(localnet_config["relayminers"]["logs"]["level"]),
+            "--set=image.repository=poktrolld",
         ],
         image_deps=["poktrolld"],
         image_keys=[("image.repository", "image.tag")],
@@ -284,6 +300,8 @@ for x in range(localnet_config["appgateservers"]["count"]):
             "--set=config.signing_key=app" + str(actor_number),
             "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
             "--set=development.delve.enabled=" + str(localnet_config["appgateservers"]["delve"]["enabled"]),
+            "--set=logLevel=" + str(localnet_config["appgateservers"]["logs"]["level"]),
+            "--set=image.repository=poktrolld",
         ],
         image_deps=["poktrolld"],
         image_keys=[("image.repository", "image.tag")],
@@ -325,6 +343,8 @@ for x in range(localnet_config["gateways"]["count"]):
             "--set=config.signing_key=gateway" + str(actor_number),
             "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
             "--set=development.delve.enabled=" + str(localnet_config["gateways"]["delve"]["enabled"]),
+            "--set=logLevel=" + str(localnet_config["gateways"]["logs"]["level"]),
+            "--set=image.repository=poktrolld",
         ],
         image_deps=["poktrolld"],
         image_keys=[("image.repository", "image.tag")],

app/app.go

@@ -302,6 +302,11 @@ func New(
 		return nil, err
 	}
 
+	// Set up poktroll telemetry using `app.toml` configuration options (in addition to cosmos-sdk telemetry config).
+	if err := telemetry.New(appOpts); err != nil {
+		return nil, err
+	}
+
 	return app, nil
 }
 

cmd/poktrolld/cmd/config.go

@@ -9,10 +9,26 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 
 	"github.com/pokt-network/poktroll/app"
+	"github.com/pokt-network/poktroll/telemetry"
 )
 
 var once sync.Once
 
+// PoktrollAdditionalConfig represents a poktroll-specific part of `app.toml` file.
+// See the `customAppConfigTemplate()` for additional information about each setting.
+type PoktrollAdditionalConfig struct {
+	Telemetry telemetry.PoktrollTelemetryConfig `mapstructure:"telemetry"`
+}
+
+// poktrollAdditionalConfigDefaults sets default values to render in `app.toml`.
+func poktrollAdditionalConfigDefaults() PoktrollAdditionalConfig {
+	return PoktrollAdditionalConfig{
+		Telemetry: telemetry.PoktrollTelemetryConfig{
+			CardinalityLevel: "medium",
+		},
+	}
+}
+
 func InitSDKConfig() {
 	once.Do(func() {
 		checkOrInitSDKConfig()
@@ -90,6 +106,7 @@ func initAppConfig() (string, interface{}) {
 	// The following code snippet is just for reference.
 	type CustomAppConfig struct {
 		serverconfig.Config `mapstructure:",squash"`
+		Poktroll            PoktrollAdditionalConfig `mapstructure:"poktroll"`
 	}
 
 	// Optionally allow the chain developer to overwrite the SDK's default
@@ -113,27 +130,46 @@ func initAppConfig() (string, interface{}) {
 	srvCfg.MinGasPrices = "0.000000001upokt" // Also adjust ignite's `config.yml`.
 	srvCfg.Mempool.MaxTxs = 10000
 	srvCfg.Telemetry.Enabled = true
-	srvCfg.Telemetry.PrometheusRetentionTime = 60 // in seconds. This turns on Prometheus support.
+	// Positive value turns on prometheus support. Prometheus metrics are removed from the exporter when retention time is reached.
+	srvCfg.Telemetry.PrometheusRetentionTime = 60 * 60 * 24 // in seconds.
 	srvCfg.Telemetry.MetricsSink = "mem"
 	srvCfg.Pruning = "nothing" // archiving node by default
 	srvCfg.API.Enable = true
 	srvCfg.GRPC.Enable = true
 	srvCfg.GRPCWeb.Enable = true
 
 	customAppConfig := CustomAppConfig{
-		Config: *srvCfg,
+		Config:   *srvCfg,
+		Poktroll: poktrollAdditionalConfigDefaults(),
 	}
 
-	customAppTemplate := serverconfig.DefaultConfigTemplate
-	// Edit the default template file
-	//
-	// customAppTemplate := serverconfig.DefaultConfigTemplate + `
-	// [wasm]
-	// # This is the maximum sdk gas (wasm and storage) that we allow for any x/wasm "smart" queries
-	// query_gas_limit = 300000
-	// # This is the number of wasm vm instances we keep cached in memory for speed-up
-	// # Warning: this is currently unstable and may lead to crashes, best to keep for 0 unless testing locally
-	// lru_size = 0`
-
-	return customAppTemplate, customAppConfig
+	return customAppConfigTemplate(), customAppConfig
+}
+
+// customAppConfigTemplate extends the default configuration `app.toml` file with our own configs. They are going to be
+// used on validators and full-nodes, and they render using default values from `poktrollAdditionalConfigDefaults()`.
+func customAppConfigTemplate() string {
+	return serverconfig.DefaultConfigTemplate + `
+		###############################################################################
+		###                               Poktroll                                  ###
+		###############################################################################
+
+		# Poktroll-specific configuration for Full Nodes and Validators.
+		[poktroll]
+
+		# Telemetry configuration in addition to the [telemetry] settings.
+		[poktroll.telemetry]
+
+		# Cardinality level for telemetry metrics collection
+		# This controls the level of detail (number of unique labels) in metrics.
+		# Options:
+		#   - "low":    Collects basic metrics with low cardinality. 
+		#              Suitable for production environments with tight performance constraints.
+		#   - "medium": Collects a moderate number of labels, balancing detail and performance.
+		#              Suitable for moderate workloads or staging environments.
+		#   - "high":   WARNING: WILL CAUSE STRESS TO YOUR MONITORING ENVIRONMENT! Collects detailed metrics with high 
+		#              cardinality, including labels with many unique values (e.g., application_id, session_id).
+		#              Recommended for debugging or testing environments.
+		cardinality-level = "{{ .Poktroll.Telemetry.CardinalityLevel }}"
+		`
 }

config.yml

@@ -91,7 +91,10 @@ validators:
       # minimum-gas-prices: 0.000000001upokt
       telemetry:
         enabled: true
-        prometheus-retention-time: "600" # seconds
+      poktroll:
+        telemetry:
+          # "high" produces a lot of timeseries. Only suitable for small networks such as LocalNet.
+          cardinality-level: high
     config:
       moniker: "validator1"
       consensus:
@@ -139,13 +142,13 @@ genesis:
             denom: upokt
     bank:
       supply:
-        - amount: "1003000204"
+        - amount: "1102000204"
           denom: upokt
       balances:
         # Application module
         - address: pokt1rl3gjgzexmplmds3tq3r3yk84zlwdl6djzgsvm
           coins:
-            - amount: "1000068" # Equals to the total of all app stakes below
+            - amount: "100000068" # Equals to the total of all app stakes below
               denom: upokt
         # Supplier module
         - address: pokt1j40dzzmn6cn9kxku7a5tjnud6hv37vesr5ccaa
@@ -171,9 +174,8 @@ genesis:
           denom: upokt
       applicationList:
         - address: pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4
-          delegatee_gateway_addresses: [
-            pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4
-          ]
+          delegatee_gateway_addresses:
+            [pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4]
           service_configs:
             - service_id: anvil
           stake:
@@ -182,9 +184,8 @@ genesis:
             amount: "100000068" # ~100 POKT
             denom: upokt
         - address: pokt184zvylazwu4queyzpl0gyz9yf5yxm2kdhh9hpm
-          delegatee_gateway_addresses: [
-            pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4
-          ]
+          delegatee_gateway_addresses:
+            [pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4]
           service_configs:
             - service_id: rest
           stake:

go.mod

@@ -82,6 +82,7 @@ require (
 require (
 	cosmossdk.io/x/tx v0.13.4
 	github.com/jhump/protoreflect v1.16.0
+	github.com/mitchellh/mapstructure v1.5.0
 )
 
 require (
@@ -224,7 +225,6 @@ require (
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect