diff --git a/README.md b/README.md index 1c8fcf4..e063777 100644 --- a/README.md +++ b/README.md @@ -97,28 +97,29 @@ GLOBAL OPTIONS: All metrics are by default prefixed by `cosmos_validator_watcher` but this can be changed through options. -Metrics (without prefix) | Description ---------------------------|------------------------------------------------------------------------- -`active_set` | Number of validators in the active set -`block_height` | Latest known block height (all nodes mixed up) -`commission` | Earned validator commission -`is_bonded` | Set to 1 if the validator is bonded -`is_jailed` | Set to 1 if the validator is jailed -`missed_blocks` | Number of missed blocks per validator (for a bonded validator) -`node_block_height` | Latest fetched block height for each node -`node_synced` | Set to 1 is the node is synced (ie. not catching-up) -`proposal_end_time` | Timestamp of the voting end time of a proposal -`proposed_blocks` | Number of proposed blocks per validator (for a bonded validator) -`rank` | Rank of the validator -`seat_price` | Min seat price to be in the active set (ie. bonded tokens of the latest validator) -`skipped_blocks` | Number of blocks skipped (ie. not tracked) since start -`solo_missed_blocks` | Number of missed blocks per validator, unless the block is missed by many other validators -`tokens` | Number of staked tokens per validator -`tracked_blocks` | Number of blocks tracked since start -`transactions` | Number of transactions since start -`validated_blocks` | Number of validated blocks per validator (for a bonded validator) -`vote` | Set to 1 if the validator has voted on a proposal -`upgrade_plan` | Block height of the upcoming upgrade (hard fork) +Metrics (without prefix) | Description +---------------------------|------------------------------------------------------------------------- +`active_set` | Number of validators in the active set +`block_height` | Latest known block height (all nodes mixed up) +`commission` | Earned validator commission +`is_bonded` | Set to 1 if the validator is bonded +`is_jailed` | Set to 1 if the validator is jailed +`missed_blocks` | Number of missed blocks per validator (for a bonded validator) +`consecutive_missed_blocks`| Number of consecutive missed blocks per validator (for a bonded validator) +`node_block_height` | Latest fetched block height for each node +`node_synced` | Set to 1 is the node is synced (ie. not catching-up) +`proposal_end_time` | Timestamp of the voting end time of a proposal +`proposed_blocks` | Number of proposed blocks per validator (for a bonded validator) +`rank` | Rank of the validator +`seat_price` | Min seat price to be in the active set (ie. bonded tokens of the latest validator) +`skipped_blocks` | Number of blocks skipped (ie. not tracked) since start +`solo_missed_blocks` | Number of missed blocks per validator, unless the block is missed by many other validators +`tokens` | Number of staked tokens per validator +`tracked_blocks` | Number of blocks tracked since start +`transactions` | Number of transactions since start +`validated_blocks` | Number of validated blocks per validator (for a bonded validator) +`vote` | Set to 1 if the validator has voted on a proposal +`upgrade_plan` | Block height of the upcoming upgrade (hard fork) ## ❓FAQ diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index bed1aec..ee0b3b3 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -19,16 +19,17 @@ type Metrics struct { UpgradePlan *prometheus.GaugeVec // Validator metrics - Rank *prometheus.GaugeVec - ProposedBlocks *prometheus.CounterVec - ValidatedBlocks *prometheus.CounterVec - MissedBlocks *prometheus.CounterVec - SoloMissedBlocks *prometheus.CounterVec - Tokens *prometheus.GaugeVec - IsBonded *prometheus.GaugeVec - IsJailed *prometheus.GaugeVec - Commission *prometheus.GaugeVec - Vote *prometheus.GaugeVec + Rank *prometheus.GaugeVec + ProposedBlocks *prometheus.CounterVec + ValidatedBlocks *prometheus.CounterVec + MissedBlocks *prometheus.CounterVec + SoloMissedBlocks *prometheus.CounterVec + ConsecutiveMissedBlocks *prometheus.GaugeVec + Tokens *prometheus.GaugeVec + IsBonded *prometheus.GaugeVec + IsJailed *prometheus.GaugeVec + Commission *prometheus.GaugeVec + Vote *prometheus.GaugeVec // Node metrics NodeBlockHeight *prometheus.GaugeVec @@ -102,6 +103,14 @@ func New(namespace string) *Metrics { }, []string{"chain_id", "address", "name"}, ), + ConsecutiveMissedBlocks: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "consecutive_missed_blocks", + Help: "Number of consecutive missed blocks per validator (for a bonded validator)", + }, + []string{"chain_id", "address", "name"}, + ), TrackedBlocks: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, @@ -215,6 +224,7 @@ func (m *Metrics) Register() { m.Registry.MustRegister(m.ValidatedBlocks) m.Registry.MustRegister(m.MissedBlocks) m.Registry.MustRegister(m.SoloMissedBlocks) + m.Registry.MustRegister(m.ConsecutiveMissedBlocks) m.Registry.MustRegister(m.TrackedBlocks) m.Registry.MustRegister(m.Transactions) m.Registry.MustRegister(m.SkippedBlocks) diff --git a/pkg/watcher/block.go b/pkg/watcher/block.go index 14ce721..69a52c9 100644 --- a/pkg/watcher/block.go +++ b/pkg/watcher/block.go @@ -172,6 +172,7 @@ func (w *BlockWatcher) handleBlockInfo(block *BlockInfo) { w.metrics.ValidatedBlocks.WithLabelValues(chainId, val.Address, val.Name) w.metrics.MissedBlocks.WithLabelValues(chainId, val.Address, val.Name) w.metrics.SoloMissedBlocks.WithLabelValues(chainId, val.Address, val.Name) + w.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainId, val.Address, val.Name) } w.metrics.SkippedBlocks.WithLabelValues(chainId) @@ -194,12 +195,15 @@ func (w *BlockWatcher) handleBlockInfo(block *BlockInfo) { icon = "👑" w.metrics.ProposedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() + w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0) } else if res.Signed { icon = "✅" w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() + w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0) } else if res.Bonded { icon = "❌" w.metrics.MissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() + w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc() // Check if solo missed block if block.SignedRatio().GreaterThan(decimal.NewFromFloat(0.66)) { diff --git a/pkg/watcher/block_test.go b/pkg/watcher/block_test.go index 629b4e2..521cb29 100644 --- a/pkg/watcher/block_test.go +++ b/pkg/watcher/block_test.go @@ -137,9 +137,11 @@ func TestBlockWatcher(t *testing.T) { assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ValidatedBlocks)) assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.MissedBlocks)) assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.SoloMissedBlocks)) + assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ConsecutiveMissedBlocks)) assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.ProposedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) assert.Equal(t, float64(3), testutil.ToFloat64(blockWatcher.metrics.ValidatedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.MissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.SoloMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) + assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName))) }) }