feat: consecutive missed blocks

README.md

@@ -97,28 +97,29 @@ GLOBAL OPTIONS:
 
 All metrics are by default prefixed by `cosmos_validator_watcher` but this can be changed through options.
 
-Metrics (without prefix)  | Description
---------------------------|-------------------------------------------------------------------------
-`active_set`              | Number of validators in the active set
-`block_height`            | Latest known block height (all nodes mixed up)
-`commission`              | Earned validator commission
-`is_bonded`               | Set to 1 if the validator is bonded
-`is_jailed`               | Set to 1 if the validator is jailed
-`missed_blocks`           | Number of missed blocks per validator (for a bonded validator)
-`node_block_height`       | Latest fetched block height for each node
-`node_synced`             | Set to 1 is the node is synced (ie. not catching-up)
-`proposal_end_time`       | Timestamp of the voting end time of a proposal
-`proposed_blocks`         | Number of proposed blocks per validator (for a bonded validator)
-`rank`                    | Rank of the validator
-`seat_price`              | Min seat price to be in the active set (ie. bonded tokens of the latest validator)
-`skipped_blocks`          | Number of blocks skipped (ie. not tracked) since start
-`solo_missed_blocks`      | Number of missed blocks per validator, unless the block is missed by many other validators
-`tokens`                  | Number of staked tokens per validator
-`tracked_blocks`          | Number of blocks tracked since start
-`transactions`            | Number of transactions since start
-`validated_blocks`        | Number of validated blocks per validator (for a bonded validator)
-`vote`                    | Set to 1 if the validator has voted on a proposal
-`upgrade_plan`            | Block height of the upcoming upgrade (hard fork)
+Metrics (without prefix)   | Description
+---------------------------|-------------------------------------------------------------------------
+`active_set`               | Number of validators in the active set
+`block_height`             | Latest known block height (all nodes mixed up)
+`commission`               | Earned validator commission
+`is_bonded`                | Set to 1 if the validator is bonded
+`is_jailed`                | Set to 1 if the validator is jailed
+`missed_blocks`            | Number of missed blocks per validator (for a bonded validator)
+`consecutive_missed_blocks`| Number of consecutive missed blocks per validator (for a bonded validator)
+`node_block_height`        | Latest fetched block height for each node
+`node_synced`              | Set to 1 is the node is synced (ie. not catching-up)
+`proposal_end_time`        | Timestamp of the voting end time of a proposal
+`proposed_blocks`          | Number of proposed blocks per validator (for a bonded validator)
+`rank`                     | Rank of the validator
+`seat_price`               | Min seat price to be in the active set (ie. bonded tokens of the latest validator)
+`skipped_blocks`           | Number of blocks skipped (ie. not tracked) since start
+`solo_missed_blocks`       | Number of missed blocks per validator, unless the block is missed by many other validators
+`tokens`                   | Number of staked tokens per validator
+`tracked_blocks`           | Number of blocks tracked since start
+`transactions`             | Number of transactions since start
+`validated_blocks`         | Number of validated blocks per validator (for a bonded validator)
+`vote`                     | Set to 1 if the validator has voted on a proposal
+`upgrade_plan`             | Block height of the upcoming upgrade (hard fork)
 
 
 ## ❓FAQ

pkg/metrics/metrics.go

@@ -19,16 +19,17 @@ type Metrics struct {
 	UpgradePlan     *prometheus.GaugeVec
 
 	// Validator metrics
-	Rank             *prometheus.GaugeVec
-	ProposedBlocks   *prometheus.CounterVec
-	ValidatedBlocks  *prometheus.CounterVec
-	MissedBlocks     *prometheus.CounterVec
-	SoloMissedBlocks *prometheus.CounterVec
-	Tokens           *prometheus.GaugeVec
-	IsBonded         *prometheus.GaugeVec
-	IsJailed         *prometheus.GaugeVec
-	Commission       *prometheus.GaugeVec
-	Vote             *prometheus.GaugeVec
+	Rank             		*prometheus.GaugeVec
+	ProposedBlocks   		*prometheus.CounterVec
+	ValidatedBlocks  		*prometheus.CounterVec
+	MissedBlocks     		*prometheus.CounterVec
+	SoloMissedBlocks 		*prometheus.CounterVec
+	ConsecutiveMissedBlocks *prometheus.GaugeVec
+	Tokens           		*prometheus.GaugeVec
+	IsBonded         		*prometheus.GaugeVec
+	IsJailed         		*prometheus.GaugeVec
+	Commission       		*prometheus.GaugeVec
+	Vote             		*prometheus.GaugeVec
 
 	// Node metrics
 	NodeBlockHeight *prometheus.GaugeVec
@@ -102,6 +103,14 @@ func New(namespace string) *Metrics {
 			},
 			[]string{"chain_id", "address", "name"},
 		),
+		ConsecutiveMissedBlocks: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Name:      "consecutive_missed_blocks",
+				Help:      "Number of consecutive missed blocks per validator (for a bonded validator)",
+			},
+			[]string{"chain_id", "address", "name"},
+		),
 		TrackedBlocks: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Namespace: namespace,
@@ -215,6 +224,7 @@ func (m *Metrics) Register() {
 	m.Registry.MustRegister(m.ValidatedBlocks)
 	m.Registry.MustRegister(m.MissedBlocks)
 	m.Registry.MustRegister(m.SoloMissedBlocks)
+	m.Registry.MustRegister(m.ConsecutiveMissedBlocks)
 	m.Registry.MustRegister(m.TrackedBlocks)
 	m.Registry.MustRegister(m.Transactions)
 	m.Registry.MustRegister(m.SkippedBlocks)

pkg/watcher/block.go

@@ -172,6 +172,7 @@ func (w *BlockWatcher) handleBlockInfo(block *BlockInfo) {
 		w.metrics.ValidatedBlocks.WithLabelValues(chainId, val.Address, val.Name)
 		w.metrics.MissedBlocks.WithLabelValues(chainId, val.Address, val.Name)
 		w.metrics.SoloMissedBlocks.WithLabelValues(chainId, val.Address, val.Name)
+		w.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainId, val.Address, val.Name)
 	}
 	w.metrics.SkippedBlocks.WithLabelValues(chainId)
 
@@ -194,12 +195,15 @@ func (w *BlockWatcher) handleBlockInfo(block *BlockInfo) {
 			icon = "👑"
 			w.metrics.ProposedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
 			w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
+			w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0)
 		} else if res.Signed {
 			icon = "✅"
 			w.metrics.ValidatedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
+			w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Set(0)
 		} else if res.Bonded {
 			icon = "❌"
 			w.metrics.MissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
+			w.metrics.ConsecutiveMissedBlocks.WithLabelValues(block.ChainID, res.Address, res.Label).Inc()
 
 			// Check if solo missed block
 			if block.SignedRatio().GreaterThan(decimal.NewFromFloat(0.66)) {

pkg/watcher/block_test.go

@@ -137,9 +137,11 @@ func TestBlockWatcher(t *testing.T) {
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.ValidatedBlocks))
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.MissedBlocks))
 		assert.Equal(t, 1, testutil.CollectAndCount(blockWatcher.metrics.SoloMissedBlocks))
+		assert.Equal(t, 0, testutil.CollectAndCount(blockWatcher.metrics.ConsecutiveMissedBlocks))
 		assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.ProposedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 		assert.Equal(t, float64(3), testutil.ToFloat64(blockWatcher.metrics.ValidatedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 		assert.Equal(t, float64(1), testutil.ToFloat64(blockWatcher.metrics.MissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 		assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.SoloMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
+		assert.Equal(t, float64(0), testutil.ToFloat64(blockWatcher.metrics.ConsecutiveMissedBlocks.WithLabelValues(chainID, kilnAddress, kilnName)))
 	})
 }