add metrics to count blocks and events

metrics/metrics.go

@@ -12,8 +12,10 @@ import (
 
 const (
 	// Monitor
-	MetricGnfdSavedBlock = "gnfd_saved_block"
-	MetricGnfdSavedEvent = "gnfd_saved_event"
+	MetricGnfdSavedBlock      = "gnfd_saved_block"
+	MetricGnfdSavedBlockCount = "gnfd_saved_block_count"
+	MetricGnfdSavedEvent      = "gnfd_saved_event"
+	MetricGnfdSavedEventCount = "gnfd_saved_event_count"
 
 	// Verifier
 	MetricVerifiedChallenges       = "verified_challenges"
@@ -63,13 +65,27 @@ func NewMetricService(config *config.Config) *MetricService {
 	ms[MetricGnfdSavedBlock] = gnfdSavedBlockMetric
 	prometheus.MustRegister(gnfdSavedBlockMetric)
 
+	gnfdSavedBlockCountMetric := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: MetricGnfdSavedBlockCount,
+		Help: "Saved block count for Greenfield in database",
+	})
+	ms[MetricGnfdSavedBlockCount] = gnfdSavedBlockCountMetric
+	prometheus.MustRegister(gnfdSavedBlockCountMetric)
+
 	gnfdSavedEventMetric := prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: MetricGnfdSavedEvent,
 		Help: "Saved event challengeId in database",
 	})
 	ms[MetricGnfdSavedEvent] = gnfdSavedEventMetric
 	prometheus.MustRegister(gnfdSavedEventMetric)
 
+	gnfdSavedEventCountMetric := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: MetricGnfdSavedEventCount,
+		Help: "Saved gnfd event count in database",
+	})
+	ms[MetricGnfdSavedEventCount] = gnfdSavedEventCountMetric
+	prometheus.MustRegister(gnfdSavedEventCountMetric)
+
 	// Hash Verifier
 	verifiedChallengesMetric := prometheus.NewCounter(prometheus.CounterOpts{
 		Name: MetricVerifiedChallenges,
@@ -101,7 +117,7 @@ func NewMetricService(config *config.Config) *MetricService {
 
 	heartbeatEventsMetric := prometheus.NewCounter(prometheus.CounterOpts{
 		Name: MetricHeartbeatEvents,
-		Help: "Succeeded challenges in database",
+		Help: "Heartbeat challenges",
 	})
 	ms[MetricHeartbeatEvents] = heartbeatEventsMetric
 	prometheus.MustRegister(heartbeatEventsMetric)
@@ -228,10 +244,18 @@ func (m *MetricService) SetGnfdSavedBlock(height uint64) {
 	m.MetricsMap[MetricGnfdSavedBlock].(prometheus.Gauge).Set(float64(height))
 }
 
+func (m *MetricService) IncGnfdSavedBlockCount() {
+	m.MetricsMap[MetricGnfdSavedBlockCount].(prometheus.Counter).Inc()
+}
+
 func (m *MetricService) SetGnfdSavedEvent(challengeId uint64) {
 	m.MetricsMap[MetricGnfdSavedEvent].(prometheus.Gauge).Set(float64(challengeId))
 }
 
+func (m *MetricService) IncGnfdSavedEventCount() {
+	m.MetricsMap[MetricGnfdSavedEventCount].(prometheus.Counter).Inc()
+}
+
 // Hash Verifier
 func (m *MetricService) IncVerifiedChallenges() {
 	m.MetricsMap[MetricVerifiedChallenges].(prometheus.Counter).Inc()

monitor/monitor.go

@@ -159,8 +159,10 @@ func (m *Monitor) monitorChallengeEvents(block *tmtypes.Block, blockResults *cty
 	for _, event := range events {
 		logging.Logger.Debugf("monitor event saved for challengeId: %d %s", event.ChallengeId, time.Now().Format("15:04:05.000000"))
 		m.metricService.SetGnfdSavedEvent(event.ChallengeId)
+		m.metricService.IncGnfdSavedEventCount()
 	}
 	m.metricService.SetGnfdSavedBlock(b.Height)
+	m.metricService.IncGnfdSavedBlockCount()
 	if err != nil {
 		return err
 	}

submitter/tx_submitter.go

@@ -61,6 +61,7 @@ func (s *TxSubmitter) SubmitTransactionLoop() {
 		currentHeight := s.executor.GetCachedBlockHeight()
 		events, err := s.FetchEventsForSubmit(currentHeight)
 		if err != nil {
+			logging.Logger.Infof("submitter error metrics increased at fetch events")
 			s.metricService.IncSubmitterErr()
 			logging.Logger.Errorf("tx submitter failed to fetch events for submitting", err)
 			continue
@@ -77,7 +78,7 @@ func (s *TxSubmitter) SubmitTransactionLoop() {
 			}
 			err = s.submitForSingleEvent(event, attestPeriodEnd)
 			if err != nil {
-				logging.Logger.Errorf("tx submitter err", err)
+				logging.Logger.Errorf("tx submitter ran into an error while trying to attest, err=%+v", err.Error())
 				continue
 			}
 			time.Sleep(TxSubmitInterval)
@@ -113,6 +114,7 @@ func (s *TxSubmitter) submitForSingleEvent(event *model.Event, attestPeriodEnd u
 	// Calculate event hash and use it to fetch votes and validator bitset
 	aggregatedSignature, valBitSet, err := s.getSignatureAndBitSet(event)
 	if err != nil {
+		logging.Logger.Infof("submitter error metrics increased at calculating event hash")
 		s.metricService.IncSubmitterErr()
 		return err
 	}
@@ -160,6 +162,7 @@ func (s *TxSubmitter) submitTransactionLoop(event *model.Event, attestPeriodEnd
 		}
 
 		if submittedAttempts > common.MaxSubmitAttempts {
+			logging.Logger.Infof("submitter metrics increased at submit attempts")
 			s.metricService.IncSubmitterErr()
 			return fmt.Errorf("submitter exceeded max submit attempts for challengeId: %d", event.ChallengeId)
 		}