[MM-57066][MM-57329] Added metrics for all notification stopping poin…

…ts, consolidated categories between metrics and logging (mattermost#26799)

* [MM-57066] Add metric counters for notification events

* Some small changes

* Account for Metrics() sometimes being nil

* Fix test (again)

* Fix more tests

* A few changes from testing - added success counter

* Missed a mock

* Lint

* Add feature flag for notification monitoring

server/channels/api4/system.go

@@ -641,13 +641,16 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	c.App.CountNotificationAck(model.NotificationTypePush)
+
 	err := c.App.SendAckToPushProxy(&ack)
 	if ack.IsIdLoaded {
 		if err != nil {
+			c.App.CountNotificationReason(model.NotificationStatusError, model.NotificationTypePush, model.NotificationReasonPushProxySendError)
 			c.App.NotificationsLog().Error("Notification ack not sent to push proxy",
-				mlog.String("type", model.TypePush),
-				mlog.String("status", model.StatusServerError),
-				mlog.String("reason", model.ReasonServerError),
+				mlog.String("type", model.NotificationTypePush),
+				mlog.String("status", model.NotificationStatusError),
+				mlog.String("reason", model.NotificationReasonPushProxySendError),
 				mlog.String("ack_id", ack.Id),
 				mlog.String("push_type", ack.NotificationType),
 				mlog.String("post_id", ack.PostId),
@@ -656,6 +659,8 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) {
 				mlog.Int("received_at", ack.ClientReceivedAt),
 				mlog.Err(err),
 			)
+		} else {
+			c.App.CountNotificationReason(model.NotificationStatusSuccess, model.NotificationTypePush, model.NotificationReason(""))
 		}
 		// Return post data only when PostId is passed.
 		if ack.PostId != "" && ack.NotificationType == model.PushTypeMessage {
@@ -687,6 +692,7 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	c.App.CountNotificationReason(model.NotificationStatusSuccess, model.NotificationTypePush, model.NotificationReason(""))
 	ReturnStatusOK(w)
 }
 

server/channels/app/expirynotify.go

@@ -24,10 +24,11 @@ func (a *App) NotifySessionsExpired() error {
 	// Get all mobile sessions that expired within the last hour.
 	sessions, err := a.ch.srv.Store().Session().GetSessionsExpired(OneHourMillis, true, true)
 	if err != nil {
+		a.CountNotificationReason(model.NotificationStatusError, model.NotificationTypePush, model.NotificationReasonFetchError)
 		a.NotificationsLog().Error("Cannot get sessions expired",
-			mlog.String("type", model.TypePush),
-			mlog.String("status", model.StatusServerError),
-			mlog.String("reason", model.ReasonFetchError),
+			mlog.String("type", model.NotificationTypePush),
+			mlog.String("status", model.NotificationStatusError),
+			mlog.String("reason", model.NotificationReasonFetchError),
 			mlog.Err(err),
 		)
 		return model.NewAppError("NotifySessionsExpired", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
@@ -46,10 +47,11 @@ func (a *App) NotifySessionsExpired() error {
 
 		errPush := a.sendToPushProxy(tmpMessage, session)
 		if errPush != nil {
+			a.CountNotificationReason(model.NotificationStatusError, model.NotificationTypePush, model.NotificationReasonPushProxySendError)
 			a.NotificationsLog().Error("Failed to send to push proxy",
-				mlog.String("type", model.TypePush),
-				mlog.String("status", model.StatusNotSent),
-				mlog.String("reason", model.ReasonPushProxyError),
+				mlog.String("type", model.NotificationTypePush),
+				mlog.String("status", model.NotificationStatusNotSent),
+				mlog.String("reason", model.NotificationReasonPushProxySendError),
 				mlog.String("ack_id", tmpMessage.AckId),
 				mlog.String("push_type", tmpMessage.Type),
 				mlog.String("user_id", session.UserId),
@@ -60,7 +62,7 @@ func (a *App) NotifySessionsExpired() error {
 		}
 
 		a.NotificationsLog().Trace("Notification sent to push proxy",
-			mlog.String("type", model.TypePush),
+			mlog.String("type", model.NotificationTypePush),
 			mlog.String("ack_id", tmpMessage.AckId),
 			mlog.String("push_type", tmpMessage.Type),
 			mlog.String("user_id", session.UserId),