From 6167465c2e87aaab65c10403244076c97ff8622b Mon Sep 17 00:00:00 2001
From: Ryan Leung <rleungx@gmail.com>
Date: Thu, 21 Sep 2023 12:09:42 +0800
Subject: [PATCH] fix metrics

Signed-off-by: Ryan Leung <rleungx@gmail.com>
---
 metrics/grafana/pd.json              | 24 ++++++++++++++++--------
 pkg/mcs/scheduling/server/cluster.go |  6 ------
 pkg/schedule/coordinator.go          |  1 +
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json
index 9d1ed90e078..bc6f0715d86 100644
--- a/metrics/grafana/pd.json
+++ b/metrics/grafana/pd.json
@@ -1636,7 +1636,7 @@
           "tableColumn": "idalloc",
           "targets": [
             {
-              "expr": "pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"idalloc\"}",
+              "expr": "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"idalloc\"})",
               "format": "time_series",
               "hide": false,
               "instant": true,
@@ -2175,7 +2175,7 @@
           "tableColumn": "tso",
           "targets": [
             {
-              "expr": "pd_cluster_tso{type=\"tso\", dc=\"global\"}",
+              "expr": "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})",
               "format": "time_series",
               "instant": true,
               "interval": "",
@@ -2479,7 +2479,7 @@
           "tableColumn": "tso",
           "targets": [
             {
-              "expr": "pd_cluster_tso{type=\"tso\", dc=\"global\"}",
+              "expr": "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})",
               "format": "time_series",
               "instant": true,
               "interval": "",
@@ -7785,7 +7785,7 @@
           "tableColumn": "",
           "targets": [
             {
-              "expr": "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}",
+              "expr": "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} != 0",
               "format": "time_series",
               "intervalFactor": 1,
               "refId": "A"
@@ -9076,7 +9076,9 @@
             "show": true,
             "sideWidth": 300,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
@@ -9171,7 +9173,9 @@
             "show": true,
             "sideWidth": 300,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
@@ -9497,7 +9501,9 @@
             "rightSide": true,
             "show": true,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
@@ -9694,7 +9700,9 @@
             "rightSide": true,
             "show": true,
             "total": false,
-            "values": true
+            "values": true,
+            "hideEmpty": true,
+            "hideZero": true
           },
           "lines": true,
           "linewidth": 1,
diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go
index 7a110eb7606..a6a3de11d53 100644
--- a/pkg/mcs/scheduling/server/cluster.go
+++ b/pkg/mcs/scheduling/server/cluster.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/pingcap/errors"
-	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/kvproto/pkg/schedulingpb"
 	"github.com/pingcap/log"
@@ -431,11 +430,6 @@ func (c *Cluster) runMetricsCollectionJob() {
 	defer c.wg.Done()
 
 	ticker := time.NewTicker(10 * time.Second)
-	failpoint.Inject("highFrequencyClusterJobs", func() {
-		ticker.Stop()
-		ticker = time.NewTicker(time.Microsecond)
-	})
-
 	defer ticker.Stop()
 
 	for {
diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go
index bcbef0a1a08..5b4ab8cdb4f 100644
--- a/pkg/schedule/coordinator.go
+++ b/pkg/schedule/coordinator.go
@@ -156,6 +156,7 @@ func (c *Coordinator) PatrolRegions() {
 			// Note: we reset the ticker here to support updating configuration dynamically.
 			ticker.Reset(c.cluster.GetCheckerConfig().GetPatrolRegionInterval())
 		case <-c.ctx.Done():
+			patrolCheckRegionsGauge.Set(0)
 			log.Info("patrol regions has been stopped")
 			return
 		}