From d67a5436e16bf0ba7b43786d79ccb4791065a03f Mon Sep 17 00:00:00 2001 From: rahulguptajss Date: Tue, 9 Jul 2024 16:12:09 +0530 Subject: [PATCH 1/6] feat: enable ChangeLog plugin to monitor metric value change --- .../changelog/{change_log.go => changelog.go} | 71 +++++++++++++++++-- ...ange_log_helper.go => changelog_helper.go} | 0 .../{change_log_test.go => changelog_test.go} | 51 ++++++++++++- docs/plugins.md | 20 +++++- .../dashboards/cmode/changelogMonitor.json | 6 +- pkg/matrix/matrix.go | 23 ++++-- pkg/matrix/metric.go | 4 ++ 7 files changed, 159 insertions(+), 16 deletions(-) rename cmd/poller/plugin/changelog/{change_log.go => changelog.go} (78%) rename cmd/poller/plugin/changelog/{change_log_helper.go => changelog_helper.go} (100%) rename cmd/poller/plugin/changelog/{change_log_test.go => changelog_test.go} (79%) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/changelog.go similarity index 78% rename from cmd/poller/plugin/changelog/change_log.go rename to cmd/poller/plugin/changelog/changelog.go index 63f0f7729..29f64ef33 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/changelog.go @@ -6,6 +6,7 @@ import ( "github.com/netapp/harvest/v2/pkg/set" "github.com/netapp/harvest/v2/pkg/tree/yaml" "github.com/netapp/harvest/v2/pkg/util" + maps2 "golang.org/x/exp/maps" "maps" "strconv" "time" @@ -110,7 +111,6 @@ func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) { // Run processes the data and generates ChangeLog instances func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { - data := dataMap[c.Object] changeLogMap, err := c.initMatrix() if err != nil { @@ -128,7 +128,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u } changeMat := changeLogMap[c.matrixName] - changeMat.SetGlobalLabels(data.GetGlobalLabels()) object := data.Object if c.changeLogConfig.Object == "" { @@ -179,7 +178,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u c.updateChangeLogLabels(object, instance, change) c.createChangeLogInstance(changeMat, change) } else { - // check for any modification + // check for any label modification cur, old := instance.CompareDiffs(prevInstance, c.changeLogConfig.Track) if len(cur) > 0 { for currentLabel, nVal := range cur { @@ -201,8 +200,33 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u c.createChangeLogInstance(changeMat, change) } } + + // check for any metric modification + metricChanges := c.CompareMetrics(data) + if changes, ok := metricChanges[key]; ok { + for metricName := range changes { + change := &Change{ + key: uuid + "_" + object + "_" + metricName, + object: object, + op: update, + labels: make(map[string]string), + track: metricName, + // Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable. + //oldValue: strconv.FormatFloat(values[0], 'f', -1, 64), + //newValue: strconv.FormatFloat(values[1], 'f', -1, 64), + time: currentTime, + } + c.updateChangeLogLabels(object, instance, change) + // add changed track and its old, new value + change.labels[track] = metricName + change.labels[oldValue] = change.oldValue + change.labels[newValue] = change.newValue + c.createChangeLogInstance(changeMat, change) + } + } } } + // create deleted instances change_log for key := range oldInstances.Iter() { prevInstance := prevMat.GetInstance(key) @@ -243,12 +267,49 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u return matricesArray, nil, nil } +// CompareMetrics compares the metrics of the current and previous instances +func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string][2]float64 { + metricChanges := make(map[string]map[string][2]float64) + prevMat := c.previousData + + met := maps2.Keys(c.previousData.GetMetrics()) + + for _, metricKey := range met { + prevMetric := prevMat.GetMetric(metricKey) + curMetric := curMat.GetMetric(metricKey) + for key, currInstance := range curMat.GetInstances() { + prevInstance := prevMat.GetInstance(key) + prevIndex := prevInstance.GetIndex() + currIndex := currInstance.GetIndex() + curVal := curMetric.GetValues()[currIndex] + prevVal := prevMetric.GetValues()[prevIndex] + if prevInstance != nil { + if curVal != prevVal { + if _, ok := metricChanges[key]; !ok { + metricChanges[key] = make(map[string][2]float64) + } + metricChanges[key][metricKey] = [2]float64{prevVal, curVal} + } + } + } + } + return metricChanges +} + // copyPreviousData creates a copy of the previous data for comparison func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) { labels := c.changeLogConfig.PublishLabels - labels = append(labels, c.changeLogConfig.Track...) + var met []string + for _, t := range c.changeLogConfig.Track { + _, ok := cur.GetMetrics()[t] + if ok { + met = append(met, t) + } else { + labels = append(labels, t) + } + } labels = append(labels, "uuid") - c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels}) + c.previousData = cur.Clone(matrix.With{Data: true, Metrics: true, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met}) } // createChangeLogInstance creates a new ChangeLog instance with the given change data diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/changelog_helper.go similarity index 100% rename from cmd/poller/plugin/changelog/change_log_helper.go rename to cmd/poller/plugin/changelog/changelog_helper.go diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/changelog_test.go similarity index 79% rename from cmd/poller/plugin/changelog/change_log_test.go rename to cmd/poller/plugin/changelog/changelog_test.go index 62fcbf145..b60a9333f 100644 --- a/cmd/poller/plugin/changelog/change_log_test.go +++ b/cmd/poller/plugin/changelog/changelog_test.go @@ -39,7 +39,7 @@ func createChangeLog(params, parentParams *node.Node) *ChangeLog { func newChangeLogUnsupportedTrack(object string) *ChangeLog { params := node.NewS("ChangeLog") - t := params.NewChildS("Track", "") + t := params.NewChildS("track", "") t.NewChildS("", "abcd") parentParams := node.NewS("parent") parentParams.NewChildS("object", object) @@ -47,6 +47,20 @@ func newChangeLogUnsupportedTrack(object string) *ChangeLog { return createChangeLog(params, parentParams) } +func newChangeLogMetricTrack(object string) *ChangeLog { + params := node.NewS("ChangeLog") + t := params.NewChildS("track", "") + t.NewChildS("", "svm") + t.NewChildS("", "type") + t.NewChildS("", "cpu_usage") + t1 := params.NewChildS("publish_labels", "") + t1.NewChildS("", "svm") + parentParams := node.NewS("parent") + parentParams.NewChildS("object", object) + + return createChangeLog(params, parentParams) +} + func checkChangeLogInstances(t *testing.T, o []*matrix.Matrix, expectedInstances, expectedLabels int, expectedOpLabel, opLabel string) { if len(o) == 1 { cl := o[0] @@ -94,6 +108,41 @@ func TestChangeLogModified(t *testing.T) { checkChangeLogInstances(t, o, 2, 9, update, opLabel) } +func TestChangeLogModifiedWithMetrics(t *testing.T) { + p := newChangeLogMetricTrack("svm") + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + // Add a metric to the instance + metric, _ := m.NewMetricFloat64("cpu_usage") + _ = metric.SetValueFloat64(instance, 10.0) + + _, _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("0") + instance1.SetLabel("uuid", "u1") + instance1.SetLabel("svm", "s2") + instance1.SetLabel("type", "t2") + + // Modify the metric value + metric1, _ := m1.NewMetricFloat64("cpu_usage") + _ = metric1.SetValueFloat64(instance1, 20.0) + + o, _, _ := p.Run(data1) + + checkChangeLogInstances(t, o, 3, 7, update, opLabel) +} + func TestChangeLogCreated(t *testing.T) { p := newChangeLog("svm", false) m := matrix.New("TestChangeLog", "svm", "svm") diff --git a/docs/plugins.md b/docs/plugins.md index 5b959bbb4..6a275b726 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -637,7 +637,7 @@ The ChangeLog plugin is a feature of Harvest, designed to detect and track chang Please note that the ChangeLog plugin requires the `uuid` label, which is unique, to be collected by the template. Without the `uuid` label, the plugin will not function. -The ChangeLog feature only detects changes when Harvest is up and running. It does not detect changes that occur when Harvest is down. Additionally, the plugin does not detect changes in metric values. +The ChangeLog feature only detects changes when Harvest is up and running. It does not detect changes that occur when Harvest is down. Additionally, the plugin does not detect changes in metric values by default, but it can be configured to do so. ## Enabling the Plugin @@ -735,7 +735,6 @@ change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter=" When an object is deleted, the ChangeLog plugin will publish a metric with the following labels: - | Label | Description | |--------------|-----------------------------------------------------------------------------| | object | name of the ONTAP object that was changed | @@ -748,6 +747,23 @@ Example of metric shape for object deletion: change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="2", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="delete", style="flexvol", svm="harvest", volume="harvest_demo"} 1698735708 ``` +### Metric Value Changes + +When a metric value changes, the ChangeLog plugin will publish a metric with the following labels. Note that old and new values are not tracked to avoid impacting cardinality in Prometheus: + +| Label | Description | +|--------------|-----------------------------------------------------------------------------| +| object | name of the ONTAP object that was changed | +| op | type of change that was made | +| track | property of the object which was modified | +| metric value | timestamp when Harvest captured the change. 1698735800 in the example below | + +Example of metric shape for metric value change: + +``` +change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="read_latency", svm="harvest", volume="harvest_demo"} 1698735800 +``` + ## Viewing the Metrics You can view the metrics published by the ChangeLog plugin in the `ChangeLog Monitor` dashboard in `Grafana`. This dashboard provides a visual representation of the changes tracked by the plugin for volume, svm, and node objects. \ No newline at end of file diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index d8523d156..1057fb049 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -304,7 +304,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made.", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the property after the change was made \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.", "fieldConfig": { "defaults": { "color": { @@ -718,7 +718,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made.", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the property after the change was made \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.", "fieldConfig": { "defaults": { "color": { @@ -1158,7 +1158,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made.", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the property after the change was made \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.", "fieldConfig": { "defaults": { "color": { diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go index 3836c98e0..492710678 100644 --- a/pkg/matrix/matrix.go +++ b/pkg/matrix/matrix.go @@ -38,6 +38,7 @@ type With struct { ExportInstances bool PartialInstances bool Labels []string + MetricsNames []string } func New(uuid, object string, identifier string) *Matrix { @@ -99,11 +100,23 @@ func (m *Matrix) Clone(with With) *Matrix { } if with.Metrics { - clone.metrics = make(map[string]*Metric, len(m.GetMetrics())) - for key, metric := range m.GetMetrics() { - c := metric.Clone(with.Data) - clone.metrics[key] = c - clone.displayMetrics[c.GetName()] = key + if len(with.MetricsNames) > 0 { + clone.metrics = make(map[string]*Metric, len(with.MetricsNames)) + for _, metricName := range with.MetricsNames { + metric, ok := m.GetMetrics()[metricName] + if ok { + c := metric.Clone(with.Data) + clone.metrics[metricName] = c + clone.displayMetrics[c.GetName()] = metricName + } + } + } else { + clone.metrics = make(map[string]*Metric, len(m.GetMetrics())) + for key, metric := range m.GetMetrics() { + c := metric.Clone(with.Data) + clone.metrics[key] = c + clone.displayMetrics[c.GetName()] = key + } } } else { clone.metrics = make(map[string]*Metric) diff --git a/pkg/matrix/metric.go b/pkg/matrix/metric.go index 60eacb643..815e44126 100644 --- a/pkg/matrix/metric.go +++ b/pkg/matrix/metric.go @@ -146,6 +146,10 @@ func (m *Metric) SetValueNAN(i *Instance) { m.record[i.index] = false } +func (m *Metric) GetValues() []float64 { + return m.values +} + // Storage resizing methods func (m *Metric) Reset(size int) { From 3b1f19741f35d134ec8e50611bd44f1465a691a8 Mon Sep 17 00:00:00 2001 From: rahulguptajss Date: Tue, 9 Jul 2024 20:26:49 +0530 Subject: [PATCH 2/6] feat: enable ChangeLog plugin to monitor metric value change --- cmd/poller/plugin/changelog/changelog.go | 12 ++++++++---- pkg/matrix/matrix.go | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/cmd/poller/plugin/changelog/changelog.go b/cmd/poller/plugin/changelog/changelog.go index 29f64ef33..4952ade28 100644 --- a/cmd/poller/plugin/changelog/changelog.go +++ b/cmd/poller/plugin/changelog/changelog.go @@ -301,15 +301,19 @@ func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) { labels := c.changeLogConfig.PublishLabels var met []string for _, t := range c.changeLogConfig.Track { - _, ok := cur.GetMetrics()[t] - if ok { - met = append(met, t) + mKey := cur.DisplayMetricKey(t) + if mKey != "" { + met = append(met, mKey) } else { labels = append(labels, t) } } labels = append(labels, "uuid") - c.previousData = cur.Clone(matrix.With{Data: true, Metrics: true, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met}) + withMetrics := false + if len(met) > 0 { + withMetrics = true + } + c.previousData = cur.Clone(matrix.With{Data: true, Metrics: withMetrics, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met}) } // createChangeLogInstance creates a new ChangeLog instance with the given change data diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go index 492710678..ab39dae7e 100644 --- a/pkg/matrix/matrix.go +++ b/pkg/matrix/matrix.go @@ -140,6 +140,13 @@ func (m *Matrix) DisplayMetric(name string) *Metric { return nil } +func (m *Matrix) DisplayMetricKey(name string) string { + if metricKey, has := m.displayMetrics[name]; has { + return metricKey + } + return "" +} + func (m *Matrix) GetMetric(key string) *Metric { if metric, has := m.metrics[key]; has { return metric From 08930256796cae8ca27e1925eb847bade76af765 Mon Sep 17 00:00:00 2001 From: rahulguptajss Date: Wed, 10 Jul 2024 13:45:42 +0530 Subject: [PATCH 3/6] feat: address review comments --- cmd/poller/plugin/changelog/changelog.go | 17 ++++------ docs/plugins.md | 40 +++++++++--------------- pkg/matrix/matrix.go | 5 +-- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/cmd/poller/plugin/changelog/changelog.go b/cmd/poller/plugin/changelog/changelog.go index 4952ade28..c77c404f4 100644 --- a/cmd/poller/plugin/changelog/changelog.go +++ b/cmd/poller/plugin/changelog/changelog.go @@ -150,6 +150,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u currentTime := time.Now().Unix() + metricChanges := c.CompareMetrics(data) // loop over current instances for key, instance := range data.GetInstances() { uuid := instance.GetLabel("uuid") @@ -202,7 +203,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u } // check for any metric modification - metricChanges := c.CompareMetrics(data) if changes, ok := metricChanges[key]; ok { for metricName := range changes { change := &Change{ @@ -212,8 +212,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u labels: make(map[string]string), track: metricName, // Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable. - //oldValue: strconv.FormatFloat(values[0], 'f', -1, 64), - //newValue: strconv.FormatFloat(values[1], 'f', -1, 64), time: currentTime, } c.updateChangeLogLabels(object, instance, change) @@ -288,7 +286,7 @@ func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string] if _, ok := metricChanges[key]; !ok { metricChanges[key] = make(map[string][2]float64) } - metricChanges[key][metricKey] = [2]float64{prevVal, curVal} + metricChanges[key][curMetric.GetName()] = [2]float64{prevVal, curVal} } } } @@ -302,17 +300,14 @@ func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) { var met []string for _, t := range c.changeLogConfig.Track { mKey := cur.DisplayMetricKey(t) - if mKey != "" { - met = append(met, mKey) - } else { + if mKey == "" { labels = append(labels, t) + } else { + met = append(met, mKey) } } labels = append(labels, "uuid") - withMetrics := false - if len(met) > 0 { - withMetrics = true - } + withMetrics := len(met) > 0 c.previousData = cur.Clone(matrix.With{Data: true, Metrics: withMetrics, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met}) } diff --git a/docs/plugins.md b/docs/plugins.md index 6a275b726..20d2ff930 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -675,7 +675,7 @@ By default, the plugin tracks changes in the following labels for svm, node, and Other objects are not tracked by default. -These default settings can be overwritten as needed in the relevant templates. For instance, if you want to track `junction_path` labels for Volume, you can overwrite this in the volume template. +These default settings can be overwritten as needed in the relevant templates. For instance, if you want to track `junction_path` label and `size_total` metric for Volume, you can overwrite this in the volume template. ```yaml plugins: @@ -690,6 +690,7 @@ plugins: - state - status - junction_path + - size_total ``` ## Change Types and Metrics @@ -718,19 +719,25 @@ When an existing object is modified, the ChangeLog plugin will publish a metric | Label | Description | |--------------|-----------------------------------------------------------------------------| -| object | name of the ONTAP object that was changed | -| op | type of change that was made | -| track | property of the object which was modified | -| new_value | new value of the object after the change | -| old_value | previous value of the object before the change | -| metric value | timestamp when Harvest captured the change. 1698735677 in the example below | +| `object` | Name of the ONTAP object that was changed | +| `op` | Type of change that was made | +| `track` | Property of the object which was modified | +| `new_value` | New value of the object after the change (only available for label changes and not for metric changes) | +| `old_value` | Previous value of the object before the change (only available for label changes and not for metric changes) | +| `metric value` | Timestamp when Harvest captured the change. `1698735677` in the example below | -Example of metric shape for object modification: +Example of metric shape for object modification for label: ``` change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677 ``` +Example of metric shape for metric value change: + +``` +change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="size_total", svm="harvest", volume="harvest_demo"} 1698735800 +``` + ### Object Deletion When an object is deleted, the ChangeLog plugin will publish a metric with the following labels: @@ -747,23 +754,6 @@ Example of metric shape for object deletion: change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="2", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="delete", style="flexvol", svm="harvest", volume="harvest_demo"} 1698735708 ``` -### Metric Value Changes - -When a metric value changes, the ChangeLog plugin will publish a metric with the following labels. Note that old and new values are not tracked to avoid impacting cardinality in Prometheus: - -| Label | Description | -|--------------|-----------------------------------------------------------------------------| -| object | name of the ONTAP object that was changed | -| op | type of change that was made | -| track | property of the object which was modified | -| metric value | timestamp when Harvest captured the change. 1698735800 in the example below | - -Example of metric shape for metric value change: - -``` -change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="read_latency", svm="harvest", volume="harvest_demo"} 1698735800 -``` - ## Viewing the Metrics You can view the metrics published by the ChangeLog plugin in the `ChangeLog Monitor` dashboard in `Grafana`. This dashboard provides a visual representation of the changes tracked by the plugin for volume, svm, and node objects. \ No newline at end of file diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go index ab39dae7e..8f20f6bf0 100644 --- a/pkg/matrix/matrix.go +++ b/pkg/matrix/matrix.go @@ -141,10 +141,7 @@ func (m *Matrix) DisplayMetric(name string) *Metric { } func (m *Matrix) DisplayMetricKey(name string) string { - if metricKey, has := m.displayMetrics[name]; has { - return metricKey - } - return "" + return m.displayMetrics[name] } func (m *Matrix) GetMetric(key string) *Metric { From d057e962b60aa94928df2be50ccfe250de19ec1b Mon Sep 17 00:00:00 2001 From: rahulguptajss Date: Wed, 10 Jul 2024 16:27:11 +0530 Subject: [PATCH 4/6] feat: address review comments --- cmd/poller/plugin/changelog/changelog.go | 3 ++- docs/plugins.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/poller/plugin/changelog/changelog.go b/cmd/poller/plugin/changelog/changelog.go index c77c404f4..e1ad5d62d 100644 --- a/cmd/poller/plugin/changelog/changelog.go +++ b/cmd/poller/plugin/changelog/changelog.go @@ -286,7 +286,8 @@ func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string] if _, ok := metricChanges[key]; !ok { metricChanges[key] = make(map[string][2]float64) } - metricChanges[key][curMetric.GetName()] = [2]float64{prevVal, curVal} + metName := curMat.Object + "_" + curMetric.GetName() + metricChanges[key][metName] = [2]float64{prevVal, curVal} } } } diff --git a/docs/plugins.md b/docs/plugins.md index 20d2ff930..89eed4e0d 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -735,7 +735,7 @@ change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter=" Example of metric shape for metric value change: ``` -change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="size_total", svm="harvest", volume="harvest_demo"} 1698735800 +change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="volume_size_total", svm="harvest", volume="harvest_demo"} 1698735800 ``` ### Object Deletion From 4baa2931d83d211f9ae5c39b98ac540dda76ffbb Mon Sep 17 00:00:00 2001 From: rahulguptajss Date: Thu, 11 Jul 2024 11:34:40 +0530 Subject: [PATCH 5/6] feat: address review comments --- cmd/exporters/prometheus/prometheus.go | 19 +++++ cmd/poller/plugin/changelog/changelog.go | 70 +++++++++++-------- cmd/poller/plugin/changelog/changelog_test.go | 4 +- docs/plugins.md | 5 +- 4 files changed, 63 insertions(+), 35 deletions(-) diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go index bc3e9ba3d..7b25546cb 100644 --- a/cmd/exporters/prometheus/prometheus.go +++ b/cmd/exporters/prometheus/prometheus.go @@ -24,6 +24,7 @@ package prometheus import ( "fmt" "github.com/netapp/harvest/v2/cmd/poller/exporter" + "github.com/netapp/harvest/v2/cmd/poller/plugin/changelog" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" @@ -337,6 +338,24 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, exporter.Stats) { instanceLabels := make([]string, 0) instanceLabelsSet := make(map[string]struct{}) + // The ChangeLog plugin tracks metric values and publishes the names of metrics that have changed. + // For example, it might indicate that 'volume_size_total' has been updated. + // If a global prefix for the exporter is defined, we need to amend the metric name with this prefix. + if p.globalPrefix != "" && data.Object == changelog.ObjectChangeLog { + categoryIsMetric := false + for label, value := range instance.GetLabels() { + if label == changelog.Category && value == changelog.Metric { + categoryIsMetric = true + break + } + } + if categoryIsMetric { + if value, ok := instance.GetLabels()[changelog.Track]; ok { + instance.GetLabels()[changelog.Track] = p.globalPrefix + value + } + } + } + if includeAllLabels { for label, value := range instance.GetLabels() { // temporary fix for the rarely happening duplicate labels diff --git a/cmd/poller/plugin/changelog/changelog.go b/cmd/poller/plugin/changelog/changelog.go index e1ad5d62d..a527d83be 100644 --- a/cmd/poller/plugin/changelog/changelog.go +++ b/cmd/poller/plugin/changelog/changelog.go @@ -19,16 +19,19 @@ The shape of the change_log is specific to each label change and is only applica // Constants for ChangeLog metrics and labels const ( - changeLog = "change" - objectLabel = "object" - opLabel = "op" - create = "create" - update = "update" - del = "delete" - track = "track" - oldValue = "old_value" - newValue = "new_value" - indexLabel = "index" + ObjectChangeLog = "change" + objectLabel = "object" + opLabel = "op" + create = "create" + update = "update" + del = "delete" + Track = "track" + oldValue = "old_value" + newValue = "new_value" + indexLabel = "index" + Metric = "metric" + Label = "label" + Category = "category" ) // Metrics to be used in ChangeLog @@ -56,6 +59,7 @@ type Change struct { oldValue string newValue string time int64 + category string } // New initializes a new instance of the ChangeLog plugin @@ -72,7 +76,7 @@ func (c *ChangeLog) Init() error { } object := c.ParentParams.GetChildS("object") - c.matrixName = object.GetContentS() + "_" + changeLog + c.matrixName = object.GetContentS() + "_" + ObjectChangeLog return c.populateChangeLogConfig() } @@ -95,7 +99,7 @@ func (c *ChangeLog) populateChangeLogConfig() error { // initMatrix initializes a new matrix with the given name func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) { changeLogMap := make(map[string]*matrix.Matrix) - changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName) + changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, ObjectChangeLog, c.matrixName) for _, changeLogMatrix := range changeLogMap { changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions()) } @@ -189,13 +193,15 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u op: update, labels: make(map[string]string), track: currentLabel, + category: Label, oldValue: old[currentLabel], newValue: nVal, time: currentTime, } c.updateChangeLogLabels(object, instance, change) - // add changed track and its old, new value - change.labels[track] = currentLabel + // add changed Track and its old, new value + change.labels[Category] = change.category + change.labels[Track] = currentLabel change.labels[oldValue] = change.oldValue change.labels[newValue] = nVal c.createChangeLogInstance(changeMat, change) @@ -206,17 +212,19 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u if changes, ok := metricChanges[key]; ok { for metricName := range changes { change := &Change{ - key: uuid + "_" + object + "_" + metricName, - object: object, - op: update, - labels: make(map[string]string), - track: metricName, + key: uuid + "_" + object + "_" + metricName, + object: object, + op: update, + labels: make(map[string]string), + track: metricName, + category: Metric, // Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable. time: currentTime, } c.updateChangeLogLabels(object, instance, change) - // add changed track and its old, new value - change.labels[track] = metricName + // add changed Track and its old, new value + change.labels[Category] = change.category + change.labels[Track] = metricName change.labels[oldValue] = change.oldValue change.labels[newValue] = change.newValue c.createChangeLogInstance(changeMat, change) @@ -266,10 +274,9 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u } // CompareMetrics compares the metrics of the current and previous instances -func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string][2]float64 { - metricChanges := make(map[string]map[string][2]float64) +func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string]struct{} { + metricChanges := make(map[string]map[string]struct{}) prevMat := c.previousData - met := maps2.Keys(c.previousData.GetMetrics()) for _, metricKey := range met { @@ -277,18 +284,19 @@ func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string] curMetric := curMat.GetMetric(metricKey) for key, currInstance := range curMat.GetInstances() { prevInstance := prevMat.GetInstance(key) + if prevInstance == nil { + continue + } prevIndex := prevInstance.GetIndex() currIndex := currInstance.GetIndex() curVal := curMetric.GetValues()[currIndex] prevVal := prevMetric.GetValues()[prevIndex] - if prevInstance != nil { - if curVal != prevVal { - if _, ok := metricChanges[key]; !ok { - metricChanges[key] = make(map[string][2]float64) - } - metName := curMat.Object + "_" + curMetric.GetName() - metricChanges[key][metName] = [2]float64{prevVal, curVal} + if curVal != prevVal { + if _, ok := metricChanges[key]; !ok { + metricChanges[key] = make(map[string]struct{}) } + metName := curMat.Object + "_" + curMetric.GetName() + metricChanges[key][metName] = struct{}{} } } } diff --git a/cmd/poller/plugin/changelog/changelog_test.go b/cmd/poller/plugin/changelog/changelog_test.go index b60a9333f..2f503d5b3 100644 --- a/cmd/poller/plugin/changelog/changelog_test.go +++ b/cmd/poller/plugin/changelog/changelog_test.go @@ -105,7 +105,7 @@ func TestChangeLogModified(t *testing.T) { o, _, _ := p.Run(data1) - checkChangeLogInstances(t, o, 2, 9, update, opLabel) + checkChangeLogInstances(t, o, 2, 10, update, opLabel) } func TestChangeLogModifiedWithMetrics(t *testing.T) { @@ -140,7 +140,7 @@ func TestChangeLogModifiedWithMetrics(t *testing.T) { o, _, _ := p.Run(data1) - checkChangeLogInstances(t, o, 3, 7, update, opLabel) + checkChangeLogInstances(t, o, 3, 8, update, opLabel) } func TestChangeLogCreated(t *testing.T) { diff --git a/docs/plugins.md b/docs/plugins.md index 89eed4e0d..1a52aca48 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -725,17 +725,18 @@ When an existing object is modified, the ChangeLog plugin will publish a metric | `new_value` | New value of the object after the change (only available for label changes and not for metric changes) | | `old_value` | Previous value of the object before the change (only available for label changes and not for metric changes) | | `metric value` | Timestamp when Harvest captured the change. `1698735677` in the example below | +| `category` | Type of the change, indicating whether it is a `metric` or a `label` change | Example of metric shape for object modification for label: ``` -change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677 +change_log{aggr="umeng_aff300_aggr2", category="label", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677 ``` Example of metric shape for metric value change: ``` -change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="volume_size_total", svm="harvest", volume="harvest_demo"} 1698735800 +change_log{aggr="umeng_aff300_aggr2", category="metric", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="volume_size_total", svm="harvest", volume="harvest_demo"} 1698735800 ``` ### Object Deletion From 1b6736cb4035cb0ad1ce176815356c0048323940 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 15 Jul 2024 01:03:17 -0400 Subject: [PATCH 6/6] feat: enable ChangeLog plugin to monitor metric value change (#3046) --- cmd/exporters/prometheus/prometheus.go | 15 ++-- cmd/exporters/prometheus/prometheus_test.go | 85 ++++++++++++++++++++- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go index 7b25546cb..2a2e3df81 100644 --- a/cmd/exporters/prometheus/prometheus.go +++ b/cmd/exporters/prometheus/prometheus.go @@ -342,16 +342,11 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, exporter.Stats) { // For example, it might indicate that 'volume_size_total' has been updated. // If a global prefix for the exporter is defined, we need to amend the metric name with this prefix. if p.globalPrefix != "" && data.Object == changelog.ObjectChangeLog { - categoryIsMetric := false - for label, value := range instance.GetLabels() { - if label == changelog.Category && value == changelog.Metric { - categoryIsMetric = true - break - } - } - if categoryIsMetric { - if value, ok := instance.GetLabels()[changelog.Track]; ok { - instance.GetLabels()[changelog.Track] = p.globalPrefix + value + if categoryValue, ok := instance.GetLabels()[changelog.Category]; ok { + if categoryValue == changelog.Metric { + if tracked, ok := instance.GetLabels()[changelog.Track]; ok { + instance.GetLabels()[changelog.Track] = p.globalPrefix + tracked + } } } } diff --git a/cmd/exporters/prometheus/prometheus_test.go b/cmd/exporters/prometheus/prometheus_test.go index e2af74ac0..34e67b9ed 100644 --- a/cmd/exporters/prometheus/prometheus_test.go +++ b/cmd/exporters/prometheus/prometheus_test.go @@ -6,6 +6,7 @@ package prometheus import ( "bytes" + "github.com/google/go-cmp/cmp" "github.com/netapp/harvest/v2/cmd/poller/exporter" "github.com/netapp/harvest/v2/cmd/poller/options" "github.com/netapp/harvest/v2/pkg/conf" @@ -88,6 +89,32 @@ func BenchmarkEscape(b *testing.B) { } } +func setUpChangeMatrix() *matrix.Matrix { + m := matrix.New("change", "change", "change") + // Create a metric with a metric value change + log, _ := m.NewMetricUint64("log") + instance, _ := m.NewInstance("A") + _ = log.SetValueInt64(instance, 3) + instance.SetLabel("category", "metric") + instance.SetLabel("cluster", "umeng-aff300-01-02") + instance.SetLabel("object", "volume") + instance.SetLabel("op", "metric_change") + instance.SetLabel("track", "volume_size_total") + + // Create a metric with a label change + instance2, _ := m.NewInstance("B") + _ = log.SetValueInt64(instance2, 3) + instance2.SetLabel("category", "label") + instance2.SetLabel("cluster", "umeng-aff300-01-02") + instance2.SetLabel("new_value", "offline") + instance2.SetLabel("object", "volume") + instance2.SetLabel("old_value", "online") + instance2.SetLabel("op", "update") + instance2.SetLabel("track", "state") + + return m +} + func setUpMatrix(object string) *matrix.Matrix { m := matrix.New("bike", object, "bike") speed, _ := m.NewMetricUint64("max_speed") @@ -115,7 +142,7 @@ max_speed{} 3`}, for _, tt := range tests { t.Run(tt.prefix, func(t *testing.T) { - p, err := setupPrometheusExporter() + p, err := setUpPrometheusExporter("") if err != nil { t.Errorf("expected nil, got %v", err) } @@ -142,14 +169,66 @@ max_speed{} 3`}, } } -func setupPrometheusExporter() (exporter.Exporter, error) { +func TestGlobalPrefixWithChangelog(t *testing.T) { + + type test struct { + prefix string + want string + } + + tests := []test{ + {"prefix", ` +netapp_change_log{category="label",cluster="umeng-aff300-01-02",new_value="offline",object="volume",old_value="online",op="update",track="state"} 3 +netapp_change_log{category="metric",cluster="umeng-aff300-01-02",object="volume",op="metric_change",track="netapp_volume_size_total"} 3`}, + } + + for _, tt := range tests { + t.Run(tt.prefix, func(t *testing.T) { + p, err := setUpPrometheusExporter("netapp") + + if err != nil { + t.Errorf("expected nil, got %v", err) + } + m := setUpChangeMatrix() + + _, err = p.Export(m) + if err != nil { + t.Errorf("expected nil, got %v", err) + } + + prom := p.(*Prometheus) + var lines []string + for _, metrics := range prom.cache.Get() { + for _, metric := range metrics { + lines = append(lines, string(metric)) + } + } + + slices.Sort(lines) + diff := cmp.Diff(strings.TrimSpace(tt.want), strings.Join(lines, "\n")) + if diff != "" { + t.Errorf("Mismatch (-got +want):\n%s", diff) + } + }) + } +} + +func setUpPrometheusExporter(prefix string) (exporter.Exporter, error) { + absExp := exporter.New( "Prometheus", "prom1", &options.Options{PromPort: 1}, - conf.Exporter{IsTest: true}, + conf.Exporter{ + IsTest: true, + SortLabels: true, + }, nil, ) + + if prefix != "" { + absExp.Params.GlobalPrefix = &prefix + } p := New(absExp) err := p.Init() return p, err