Skip to content

Commit

Permalink
feat: enable ChangeLog plugin to monitor metric value change (#3041)
Browse files Browse the repository at this point in the history 
* feat: enable ChangeLog plugin to monitor metric value change

* feat: enable ChangeLog plugin to monitor metric value change

* feat: address review comments

* feat: address review comments

* feat: address review comments

* feat: enable ChangeLog plugin to monitor metric value change (#3046)

---------

Co-authored-by: Chris Grindstaff <[email protected]>
  • Loading branch information
@rahulguptajss @cgrinds
rahulguptajss and cgrinds authored Jul 16, 2024
1 parent 22b322c commit ded2a1e
Show file tree
Hide file tree
Showing 9 changed files with 282 additions and 43 deletions.
14 changes: 14 additions & 0 deletions cmd/exporters/prometheus/prometheus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ package prometheus
import (
"fmt"
"github.com/netapp/harvest/v2/cmd/poller/exporter"
"github.com/netapp/harvest/v2/cmd/poller/plugin/changelog"
"github.com/netapp/harvest/v2/pkg/errs"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/set"
Expand Down Expand Up @@ -323,6 +324,19 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, exporter.Stats) {
instanceLabels := make([]string, 0)
instanceLabelsSet := make(map[string]struct{})

// The ChangeLog plugin tracks metric values and publishes the names of metrics that have changed.
// For example, it might indicate that 'volume_size_total' has been updated.
// If a global prefix for the exporter is defined, we need to amend the metric name with this prefix.
if p.globalPrefix != "" && data.Object == changelog.ObjectChangeLog {
if categoryValue, ok := instance.GetLabels()[changelog.Category]; ok {
if categoryValue == changelog.Metric {
if tracked, ok := instance.GetLabels()[changelog.Track]; ok {
instance.GetLabels()[changelog.Track] = p.globalPrefix + tracked
}
}
}
}

if includeAllLabels {
for label, value := range instance.GetLabels() {
// temporary fix for the rarely happening duplicate labels
Expand Down
85 changes: 82 additions & 3 deletions cmd/exporters/prometheus/prometheus_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package prometheus

import (
"bytes"
"github.com/google/go-cmp/cmp"
"github.com/netapp/harvest/v2/cmd/poller/exporter"
"github.com/netapp/harvest/v2/cmd/poller/options"
"github.com/netapp/harvest/v2/pkg/conf"
Expand Down Expand Up @@ -88,6 +89,32 @@ func BenchmarkEscape(b *testing.B) {
}
}

func setUpChangeMatrix() *matrix.Matrix {
m := matrix.New("change", "change", "change")
// Create a metric with a metric value change
log, _ := m.NewMetricUint64("log")
instance, _ := m.NewInstance("A")
_ = log.SetValueInt64(instance, 3)
instance.SetLabel("category", "metric")
instance.SetLabel("cluster", "umeng-aff300-01-02")
instance.SetLabel("object", "volume")
instance.SetLabel("op", "metric_change")
instance.SetLabel("track", "volume_size_total")

// Create a metric with a label change
instance2, _ := m.NewInstance("B")
_ = log.SetValueInt64(instance2, 3)
instance2.SetLabel("category", "label")
instance2.SetLabel("cluster", "umeng-aff300-01-02")
instance2.SetLabel("new_value", "offline")
instance2.SetLabel("object", "volume")
instance2.SetLabel("old_value", "online")
instance2.SetLabel("op", "update")
instance2.SetLabel("track", "state")

return m
}

func setUpMatrix(object string) *matrix.Matrix {
m := matrix.New("bike", object, "bike")
speed, _ := m.NewMetricUint64("max_speed")
Expand Down Expand Up @@ -115,7 +142,7 @@ max_speed{} 3`},

for _, tt := range tests {
t.Run(tt.prefix, func(t *testing.T) {
p, err := setupPrometheusExporter()
p, err := setUpPrometheusExporter("")
if err != nil {
t.Errorf("expected nil, got %v", err)
}
Expand All @@ -142,14 +169,66 @@ max_speed{} 3`},
}
}

func setupPrometheusExporter() (exporter.Exporter, error) {
func TestGlobalPrefixWithChangelog(t *testing.T) {

type test struct {
prefix string
want string
}

tests := []test{
{"prefix", `
netapp_change_log{category="label",cluster="umeng-aff300-01-02",new_value="offline",object="volume",old_value="online",op="update",track="state"} 3
netapp_change_log{category="metric",cluster="umeng-aff300-01-02",object="volume",op="metric_change",track="netapp_volume_size_total"} 3`},
}

for _, tt := range tests {
t.Run(tt.prefix, func(t *testing.T) {
p, err := setUpPrometheusExporter("netapp")

if err != nil {
t.Errorf("expected nil, got %v", err)
}
m := setUpChangeMatrix()

_, err = p.Export(m)
if err != nil {
t.Errorf("expected nil, got %v", err)
}

prom := p.(*Prometheus)
var lines []string
for _, metrics := range prom.cache.Get() {
for _, metric := range metrics {
lines = append(lines, string(metric))
}
}

slices.Sort(lines)
diff := cmp.Diff(strings.TrimSpace(tt.want), strings.Join(lines, "\n"))
if diff != "" {
t.Errorf("Mismatch (-got +want):\n%s", diff)
}
})
}
}

func setUpPrometheusExporter(prefix string) (exporter.Exporter, error) {

absExp := exporter.New(
"Prometheus",
"prom1",
&options.Options{PromPort: 1},
conf.Exporter{IsTest: true},
conf.Exporter{
IsTest: true,
SortLabels: true,
},
nil,
)

if prefix != "" {
absExp.Params.GlobalPrefix = &prefix
}
p := New(absExp)
err := p.Init()
return p, err
Expand Down
107 changes: 88 additions & 19 deletions cmd/poller/plugin/changelog/change_log.go → cmd/poller/plugin/changelog/changelog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/netapp/harvest/v2/pkg/set"
"github.com/netapp/harvest/v2/pkg/tree/yaml"
"github.com/netapp/harvest/v2/pkg/util"
maps2 "golang.org/x/exp/maps"
"maps"
"strconv"
"time"
Expand All @@ -18,16 +19,19 @@ The shape of the change_log is specific to each label change and is only applica

// Constants for ChangeLog metrics and labels
const (
changeLog = "change"
objectLabel = "object"
opLabel = "op"
create = "create"
update = "update"
del = "delete"
track = "track"
oldValue = "old_value"
newValue = "new_value"
indexLabel = "index"
ObjectChangeLog = "change"
objectLabel = "object"
opLabel = "op"
create = "create"
update = "update"
del = "delete"
Track = "track"
oldValue = "old_value"
newValue = "new_value"
indexLabel = "index"
Metric = "metric"
Label = "label"
Category = "category"
)

// Metrics to be used in ChangeLog
Expand Down Expand Up @@ -55,6 +59,7 @@ type Change struct {
oldValue string
newValue string
time int64
category string
}

// New initializes a new instance of the ChangeLog plugin
Expand All @@ -71,7 +76,7 @@ func (c *ChangeLog) Init() error {
}

object := c.ParentParams.GetChildS("object")
c.matrixName = object.GetContentS() + "_" + changeLog
c.matrixName = object.GetContentS() + "_" + ObjectChangeLog

return c.populateChangeLogConfig()
}
Expand All @@ -94,7 +99,7 @@ func (c *ChangeLog) populateChangeLogConfig() error {
// initMatrix initializes a new matrix with the given name
func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) {
changeLogMap := make(map[string]*matrix.Matrix)
changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName)
changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, ObjectChangeLog, c.matrixName)
for _, changeLogMatrix := range changeLogMap {
changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions())
}
Expand All @@ -110,7 +115,6 @@ func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) {

// Run processes the data and generates ChangeLog instances
func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {

data := dataMap[c.Object]
changeLogMap, err := c.initMatrix()
if err != nil {
Expand All @@ -128,7 +132,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
}

changeMat := changeLogMap[c.matrixName]

changeMat.SetGlobalLabels(data.GetGlobalLabels())
object := data.Object
if c.changeLogConfig.Object == "" {
Expand All @@ -151,6 +154,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u

currentTime := time.Now().Unix()

metricChanges := c.CompareMetrics(data)
// loop over current instances
for key, instance := range data.GetInstances() {
uuid := instance.GetLabel("uuid")
Expand Down Expand Up @@ -179,7 +183,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
c.updateChangeLogLabels(object, instance, change)
c.createChangeLogInstance(changeMat, change)
} else {
// check for any modification
// check for any label modification
cur, old := instance.CompareDiffs(prevInstance, c.changeLogConfig.Track)
if len(cur) > 0 {
for currentLabel, nVal := range cur {
Expand All @@ -189,20 +193,46 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
op: update,
labels: make(map[string]string),
track: currentLabel,
category: Label,
oldValue: old[currentLabel],
newValue: nVal,
time: currentTime,
}
c.updateChangeLogLabels(object, instance, change)
// add changed track and its old, new value
change.labels[track] = currentLabel
// add changed Track and its old, new value
change.labels[Category] = change.category
change.labels[Track] = currentLabel
change.labels[oldValue] = change.oldValue
change.labels[newValue] = nVal
c.createChangeLogInstance(changeMat, change)
}
}

// check for any metric modification
if changes, ok := metricChanges[key]; ok {
for metricName := range changes {
change := &Change{
key: uuid + "_" + object + "_" + metricName,
object: object,
op: update,
labels: make(map[string]string),
track: metricName,
category: Metric,
// Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable.
time: currentTime,
}
c.updateChangeLogLabels(object, instance, change)
// add changed Track and its old, new value
change.labels[Category] = change.category
change.labels[Track] = metricName
change.labels[oldValue] = change.oldValue
change.labels[newValue] = change.newValue
c.createChangeLogInstance(changeMat, change)
}
}
}
}

// create deleted instances change_log
for key := range oldInstances.Iter() {
prevInstance := prevMat.GetInstance(key)
Expand Down Expand Up @@ -243,12 +273,51 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
return matricesArray, nil, nil
}

// CompareMetrics compares the metrics of the current and previous instances
func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string]struct{} {
metricChanges := make(map[string]map[string]struct{})
prevMat := c.previousData
met := maps2.Keys(c.previousData.GetMetrics())

for _, metricKey := range met {
prevMetric := prevMat.GetMetric(metricKey)
curMetric := curMat.GetMetric(metricKey)
for key, currInstance := range curMat.GetInstances() {
prevInstance := prevMat.GetInstance(key)
if prevInstance == nil {
continue
}
prevIndex := prevInstance.GetIndex()
currIndex := currInstance.GetIndex()
curVal := curMetric.GetValues()[currIndex]
prevVal := prevMetric.GetValues()[prevIndex]
if curVal != prevVal {
if _, ok := metricChanges[key]; !ok {
metricChanges[key] = make(map[string]struct{})
}
metName := curMat.Object + "_" + curMetric.GetName()
metricChanges[key][metName] = struct{}{}
}
}
}
return metricChanges
}

// copyPreviousData creates a copy of the previous data for comparison
func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) {
labels := c.changeLogConfig.PublishLabels
labels = append(labels, c.changeLogConfig.Track...)
var met []string
for _, t := range c.changeLogConfig.Track {
mKey := cur.DisplayMetricKey(t)
if mKey == "" {
labels = append(labels, t)
} else {
met = append(met, mKey)
}
}
labels = append(labels, "uuid")
c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels})
withMetrics := len(met) > 0
c.previousData = cur.Clone(matrix.With{Data: true, Metrics: withMetrics, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met})
}

// createChangeLogInstance creates a new ChangeLog instance with the given change data
Expand Down
0 ...ler/plugin/changelog/change_log_helper.go → ...ller/plugin/changelog/changelog_helper.go
View file
File renamed without changes.
Loading

0 comments on commit ded2a1e

Please sign in to comment.