From c68dcc8ef70ff08954a06c343ce17765d34a369d Mon Sep 17 00:00:00 2001 From: Mateusz Sekara Date: Thu, 12 Dec 2024 15:28:41 +0100 Subject: [PATCH] CCIP-4448 Track observation/outcome length in bytes (#15656) * Track observation/outcome length * Track observation/outcome length * Post review fixes --- core/services/ocr3/promwrapper/factory.go | 1 + core/services/ocr3/promwrapper/plugin.go | 20 +++++++++- core/services/ocr3/promwrapper/plugin_test.go | 40 ++++++++++++++----- core/services/ocr3/promwrapper/types.go | 7 ++++ 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/core/services/ocr3/promwrapper/factory.go b/core/services/ocr3/promwrapper/factory.go index 6518cea3c0d..e369b3260ef 100644 --- a/core/services/ocr3/promwrapper/factory.go +++ b/core/services/ocr3/promwrapper/factory.go @@ -47,6 +47,7 @@ func (r ReportingPluginFactory[RI]) NewReportingPlugin(ctx context.Context, conf config.ConfigDigest.String(), promOCR3ReportsGenerated, promOCR3Durations, + promOCR3Sizes, promOCR3PluginStatus, ) return wrapped, info, err diff --git a/core/services/ocr3/promwrapper/plugin.go b/core/services/ocr3/promwrapper/plugin.go index dcee5050d1e..aa5fb87a6ee 100644 --- a/core/services/ocr3/promwrapper/plugin.go +++ b/core/services/ocr3/promwrapper/plugin.go @@ -21,6 +21,7 @@ type reportingPlugin[RI any] struct { // Prometheus components for tracking metrics reportsGenerated *prometheus.CounterVec durations *prometheus.HistogramVec + sizes *prometheus.CounterVec status *prometheus.GaugeVec } @@ -31,6 +32,7 @@ func newReportingPlugin[RI any]( configDigest string, reportsGenerated *prometheus.CounterVec, durations *prometheus.HistogramVec, + sizes *prometheus.CounterVec, status *prometheus.GaugeVec, ) *reportingPlugin[RI] { return &reportingPlugin[RI]{ @@ -40,6 +42,7 @@ func newReportingPlugin[RI any]( configDigest: configDigest, reportsGenerated: reportsGenerated, durations: durations, + sizes: sizes, status: status, } } @@ -51,9 +54,11 @@ func (p *reportingPlugin[RI]) Query(ctx context.Context, outctx ocr3types.Outcom } func (p *reportingPlugin[RI]) Observation(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query) (ocrtypes.Observation, error) { - return withObservedExecution(p, observation, func() (ocrtypes.Observation, error) { + result, err := withObservedExecution(p, observation, func() (ocrtypes.Observation, error) { return p.ReportingPlugin.Observation(ctx, outctx, query) }) + p.trackSize(observation, len(result), err) + return result, err } func (p *reportingPlugin[RI]) ValidateObservation(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query, ao ocrtypes.AttributedObservation) error { @@ -65,9 +70,11 @@ func (p *reportingPlugin[RI]) ValidateObservation(ctx context.Context, outctx oc } func (p *reportingPlugin[RI]) Outcome(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query, aos []ocrtypes.AttributedObservation) (ocr3types.Outcome, error) { - return withObservedExecution(p, outcome, func() (ocr3types.Outcome, error) { + result, err := withObservedExecution(p, outcome, func() (ocr3types.Outcome, error) { return p.ReportingPlugin.Outcome(ctx, outctx, query, aos) }) + p.trackSize(outcome, len(result), err) + return result, err } func (p *reportingPlugin[RI]) Reports(ctx context.Context, seqNr uint64, outcome ocr3types.Outcome) ([]ocr3types.ReportPlus[RI], error) { @@ -111,6 +118,15 @@ func (p *reportingPlugin[RI]) updateStatus(status bool) { Set(float64(boolToInt(status))) } +func (p *reportingPlugin[RI]) trackSize(function functionType, size int, err error) { + if err != nil { + return + } + p.sizes. + WithLabelValues(p.chainID, p.plugin, string(function)). + Add(float64(size)) +} + func boolToInt(arg bool) int { if arg { return 1 diff --git a/core/services/ocr3/promwrapper/plugin_test.go b/core/services/ocr3/promwrapper/plugin_test.go index 9a7b6f2e648..a10a467799f 100644 --- a/core/services/ocr3/promwrapper/plugin_test.go +++ b/core/services/ocr3/promwrapper/plugin_test.go @@ -17,17 +17,20 @@ import ( ) func Test_ReportsGeneratedGauge(t *testing.T) { + pluginObservationSize := 5 + pluginOutcomeSize := 3 + plugin1 := newReportingPlugin( fakePlugin[uint]{reports: make([]ocr3types.ReportPlus[uint], 2)}, - "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, + "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus, ) plugin2 := newReportingPlugin( - fakePlugin[bool]{reports: make([]ocr3types.ReportPlus[bool], 10)}, - "solana", "different_plugin", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, + fakePlugin[bool]{reports: make([]ocr3types.ReportPlus[bool], 10), observationSize: pluginObservationSize, outcomeSize: pluginOutcomeSize}, + "solana", "different_plugin", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus, ) plugin3 := newReportingPlugin( fakePlugin[string]{err: errors.New("error")}, - "1234", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, + "1234", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus, ) r1, err := plugin1.Reports(tests.Context(t), 1, nil) @@ -64,20 +67,33 @@ func Test_ReportsGeneratedGauge(t *testing.T) { require.NoError(t, plugin1.Close()) pluginHealth = testutil.ToFloat64(promOCR3PluginStatus.WithLabelValues("123", "empty", "abc")) require.Equal(t, 0, int(pluginHealth)) + + iterations := 10 + for i := 0; i < iterations; i++ { + _, err1 := plugin2.Outcome(tests.Context(t), ocr3types.OutcomeContext{}, nil, nil) + require.NoError(t, err1) + } + _, err1 := plugin2.Observation(tests.Context(t), ocr3types.OutcomeContext{}, nil) + require.NoError(t, err1) + + outcomesLen := testutil.ToFloat64(promOCR3Sizes.WithLabelValues("solana", "different_plugin", "outcome")) + require.Equal(t, pluginOutcomeSize*iterations, int(outcomesLen)) + observationLen := testutil.ToFloat64(promOCR3Sizes.WithLabelValues("solana", "different_plugin", "observation")) + require.Equal(t, pluginObservationSize, int(observationLen)) } func Test_DurationHistograms(t *testing.T) { plugin1 := newReportingPlugin( fakePlugin[uint]{}, - "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, + "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus, ) plugin2 := newReportingPlugin( fakePlugin[uint]{err: errors.New("error")}, - "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, + "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus, ) plugin3 := newReportingPlugin( fakePlugin[uint]{}, - "solana", "commit", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, + "solana", "commit", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus, ) for _, p := range []*reportingPlugin[uint]{plugin1, plugin2, plugin3} { @@ -102,8 +118,10 @@ func Test_DurationHistograms(t *testing.T) { } type fakePlugin[RI any] struct { - reports []ocr3types.ReportPlus[RI] - err error + reports []ocr3types.ReportPlus[RI] + observationSize int + outcomeSize int + err error } func (f fakePlugin[RI]) Query(context.Context, ocr3types.OutcomeContext) (ocrtypes.Query, error) { @@ -117,7 +135,7 @@ func (f fakePlugin[RI]) Observation(context.Context, ocr3types.OutcomeContext, o if f.err != nil { return nil, f.err } - return ocrtypes.Observation{}, nil + return make([]byte, f.observationSize), nil } func (f fakePlugin[RI]) ValidateObservation(context.Context, ocr3types.OutcomeContext, ocrtypes.Query, ocrtypes.AttributedObservation) error { @@ -132,7 +150,7 @@ func (f fakePlugin[RI]) Outcome(context.Context, ocr3types.OutcomeContext, ocrty if f.err != nil { return nil, f.err } - return ocr3types.Outcome{}, nil + return make([]byte, f.outcomeSize), nil } func (f fakePlugin[RI]) Reports(context.Context, uint64, ocr3types.Outcome) ([]ocr3types.ReportPlus[RI], error) { diff --git a/core/services/ocr3/promwrapper/types.go b/core/services/ocr3/promwrapper/types.go index 2fa29dcdf20..59468358783 100644 --- a/core/services/ocr3/promwrapper/types.go +++ b/core/services/ocr3/promwrapper/types.go @@ -48,6 +48,13 @@ var ( }, []string{"chainID", "plugin", "function", "success"}, ) + promOCR3Sizes = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ocr3_reporting_plugin_data_sizes", + Help: "Tracks the size of the data produced by OCR3 plugin in bytes (e.g. reports, observations etc.)", + }, + []string{"chainID", "plugin", "function"}, + ) promOCR3PluginStatus = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "ocr3_reporting_plugin_status",