Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MQE: Add support for histogram_quantile #9929

Open
wants to merge 40 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
d293f3a
MQE: Add support for histogram_quantile
jhesketh Oct 23, 2024
b471cea
Update comments
jhesketh Nov 18, 2024
5abeee0
Copy in quantile functions instead of exporting them from Prometheus
jhesketh Nov 18, 2024
c2ac252
Fix lint
jhesketh Nov 18, 2024
bb84e60
Address review feedback
jhesketh Nov 18, 2024
016d546
Fix tests
jhesketh Nov 18, 2024
c2c54ac
Address review feedback (part 1)
jhesketh Nov 20, 2024
2084871
Add upstream quantile tests
jhesketh Nov 20, 2024
853312b
Address review feedback (part 2)
jhesketh Nov 20, 2024
c3ef405
Use seriesGroupPair struct
jhesketh Nov 20, 2024
0ecdfd6
Only set classicHistogramGroup if le exists
jhesketh Nov 20, 2024
91734fb
Sort the output groups
jhesketh Nov 20, 2024
6aa3dda
Merge branch 'main' into jhesketh/mqe-histogram-quantile
jhesketh Nov 26, 2024
042d776
Add extra test case
jhesketh Nov 26, 2024
29e883e
Fix annotations
jhesketh Nov 26, 2024
ca9e43a
Use a pool for point buckets
jhesketh Nov 26, 2024
974a238
Fix sorting of groups and add test
jhesketh Nov 26, 2024
0800b04
Merge branch 'main' into jhesketh/mqe-histogram-quantile
jhesketh Nov 26, 2024
1bcf411
Add support for mangling buckets
jhesketh Nov 26, 2024
2c64ab7
Fix lint
jhesketh Nov 26, 2024
bf2e183
Add feature toggle for classic histogrms
jhesketh Nov 26, 2024
23c8dff
Update docs
jhesketh Nov 26, 2024
9ad9857
Add benchmark
jhesketh Nov 26, 2024
6728390
Tidy up doc wording
jhesketh Nov 26, 2024
2892670
Optimise group sorting
jhesketh Nov 26, 2024
448118c
Address review feedback
jhesketh Nov 26, 2024
2de496b
Update note on ph values
jhesketh Nov 27, 2024
64c2866
Use pool for bucket slices
jhesketh Nov 27, 2024
eea483c
Also pool buckets themselves
jhesketh Nov 27, 2024
f5dc444
Avoid heap-allocating the point buckets
jhesketh Nov 27, 2024
1ee4ec6
Remove pointBucketsIndex
jhesketh Nov 27, 2024
95cc86d
Remove unused pools
jhesketh Nov 27, 2024
29173d6
benchmark native histograms
jhesketh Nov 27, 2024
f6b334b
Tidy up naming since this will only be for one function
jhesketh Nov 27, 2024
4521922
Rename files
jhesketh Nov 27, 2024
502d6b7
Merge branch 'main' into jhesketh/mqe-histogram-quantile
jhesketh Nov 28, 2024
341ac00
Deduplicate and merge results consistent with prom3
jhesketh Nov 28, 2024
4fa411a
Re-enable upstream tests
jhesketh Nov 28, 2024
a177e99
Fix quantile funtion in line with v3
jhesketh Nov 28, 2024
5c1c901
Update CHANGELOG
jhesketh Nov 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pkg/streamingpromql/benchmarks/comparison_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func BenchmarkQuery(b *testing.B) {
prometheusResult, prometheusClose := c.Run(ctx, b, start, end, interval, prometheusEngine, q)
mimirResult, mimirClose := c.Run(ctx, b, start, end, interval, mimirEngine, q)

testutils.RequireEqualResults(b, c.Expr, prometheusResult, mimirResult)
testutils.RequireEqualResults(b, c.Expr, prometheusResult, mimirResult, false)

prometheusClose()
mimirClose()
Expand Down Expand Up @@ -108,7 +108,7 @@ func TestBothEnginesReturnSameResultsForBenchmarkQueries(t *testing.T) {
prometheusResult, prometheusClose := c.Run(ctx, t, start, end, interval, prometheusEngine, q)
mimirResult, mimirClose := c.Run(ctx, t, start, end, interval, mimirEngine, q)

testutils.RequireEqualResults(t, c.Expr, prometheusResult, mimirResult)
testutils.RequireEqualResults(t, c.Expr, prometheusResult, mimirResult, false)

prometheusClose()
mimirClose()
Expand Down
331 changes: 242 additions & 89 deletions pkg/streamingpromql/engine_test.go

Large diffs are not rendered by default.

110 changes: 67 additions & 43 deletions pkg/streamingpromql/functions.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,53 +286,77 @@ func RoundFunctionOperatorFactory() InstantVectorFunctionOperatorFactory {
}
}

func HistogramQuantileOperatorFactory() InstantVectorFunctionOperatorFactory {
return func(args []types.Operator, memoryConsumptionTracker *limiting.MemoryConsumptionTracker, annotations *annotations.Annotations, expressionPosition posrange.PositionRange, timeRange types.QueryTimeRange) (types.InstantVectorOperator, error) {
if len(args) != 2 {
// Should be caught by the PromQL parser, but we check here for safety.
return nil, fmt.Errorf("expected exactly 2 argument for histogram_quantile, got %v", len(args))
}

ph, ok := args[0].(types.ScalarOperator)
if !ok {
// Should be caught by the PromQL parser, but we check here for safety.
return nil, fmt.Errorf("expected a scalar for 1st argument for histogram_quantile, got %T", args[0])
}

inner, ok := args[1].(types.InstantVectorOperator)
if !ok {
// Should be caught by the PromQL parser, but we check here for safety.
return nil, fmt.Errorf("expected an instant vector for 2nd argument for histogram_quantile, got %T", args[1])
}

return functions.NewHistogramFunctionOverInstantVector(ph, inner, memoryConsumptionTracker, annotations, expressionPosition, timeRange), nil
}
}

// These functions return an instant-vector.
var instantVectorFunctionOperatorFactories = map[string]InstantVectorFunctionOperatorFactory{
// Please keep this list sorted alphabetically.

"abs": InstantVectorTransformationFunctionOperatorFactory("abs", functions.Abs),
"acos": InstantVectorTransformationFunctionOperatorFactory("acos", functions.Acos),
"acosh": InstantVectorTransformationFunctionOperatorFactory("acosh", functions.Acosh),
"asin": InstantVectorTransformationFunctionOperatorFactory("asin", functions.Asin),
"asinh": InstantVectorTransformationFunctionOperatorFactory("asinh", functions.Asinh),
"atan": InstantVectorTransformationFunctionOperatorFactory("atan", functions.Atan),
"atanh": InstantVectorTransformationFunctionOperatorFactory("atanh", functions.Atanh),
"avg_over_time": FunctionOverRangeVectorOperatorFactory("avg_over_time", functions.AvgOverTime),
"ceil": InstantVectorTransformationFunctionOperatorFactory("ceil", functions.Ceil),
"changes": FunctionOverRangeVectorOperatorFactory("changes", functions.Changes),
"clamp": ClampFunctionOperatorFactory(),
"clamp_max": ClampMinMaxFunctionOperatorFactory("clamp_max", false),
"clamp_min": ClampMinMaxFunctionOperatorFactory("clamp_min", true),
"cos": InstantVectorTransformationFunctionOperatorFactory("cos", functions.Cos),
"cosh": InstantVectorTransformationFunctionOperatorFactory("cosh", functions.Cosh),
"count_over_time": FunctionOverRangeVectorOperatorFactory("count_over_time", functions.CountOverTime),
"deg": InstantVectorTransformationFunctionOperatorFactory("deg", functions.Deg),
"deriv": FunctionOverRangeVectorOperatorFactory("deriv", functions.Deriv),
"exp": InstantVectorTransformationFunctionOperatorFactory("exp", functions.Exp),
"floor": InstantVectorTransformationFunctionOperatorFactory("floor", functions.Floor),
"histogram_count": InstantVectorTransformationFunctionOperatorFactory("histogram_count", functions.HistogramCount),
"histogram_sum": InstantVectorTransformationFunctionOperatorFactory("histogram_sum", functions.HistogramSum),
"increase": FunctionOverRangeVectorOperatorFactory("increase", functions.Increase),
"label_replace": LabelReplaceFunctionOperatorFactory(),
"last_over_time": FunctionOverRangeVectorOperatorFactory("last_over_time", functions.LastOverTime),
"ln": InstantVectorTransformationFunctionOperatorFactory("ln", functions.Ln),
"log10": InstantVectorTransformationFunctionOperatorFactory("log10", functions.Log10),
"log2": InstantVectorTransformationFunctionOperatorFactory("log2", functions.Log2),
"max_over_time": FunctionOverRangeVectorOperatorFactory("max_over_time", functions.MaxOverTime),
"min_over_time": FunctionOverRangeVectorOperatorFactory("min_over_time", functions.MinOverTime),
"present_over_time": FunctionOverRangeVectorOperatorFactory("present_over_time", functions.PresentOverTime),
"rad": InstantVectorTransformationFunctionOperatorFactory("rad", functions.Rad),
"rate": FunctionOverRangeVectorOperatorFactory("rate", functions.Rate),
"resets": FunctionOverRangeVectorOperatorFactory("resets", functions.Resets),
"round": RoundFunctionOperatorFactory(),
"sgn": InstantVectorTransformationFunctionOperatorFactory("sgn", functions.Sgn),
"sin": InstantVectorTransformationFunctionOperatorFactory("sin", functions.Sin),
"sinh": InstantVectorTransformationFunctionOperatorFactory("sinh", functions.Sinh),
"sqrt": InstantVectorTransformationFunctionOperatorFactory("sqrt", functions.Sqrt),
"sum_over_time": FunctionOverRangeVectorOperatorFactory("sum_over_time", functions.SumOverTime),
"tan": InstantVectorTransformationFunctionOperatorFactory("tan", functions.Tan),
"tanh": InstantVectorTransformationFunctionOperatorFactory("tanh", functions.Tanh),
"vector": scalarToInstantVectorOperatorFactory,
"abs": InstantVectorTransformationFunctionOperatorFactory("abs", functions.Abs),
"acos": InstantVectorTransformationFunctionOperatorFactory("acos", functions.Acos),
"acosh": InstantVectorTransformationFunctionOperatorFactory("acosh", functions.Acosh),
"asin": InstantVectorTransformationFunctionOperatorFactory("asin", functions.Asin),
"asinh": InstantVectorTransformationFunctionOperatorFactory("asinh", functions.Asinh),
"atan": InstantVectorTransformationFunctionOperatorFactory("atan", functions.Atan),
"atanh": InstantVectorTransformationFunctionOperatorFactory("atanh", functions.Atanh),
"avg_over_time": FunctionOverRangeVectorOperatorFactory("avg_over_time", functions.AvgOverTime),
"ceil": InstantVectorTransformationFunctionOperatorFactory("ceil", functions.Ceil),
"changes": FunctionOverRangeVectorOperatorFactory("changes", functions.Changes),
"clamp": ClampFunctionOperatorFactory(),
"clamp_max": ClampMinMaxFunctionOperatorFactory("clamp_max", false),
"clamp_min": ClampMinMaxFunctionOperatorFactory("clamp_min", true),
"cos": InstantVectorTransformationFunctionOperatorFactory("cos", functions.Cos),
"cosh": InstantVectorTransformationFunctionOperatorFactory("cosh", functions.Cosh),
"count_over_time": FunctionOverRangeVectorOperatorFactory("count_over_time", functions.CountOverTime),
"deg": InstantVectorTransformationFunctionOperatorFactory("deg", functions.Deg),
"deriv": FunctionOverRangeVectorOperatorFactory("deriv", functions.Deriv),
"exp": InstantVectorTransformationFunctionOperatorFactory("exp", functions.Exp),
"floor": InstantVectorTransformationFunctionOperatorFactory("floor", functions.Floor),
"histogram_count": InstantVectorTransformationFunctionOperatorFactory("histogram_count", functions.HistogramCount),
"histogram_quantile": HistogramQuantileOperatorFactory(),
"histogram_sum": InstantVectorTransformationFunctionOperatorFactory("histogram_sum", functions.HistogramSum),
"increase": FunctionOverRangeVectorOperatorFactory("increase", functions.Increase),
"label_replace": LabelReplaceFunctionOperatorFactory(),
"last_over_time": FunctionOverRangeVectorOperatorFactory("last_over_time", functions.LastOverTime),
"ln": InstantVectorTransformationFunctionOperatorFactory("ln", functions.Ln),
"log10": InstantVectorTransformationFunctionOperatorFactory("log10", functions.Log10),
"log2": InstantVectorTransformationFunctionOperatorFactory("log2", functions.Log2),
"max_over_time": FunctionOverRangeVectorOperatorFactory("max_over_time", functions.MaxOverTime),
"min_over_time": FunctionOverRangeVectorOperatorFactory("min_over_time", functions.MinOverTime),
"present_over_time": FunctionOverRangeVectorOperatorFactory("present_over_time", functions.PresentOverTime),
"rad": InstantVectorTransformationFunctionOperatorFactory("rad", functions.Rad),
"rate": FunctionOverRangeVectorOperatorFactory("rate", functions.Rate),
"resets": FunctionOverRangeVectorOperatorFactory("resets", functions.Resets),
"round": RoundFunctionOperatorFactory(),
"sgn": InstantVectorTransformationFunctionOperatorFactory("sgn", functions.Sgn),
"sin": InstantVectorTransformationFunctionOperatorFactory("sin", functions.Sin),
"sinh": InstantVectorTransformationFunctionOperatorFactory("sinh", functions.Sinh),
"sqrt": InstantVectorTransformationFunctionOperatorFactory("sqrt", functions.Sqrt),
"sum_over_time": FunctionOverRangeVectorOperatorFactory("sum_over_time", functions.SumOverTime),
"tan": InstantVectorTransformationFunctionOperatorFactory("tan", functions.Tan),
"tanh": InstantVectorTransformationFunctionOperatorFactory("tanh", functions.Tanh),
"vector": scalarToInstantVectorOperatorFactory,
}

func RegisterInstantVectorFunctionOperatorFactory(functionName string, factory InstantVectorFunctionOperatorFactory) error {
Expand Down
Loading
Loading