From be1bd9615a1e8f1e951794eaeccf40d17f39db47 Mon Sep 17 00:00:00 2001 From: chyezh Date: Tue, 6 Feb 2024 22:00:49 +0800 Subject: [PATCH] enhance: add configurable memory index load predict memory usage factor (#30563) pr: #30561 related pr: #30475 Signed-off-by: chyezh --- internal/querynodev2/segments/segment_loader.go | 6 +++--- pkg/util/paramtable/component_param.go | 10 ++++++++++ pkg/util/paramtable/component_param_test.go | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index 40f28da8b0642..ab0f9bd11b840 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -918,7 +918,7 @@ func GetIndexResourceUsage(indexInfo *querypb.FieldIndexInfo) (uint64, uint64, e return uint64(neededMemSize), uint64(neededDiskSize), nil } - factor := uint64(1) + factor := float64(1) var isLoadWithDisk bool GetDynamicPool().Submit(func() (any, error) { @@ -930,10 +930,10 @@ func GetIndexResourceUsage(indexInfo *querypb.FieldIndexInfo) (uint64, uint64, e }).Await() if !isLoadWithDisk { - factor = 2 + factor = paramtable.Get().QueryNodeCfg.MemoryIndexLoadPredictMemoryUsageFactor.GetAsFloat() } - return uint64(indexInfo.IndexSize) * factor, 0, nil + return uint64(float64(indexInfo.IndexSize) * factor), 0, nil } // checkSegmentSize checks whether the memory & disk is sufficient to load the segments with given concurrency, diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 66f311b269ee6..a051d6c2a646b 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -1807,6 +1807,8 @@ type queryNodeConfig struct { CGOPoolSizeRatio ParamItem `refreshable:"false"` EnableWorkerSQCostMetrics ParamItem `refreshable:"true"` + + MemoryIndexLoadPredictMemoryUsageFactor ParamItem `refreshable:"true"` } func (p *queryNodeConfig) init(base *BaseTable) { @@ -2214,6 +2216,14 @@ Max read concurrency must greater than or equal to 1, and less than or equal to Doc: "whether use worker's cost to measure delegator's workload", } p.EnableWorkerSQCostMetrics.Init(base.mgr) + + p.MemoryIndexLoadPredictMemoryUsageFactor = ParamItem{ + Key: "queryNode.memoryIndexLoadPredictMemoryUsageFactor", + Version: "2.3.8", + DefaultValue: "2.5", // HNSW index needs more memory to load. + Doc: "memory usage prediction factor for memory index loaded", + } + p.MemoryIndexLoadPredictMemoryUsageFactor.Init(base.mgr) } // ///////////////////////////////////////////////////////////////////////////// diff --git a/pkg/util/paramtable/component_param_test.go b/pkg/util/paramtable/component_param_test.go index e6506624e96ad..16ee3d085453e 100644 --- a/pkg/util/paramtable/component_param_test.go +++ b/pkg/util/paramtable/component_param_test.go @@ -360,6 +360,10 @@ func TestComponentParam(t *testing.T) { assert.Equal(t, false, Params.EnableWorkerSQCostMetrics.GetAsBool()) + assert.Equal(t, 2.5, Params.MemoryIndexLoadPredictMemoryUsageFactor.GetAsFloat()) + params.Save("queryNode.memoryIndexLoadPredictMemoryUsageFactor", "2.0") + assert.Equal(t, 2.0, Params.MemoryIndexLoadPredictMemoryUsageFactor.GetAsFloat()) + params.Save("querynode.gracefulStopTimeout", "100") assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second)) })