enhance: Use common gc config (#36668) (#36670)

Use the GC config from `common` and remove the GC config from
`queryNode`.

issue: https://github.com/milvus-io/milvus/issues/36667

pr: https://github.com/milvus-io/milvus/pull/36668

related pr: https://github.com/milvus-io/milvus/pull/34949

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
yihao.dai 2024-10-09 19:49:20 +08:00 committed by GitHub
parent 3a80d1f602
commit 9cb5396cf6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 17 additions and 44 deletions

View File

@ -402,10 +402,10 @@ func (mr *MilvusRoles) Run() {
action := func(GOGC uint32) { action := func(GOGC uint32) {
debug.SetGCPercent(int(GOGC)) debug.SetGCPercent(int(GOGC))
} }
gc.NewTuner(paramtable.Get().CommonCfg.OverloadedMemoryThresholdPercentage.GetAsFloat(), uint32(paramtable.Get().QueryNodeCfg.MinimumGOGCConfig.GetAsInt()), uint32(paramtable.Get().QueryNodeCfg.MaximumGOGCConfig.GetAsInt()), action) gc.NewTuner(paramtable.Get().CommonCfg.OverloadedMemoryThresholdPercentage.GetAsFloat(), uint32(paramtable.Get().CommonCfg.MinimumGOGCConfig.GetAsInt()), uint32(paramtable.Get().CommonCfg.MaximumGOGCConfig.GetAsInt()), action)
} else { } else {
action := func(uint32) {} action := func(uint32) {}
gc.NewTuner(paramtable.Get().CommonCfg.OverloadedMemoryThresholdPercentage.GetAsFloat(), uint32(paramtable.Get().QueryNodeCfg.MinimumGOGCConfig.GetAsInt()), uint32(paramtable.Get().QueryNodeCfg.MaximumGOGCConfig.GetAsInt()), action) gc.NewTuner(paramtable.Get().CommonCfg.OverloadedMemoryThresholdPercentage.GetAsFloat(), uint32(paramtable.Get().CommonCfg.MinimumGOGCConfig.GetAsInt()), uint32(paramtable.Get().CommonCfg.MaximumGOGCConfig.GetAsInt()), action)
} }
} }

View File

@ -63,7 +63,7 @@ func (suite *HandlersSuite) SetupTest() {
var err error var err error
paramtable.Init() paramtable.Init()
suite.params = paramtable.Get() suite.params = paramtable.Get()
suite.params.Save(suite.params.QueryNodeCfg.GCEnabled.Key, "false") suite.params.Save(suite.params.CommonCfg.GCEnabled.Key, "false")
// mock factory // mock factory
suite.factory = dependency.NewMockFactory(suite.T()) suite.factory = dependency.NewMockFactory(suite.T())

View File

@ -71,7 +71,7 @@ func (suite *LocalWorkerTestSuite) BeforeTest(suiteName, testName string) {
paramtable.Init() paramtable.Init()
suite.params = paramtable.Get() suite.params = paramtable.Get()
// close GC at test to avoid data race // close GC at test to avoid data race
suite.params.Save(suite.params.QueryNodeCfg.GCEnabled.Key, "false") suite.params.Save(suite.params.CommonCfg.GCEnabled.Key, "false")
suite.ctx, suite.cancel = context.WithCancel(context.Background()) suite.ctx, suite.cancel = context.WithCancel(context.Background())
// init node // init node

View File

@ -65,7 +65,7 @@ func (suite *QueryNodeSuite) SetupTest() {
var err error var err error
paramtable.Init() paramtable.Init()
suite.params = paramtable.Get() suite.params = paramtable.Get()
suite.params.Save(suite.params.QueryNodeCfg.GCEnabled.Key, "false") suite.params.Save(suite.params.CommonCfg.GCEnabled.Key, "false")
// mock factory // mock factory
suite.factory = dependency.NewMockFactory(suite.T()) suite.factory = dependency.NewMockFactory(suite.T())

View File

@ -94,7 +94,7 @@ func (suite *ServiceSuite) SetupSuite() {
// collection and segments data // collection and segments data
// init param // init param
paramtable.Init() paramtable.Init()
paramtable.Get().Save(paramtable.Get().QueryNodeCfg.GCEnabled.Key, "false") paramtable.Get().Save(paramtable.Get().CommonCfg.GCEnabled.Key, "false")
suite.rootPath = suite.T().Name() suite.rootPath = suite.T().Name()
suite.collectionID = 111 suite.collectionID = 111

View File

@ -254,11 +254,13 @@ type commonConfig struct {
UseVectorAsClusteringKey ParamItem `refreshable:"true"` UseVectorAsClusteringKey ParamItem `refreshable:"true"`
EnableVectorClusteringKey ParamItem `refreshable:"true"` EnableVectorClusteringKey ParamItem `refreshable:"true"`
GCEnabled ParamItem `refreshable:"false"` // GC
GCHelperEnabled ParamItem `refreshable:"false"` GCEnabled ParamItem `refreshable:"false"`
GCHelperEnabled ParamItem `refreshable:"false"`
MaximumGOGCConfig ParamItem `refreshable:"false"`
MinimumGOGCConfig ParamItem `refreshable:"false"`
OverloadedMemoryThresholdPercentage ParamItem `refreshable:"false"` OverloadedMemoryThresholdPercentage ParamItem `refreshable:"false"`
MaximumGOGCConfig ParamItem `refreshable:"false"`
MinimumGOGCConfig ParamItem `refreshable:"false"`
ReadOnlyPrivileges ParamItem `refreshable:"false"` ReadOnlyPrivileges ParamItem `refreshable:"false"`
ReadWritePrivileges ParamItem `refreshable:"false"` ReadWritePrivileges ParamItem `refreshable:"false"`
AdminPrivileges ParamItem `refreshable:"false"` AdminPrivileges ParamItem `refreshable:"false"`
@ -850,6 +852,7 @@ This helps Milvus-CDC synchronize incremental data`,
p.GCEnabled = ParamItem{ p.GCEnabled = ParamItem{
Key: "common.gcenabled", Key: "common.gcenabled",
FallbackKeys: []string{"queryNode.gcenabled"},
Version: "2.4.7", Version: "2.4.7",
DefaultValue: "true", DefaultValue: "true",
} }
@ -857,6 +860,7 @@ This helps Milvus-CDC synchronize incremental data`,
p.GCHelperEnabled = ParamItem{ p.GCHelperEnabled = ParamItem{
Key: "common.gchelper.enabled", Key: "common.gchelper.enabled",
FallbackKeys: []string{"queryNode.gchelper.enabled"},
Version: "2.4.7", Version: "2.4.7",
DefaultValue: "true", DefaultValue: "true",
} }
@ -875,6 +879,7 @@ This helps Milvus-CDC synchronize incremental data`,
p.MaximumGOGCConfig = ParamItem{ p.MaximumGOGCConfig = ParamItem{
Key: "common.gchelper.maximumGoGC", Key: "common.gchelper.maximumGoGC",
FallbackKeys: []string{"queryNode.gchelper.maximumGoGC"},
Version: "2.4.7", Version: "2.4.7",
DefaultValue: "200", DefaultValue: "200",
} }
@ -882,6 +887,7 @@ This helps Milvus-CDC synchronize incremental data`,
p.MinimumGOGCConfig = ParamItem{ p.MinimumGOGCConfig = ParamItem{
Key: "common.gchelper.minimumGoGC", Key: "common.gchelper.minimumGoGC",
FallbackKeys: []string{"queryNode.gchelper.minimumGoGC"},
Version: "2.4.7", Version: "2.4.7",
DefaultValue: "30", DefaultValue: "30",
} }
@ -2371,12 +2377,7 @@ type queryNodeConfig struct {
TopKMergeRatio ParamItem `refreshable:"true"` TopKMergeRatio ParamItem `refreshable:"true"`
CPURatio ParamItem `refreshable:"true"` CPURatio ParamItem `refreshable:"true"`
MaxTimestampLag ParamItem `refreshable:"true"` MaxTimestampLag ParamItem `refreshable:"true"`
GCEnabled ParamItem `refreshable:"true"` GracefulStopTimeout ParamItem `refreshable:"false"`
GCHelperEnabled ParamItem `refreshable:"false"`
MinimumGOGCConfig ParamItem `refreshable:"false"`
MaximumGOGCConfig ParamItem `refreshable:"false"`
GracefulStopTimeout ParamItem `refreshable:"false"`
// delete buffer // delete buffer
MaxSegmentDeleteBuffer ParamItem `refreshable:"false"` MaxSegmentDeleteBuffer ParamItem `refreshable:"false"`
@ -2925,34 +2926,6 @@ Max read concurrency must greater than or equal to 1, and less than or equal to
} }
p.MaxTimestampLag.Init(base.mgr) p.MaxTimestampLag.Init(base.mgr)
p.GCEnabled = ParamItem{
Key: "queryNode.gcenabled",
Version: "2.3.0",
DefaultValue: "true",
}
p.GCEnabled.Init(base.mgr)
p.GCHelperEnabled = ParamItem{
Key: "queryNode.gchelper.enabled",
Version: "2.0.0",
DefaultValue: "true",
}
p.GCHelperEnabled.Init(base.mgr)
p.MaximumGOGCConfig = ParamItem{
Key: "queryNode.gchelper.maximumGoGC",
Version: "2.0.0",
DefaultValue: "200",
}
p.MaximumGOGCConfig.Init(base.mgr)
p.MinimumGOGCConfig = ParamItem{
Key: "queryNode.gchelper.minimumGoGC",
Version: "2.0.0",
DefaultValue: "30",
}
p.MinimumGOGCConfig.Init(base.mgr)
p.GracefulStopTimeout = ParamItem{ p.GracefulStopTimeout = ParamItem{
Key: "queryNode.gracefulStopTimeout", Key: "queryNode.gracefulStopTimeout",
Version: "2.2.1", Version: "2.2.1",