test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 02:48:45 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Make autoindex config clean (#26732)

Browse Source 
Signed-off-by: chasingegg <chao.gao@zilliz.com>
This commit is contained in:
Gao 2023-08-31 10:25:01 +08:00 committed by GitHub
parent 41d37409e9
commit 7e36f819b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 28 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
internal/querynodev2/server.go
View File

@ -493,10 +493,10 @@ func (node *QueryNode) initHook() error {
if !ok {
return fmt.Errorf("fail to convert the `Hook` interface")
}
if err = hoo.Init(paramtable.Get().HookCfg.QueryNodePluginConfig.GetValue()); err != nil {
if err = hoo.Init(paramtable.Get().AutoIndexConfig.AutoIndexSearchConfig.GetValue()); err != nil {
return fmt.Errorf("fail to init configs for the hook, error: %s", err.Error())
}
if err = hoo.InitTuningConfig(paramtable.Get().HookCfg.QueryNodePluginTuningConfig.GetValue()); err != nil {
if err = hoo.InitTuningConfig(paramtable.Get().AutoIndexConfig.AutoIndexTuningConfig.GetValue()); err != nil {
return fmt.Errorf("fail to init tuning configs for the hook, error: %s", err.Error())
}
@ -515,8 +515,8 @@ func (node *QueryNode) handleQueryHookEvent() {
}
}
onEvent2 := func(event *config.Event) {
if node.queryHook != nil && strings.HasPrefix(event.Key, paramtable.Get().HookCfg.QueryNodePluginTuningConfig.KeyPrefix) {
realKey := strings.TrimPrefix(event.Key, paramtable.Get().HookCfg.QueryNodePluginTuningConfig.KeyPrefix)
if node.queryHook != nil && strings.HasPrefix(event.Key, paramtable.Get().AutoIndexConfig.AutoIndexTuningConfig.KeyPrefix) {
realKey := strings.TrimPrefix(event.Key, paramtable.Get().AutoIndexConfig.AutoIndexTuningConfig.KeyPrefix)
if event.EventType == config.CreateType || event.EventType == config.UpdateType {
if err := node.queryHook.InitTuningConfig(map[string]string{realKey: event.Value}); err != nil {
log.Warn("failed to refresh hook tuning config", zap.Error(err))
@ -528,7 +528,7 @@ func (node *QueryNode) handleQueryHookEvent() {
}
}
}
paramtable.Get().Watch(paramtable.Get().HookCfg.QueryNodePluginConfig.Key, config.NewHandler("queryHook", onEvent))
paramtable.Get().Watch(paramtable.Get().AutoIndexConfig.AutoIndexSearchConfig.Key, config.NewHandler("queryHook", onEvent))
paramtable.Get().WatchKeyPrefix(paramtable.Get().HookCfg.QueryNodePluginTuningConfig.KeyPrefix, config.NewHandler("queryHook2", onEvent2))
paramtable.Get().WatchKeyPrefix(paramtable.Get().AutoIndexConfig.AutoIndexTuningConfig.KeyPrefix, config.NewHandler("queryHook2", onEvent2))
}

22
pkg/util/paramtable/autoindex_param.go
View File

@ -31,10 +31,12 @@ import (
type autoIndexConfig struct {
Enable ParamItem `refreshable:"true"`
IndexParams ParamItem `refreshable:"true"`
ExtraParams ParamItem `refreshable:"true"`
IndexType ParamItem `refreshable:"true"`
AutoIndexTypeName ParamItem `refreshable:"true"`
IndexParams ParamItem `refreshable:"true"`
ExtraParams ParamItem `refreshable:"true"`
IndexType ParamItem `refreshable:"true"`
AutoIndexTypeName ParamItem `refreshable:"true"`
AutoIndexSearchConfig ParamItem `refreshable:"true"`
AutoIndexTuningConfig ParamGroup `refreshable:"true"`
}
func (p *autoIndexConfig) init(base *BaseTable) {
@ -77,6 +79,18 @@ func (p *autoIndexConfig) init(base *BaseTable) {
}
p.AutoIndexTypeName.Init(base.mgr)
p.AutoIndexSearchConfig = ParamItem{
Key: "autoindex.params.search",
Version: "2.2.0",
}
p.AutoIndexSearchConfig.Init(base.mgr)
p.AutoIndexTuningConfig = ParamGroup{
KeyPrefix: "autoindex.params.tuning.",
Version: "2.3.0",
}
p.AutoIndexTuningConfig.Init(base.mgr)
p.panicIfNotValidAndSetDefaultMetricType(base.mgr)
}

27
pkg/util/paramtable/autoindex_param_test.go
View File

@ -66,33 +66,6 @@ func TestAutoIndexParams_build(t *testing.T) {
assert.Equal(t, "IVF_FLAT", CParams.AutoIndexConfig.IndexType.GetValue())
assert.Equal(t, strconv.Itoa(map2["nlist"].(int)), CParams.AutoIndexConfig.IndexParams.GetAsJSONMap()["nlist"])
})
// t.Run("test parseBuildParams miss total", func(t *testing.T) {
// defer func() {
// if r := recover(); r == nil {
// t.Errorf("The code did not panic")
// }
// }()
// CParams.Save(CParams.AutoIndexConfig.IndexParams.Key, "")
//
// })
//
// t.Run("test parseBuildParams miss index_type", func(t *testing.T) {
// defer func() {
// if r := recover(); r == nil {
// t.Errorf("The code did not panic")
// }
// }()
// var err error
// map1 := map[string]any{
// "M": 48,
// "efConstruction": 500,
// }
// var jsonStrBytes []byte
// jsonStrBytes, err = json.Marshal(map1)
// assert.NoError(t, err)
// CParams.Save(CParams.AutoIndexConfig.IndexParams.Key, string(jsonStrBytes))
// })
}
func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {

2
pkg/util/paramtable/component_param.go
View File

@ -118,7 +118,7 @@ func (p *ComponentParam) init() {
p.IndexNodeCfg.init(&p.BaseTable)
p.HTTPCfg.init(&p.BaseTable)
p.LogCfg.init(&p.BaseTable)
p.HookCfg.init(&p.BaseTable)
p.HookCfg.init()
p.RootCoordGrpcServerCfg.Init("rootCoord", &p.BaseTable)
p.ProxyGrpcServerCfg.Init("proxy", &p.BaseTable)

20
pkg/util/paramtable/hook_config.go
View File

@ -11,13 +11,11 @@ const hookYamlFile = "hook.yaml"
type hookConfig struct {
hookBase *BaseTable
SoPath ParamItem `refreshable:"false"`
SoConfig ParamGroup `refreshable:"true"`
QueryNodePluginConfig ParamItem `refreshable:"true"`
QueryNodePluginTuningConfig ParamGroup `refreshable:"true"`
SoPath ParamItem `refreshable:"false"`
SoConfig ParamGroup `refreshable:"true"`
}
func (h *hookConfig) init(base *BaseTable) {
func (h *hookConfig) init() {
hookBase := &BaseTable{YamlFiles: []string{hookYamlFile}}
hookBase.init(2)
h.hookBase = hookBase
@ -36,18 +34,6 @@ func (h *hookConfig) init(base *BaseTable) {
Version: "2.2.0",
}
h.SoConfig.Init(hookBase.mgr)
h.QueryNodePluginConfig = ParamItem{
Key: "autoindex.params.search",
Version: "2.3.0",
}
h.QueryNodePluginConfig.Init(base.mgr)
h.QueryNodePluginTuningConfig = ParamGroup{
KeyPrefix: "autoindex.params.tuning.",
Version: "2.3.0",
}
h.QueryNodePluginTuningConfig.Init(base.mgr)
}
func (h *hookConfig) WatchHookWithPrefix(ident string, keyPrefix string, onEvent func(*config.Event)) {