test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 02:48:45 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: rewrite index params for compatibility (#35788)

Browse Source 
#32900

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2024-09-02 19:41:03 +08:00 committed by GitHub
parent 3698c53a72
commit 325f1987d9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 160 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
internal/datacoord/index_meta.go
View File

@ -210,6 +210,7 @@ func checkParams(fieldIndex *model.Index, req *indexpb.CreateIndexRequest) bool
for i, param2 := range req.GetUserIndexParams() {
if param2.Key == param1.Key && param2.Value == param1.Value {
exist = true
break
} else if param1.Key == common.MetricTypeKey && param2.Key == param1.Key && useAutoIndex && !req.GetUserAutoindexMetricTypeSpecified() {
// when users use autoindex, metric type is the only thing they can specify
// if they do not specify metric type, will use autoindex default metric type
@ -225,6 +226,7 @@ func checkParams(fieldIndex *model.Index, req *indexpb.CreateIndexRequest) bool
}
}
exist = true
break
}
}
if !exist {
@ -232,7 +234,24 @@ func checkParams(fieldIndex *model.Index, req *indexpb.CreateIndexRequest) bool
break
}
}
// Check whether new index type match old, if not, only
// allow autoindex config changed when upgraded to new config
// using store meta config to rewrite new config
if !notEq && req.GetIsAutoIndex() && useAutoIndex {
for _, param1 := range fieldIndex.IndexParams {
if param1.Key == common.IndexTypeKey &&
indexparamcheck.IsScalarIndexType(param1.Value) {
for _, param2 := range req.GetIndexParams() {
if param1.Key == param2.Key && param1.Value != param2.Value {
req.IndexParams = make([]*commonpb.KeyValuePair, len(fieldIndex.IndexParams))
copy(req.IndexParams, fieldIndex.IndexParams)
break
}
}
}
}
}
log.Info("final request", zap.Any("create index request", req.String()))
return !notEq
}
@ -254,8 +273,10 @@ func (m *indexMeta) CanCreateIndex(req *indexpb.CreateIndexRequest) (UniqueID, e
}
errMsg := "at most one distinct index is allowed per field"
log.Warn(errMsg,
zap.String("source index", fmt.Sprintf("{index_name: %s, field_id: %d, index_params: %v, type_params: %v}", index.IndexName, index.FieldID, index.IndexParams, index.TypeParams)),
zap.String("current index", fmt.Sprintf("{index_name: %s, field_id: %d, index_params: %v, type_params: %v}", req.GetIndexName(), req.GetFieldID(), req.GetIndexParams(), req.GetTypeParams())))
zap.String("source index", fmt.Sprintf("{index_name: %s, field_id: %d, index_params: %v, user_params: %v, type_params: %v}",
index.IndexName, index.FieldID, index.IndexParams, index.UserIndexParams, index.TypeParams)),
zap.String("current index", fmt.Sprintf("{index_name: %s, field_id: %d, index_params: %v, user_params: %v, type_params: %v}",
req.GetIndexName(), req.GetFieldID(), req.GetIndexParams(), req.GetUserIndexParams(), req.GetTypeParams())))
return 0, fmt.Errorf("CreateIndex failed: %s", errMsg)
}
if req.FieldID == index.FieldID {

130
internal/datacoord/index_meta_test.go
View File

@ -78,6 +78,136 @@ func TestReloadFromKV(t *testing.T) {
})
}
func TestMeta_ScalarAutoIndex(t *testing.T) {
var (
collID = UniqueID(1)
indexID = UniqueID(10)
fieldID = UniqueID(100)
indexName = "_default_idx"
typeParams = []*commonpb.KeyValuePair{}
indexParams = []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: "HYBRID",
},
}
userIndexParams = []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: common.AutoIndexName,
},
}
)
catalog := catalogmocks.NewDataCoordCatalog(t)
m := newSegmentIndexMeta(catalog)
req := &indexpb.CreateIndexRequest{
CollectionID: collID,
FieldID: fieldID,
IndexName: indexName,
TypeParams: typeParams,
IndexParams: indexParams,
Timestamp: 0,
IsAutoIndex: true,
UserIndexParams: userIndexParams,
}
t.Run("user index params consistent", func(t *testing.T) {
m.indexes[collID] = map[UniqueID]*model.Index{
indexID: {
TenantID: "",
CollectionID: collID,
FieldID: fieldID,
IndexID: indexID,
IndexName: indexName,
IsDeleted: false,
CreateTime: 10,
TypeParams: typeParams,
IndexParams: indexParams,
IsAutoIndex: false,
UserIndexParams: userIndexParams,
},
}
tmpIndexID, err := m.CanCreateIndex(req)
assert.NoError(t, err)
assert.Equal(t, int64(indexID), tmpIndexID)
})
t.Run("user index params not consistent", func(t *testing.T) {
m.indexes[collID] = map[UniqueID]*model.Index{
indexID: {
TenantID: "",
CollectionID: collID,
FieldID: fieldID,
IndexID: indexID,
IndexName: indexName,
IsDeleted: false,
CreateTime: 10,
TypeParams: typeParams,
IndexParams: indexParams,
IsAutoIndex: false,
UserIndexParams: userIndexParams,
},
}
req.UserIndexParams = append(req.UserIndexParams, &commonpb.KeyValuePair{Key: "bitmap_cardinality_limit", Value: "1000"})
tmpIndexID, err := m.CanCreateIndex(req)
assert.Error(t, err)
assert.Equal(t, int64(0), tmpIndexID)
req.UserIndexParams = append(req.UserIndexParams, &commonpb.KeyValuePair{Key: "bitmap_cardinality_limit", Value: "500"})
tmpIndexID, err = m.CanCreateIndex(req)
assert.Error(t, err)
assert.Equal(t, int64(0), tmpIndexID)
})
req = &indexpb.CreateIndexRequest{
CollectionID: collID,
FieldID: fieldID,
IndexName: indexName,
TypeParams: typeParams,
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: "HYBRID",
}},
Timestamp: 0,
IsAutoIndex: true,
UserIndexParams: userIndexParams,
}
t.Run("index param rewrite", func(t *testing.T) {
m.indexes[collID] = map[UniqueID]*model.Index{
indexID: {
TenantID: "",
CollectionID: collID,
FieldID: fieldID,
IndexID: indexID,
IndexName: indexName,
IsDeleted: false,
CreateTime: 10,
TypeParams: typeParams,
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: "INVERTED",
},
},
IsAutoIndex: false,
UserIndexParams: userIndexParams,
},
}
tmpIndexID, err := m.CanCreateIndex(req)
assert.NoError(t, err)
assert.Equal(t, int64(indexID), tmpIndexID)
newIndexParams := req.GetIndexParams()
assert.Equal(t, len(newIndexParams), 1)
assert.Equal(t, newIndexParams[0].Key, common.IndexTypeKey)
assert.Equal(t, newIndexParams[0].Value, "INVERTED")
})
}
func TestMeta_CanCreateIndex(t *testing.T) {
var (
collID = UniqueID(1)

1
internal/proxy/task_index.go
View File

@ -197,6 +197,7 @@ func (cit *createIndexTask) parseIndexParams() error {
}
indexParamsMap[common.IndexTypeKey] = indexType
cit.isAutoIndex = true
}
} else {
specifyIndexType, exist := indexParamsMap[common.IndexTypeKey]

5
pkg/util/indexparamcheck/index_type.go
View File

@ -52,6 +52,11 @@ const (
AutoIndex IndexType = "AUTOINDEX"
)
func IsScalarIndexType(indexType IndexType) bool {
return indexType == IndexSTLSORT || indexType == IndexTRIE || indexType == IndexTrie ||
indexType == IndexBitmap || indexType == IndexHybrid || indexType == IndexINVERTED
}
func IsGpuIndex(indexType IndexType) bool {
return indexType == IndexGpuBF ||
indexType == IndexRaftIvfFlat ||