test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 11:29:48 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Suport to create flat index (#19398)

Browse Source 
Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>
This commit is contained in:
cai.zhang 2022-09-29 09:32:55 +08:00 committed by GitHub
parent 2e0aa0afd7
commit 3a770f7dd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/indexcoord/const.go
View File

@ -21,6 +21,7 @@ const (
IndexAddTaskName = "IndexAddTask"
CreateIndexTaskName = "CreateIndexTask"
flatIndex = "FLAT"
diskAnnIndex = "DISKANN"
invalidIndex = "invalid"
)

6
internal/indexcoord/index_builder.go
View File

@ -212,8 +212,9 @@ func (ib *indexBuilder) process(buildID UniqueID) bool {
deleteFunc(buildID)
return true
}
if meta.NumRows < Params.IndexCoordCfg.MinSegmentNumRowsToEnableIndex {
log.Ctx(ib.ctx).Debug("segment num rows is too few, no need to build index", zap.Int64("buildID", buildID),
indexParams := ib.meta.GetIndexParams(meta.CollectionID, meta.IndexID)
if getIndexType(indexParams) == flatIndex || meta.NumRows < Params.IndexCoordCfg.MinSegmentNumRowsToEnableIndex {
log.Ctx(ib.ctx).Debug("segment does not need index really", zap.Int64("buildID", buildID),
zap.Int64("segID", meta.SegmentID), zap.Int64("num rows", meta.NumRows))
if err := ib.meta.FinishTask(&indexpb.IndexTaskInfo{
BuildID: buildID,
@ -271,7 +272,6 @@ func (ib *indexBuilder) process(buildID UniqueID) bool {
}
typeParams := ib.meta.GetTypeParams(meta.CollectionID, meta.IndexID)
indexParams := ib.meta.GetIndexParams(meta.CollectionID, meta.IndexID)
var storageConfig *indexpb.StorageConfig
if Params.CommonCfg.StorageType == "local" {

2
internal/indexcoord/task.go
View File

@ -127,7 +127,7 @@ func (cit *CreateIndexTask) PreExecute(ctx context.Context) error {
log.Info("IndexCoord CreateIndexTask PreExecute", zap.Int64("collectionID", cit.req.CollectionID),
zap.Int64("fieldID", cit.req.FieldID), zap.String("indexName", cit.req.IndexName))
// TODO: check index type is disk index.
if GetIndexType(cit.req.GetIndexParams()) == diskAnnIndex && !cit.indexCoordClient.nodeManager.ClientSupportDisk() {
if getIndexType(cit.req.GetIndexParams()) == diskAnnIndex && !cit.indexCoordClient.nodeManager.ClientSupportDisk() {
return errors.New("all IndexNodes do not support disk indexes, please verify")
}
return nil

2
internal/indexcoord/util.go
View File

@ -74,7 +74,7 @@ func buildHandoffKey(collID, partID, segID UniqueID) string {
return fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, collID, partID, segID)
}
func GetIndexType(indexParams []*commonpb.KeyValuePair) string {
func getIndexType(indexParams []*commonpb.KeyValuePair) string {
for _, param := range indexParams {
if param.Key == "index_type" {
return param.Value

1
internal/proxy/task.go
View File

@ -1191,6 +1191,7 @@ func parseIndexParams(m []*commonpb.KeyValuePair) (map[string]string, error) {
}
_, exist := indexParams["index_type"] // TODO(dragondriver): change `index_type` to const variable
if !exist {
//return nil, errors.New("there is no index_type in index params")
indexParams["index_type"] = indexparamcheck.IndexFaissIvfPQ // IVF_PQ is the default index type
}
return indexParams, nil