test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 03:18:29 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

start background go routine to check if all the segments have indexed (#6290)

Browse Source 
Signed-off-by: yefu.chen <yefu.chen@zilliz.com>
This commit is contained in:
neza2017 2021-07-03 17:54:25 +08:00 committed by GitHub
parent a0430287a5
commit f8fa025dcc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 107 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
internal/rootcoord/meta_table.go
View File

@ -964,3 +964,29 @@ func (mt *metaTable) GetIndexByID(indexID typeutil.UniqueID) (*pb.IndexInfo, err
}
return &indexInfo, nil
}
func (mt *metaTable) dupMeta() (
map[typeutil.UniqueID]pb.CollectionInfo,
map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo,
map[typeutil.UniqueID]pb.IndexInfo,
) {
mt.ddLock.RLock()
defer mt.ddLock.RUnlock()
collID2Meta := map[typeutil.UniqueID]pb.CollectionInfo{}
segID2IndexMeta := map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo{}
indexID2Meta := map[typeutil.UniqueID]pb.IndexInfo{}
for k, v := range mt.collID2Meta {
collID2Meta[k] = v
}
for k, v := range mt.segID2IndexMeta {
segID2IndexMeta[k] = map[typeutil.UniqueID]pb.SegmentIndexInfo{}
for k2, v2 := range v {
segID2IndexMeta[k][k2] = v2
}
}
for k, v := range mt.indexID2Meta {
indexID2Meta[k] = v
}
return collID2Meta, segID2IndexMeta, indexID2Meta
}

83
internal/rootcoord/root_coord.go
View File

@ -281,8 +281,86 @@ func (c *Core) sessionLoop() {
}
}
func (c *Core) watchProxyLoop() {
func (c *Core) checkFlushedSegmentsLoop() {
ticker := time.NewTicker(10 * time.Minute)
for {
select {
case <-c.ctx.Done():
log.Debug("RootCoord context done,exit checkFlushedSegmentsLoop")
return
case <-ticker.C:
log.Debug("check flushed segments")
collID2Meta, segID2IndexMeta, indexID2Meta := c.MetaTable.dupMeta()
for _, collMeta := range collID2Meta {
if len(collMeta.FieldIndexes) == 0 {
continue
}
for _, partID := range collMeta.PartitionIDs {
ctx2, cancel2 := context.WithTimeout(c.ctx, 3*time.Minute)
segIDs, err := c.CallGetFlushedSegmentsService(ctx2, collMeta.ID, partID)
if err != nil {
log.Debug("get flushed segments from data coord failed",
zap.Int64("collection id", collMeta.ID),
zap.Int64("partition id", partID),
zap.Error(err))
} else {
for _, segID := range segIDs {
indexInfos := []*etcdpb.FieldIndexInfo{}
indexMeta, ok := segID2IndexMeta[segID]
if !ok {
indexInfos = append(indexInfos, collMeta.FieldIndexes...)
} else {
for _, idx := range collMeta.FieldIndexes {
if _, ok := indexMeta[idx.IndexID]; !ok {
indexInfos = append(indexInfos, idx)
}
}
}
for _, idxInfo := range indexInfos {
field, err := GetFieldSchemaByID(&collMeta, idxInfo.FiledID)
if err != nil {
log.Debug("GetFieldSchemaByID",
zap.Any("collection_meta", collMeta),
zap.Int64("field id", idxInfo.FiledID))
continue
}
indexMeta, ok := indexID2Meta[idxInfo.IndexID]
if !ok {
log.Debug("index meta not exist", zap.Int64("index_id", idxInfo.IndexID))
continue
}
info := etcdpb.SegmentIndexInfo{
SegmentID: segID,
FieldID: idxInfo.FiledID,
IndexID: idxInfo.IndexID,
EnableIndex: false,
}
log.Debug("build index by background checker",
zap.Int64("segment_id", segID),
zap.Int64("index_id", indexMeta.IndexID),
zap.Int64("collection_id", collMeta.ID))
info.BuildID, err = c.BuildIndex(ctx2, segID, field, &indexMeta, false)
if err != nil {
log.Debug("build index failed",
zap.Int64("segment_id", segID),
zap.Int64("field_id", field.FieldID),
zap.Int64("index_id", indexMeta.IndexID))
continue
}
if info.BuildID != 0 {
info.EnableIndex = true
}
if _, err := c.MetaTable.AddIndex(&info, collMeta.ID, partID); err != nil {
log.Debug("Add index into meta table failed", zap.Int64("collection_id", collMeta.ID), zap.Int64("index_id", info.IndexID), zap.Int64("build_id", info.BuildID), zap.Error(err))
}
}
}
}
cancel2()
}
}
}
}
}
func (c *Core) getSegments(ctx context.Context, collID typeutil.UniqueID) (map[typeutil.UniqueID]typeutil.UniqueID, error) {
@ -1028,6 +1106,7 @@ func (c *Core) Start() error {
go c.tsLoop()
go c.sessionLoop()
go c.chanTimeTick.StartWatch()
go c.checkFlushedSegmentsLoop()
c.stateCode.Store(internalpb.StateCode_Healthy)
})
log.Debug(typeutil.RootCoordRole, zap.String("State Code", internalpb.StateCode_name[int32(internalpb.StateCode_Healthy)]))