2022-07-07 17:44:21 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2022-07-07 14:44:21 +08:00
|
|
|
package indexcoord
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2022-09-23 09:58:51 +08:00
|
|
|
"errors"
|
2022-08-25 19:32:53 +08:00
|
|
|
"path"
|
2022-09-09 15:52:35 +08:00
|
|
|
"sort"
|
2022-07-07 14:44:21 +08:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2022-08-25 15:48:54 +08:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2022-10-16 20:49:27 +08:00
|
|
|
"github.com/milvus-io/milvus-proto/go-api/commonpb"
|
2022-09-09 15:52:35 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/common"
|
2022-07-07 14:44:21 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
|
|
|
"github.com/milvus-io/milvus/internal/proto/indexpb"
|
|
|
|
)
|
|
|
|
|
|
|
|
type indexBuilder struct {
|
|
|
|
ctx context.Context
|
|
|
|
cancel context.CancelFunc
|
|
|
|
|
|
|
|
wg sync.WaitGroup
|
|
|
|
taskMutex sync.RWMutex
|
|
|
|
scheduleDuration time.Duration
|
|
|
|
|
|
|
|
// TODO @xiaocai2333: use priority queue
|
2022-09-09 15:52:35 +08:00
|
|
|
tasks map[int64]indexTaskState
|
|
|
|
notifyChan chan struct{}
|
2022-07-07 14:44:21 +08:00
|
|
|
|
|
|
|
ic *IndexCoord
|
|
|
|
|
|
|
|
meta *metaTable
|
|
|
|
}
|
|
|
|
|
|
|
|
func newIndexBuilder(ctx context.Context, ic *IndexCoord, metaTable *metaTable, aliveNodes []UniqueID) *indexBuilder {
|
|
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
|
|
|
|
|
|
ib := &indexBuilder{
|
|
|
|
ctx: ctx,
|
|
|
|
cancel: cancel,
|
|
|
|
meta: metaTable,
|
|
|
|
ic: ic,
|
2022-09-09 15:52:35 +08:00
|
|
|
tasks: make(map[int64]indexTaskState),
|
|
|
|
notifyChan: make(chan struct{}, 1),
|
2022-09-15 15:44:31 +08:00
|
|
|
scheduleDuration: time.Second,
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
ib.reloadFromKV(aliveNodes)
|
|
|
|
return ib
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) Start() {
|
|
|
|
ib.wg.Add(1)
|
|
|
|
go ib.schedule()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) Stop() {
|
|
|
|
ib.cancel()
|
|
|
|
ib.wg.Wait()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) reloadFromKV(aliveNodes []UniqueID) {
|
|
|
|
metas := ib.meta.GetAllIndexMeta()
|
|
|
|
for build, indexMeta := range metas {
|
|
|
|
// deleted, need to release lock and clean meta
|
2022-08-25 15:48:54 +08:00
|
|
|
if indexMeta.IsDeleted || ib.meta.IsIndexDeleted(indexMeta.CollectionID, indexMeta.IndexID) {
|
2022-07-07 14:44:21 +08:00
|
|
|
if indexMeta.NodeID != 0 {
|
|
|
|
ib.tasks[build] = indexTaskDeleted
|
|
|
|
}
|
2022-08-25 15:48:54 +08:00
|
|
|
} else if indexMeta.IndexState == commonpb.IndexState_Unissued && indexMeta.NodeID == 0 {
|
2022-07-07 14:44:21 +08:00
|
|
|
// unissued, need to acquire lock and assign task
|
|
|
|
ib.tasks[build] = indexTaskInit
|
2022-08-25 15:48:54 +08:00
|
|
|
} else if indexMeta.IndexState == commonpb.IndexState_Unissued && indexMeta.NodeID != 0 {
|
2022-07-07 14:44:21 +08:00
|
|
|
// retry, need to release lock and reassign task
|
|
|
|
// need to release reference lock
|
|
|
|
ib.tasks[build] = indexTaskRetry
|
2022-08-25 15:48:54 +08:00
|
|
|
} else if indexMeta.IndexState == commonpb.IndexState_InProgress {
|
2022-07-07 14:44:21 +08:00
|
|
|
// need to check IndexNode is still alive.
|
|
|
|
alive := false
|
|
|
|
for _, nodeID := range aliveNodes {
|
|
|
|
if nodeID == indexMeta.NodeID {
|
|
|
|
alive = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !alive {
|
|
|
|
// IndexNode is down, need to retry
|
|
|
|
ib.tasks[build] = indexTaskRetry
|
|
|
|
} else {
|
|
|
|
// in_progress, nothing to do
|
|
|
|
ib.tasks[build] = indexTaskInProgress
|
|
|
|
}
|
2022-08-25 15:48:54 +08:00
|
|
|
} else if indexMeta.IndexState == commonpb.IndexState_Finished || indexMeta.IndexState == commonpb.IndexState_Failed {
|
2022-07-07 14:44:21 +08:00
|
|
|
if indexMeta.NodeID != 0 {
|
|
|
|
// task is done, but the lock has not been released, need to release.
|
|
|
|
ib.tasks[build] = indexTaskDone
|
|
|
|
}
|
|
|
|
// else: task is done, and lock has been released, no need to add to index builder.
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-09 15:52:35 +08:00
|
|
|
// notify is an unblocked notify function
|
|
|
|
func (ib *indexBuilder) notify() {
|
|
|
|
select {
|
|
|
|
case ib.notifyChan <- struct{}{}:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-07 14:44:21 +08:00
|
|
|
func (ib *indexBuilder) enqueue(buildID UniqueID) {
|
2022-09-09 15:52:35 +08:00
|
|
|
defer ib.notify()
|
|
|
|
|
2022-07-07 14:44:21 +08:00
|
|
|
ib.taskMutex.Lock()
|
|
|
|
defer ib.taskMutex.Unlock()
|
2022-09-15 15:44:31 +08:00
|
|
|
if _, ok := ib.tasks[buildID]; !ok {
|
|
|
|
ib.tasks[buildID] = indexTaskInit
|
|
|
|
}
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) schedule() {
|
2022-09-09 15:52:35 +08:00
|
|
|
// receive notifyChan
|
2022-07-07 14:44:21 +08:00
|
|
|
// time ticker
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("index builder schedule loop start")
|
2022-07-07 14:44:21 +08:00
|
|
|
defer ib.wg.Done()
|
|
|
|
ticker := time.NewTicker(ib.scheduleDuration)
|
|
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ib.ctx.Done():
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Warn("index builder ctx done")
|
2022-07-07 14:44:21 +08:00
|
|
|
return
|
2022-09-09 15:52:35 +08:00
|
|
|
case _, ok := <-ib.notifyChan:
|
2022-07-07 14:44:21 +08:00
|
|
|
if ok {
|
2022-09-09 15:52:35 +08:00
|
|
|
ib.run()
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
// !ok means indexBuild is closed.
|
2022-07-07 14:44:21 +08:00
|
|
|
case <-ticker.C:
|
2022-09-09 15:52:35 +08:00
|
|
|
ib.run()
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-09 15:52:35 +08:00
|
|
|
func (ib *indexBuilder) run() {
|
|
|
|
ib.taskMutex.RLock()
|
|
|
|
buildIDs := make([]UniqueID, 0, len(ib.tasks))
|
|
|
|
for tID := range ib.tasks {
|
|
|
|
buildIDs = append(buildIDs, tID)
|
|
|
|
}
|
|
|
|
ib.taskMutex.RUnlock()
|
|
|
|
|
|
|
|
sort.Slice(buildIDs, func(i, j int) bool {
|
|
|
|
return buildIDs[i] < buildIDs[j]
|
|
|
|
})
|
2022-09-15 15:44:31 +08:00
|
|
|
if len(buildIDs) > 0 {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("index builder task schedule", zap.Int("task num", len(buildIDs)))
|
2022-09-15 15:44:31 +08:00
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
for _, buildID := range buildIDs {
|
2022-09-24 13:40:52 +08:00
|
|
|
ok := ib.process(buildID)
|
|
|
|
if !ok {
|
|
|
|
log.Ctx(ib.ctx).Debug("there is no IndexNode available or etcd is not serviceable, wait a minute...")
|
|
|
|
break
|
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-24 13:40:52 +08:00
|
|
|
func (ib *indexBuilder) process(buildID UniqueID) bool {
|
2022-09-09 15:52:35 +08:00
|
|
|
ib.taskMutex.RLock()
|
2022-07-07 14:44:21 +08:00
|
|
|
state := ib.tasks[buildID]
|
2022-09-09 15:52:35 +08:00
|
|
|
ib.taskMutex.RUnlock()
|
|
|
|
|
|
|
|
updateStateFunc := func(buildID UniqueID, state indexTaskState) {
|
|
|
|
ib.taskMutex.Lock()
|
|
|
|
defer ib.taskMutex.Unlock()
|
|
|
|
ib.tasks[buildID] = state
|
|
|
|
}
|
|
|
|
|
|
|
|
deleteFunc := func(buildID UniqueID) {
|
|
|
|
ib.taskMutex.Lock()
|
|
|
|
defer ib.taskMutex.Unlock()
|
|
|
|
delete(ib.tasks, buildID)
|
|
|
|
}
|
|
|
|
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedDebug(10, "index task is processing", zap.Int64("buildID", buildID), zap.String("task state", state.String()))
|
2022-07-07 14:44:21 +08:00
|
|
|
meta, exist := ib.meta.GetMeta(buildID)
|
2022-09-09 15:52:35 +08:00
|
|
|
if !exist {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedDebug(10, "index task has not exist in meta table, remove task", zap.Int64("buildID", buildID))
|
2022-09-09 15:52:35 +08:00
|
|
|
deleteFunc(buildID)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-09-09 15:52:35 +08:00
|
|
|
}
|
2022-07-07 14:44:21 +08:00
|
|
|
|
|
|
|
switch state {
|
|
|
|
case indexTaskInit:
|
2022-09-09 15:52:35 +08:00
|
|
|
if !ib.meta.NeedIndex(meta.CollectionID, meta.IndexID) {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Debug("task is no need to build index, remove it", zap.Int64("buildID", buildID))
|
2022-09-09 15:52:35 +08:00
|
|
|
deleteFunc(buildID)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-09-09 15:52:35 +08:00
|
|
|
}
|
2022-09-29 09:32:55 +08:00
|
|
|
indexParams := ib.meta.GetIndexParams(meta.CollectionID, meta.IndexID)
|
|
|
|
if getIndexType(indexParams) == flatIndex || meta.NumRows < Params.IndexCoordCfg.MinSegmentNumRowsToEnableIndex {
|
|
|
|
log.Ctx(ib.ctx).Debug("segment does not need index really", zap.Int64("buildID", buildID),
|
2022-09-24 13:40:52 +08:00
|
|
|
zap.Int64("segID", meta.SegmentID), zap.Int64("num rows", meta.NumRows))
|
2022-09-09 15:52:35 +08:00
|
|
|
if err := ib.meta.FinishTask(&indexpb.IndexTaskInfo{
|
|
|
|
BuildID: buildID,
|
|
|
|
State: commonpb.IndexState_Finished,
|
2022-10-19 16:55:27 +08:00
|
|
|
IndexFileKeys: nil,
|
2022-09-09 15:52:35 +08:00
|
|
|
SerializedSize: 0,
|
|
|
|
FailReason: "",
|
|
|
|
}); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "IndexCoord update index state fail", zap.Int64("buildID", buildID), zap.Error(err))
|
|
|
|
return false
|
2022-09-09 15:52:35 +08:00
|
|
|
}
|
|
|
|
updateStateFunc(buildID, indexTaskDone)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
2022-07-07 14:44:21 +08:00
|
|
|
// peek client
|
|
|
|
// if all IndexNodes are executing task, wait for one of them to finish the task.
|
|
|
|
nodeID, client := ib.ic.nodeManager.PeekClient(meta)
|
|
|
|
if client == nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedDebug(10, "index builder peek client error, there is no available")
|
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
// update version and set nodeID
|
|
|
|
if err := ib.meta.UpdateVersion(buildID, nodeID); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder update index version failed", zap.Int64("build", buildID), zap.Error(err))
|
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// acquire lock
|
2022-08-25 15:48:54 +08:00
|
|
|
if err := ib.ic.tryAcquireSegmentReferLock(ib.ctx, buildID, nodeID, []UniqueID{meta.SegmentID}); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder acquire segment reference lock failed", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID), zap.Error(err))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskRetry)
|
2022-09-24 13:40:52 +08:00
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-23 09:58:51 +08:00
|
|
|
info, err := ib.ic.pullSegmentInfo(ib.ctx, meta.SegmentID)
|
2022-08-25 15:48:54 +08:00
|
|
|
if err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "IndexCoord get segment info from DataCoord fail", zap.Int64("segID", meta.SegmentID),
|
2022-08-25 15:48:54 +08:00
|
|
|
zap.Int64("buildID", buildID), zap.Error(err))
|
2022-09-23 09:58:51 +08:00
|
|
|
if errors.Is(err, ErrSegmentNotFound) {
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskDeleted)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-09-09 15:52:35 +08:00
|
|
|
}
|
|
|
|
updateStateFunc(buildID, indexTaskRetry)
|
2022-09-24 13:40:52 +08:00
|
|
|
return false
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
binLogs := make([]string, 0)
|
2022-09-23 09:58:51 +08:00
|
|
|
fieldID := ib.meta.GetFieldIDByIndexID(meta.CollectionID, meta.IndexID)
|
|
|
|
for _, fieldBinLog := range info.GetBinlogs() {
|
|
|
|
if fieldBinLog.GetFieldID() == fieldID {
|
|
|
|
for _, binLog := range fieldBinLog.GetBinlogs() {
|
|
|
|
binLogs = append(binLogs, binLog.LogPath)
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
2022-09-23 09:58:51 +08:00
|
|
|
break
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-09 15:52:35 +08:00
|
|
|
typeParams := ib.meta.GetTypeParams(meta.CollectionID, meta.IndexID)
|
2022-08-25 15:48:54 +08:00
|
|
|
|
2022-09-06 17:19:11 +08:00
|
|
|
var storageConfig *indexpb.StorageConfig
|
|
|
|
if Params.CommonCfg.StorageType == "local" {
|
|
|
|
storageConfig = &indexpb.StorageConfig{
|
|
|
|
RootPath: Params.LocalStorageCfg.Path,
|
|
|
|
StorageType: Params.CommonCfg.StorageType,
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
storageConfig = &indexpb.StorageConfig{
|
2022-08-25 15:48:54 +08:00
|
|
|
Address: Params.MinioCfg.Address,
|
|
|
|
AccessKeyID: Params.MinioCfg.AccessKeyID,
|
|
|
|
SecretAccessKey: Params.MinioCfg.SecretAccessKey,
|
|
|
|
UseSSL: Params.MinioCfg.UseSSL,
|
|
|
|
BucketName: Params.MinioCfg.BucketName,
|
|
|
|
RootPath: Params.MinioCfg.RootPath,
|
|
|
|
UseIAM: Params.MinioCfg.UseIAM,
|
|
|
|
IAMEndpoint: Params.MinioCfg.IAMEndpoint,
|
2022-09-06 17:19:11 +08:00
|
|
|
StorageType: Params.CommonCfg.StorageType,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
req := &indexpb.CreateJobRequest{
|
|
|
|
ClusterID: Params.CommonCfg.ClusterPrefix,
|
2022-08-25 19:32:53 +08:00
|
|
|
IndexFilePrefix: path.Join(ib.ic.chunkManager.RootPath(), common.SegmentIndexPath),
|
2022-08-25 15:48:54 +08:00
|
|
|
BuildID: buildID,
|
|
|
|
DataPaths: binLogs,
|
|
|
|
IndexVersion: meta.IndexVersion + 1,
|
2022-09-06 17:19:11 +08:00
|
|
|
StorageConfig: storageConfig,
|
2022-08-25 15:48:54 +08:00
|
|
|
IndexParams: indexParams,
|
|
|
|
TypeParams: typeParams,
|
2022-09-06 17:19:11 +08:00
|
|
|
NumRows: meta.NumRows,
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedDebug(10, "assign task to indexNode", zap.Int64("buildID", buildID), zap.Int64("nodeID", nodeID))
|
2022-07-07 14:44:21 +08:00
|
|
|
if err := ib.ic.assignTask(client, req); err != nil {
|
|
|
|
// need to release lock then reassign, so set task state to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder assign task to IndexNode failed", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID), zap.Error(err))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskRetry)
|
2022-09-24 13:40:52 +08:00
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
// update index meta state to InProgress
|
|
|
|
if err := ib.meta.BuildIndex(buildID); err != nil {
|
|
|
|
// need to release lock then reassign, so set task state to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder update index meta to InProgress failed", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID), zap.Error(err))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskRetry)
|
2022-09-24 13:40:52 +08:00
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedDebug(10, "index task assigned success", zap.Int64("buildID", buildID), zap.Int64("nodeID", nodeID))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskInProgress)
|
2022-07-07 14:44:21 +08:00
|
|
|
|
|
|
|
case indexTaskDone:
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedDebug(10, "index task has done", zap.Int64("buildID", buildID))
|
2022-09-09 15:52:35 +08:00
|
|
|
if !ib.meta.NeedIndex(meta.CollectionID, meta.IndexID) {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Debug("task is no need to build index, remove it", zap.Int64("buildID", buildID))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskDeleted)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if !ib.dropIndexTask(buildID, meta.NodeID) {
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
if err := ib.releaseLockAndResetNode(buildID, meta.NodeID); err != nil {
|
2022-07-07 14:44:21 +08:00
|
|
|
// release lock failed, no need to modify state, wait to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder try to release reference lock failed", zap.Error(err))
|
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
deleteFunc(buildID)
|
2022-07-07 14:44:21 +08:00
|
|
|
case indexTaskRetry:
|
2022-09-09 15:52:35 +08:00
|
|
|
if !ib.meta.NeedIndex(meta.CollectionID, meta.IndexID) {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Debug("task is no need to build index, remove it", zap.Int64("buildID", buildID))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskDeleted)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
if err := ib.releaseLockAndResetTask(buildID, meta.NodeID); err != nil {
|
2022-07-07 14:44:21 +08:00
|
|
|
// release lock failed, no need to modify state, wait to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder try to release reference lock failed", zap.Error(err))
|
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskInit)
|
2022-07-07 14:44:21 +08:00
|
|
|
|
|
|
|
case indexTaskDeleted:
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Debug("index task state is deleted, try to release reference lock", zap.Int64("buildID", buildID))
|
2022-09-15 15:44:31 +08:00
|
|
|
// TODO: delete after QueryCoordV2
|
|
|
|
if err := ib.meta.MarkSegmentsIndexAsDeletedByBuildID([]int64{buildID}); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
return false
|
2022-09-15 15:44:31 +08:00
|
|
|
}
|
|
|
|
if meta.NodeID != 0 {
|
2022-08-25 15:48:54 +08:00
|
|
|
if !ib.dropIndexTask(buildID, meta.NodeID) {
|
2022-09-23 09:58:51 +08:00
|
|
|
log.Ctx(ib.ctx).Warn("index task state is deleted and drop index job for node fail", zap.Int64("build", buildID),
|
|
|
|
zap.Int64("nodeID", meta.NodeID))
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
if err := ib.releaseLockAndResetNode(buildID, meta.NodeID); err != nil {
|
2022-07-07 14:44:21 +08:00
|
|
|
// release lock failed, no need to modify state, wait to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).RatedWarn(10, "index builder try to release reference lock failed", zap.Error(err))
|
|
|
|
return false
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// reset nodeID success, remove task.
|
2022-09-09 15:52:35 +08:00
|
|
|
deleteFunc(buildID)
|
2022-08-25 15:48:54 +08:00
|
|
|
|
|
|
|
default:
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Debug("index task is in progress", zap.Int64("buildID", buildID),
|
2022-08-25 15:48:54 +08:00
|
|
|
zap.String("state", meta.IndexState.String()))
|
2022-09-09 15:52:35 +08:00
|
|
|
if !ib.meta.NeedIndex(meta.CollectionID, meta.IndexID) {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Debug("task is no need to build index, remove it", zap.Int64("buildID", buildID))
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, indexTaskDeleted)
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
updateStateFunc(buildID, ib.getTaskState(buildID, meta.NodeID))
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
2022-09-24 13:40:52 +08:00
|
|
|
return true
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
|
2022-08-25 15:48:54 +08:00
|
|
|
func (ib *indexBuilder) getTaskState(buildID, nodeID UniqueID) indexTaskState {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("IndexCoord indexBuilder get index task state", zap.Int64("buildID", buildID), zap.Int64("nodeID", nodeID))
|
2022-08-25 15:48:54 +08:00
|
|
|
client, exist := ib.ic.nodeManager.GetClientByID(nodeID)
|
|
|
|
if exist {
|
|
|
|
response, err := client.QueryJobs(ib.ctx, &indexpb.QueryJobsRequest{
|
2022-09-06 17:19:11 +08:00
|
|
|
ClusterID: Params.CommonCfg.ClusterPrefix,
|
2022-08-25 15:48:54 +08:00
|
|
|
BuildIDs: []int64{buildID},
|
|
|
|
})
|
|
|
|
if err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("IndexCoord get jobs info from IndexNode fail", zap.Int64("nodeID", nodeID),
|
2022-08-25 15:48:54 +08:00
|
|
|
zap.Error(err))
|
|
|
|
return indexTaskInProgress
|
|
|
|
}
|
|
|
|
if response.Status.ErrorCode != commonpb.ErrorCode_Success {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("IndexCoord get jobs info from IndexNode fail", zap.Int64("nodeID", nodeID),
|
2022-08-25 15:48:54 +08:00
|
|
|
zap.Int64("buildID", buildID), zap.String("fail reason", response.Status.Reason))
|
|
|
|
return indexTaskInProgress
|
|
|
|
}
|
|
|
|
|
|
|
|
// indexInfos length is always one.
|
|
|
|
for _, info := range response.IndexInfos {
|
2022-09-09 15:52:35 +08:00
|
|
|
if info.BuildID == buildID {
|
|
|
|
if info.State == commonpb.IndexState_Failed || info.State == commonpb.IndexState_Finished {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("this task has been finished", zap.Int64("buildID", info.BuildID),
|
2022-09-09 15:52:35 +08:00
|
|
|
zap.String("index state", info.State.String()))
|
|
|
|
if err := ib.meta.FinishTask(info); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("IndexCoord update index state fail", zap.Int64("buildID", info.BuildID),
|
2022-09-09 15:52:35 +08:00
|
|
|
zap.String("index state", info.State.String()), zap.Error(err))
|
|
|
|
return indexTaskInProgress
|
|
|
|
}
|
|
|
|
return indexTaskDone
|
|
|
|
} else if info.State == commonpb.IndexState_Retry || info.State == commonpb.IndexState_IndexStateNone {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("this task should be retry", zap.Int64("buildID", buildID), zap.String("fail reason", info.FailReason))
|
2022-09-09 15:52:35 +08:00
|
|
|
return indexTaskRetry
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
2022-09-09 15:52:35 +08:00
|
|
|
return indexTaskInProgress
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
}
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("this task should be retry, indexNode does not have this task", zap.Int64("buildID", buildID),
|
2022-09-09 15:52:35 +08:00
|
|
|
zap.Int64("nodeID", nodeID))
|
|
|
|
return indexTaskRetry
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
// !exist --> node down
|
2022-09-09 15:52:35 +08:00
|
|
|
return indexTaskRetry
|
2022-08-25 15:48:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) dropIndexTask(buildID, nodeID UniqueID) bool {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("IndexCoord notify IndexNode drop the index task", zap.Int64("buildID", buildID), zap.Int64("nodeID", nodeID))
|
2022-08-25 15:48:54 +08:00
|
|
|
client, exist := ib.ic.nodeManager.GetClientByID(nodeID)
|
|
|
|
if exist {
|
|
|
|
status, err := client.DropJobs(ib.ctx, &indexpb.DropJobsRequest{
|
2022-09-06 17:19:11 +08:00
|
|
|
ClusterID: Params.CommonCfg.ClusterPrefix,
|
2022-08-25 15:48:54 +08:00
|
|
|
BuildIDs: []UniqueID{buildID},
|
|
|
|
})
|
|
|
|
if err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Warn("IndexCoord notify IndexNode drop the index task fail", zap.Int64("buildID", buildID),
|
2022-08-25 15:48:54 +08:00
|
|
|
zap.Int64("nodeID", nodeID), zap.Error(err))
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if status.ErrorCode != commonpb.ErrorCode_Success {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Warn("IndexCoord notify IndexNode drop the index task fail", zap.Int64("buildID", buildID),
|
2022-08-25 15:48:54 +08:00
|
|
|
zap.Int64("nodeID", nodeID), zap.String("fail reason", status.Reason))
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2022-07-07 14:44:21 +08:00
|
|
|
func (ib *indexBuilder) releaseLockAndResetNode(buildID UniqueID, nodeID UniqueID) error {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("release segment reference lock and reset nodeID", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID))
|
|
|
|
if err := ib.ic.tryReleaseSegmentReferLock(ib.ctx, buildID, nodeID); err != nil {
|
|
|
|
// release lock failed, no need to modify state, wait to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("index builder try to release reference lock failed", zap.Error(err))
|
2022-07-07 14:44:21 +08:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := ib.meta.ResetNodeID(buildID); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("index builder try to reset nodeID failed", zap.Error(err))
|
2022-07-07 14:44:21 +08:00
|
|
|
return err
|
|
|
|
}
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("release segment reference lock and reset nodeID success", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID))
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) releaseLockAndResetTask(buildID UniqueID, nodeID UniqueID) error {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("release segment reference lock and reset task", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID))
|
|
|
|
if nodeID != 0 {
|
|
|
|
if err := ib.ic.tryReleaseSegmentReferLock(ib.ctx, buildID, nodeID); err != nil {
|
|
|
|
// release lock failed, no need to modify state, wait to retry
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("index builder try to release reference lock failed", zap.Error(err))
|
2022-07-07 14:44:21 +08:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err := ib.meta.ResetMeta(buildID); err != nil {
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Error("index builder try to reset task failed", zap.Error(err))
|
2022-07-07 14:44:21 +08:00
|
|
|
return err
|
|
|
|
}
|
2022-09-24 13:40:52 +08:00
|
|
|
log.Ctx(ib.ctx).Info("release segment reference lock and reset task success", zap.Int64("buildID", buildID),
|
2022-07-07 14:44:21 +08:00
|
|
|
zap.Int64("nodeID", nodeID))
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ib *indexBuilder) nodeDown(nodeID UniqueID) {
|
2022-09-09 15:52:35 +08:00
|
|
|
defer ib.notify()
|
|
|
|
|
|
|
|
metas := ib.meta.GetMetasByNodeID(nodeID)
|
|
|
|
|
2022-07-07 14:44:21 +08:00
|
|
|
ib.taskMutex.Lock()
|
|
|
|
defer ib.taskMutex.Unlock()
|
|
|
|
|
|
|
|
for _, meta := range metas {
|
2022-08-25 15:48:54 +08:00
|
|
|
if ib.tasks[meta.BuildID] != indexTaskDone {
|
|
|
|
ib.tasks[meta.BuildID] = indexTaskRetry
|
2022-07-07 14:44:21 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|