mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-03 04:19:18 +08:00
a4439cc911
- Implement flusher to: - Manage the pipelines (creation, deletion, etc.) - Manage the segment write buffer - Manage sync operation (including receive flushMsg and execute flush) - Add a new `GetChannelRecoveryInfo` RPC in DataCoord. - Reorganize packages: `flushcommon` and `datanode`. issue: https://github.com/milvus-io/milvus/issues/33285 --------- Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
123 lines
3.8 KiB
Go
123 lines
3.8 KiB
Go
package syncmgr
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
|
"github.com/milvus-io/milvus/internal/allocator"
|
|
"github.com/milvus-io/milvus/internal/flushcommon/metacache"
|
|
"github.com/milvus-io/milvus/internal/storage"
|
|
"github.com/milvus-io/milvus/pkg/config"
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
"github.com/milvus-io/milvus/pkg/util/conc"
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
|
)
|
|
|
|
type SyncManagerOption struct {
|
|
chunkManager storage.ChunkManager
|
|
allocator allocator.Interface
|
|
parallelTask int
|
|
}
|
|
|
|
type SyncMeta struct {
|
|
collectionID int64
|
|
partitionID int64
|
|
segmentID int64
|
|
channelName string
|
|
schema *schemapb.CollectionSchema
|
|
checkpoint *msgpb.MsgPosition
|
|
tsFrom typeutil.Timestamp
|
|
tsTo typeutil.Timestamp
|
|
|
|
metacache metacache.MetaCache
|
|
}
|
|
|
|
// SyncManager is the interface for sync manager.
|
|
// it processes the sync tasks inside and changes the meta.
|
|
//
|
|
//go:generate mockery --name=SyncManager --structname=MockSyncManager --output=./ --filename=mock_sync_manager.go --with-expecter --inpackage
|
|
type SyncManager interface {
|
|
// SyncData is the method to submit sync task.
|
|
SyncData(ctx context.Context, task Task, callbacks ...func(error) error) *conc.Future[struct{}]
|
|
}
|
|
|
|
type syncManager struct {
|
|
*keyLockDispatcher[int64]
|
|
chunkManager storage.ChunkManager
|
|
|
|
tasks *typeutil.ConcurrentMap[string, Task]
|
|
}
|
|
|
|
func NewSyncManager(chunkManager storage.ChunkManager) SyncManager {
|
|
params := paramtable.Get()
|
|
initPoolSize := params.DataNodeCfg.MaxParallelSyncMgrTasks.GetAsInt()
|
|
dispatcher := newKeyLockDispatcher[int64](initPoolSize)
|
|
log.Info("sync manager initialized", zap.Int("initPoolSize", initPoolSize))
|
|
|
|
syncMgr := &syncManager{
|
|
keyLockDispatcher: dispatcher,
|
|
chunkManager: chunkManager,
|
|
tasks: typeutil.NewConcurrentMap[string, Task](),
|
|
}
|
|
// setup config update watcher
|
|
params.Watch(params.DataNodeCfg.MaxParallelSyncMgrTasks.Key, config.NewHandler("datanode.syncmgr.poolsize", syncMgr.resizeHandler))
|
|
return syncMgr
|
|
}
|
|
|
|
func (mgr *syncManager) resizeHandler(evt *config.Event) {
|
|
if evt.HasUpdated {
|
|
log := log.Ctx(context.Background()).With(
|
|
zap.String("key", evt.Key),
|
|
zap.String("value", evt.Value),
|
|
)
|
|
size, err := strconv.ParseInt(evt.Value, 10, 64)
|
|
if err != nil {
|
|
log.Warn("failed to parse new datanode syncmgr pool size", zap.Error(err))
|
|
return
|
|
}
|
|
err = mgr.keyLockDispatcher.workerPool.Resize(int(size))
|
|
if err != nil {
|
|
log.Warn("failed to resize datanode syncmgr pool size", zap.String("key", evt.Key), zap.String("value", evt.Value), zap.Error(err))
|
|
return
|
|
}
|
|
log.Info("sync mgr pool size updated", zap.Int64("newSize", size))
|
|
}
|
|
}
|
|
|
|
func (mgr *syncManager) SyncData(ctx context.Context, task Task, callbacks ...func(error) error) *conc.Future[struct{}] {
|
|
switch t := task.(type) {
|
|
case *SyncTask:
|
|
t.WithChunkManager(mgr.chunkManager)
|
|
}
|
|
|
|
return mgr.safeSubmitTask(ctx, task, callbacks...)
|
|
}
|
|
|
|
// safeSubmitTask submits task to SyncManager
|
|
func (mgr *syncManager) safeSubmitTask(ctx context.Context, task Task, callbacks ...func(error) error) *conc.Future[struct{}] {
|
|
taskKey := fmt.Sprintf("%d-%d", task.SegmentID(), task.Checkpoint().GetTimestamp())
|
|
mgr.tasks.Insert(taskKey, task)
|
|
|
|
key := task.SegmentID()
|
|
return mgr.submit(ctx, key, task, callbacks...)
|
|
}
|
|
|
|
func (mgr *syncManager) submit(ctx context.Context, key int64, task Task, callbacks ...func(error) error) *conc.Future[struct{}] {
|
|
handler := func(err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
task.HandleError(err)
|
|
return err
|
|
}
|
|
callbacks = append([]func(error) error{handler}, callbacks...)
|
|
log.Info("sync mgr sumbit task with key", zap.Int64("key", key))
|
|
return mgr.Submit(ctx, key, task, callbacks...)
|
|
}
|