mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-04 04:49:08 +08:00
a68b32134a
See also #27675 #30469 For a sync task, the segment could be compacted during sync task. In previous implementation, this sync task will hold only the old segment id as KeyLock, in which case compaction on compacted to segment may run in parallel with delta sync of this sync task. This PR introduces sync target segment verification logic. It shall check target segment lock it's holding beforing actually syncing logic. If this check failed, sync task shall return`errTargetSegementNotMatch` error and make manager re-fetch the current target segment id. Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
45 lines
1.0 KiB
Go
45 lines
1.0 KiB
Go
package syncmgr
|
|
|
|
import (
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
|
"github.com/milvus-io/milvus/pkg/util/conc"
|
|
"github.com/milvus-io/milvus/pkg/util/lock"
|
|
)
|
|
|
|
type Task interface {
|
|
SegmentID() int64
|
|
CalcTargetSegment() (int64, error)
|
|
Checkpoint() *msgpb.MsgPosition
|
|
StartPosition() *msgpb.MsgPosition
|
|
ChannelName() string
|
|
Run() error
|
|
}
|
|
|
|
type keyLockDispatcher[K comparable] struct {
|
|
keyLock *lock.KeyLock[K]
|
|
workerPool *conc.Pool[error]
|
|
}
|
|
|
|
func newKeyLockDispatcher[K comparable](maxParallel int) *keyLockDispatcher[K] {
|
|
dispatcher := &keyLockDispatcher[K]{
|
|
workerPool: conc.NewPool[error](maxParallel, conc.WithPreAlloc(false)),
|
|
keyLock: lock.NewKeyLock[K](),
|
|
}
|
|
return dispatcher
|
|
}
|
|
|
|
func (d *keyLockDispatcher[K]) Submit(key K, t Task, callbacks ...func(error)) *conc.Future[error] {
|
|
d.keyLock.Lock(key)
|
|
|
|
return d.workerPool.Submit(func() (error, error) {
|
|
defer d.keyLock.Unlock(key)
|
|
err := t.Run()
|
|
|
|
for _, callback := range callbacks {
|
|
callback(err)
|
|
}
|
|
|
|
return err, nil
|
|
})
|
|
}
|