test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: Skip update index for L0 segment (#34099)

Browse Source 
try to update index for l0 segment, will failed by `index not found`

This PR skip update index for l0 segment

Signed-off-by: Wei Liu <wei.liu@zilliz.com>
This commit is contained in:
wei liu 2024-07-01 10:26:06 +08:00 committed by GitHub
parent 0426390f06
commit f7ecafe77d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 39 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/querycoordv2/checkers/balance_checker.go
View File

@ -43,12 +43,12 @@ type BalanceChecker struct {
nodeManager *session.NodeManager
normalBalanceCollectionsCurrentRound typeutil.UniqueSet
scheduler task.Scheduler
targetMgr *meta.TargetManager
targetMgr meta.TargetManagerInterface
getBalancerFunc GetBalancerFunc
}
func NewBalanceChecker(meta *meta.Meta,
targetMgr *meta.TargetManager,
targetMgr meta.TargetManagerInterface,
nodeMgr *session.NodeManager,
scheduler task.Scheduler,
getBalancerFunc GetBalancerFunc,

4
internal/querycoordv2/checkers/channel_checker.go
View File

@ -38,7 +38,7 @@ type ChannelChecker struct {
*checkerActivation
meta *meta.Meta
dist *meta.DistributionManager
targetMgr *meta.TargetManager
targetMgr meta.TargetManagerInterface
nodeMgr *session.NodeManager
getBalancerFunc GetBalancerFunc
}
@ -46,7 +46,7 @@ type ChannelChecker struct {
func NewChannelChecker(
meta *meta.Meta,
dist *meta.DistributionManager,
targetMgr *meta.TargetManager,
targetMgr meta.TargetManagerInterface,
nodeMgr *session.NodeManager,
getBalancerFunc GetBalancerFunc,
) *ChannelChecker {

6
internal/querycoordv2/checkers/controller.go
View File

@ -42,7 +42,7 @@ type CheckerController struct {
manualCheckChs map[utils.CheckerType]chan struct{}
meta *meta.Meta
dist *meta.DistributionManager
targetMgr *meta.TargetManager
targetMgr meta.TargetManagerInterface
broker meta.Broker
nodeMgr *session.NodeManager
balancer balance.Balance
@ -56,7 +56,7 @@ type CheckerController struct {
func NewCheckerController(
meta *meta.Meta,
dist *meta.DistributionManager,
targetMgr *meta.TargetManager,
targetMgr meta.TargetManagerInterface,
nodeMgr *session.NodeManager,
scheduler task.Scheduler,
broker meta.Broker,
@ -68,7 +68,7 @@ func NewCheckerController(
utils.ChannelChecker: NewChannelChecker(meta, dist, targetMgr, nodeMgr, getBalancerFunc),
utils.SegmentChecker: NewSegmentChecker(meta, dist, targetMgr, nodeMgr, getBalancerFunc),
utils.BalanceChecker: NewBalanceChecker(meta, targetMgr, nodeMgr, scheduler, getBalancerFunc),
utils.IndexChecker: NewIndexChecker(meta, dist, broker, nodeMgr),
utils.IndexChecker: NewIndexChecker(meta, dist, broker, nodeMgr, targetMgr),
// todo temporary work around must fix
// utils.LeaderChecker: NewLeaderChecker(meta, dist, targetMgr, nodeMgr, true),
utils.LeaderChecker: NewLeaderChecker(meta, dist, targetMgr, nodeMgr),

12
internal/querycoordv2/checkers/index_checker.go
View File

@ -23,6 +23,7 @@ import (
"github.com/samber/lo"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
@ -43,6 +44,8 @@ type IndexChecker struct {
dist *meta.DistributionManager
broker meta.Broker
nodeMgr *session.NodeManager
targetMgr meta.TargetManagerInterface
}
func NewIndexChecker(
@ -50,6 +53,7 @@ func NewIndexChecker(
dist *meta.DistributionManager,
broker meta.Broker,
nodeMgr *session.NodeManager,
targetMgr meta.TargetManagerInterface,
) *IndexChecker {
return &IndexChecker{
checkerActivation: newCheckerActivation(),
@ -57,6 +61,7 @@ func NewIndexChecker(
dist: dist,
broker: broker,
nodeMgr: nodeMgr,
targetMgr: targetMgr,
}
}
@ -112,6 +117,13 @@ func (c *IndexChecker) checkReplica(ctx context.Context, collection *meta.Collec
if roNodeSet.Contain(segment.Node) {
continue
}
// skip update index for l0 segment
segmentInTarget := c.targetMgr.GetSealedSegment(collection.GetCollectionID(), segment.GetID(), meta.CurrentTargetFirst)
if segmentInTarget == nil || segmentInTarget.GetLevel() == datapb.SegmentLevel_L0 {
continue
}
missing := c.checkSegment(segment, indexInfos)
if len(missing) > 0 {
targets[segment.GetID()] = missing

22
internal/querycoordv2/checkers/index_checker_test.go
View File

@ -26,6 +26,7 @@ import (
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/metastore/kv/querycoord"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
@ -40,11 +41,12 @@ import (
type IndexCheckerSuite struct {
suite.Suite
kv kv.MetaKv
checker *IndexChecker
meta *meta.Meta
broker *meta.MockBroker
nodeMgr *session.NodeManager
kv kv.MetaKv
checker *IndexChecker
meta *meta.Meta
broker *meta.MockBroker
nodeMgr *session.NodeManager
targetMgr *meta.MockTargetManager
}
func (suite *IndexCheckerSuite) SetupSuite() {
@ -73,7 +75,15 @@ func (suite *IndexCheckerSuite) SetupTest() {
distManager := meta.NewDistributionManager()
suite.broker = meta.NewMockBroker(suite.T())
suite.checker = NewIndexChecker(suite.meta, distManager, suite.broker, suite.nodeMgr)
suite.targetMgr = meta.NewMockTargetManager(suite.T())
suite.checker = NewIndexChecker(suite.meta, distManager, suite.broker, suite.nodeMgr, suite.targetMgr)
suite.targetMgr.EXPECT().GetSealedSegment(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(cid, sid int64, i3 int32) *datapb.SegmentInfo {
return &datapb.SegmentInfo{
ID: sid,
Level: datapb.SegmentLevel_L1,
}
}).Maybe()
}
func (suite *IndexCheckerSuite) TearDownTest() {

4
internal/querycoordv2/checkers/leader_checker.go
View File

@ -37,14 +37,14 @@ type LeaderChecker struct {
*checkerActivation
meta *meta.Meta
dist *meta.DistributionManager
target *meta.TargetManager
target meta.TargetManagerInterface
nodeMgr *session.NodeManager
}
func NewLeaderChecker(
meta *meta.Meta,
dist *meta.DistributionManager,
target *meta.TargetManager,
target meta.TargetManagerInterface,
nodeMgr *session.NodeManager,
) *LeaderChecker {
return &LeaderChecker{

4
internal/querycoordv2/checkers/segment_checker.go
View File

@ -43,7 +43,7 @@ type SegmentChecker struct {
*checkerActivation
meta *meta.Meta
dist *meta.DistributionManager
targetMgr *meta.TargetManager
targetMgr meta.TargetManagerInterface
nodeMgr *session.NodeManager
getBalancerFunc GetBalancerFunc
}
@ -51,7 +51,7 @@ type SegmentChecker struct {
func NewSegmentChecker(
meta *meta.Meta,
dist *meta.DistributionManager,
targetMgr *meta.TargetManager,
targetMgr meta.TargetManagerInterface,
nodeMgr *session.NodeManager,
getBalancerFunc GetBalancerFunc,
) *SegmentChecker {