test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 02:48:45 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

fix: prevent segments got flushed multiple times (#30240)

Browse Source 
See also #30111

Segments could be "Flushed" only by `FlushSegments` grpc call from
datacoord by design. There are two possible reason to cause one segment
got flushed multiple times.

- Segment is in flushing state during multiple epoch in flowgraph
- Segment is flushed by flushTs & Flush segments

So this pr fix:

- Remove state change logic form FlushTs policy
- Change Flush segment into three stage way: Sealed->Flushing->Flushed
preventing multiple Flushed=true operations.

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2024-01-24 14:19:00 +08:00 committed by GitHub
parent 08ca0a2ca5
commit 6445880753
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/datanode/writebuffer/bf_write_buffer_test.go
View File

@ -313,7 +313,7 @@ func (s *BFWriteBufferSuite) TestAutoSync() {
syncPolicies: []SyncPolicy{
GetFullBufferPolicy(),
GetSyncStaleBufferPolicy(paramtable.Get().DataNodeCfg.SyncPeriod.GetAsDuration(time.Second)),
GetFlushingSegmentsPolicy(s.metacacheInt64),
GetSealedSegmentsPolicy(s.metacacheInt64),
},
})
s.NoError(err)
@ -394,7 +394,7 @@ func (s *BFWriteBufferSuite) TestAutoSyncWithStorageV2() {
syncPolicies: []SyncPolicy{
GetFullBufferPolicy(),
GetSyncStaleBufferPolicy(paramtable.Get().DataNodeCfg.SyncPeriod.GetAsDuration(time.Second)),
GetFlushingSegmentsPolicy(s.metacacheInt64),
GetSealedSegmentsPolicy(s.metacacheInt64),
},
})
s.NoError(err)

2
internal/datanode/writebuffer/options.go
View File

@ -41,7 +41,7 @@ func defaultWBOption(metacache metacache.MetaCache) *writeBufferOption {
GetFullBufferPolicy(),
GetSyncStaleBufferPolicy(paramtable.Get().DataNodeCfg.SyncPeriod.GetAsDuration(time.Second)),
GetCompactedSegmentsPolicy(metacache),
GetFlushingSegmentsPolicy(metacache),
GetSealedSegmentsPolicy(metacache),
},
}
}

10
internal/datanode/writebuffer/sync_policy.go
View File

@ -67,9 +67,12 @@ func GetSyncStaleBufferPolicy(staleDuration time.Duration) SyncPolicy {
}, "buffer stale")
}
func GetFlushingSegmentsPolicy(meta metacache.MetaCache) SyncPolicy {
func GetSealedSegmentsPolicy(meta metacache.MetaCache) SyncPolicy {
return wrapSelectSegmentFuncPolicy(func(_ []*segmentBuffer, _ typeutil.Timestamp) []int64 {
return meta.GetSegmentIDsBy(metacache.WithSegmentState(commonpb.SegmentState_Flushing))
ids := meta.GetSegmentIDsBy(metacache.WithSegmentState(commonpb.SegmentState_Sealed))
meta.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Flushing),
metacache.WithSegmentIDs(ids...), metacache.WithSegmentState(commonpb.SegmentState_Sealed))
return ids
}, "segment flushing")
}
@ -87,9 +90,6 @@ func GetFlushTsPolicy(flushTimestamp *atomic.Uint64, meta metacache.MetaCache) S
seg.Level() == datapb.SegmentLevel_L0
return buf.segmentID, inRange && buf.MinTimestamp() < flushTs
})
// set segment flushing
meta.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Flushing),
metacache.WithSegmentIDs(ids...), metacache.WithSegmentState(commonpb.SegmentState_Growing))
// flush all buffer
return ids

5
internal/datanode/writebuffer/sync_policy_test.go
View File

@ -73,11 +73,12 @@ func (s *SyncPolicySuite) TestSyncStalePolicy() {
s.Equal(0, len(ids), "")
}
func (s *SyncPolicySuite) TestFlushingSegmentsPolicy() {
func (s *SyncPolicySuite) TestSealedSegmentsPolicy() {
metacache := metacache.NewMockMetaCache(s.T())
policy := GetFlushingSegmentsPolicy(metacache)
policy := GetSealedSegmentsPolicy(metacache)
ids := []int64{1, 2, 3}
metacache.EXPECT().GetSegmentIDsBy(mock.Anything).Return(ids)
metacache.EXPECT().UpdateSegments(mock.Anything, mock.Anything, mock.Anything).Return()
result := policy.SelectSegments([]*segmentBuffer{}, tsoutil.ComposeTSByTime(time.Now(), 0))
s.ElementsMatch(ids, result)

8
internal/datanode/writebuffer/write_buffer.go
View File

@ -151,7 +151,7 @@ func (wb *writeBufferBase) FlushSegments(ctx context.Context, segmentIDs []int64
wb.mut.RLock()
defer wb.mut.RUnlock()
return wb.flushSegments(ctx, segmentIDs)
return wb.sealSegments(ctx, segmentIDs)
}
func (wb *writeBufferBase) SetFlushTimestamp(flushTs uint64) {
@ -250,13 +250,13 @@ func (wb *writeBufferBase) cleanupCompactedSegments() {
}
}
func (wb *writeBufferBase) flushSegments(ctx context.Context, segmentIDs []int64) error {
func (wb *writeBufferBase) sealSegments(ctx context.Context, segmentIDs []int64) error {
// mark segment flushing if segment was growing
wb.metaCache.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Flushing),
wb.metaCache.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Sealed),
metacache.WithSegmentIDs(segmentIDs...),
metacache.WithSegmentState(commonpb.SegmentState_Growing))
// mark segment flushing if segment was importing
wb.metaCache.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Flushing),
wb.metaCache.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Sealed),
metacache.WithSegmentIDs(segmentIDs...),
metacache.WithImporting())
return nil