test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 19:39:21 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

fix: Fix large growing segment (#37388)

Browse Source 
Consider the `sealProportion` factor during segment allocation.

issue: https://github.com/milvus-io/milvus/issues/37387

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
yihao.dai 2024-11-07 17:12:25 +08:00 committed by GitHub
parent 994f52fab8
commit 36bb87d9b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
internal/datacoord/segment_allocation_policy.go
View File

@ -108,7 +108,18 @@ func AllocatePolicyL1(segments []*SegmentInfo, count int64,
for _, allocation := range segment.allocations {
allocSize += allocation.NumOfRows
}
free := segment.GetMaxRowNum() - segment.GetNumOfRows() - allocSize
// When inserts are too fast, hardTimeTick may lag, causing segment to be unable to seal in time.
// To prevent allocating large segment, introducing the sealProportion factor here.
// The condition `free < 0` ensures that the allocation exceeds the minimum sealable size,
// preventing segments from remaining unsealable indefinitely.
maxRowsWithSealProportion := int64(float64(segment.GetMaxRowNum()) * paramtable.Get().DataCoordCfg.SegmentSealProportion.GetAsFloat())
free := maxRowsWithSealProportion - segment.GetNumOfRows() - allocSize
if free < 0 {
continue
}
free = segment.GetMaxRowNum() - segment.GetNumOfRows() - allocSize
if free < count {
continue
}