test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 02:48:45 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Modify segment seal policy with number of statslog files (#21957)

Browse Source 
Signed-off-by: jaime <yun.zhang@zilliz.com>
This commit is contained in:
jaime 2023-02-03 17:07:52 +08:00 committed by GitHub
parent f295caf05e
commit 748a935a11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
configs/milvus.yaml
View File

@ -291,9 +291,9 @@ dataCoord:
# `minSizeFromIdleToSealed`, Milvus will automatically seal it.
maxIdleTime: 600 # The max idle time of segment in seconds, 10*60.
minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed.
# The max number of binlog file for one segment, the segment will be sealed if
# the number of binlog file reaches to max value.
maxBinlogFileNumber: 256
# The max number of statslog file for one segment, the segment will be sealed if
# the number of statslog file reaches to max value.
maxBinlogFileNumber: 16
smallProportion: 0.5 # The segment is considered as "small segment" when its # of rows is smaller than
# (smallProportion * segment max # of rows).
compactableProportion: 0.5 # A compaction will happen on small segments if the segment after compaction will have

2
internal/datacoord/segment_allocation_policy.go
View File

@ -130,7 +130,7 @@ func sealByLifetimePolicy(lifetime time.Duration) segmentSealPolicy {
func sealByMaxBinlogFileNumberPolicy(maxBinlogFileNumber int) segmentSealPolicy {
return func(segment *SegmentInfo, ts Timestamp) bool {
logFileCounter := 0
for _, fieldBinlog := range segment.Binlogs {
for _, fieldBinlog := range segment.GetStatslogs() {
logFileCounter += len(fieldBinlog.GetBinlogs())
}

4
internal/datacoord/segment_manager_test.go
View File

@ -554,7 +554,7 @@ func TestTryToSealSegment(t *testing.T) {
segments := segmentManager.meta.segments.segments
assert.Equal(t, 1, len(segments))
for _, seg := range segments {
seg.Binlogs = []*datapb.FieldBinlog{
seg.Statslogs = []*datapb.FieldBinlog{
{
FieldID: 2,
Binlogs: []*datapb.Binlog{
@ -579,7 +579,7 @@ func TestTryToSealSegment(t *testing.T) {
segments := segmentManager.meta.segments.segments
assert.Equal(t, 1, len(segments))
for _, seg := range segments {
seg.Binlogs = []*datapb.FieldBinlog{
seg.Statslogs = []*datapb.FieldBinlog{
{
FieldID: 1,
Binlogs: []*datapb.Binlog{

4
internal/util/paramtable/component_param.go
View File

@ -1467,8 +1467,8 @@ func (p *dataCoordConfig) init(base *BaseTable) {
p.SegmentMaxBinlogFileNumber = ParamItem{
Key: "dataCoord.segment.maxBinlogFileNumber",
Version: "2.0.0",
DefaultValue: "256",
Version: "2.2.0",
DefaultValue: "16",
}
p.SegmentMaxBinlogFileNumber.Init(base.mgr)