mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-11-29 18:38:44 +08:00
258 lines
7.1 KiB
Go
258 lines
7.1 KiB
Go
package datacoord
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/samber/lo"
|
|
"github.com/stretchr/testify/suite"
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
|
"github.com/milvus-io/milvus/internal/proto/datapb"
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
)
|
|
|
|
func TestLevelZeroSegmentsViewSuite(t *testing.T) {
|
|
suite.Run(t, new(LevelZeroSegmentsViewSuite))
|
|
}
|
|
|
|
type LevelZeroSegmentsViewSuite struct {
|
|
suite.Suite
|
|
v *LevelZeroSegmentsView
|
|
}
|
|
|
|
func genTestL0SegmentView(ID UniqueID, label *CompactionGroupLabel, posTime Timestamp) *SegmentView {
|
|
return &SegmentView{
|
|
ID: ID,
|
|
label: label,
|
|
dmlPos: &msgpb.MsgPosition{Timestamp: posTime},
|
|
Level: datapb.SegmentLevel_L0,
|
|
State: commonpb.SegmentState_Flushed,
|
|
}
|
|
}
|
|
|
|
func (s *LevelZeroSegmentsViewSuite) SetupTest() {
|
|
label := &CompactionGroupLabel{
|
|
CollectionID: 1,
|
|
PartitionID: 10,
|
|
Channel: "ch-1",
|
|
}
|
|
segments := []*SegmentView{
|
|
genTestL0SegmentView(100, label, 10000),
|
|
genTestL0SegmentView(101, label, 10000),
|
|
genTestL0SegmentView(102, label, 10000),
|
|
}
|
|
|
|
targetView := &LevelZeroSegmentsView{
|
|
label, segments, &msgpb.MsgPosition{Timestamp: 10000},
|
|
}
|
|
|
|
s.True(label.Equal(targetView.GetGroupLabel()))
|
|
log.Info("LevelZeroSegmentsView", zap.String("view", targetView.String()))
|
|
|
|
s.v = targetView
|
|
}
|
|
|
|
func (s *LevelZeroSegmentsViewSuite) TestEqual() {
|
|
label := s.v.GetGroupLabel()
|
|
|
|
tests := []struct {
|
|
description string
|
|
|
|
input []*SegmentView
|
|
output bool
|
|
}{
|
|
{"Different segment numbers", []*SegmentView{genTestL0SegmentView(100, label, 10000)}, false},
|
|
{"Same number, diff segmentIDs", []*SegmentView{
|
|
genTestL0SegmentView(100, label, 10000),
|
|
genTestL0SegmentView(101, label, 10000),
|
|
genTestL0SegmentView(200, label, 10000),
|
|
}, false},
|
|
{"Same", s.v.GetSegmentsView(), true},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
s.Run(test.description, func() {
|
|
got := s.v.Equal(test.input)
|
|
s.Equal(test.output, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func (s *LevelZeroSegmentsViewSuite) TestTrigger() {
|
|
label := s.v.GetGroupLabel()
|
|
views := []*SegmentView{
|
|
genTestL0SegmentView(100, label, 20000),
|
|
genTestL0SegmentView(101, label, 10000),
|
|
genTestL0SegmentView(102, label, 30000),
|
|
genTestL0SegmentView(103, label, 40000),
|
|
}
|
|
|
|
s.v.segments = views
|
|
tests := []struct {
|
|
description string
|
|
|
|
prepSizeEach float64
|
|
prepCountEach int
|
|
prepEarliestT Timestamp
|
|
|
|
expectedSegs []UniqueID
|
|
}{
|
|
{
|
|
"No valid segments by earliest growing segment pos",
|
|
64,
|
|
20,
|
|
10000,
|
|
nil,
|
|
},
|
|
{
|
|
"Not qualified",
|
|
1,
|
|
1,
|
|
30000,
|
|
nil,
|
|
},
|
|
{
|
|
"Trigger by > TriggerDeltaSize",
|
|
8 * 1024 * 1024,
|
|
1,
|
|
30000,
|
|
[]UniqueID{100, 101},
|
|
},
|
|
{
|
|
"Trigger by > TriggerDeltaCount",
|
|
1,
|
|
10,
|
|
30000,
|
|
[]UniqueID{100, 101},
|
|
},
|
|
{
|
|
"Trigger by > maxDeltaSize",
|
|
128 * 1024 * 1024,
|
|
1,
|
|
30000,
|
|
[]UniqueID{100},
|
|
},
|
|
{
|
|
"Trigger by > maxDeltaCount",
|
|
1,
|
|
24,
|
|
30000,
|
|
[]UniqueID{100},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
s.Run(test.description, func() {
|
|
s.v.earliestGrowingSegmentPos.Timestamp = test.prepEarliestT
|
|
for _, view := range s.v.GetSegmentsView() {
|
|
if view.dmlPos.Timestamp < test.prepEarliestT {
|
|
view.DeltalogCount = test.prepCountEach
|
|
view.DeltaSize = test.prepSizeEach
|
|
view.DeltaRowCount = 1
|
|
}
|
|
}
|
|
log.Info("LevelZeroSegmentsView", zap.String("view", s.v.String()))
|
|
|
|
gotView, reason := s.v.Trigger()
|
|
if len(test.expectedSegs) == 0 {
|
|
s.Nil(gotView)
|
|
} else {
|
|
levelZeroView, ok := gotView.(*LevelZeroSegmentsView)
|
|
s.True(ok)
|
|
s.NotNil(levelZeroView)
|
|
|
|
gotSegIDs := lo.Map(levelZeroView.GetSegmentsView(), func(v *SegmentView, _ int) int64 {
|
|
return v.ID
|
|
})
|
|
s.ElementsMatch(gotSegIDs, test.expectedSegs)
|
|
log.Info("output view", zap.String("view", levelZeroView.String()), zap.String("trigger reason", reason))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func (s *LevelZeroSegmentsViewSuite) TestMinCountSizeTrigger() {
|
|
label := s.v.GetGroupLabel()
|
|
tests := []struct {
|
|
description string
|
|
segIDs []int64
|
|
segCounts []int
|
|
segSize []float64
|
|
|
|
expectedIDs []int64
|
|
}{
|
|
{"donot trigger", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{1, 1, 1}, nil},
|
|
{"trigger by count=15", []int64{100, 101, 102}, []int{5, 5, 5}, []float64{1, 1, 1}, []int64{100, 101, 102}},
|
|
{"trigger by count=10", []int64{100, 101, 102}, []int{5, 3, 2}, []float64{1, 1, 1}, []int64{100, 101, 102}},
|
|
{"trigger by count=50", []int64{100, 101, 102}, []int{32, 10, 8}, []float64{1, 1, 1}, []int64{100}},
|
|
{"trigger by size=24MB", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{8 * 1024 * 1024, 8 * 1024 * 1024, 8 * 1024 * 1024}, []int64{100, 101, 102}},
|
|
{"trigger by size=8MB", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{3 * 1024 * 1024, 3 * 1024 * 1024, 2 * 1024 * 1024}, []int64{100, 101, 102}},
|
|
{"trigger by size=128MB", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{100 * 1024 * 1024, 20 * 1024 * 1024, 8 * 1024 * 1024}, []int64{100}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
s.Run(test.description, func() {
|
|
views := []*SegmentView{}
|
|
for idx, ID := range test.segIDs {
|
|
seg := genTestL0SegmentView(ID, label, 10000)
|
|
seg.DeltaSize = test.segSize[idx]
|
|
seg.DeltalogCount = test.segCounts[idx]
|
|
|
|
views = append(views, seg)
|
|
}
|
|
|
|
picked, reason := s.v.minCountSizeTrigger(views)
|
|
s.ElementsMatch(lo.Map(picked, func(view *SegmentView, _ int) int64 {
|
|
return view.ID
|
|
}), test.expectedIDs)
|
|
|
|
if len(picked) > 0 {
|
|
s.NotEmpty(reason)
|
|
}
|
|
|
|
log.Info("test minCountSizeTrigger", zap.Any("trigger reason", reason))
|
|
})
|
|
}
|
|
}
|
|
|
|
func (s *LevelZeroSegmentsViewSuite) TestForceTrigger() {
|
|
label := s.v.GetGroupLabel()
|
|
tests := []struct {
|
|
description string
|
|
segIDs []int64
|
|
segCounts []int
|
|
segSize []float64
|
|
|
|
expectedIDs []int64
|
|
}{
|
|
{"force trigger", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{1, 1, 1}, []int64{100, 101, 102}},
|
|
{"trigger by count=15", []int64{100, 101, 102}, []int{5, 5, 5}, []float64{1, 1, 1}, []int64{100, 101, 102}},
|
|
{"trigger by count=10", []int64{100, 101, 102}, []int{5, 3, 2}, []float64{1, 1, 1}, []int64{100, 101, 102}},
|
|
{"trigger by count=50", []int64{100, 101, 102}, []int{32, 10, 8}, []float64{1, 1, 1}, []int64{100}},
|
|
{"trigger by size=24MB", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{8 * 1024 * 1024, 8 * 1024 * 1024, 8 * 1024 * 1024}, []int64{100, 101, 102}},
|
|
{"trigger by size=8MB", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{3 * 1024 * 1024, 3 * 1024 * 1024, 2 * 1024 * 1024}, []int64{100, 101, 102}},
|
|
{"trigger by size=128MB", []int64{100, 101, 102}, []int{1, 1, 1}, []float64{100 * 1024 * 1024, 20 * 1024 * 1024, 8 * 1024 * 1024}, []int64{100}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
s.Run(test.description, func() {
|
|
views := []*SegmentView{}
|
|
for idx, ID := range test.segIDs {
|
|
seg := genTestL0SegmentView(ID, label, 10000)
|
|
seg.DeltaSize = test.segSize[idx]
|
|
seg.DeltalogCount = test.segCounts[idx]
|
|
|
|
views = append(views, seg)
|
|
}
|
|
|
|
picked, reason := s.v.forceTrigger(views)
|
|
s.ElementsMatch(lo.Map(picked, func(view *SegmentView, _ int) int64 {
|
|
return view.ID
|
|
}), test.expectedIDs)
|
|
log.Info("test forceTrigger", zap.Any("trigger reason", reason))
|
|
})
|
|
}
|
|
}
|