test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

load segment with target version, avoid read redundant segment (#24929)

Browse Source 
Signed-off-by: Wei Liu <wei.liu@zilliz.com>
This commit is contained in:
wei liu 2023-06-27 11:48:45 +08:00 committed by GitHub
parent a1e6428bc7
commit 68ae199a9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1137 additions and 517 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/proto/query_coord.proto
View File

@ -490,6 +490,7 @@ message LeaderView {
map<int64, SegmentDist> segment_dist = 3; map<int64, SegmentDist> segment_dist = 3;
repeated int64 growing_segmentIDs = 4; repeated int64 growing_segmentIDs = 4;
map<int64, msg.MsgPosition> growing_segments = 5; map<int64, msg.MsgPosition> growing_segments = 5;
int64 TargetVersion = 6;
} }
message SegmentDist { message SegmentDist {
@ -547,6 +548,7 @@ enum SyncType {
Remove = 0; Remove = 0;
Set = 1; Set = 1;
Amend = 2; Amend = 2;
UpdateVersion = 3;
} }
message SyncAction { message SyncAction {
@ -556,6 +558,9 @@ message SyncAction {
int64 nodeID = 4; int64 nodeID = 4;
int64 version = 5; int64 version = 5;
SegmentLoadInfo info = 6; SegmentLoadInfo info = 6;
repeated int64 growingInTarget = 7;
repeated int64 sealedInTarget = 8;
int64 TargetVersion = 9;
} }
message SyncDistributionRequest { message SyncDistributionRequest {

633
internal/proto/querypb/query_coord.pb.go
View File

@ -220,21 +220,24 @@ func (LoadStatus) EnumDescriptor() ([]byte, []int) {
type SyncType int32 type SyncType int32
const ( const (
SyncType_Remove SyncType = 0 SyncType_Remove SyncType = 0
SyncType_Set SyncType = 1 SyncType_Set SyncType = 1
SyncType_Amend SyncType = 2 SyncType_Amend SyncType = 2
SyncType_UpdateVersion SyncType = 3
) )
var SyncType_name = map[int32]string{ var SyncType_name = map[int32]string{
0: "Remove", 0: "Remove",
1: "Set", 1: "Set",
2: "Amend", 2: "Amend",
3: "UpdateVersion",
} }
var SyncType_value = map[string]int32{ var SyncType_value = map[string]int32{
"Remove": 0, "Remove": 0,
"Set": 1, "Set": 1,
"Amend": 2, "Amend": 2,
"UpdateVersion": 3,
} }
func (x SyncType) String() string { func (x SyncType) String() string {
@ -3398,6 +3401,7 @@ type LeaderView struct {
SegmentDist map[int64]*SegmentDist `protobuf:"bytes,3,rep,name=segment_dist,json=segmentDist,proto3" json:"segment_dist,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` SegmentDist map[int64]*SegmentDist `protobuf:"bytes,3,rep,name=segment_dist,json=segmentDist,proto3" json:"segment_dist,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
GrowingSegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=growing_segmentIDs,json=growingSegmentIDs,proto3" json:"growing_segmentIDs,omitempty"` GrowingSegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=growing_segmentIDs,json=growingSegmentIDs,proto3" json:"growing_segmentIDs,omitempty"`
GrowingSegments map[int64]*msgpb.MsgPosition `protobuf:"bytes,5,rep,name=growing_segments,json=growingSegments,proto3" json:"growing_segments,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` GrowingSegments map[int64]*msgpb.MsgPosition `protobuf:"bytes,5,rep,name=growing_segments,json=growingSegments,proto3" json:"growing_segments,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
TargetVersion int64 `protobuf:"varint,6,opt,name=TargetVersion,proto3" json:"TargetVersion,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"` XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"` XXX_sizecache int32 `json:"-"`
@ -3463,6 +3467,13 @@ func (m *LeaderView) GetGrowingSegments() map[int64]*msgpb.MsgPosition {
return nil return nil
} }
func (m *LeaderView) GetTargetVersion() int64 {
if m != nil {
return m.TargetVersion
}
return 0
}
type SegmentDist struct { type SegmentDist struct {
NodeID int64 `protobuf:"varint,1,opt,name=nodeID,proto3" json:"nodeID,omitempty"` NodeID int64 `protobuf:"varint,1,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
Version int64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` Version int64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"`
@ -3864,6 +3875,9 @@ type SyncAction struct {
NodeID int64 `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"` NodeID int64 `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
Version int64 `protobuf:"varint,5,opt,name=version,proto3" json:"version,omitempty"` Version int64 `protobuf:"varint,5,opt,name=version,proto3" json:"version,omitempty"`
Info *SegmentLoadInfo `protobuf:"bytes,6,opt,name=info,proto3" json:"info,omitempty"` Info *SegmentLoadInfo `protobuf:"bytes,6,opt,name=info,proto3" json:"info,omitempty"`
GrowingInTarget []int64 `protobuf:"varint,7,rep,packed,name=growingInTarget,proto3" json:"growingInTarget,omitempty"`
SealedInTarget []int64 `protobuf:"varint,8,rep,packed,name=sealedInTarget,proto3" json:"sealedInTarget,omitempty"`
TargetVersion int64 `protobuf:"varint,9,opt,name=TargetVersion,proto3" json:"TargetVersion,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"` XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"` XXX_sizecache int32 `json:"-"`
@ -3936,6 +3950,27 @@ func (m *SyncAction) GetInfo() *SegmentLoadInfo {
return nil return nil
} }
func (m *SyncAction) GetGrowingInTarget() []int64 {
if m != nil {
return m.GrowingInTarget
}
return nil
}
func (m *SyncAction) GetSealedInTarget() []int64 {
if m != nil {
return m.SealedInTarget
}
return nil
}
func (m *SyncAction) GetTargetVersion() int64 {
if m != nil {
return m.TargetVersion
}
return 0
}
type SyncDistributionRequest struct { type SyncDistributionRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"` CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
@ -4503,297 +4538,301 @@ func init() {
func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) }
var fileDescriptor_aab7cc9a69ed26e8 = []byte{ var fileDescriptor_aab7cc9a69ed26e8 = []byte{
// 4638 bytes of a gzipped FileDescriptorProto // 4694 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3c, 0x4b, 0x6f, 0x1c, 0x47, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3c, 0x4b, 0x6c, 0x1c, 0x47,
0x7a, 0xea, 0x79, 0x71, 0xe6, 0x9b, 0x27, 0x8b, 0xa4, 0x34, 0x3b, 0x2b, 0xc9, 0x74, 0xcb, 0xb2, 0x76, 0xea, 0xf9, 0x71, 0xe6, 0xcd, 0xaf, 0x59, 0x24, 0xa5, 0xd9, 0x59, 0x49, 0xa6, 0x5b, 0xb6,
0xb9, 0xb4, 0x4d, 0x6a, 0xa9, 0x5d, 0xaf, 0x76, 0xed, 0x85, 0x23, 0x91, 0x96, 0xcc, 0xb5, 0x4c, 0xcc, 0xa5, 0x6d, 0x52, 0x4b, 0xed, 0x7a, 0xb5, 0x6b, 0x1b, 0x8e, 0x44, 0x5a, 0x32, 0xd7, 0x32,
0x73, 0x9b, 0x92, 0x36, 0x70, 0xbc, 0x3b, 0xdb, 0x9c, 0x2e, 0x0e, 0x1b, 0xec, 0xc7, 0xa8, 0xbb, 0xcd, 0x6d, 0x4a, 0xda, 0xc0, 0xf1, 0xee, 0x6c, 0x73, 0xba, 0x38, 0x6c, 0xb0, 0x3f, 0xa3, 0xee,
0x87, 0x14, 0x1d, 0x20, 0xa7, 0x5c, 0xb2, 0xc8, 0x26, 0xb9, 0x25, 0x87, 0x20, 0x87, 0x04, 0x01, 0x1e, 0x52, 0x74, 0x80, 0x9c, 0x72, 0xc9, 0x22, 0x9b, 0x04, 0xc8, 0x21, 0x39, 0x04, 0x39, 0x24,
0x36, 0x41, 0x72, 0x09, 0x12, 0x20, 0x87, 0x1c, 0x72, 0x4b, 0x90, 0x20, 0x8f, 0x5b, 0xfe, 0x40, 0x08, 0xb0, 0x09, 0x92, 0x4b, 0x90, 0x00, 0x39, 0xe4, 0x90, 0x5b, 0x82, 0x04, 0xf9, 0xdc, 0x72,
0x72, 0x08, 0x90, 0x20, 0xa7, 0x45, 0xe0, 0x5b, 0x50, 0x8f, 0x7e, 0x54, 0x77, 0x0d, 0xa7, 0xc9, 0x0e, 0x90, 0xdc, 0x02, 0xe4, 0xb4, 0x08, 0x7c, 0x0b, 0xea, 0xd3, 0x9f, 0xea, 0xae, 0xe1, 0x0c,
0x91, 0x5f, 0xc1, 0xde, 0xa6, 0xbf, 0x7a, 0x7c, 0x5f, 0x7d, 0xf5, 0xbd, 0xab, 0x6a, 0x60, 0xfe, 0x39, 0xf2, 0xda, 0x0e, 0xf6, 0x36, 0xfd, 0xea, 0xf3, 0x5e, 0xd5, 0xfb, 0xbf, 0xaa, 0x1a, 0x98,
0xe9, 0x18, 0x7b, 0xa7, 0xfd, 0x81, 0xeb, 0x7a, 0xc6, 0xda, 0xc8, 0x73, 0x03, 0x17, 0x21, 0xdb, 0x7f, 0x3a, 0xc2, 0xfe, 0x69, 0xaf, 0xef, 0x79, 0xbe, 0xb9, 0x36, 0xf4, 0xbd, 0xd0, 0x43, 0xc8,
0xb4, 0x8e, 0xc7, 0x3e, 0xfb, 0x5a, 0xa3, 0xed, 0xbd, 0xc6, 0xc0, 0xb5, 0x6d, 0xd7, 0x61, 0xb0, 0xb1, 0xec, 0xe3, 0x51, 0xc0, 0xbe, 0xd6, 0x68, 0x7b, 0xb7, 0xd1, 0xf7, 0x1c, 0xc7, 0x73, 0x19,
0x5e, 0x23, 0xd9, 0xa3, 0xd7, 0x32, 0x9d, 0x00, 0x7b, 0x8e, 0x6e, 0x85, 0xad, 0xfe, 0xe0, 0x10, 0xac, 0xdb, 0x48, 0xf7, 0xe8, 0xb6, 0x2c, 0x37, 0xc4, 0xbe, 0x6b, 0xd8, 0x51, 0x6b, 0xd0, 0x3f,
0xdb, 0x3a, 0xff, 0xaa, 0xd9, 0xfe, 0x90, 0xff, 0xec, 0x18, 0x7a, 0xa0, 0x27, 0x51, 0xf5, 0xe6, 0xc4, 0x8e, 0xc1, 0xbf, 0x6a, 0x4e, 0x30, 0xe0, 0x3f, 0x55, 0xd3, 0x08, 0x8d, 0x34, 0xaa, 0xee,
0x4d, 0xc7, 0xc0, 0xcf, 0x92, 0x20, 0xf5, 0xd7, 0x15, 0xb8, 0xbc, 0x77, 0xe8, 0x9e, 0x6c, 0xba, 0xbc, 0xe5, 0x9a, 0xf8, 0x59, 0x1a, 0xa4, 0xfd, 0xba, 0x02, 0x97, 0xf7, 0x0e, 0xbd, 0x93, 0x4d,
0x96, 0x85, 0x07, 0x81, 0xe9, 0x3a, 0xbe, 0x86, 0x9f, 0x8e, 0xb1, 0x1f, 0xa0, 0x5b, 0x50, 0xda, 0xcf, 0xb6, 0x71, 0x3f, 0xb4, 0x3c, 0x37, 0xd0, 0xf1, 0xd3, 0x11, 0x0e, 0x42, 0x74, 0x0b, 0x4a,
0xd7, 0x7d, 0xdc, 0x55, 0x96, 0x95, 0x95, 0xfa, 0xc6, 0xd5, 0x35, 0x81, 0x4e, 0x4e, 0xe0, 0xfb, 0xfb, 0x46, 0x80, 0x3b, 0xca, 0xb2, 0xb2, 0x52, 0xdf, 0xb8, 0xba, 0x26, 0xd0, 0xc9, 0x09, 0xfc,
0xfe, 0xf0, 0x9e, 0xee, 0x63, 0x8d, 0xf6, 0x44, 0x08, 0x4a, 0xc6, 0xfe, 0xf6, 0x56, 0xb7, 0xb0, 0x20, 0x18, 0xdc, 0x33, 0x02, 0xac, 0xd3, 0x9e, 0x08, 0x41, 0xc9, 0xdc, 0xdf, 0xde, 0xea, 0x14,
0xac, 0xac, 0x14, 0x35, 0xfa, 0x1b, 0xbd, 0x04, 0xcd, 0x41, 0x34, 0xf7, 0xf6, 0x96, 0xdf, 0x2d, 0x96, 0x95, 0x95, 0xa2, 0x4e, 0x7f, 0xa3, 0x97, 0xa0, 0xd9, 0x8f, 0xe7, 0xde, 0xde, 0x0a, 0x3a,
0x2e, 0x17, 0x57, 0x8a, 0x9a, 0x08, 0x54, 0x7f, 0x52, 0x80, 0x2b, 0x19, 0x32, 0xfc, 0x91, 0xeb, 0xc5, 0xe5, 0xe2, 0x4a, 0x51, 0x17, 0x81, 0xda, 0x8f, 0x0b, 0x70, 0x25, 0x47, 0x46, 0x30, 0xf4,
0xf8, 0x18, 0xdd, 0x86, 0x8a, 0x1f, 0xe8, 0xc1, 0xd8, 0xe7, 0x94, 0x7c, 0x55, 0x4a, 0xc9, 0x1e, 0xdc, 0x00, 0xa3, 0xdb, 0x50, 0x09, 0x42, 0x23, 0x1c, 0x05, 0x9c, 0x92, 0xaf, 0x4a, 0x29, 0xd9,
0xed, 0xa2, 0xf1, 0xae, 0x59, 0xb4, 0x05, 0x09, 0x5a, 0xf4, 0x75, 0x58, 0x34, 0x9d, 0xf7, 0xb1, 0xa3, 0x5d, 0x74, 0xde, 0x35, 0x8f, 0xb6, 0x20, 0x41, 0x8b, 0xbe, 0x0e, 0x8b, 0x96, 0xfb, 0x01,
0xed, 0x7a, 0xa7, 0xfd, 0x11, 0xf6, 0x06, 0xd8, 0x09, 0xf4, 0x21, 0x0e, 0x69, 0x5c, 0x08, 0xdb, 0x76, 0x3c, 0xff, 0xb4, 0x37, 0xc4, 0x7e, 0x1f, 0xbb, 0xa1, 0x31, 0xc0, 0x11, 0x8d, 0x0b, 0x51,
0x76, 0xe3, 0x26, 0xf4, 0x06, 0x5c, 0x61, 0x7b, 0xe8, 0x63, 0xef, 0xd8, 0x1c, 0xe0, 0xbe, 0x7e, 0xdb, 0x6e, 0xd2, 0x84, 0xde, 0x80, 0x2b, 0x8c, 0x87, 0x01, 0xf6, 0x8f, 0xad, 0x3e, 0xee, 0x19,
0xac, 0x9b, 0x96, 0xbe, 0x6f, 0xe1, 0x6e, 0x69, 0xb9, 0xb8, 0x52, 0xd5, 0x96, 0x68, 0xf3, 0x1e, 0xc7, 0x86, 0x65, 0x1b, 0xfb, 0x36, 0xee, 0x94, 0x96, 0x8b, 0x2b, 0x55, 0x7d, 0x89, 0x36, 0xef,
0x6b, 0xbd, 0x1b, 0x36, 0xa2, 0xaf, 0x41, 0xc7, 0xc3, 0x07, 0x1e, 0xf6, 0x0f, 0xfb, 0x23, 0xcf, 0xb1, 0xd6, 0xbb, 0x51, 0x23, 0xfa, 0x1a, 0xa8, 0x3e, 0x3e, 0xf0, 0x71, 0x70, 0xd8, 0x1b, 0xfa,
0x1d, 0x7a, 0xd8, 0xf7, 0xbb, 0x65, 0x8a, 0xa6, 0xcd, 0xe1, 0xbb, 0x1c, 0xac, 0xfe, 0xb1, 0x02, 0xde, 0xc0, 0xc7, 0x41, 0xd0, 0x29, 0x53, 0x34, 0x6d, 0x0e, 0xdf, 0xe5, 0x60, 0xed, 0x4f, 0x14,
0x4b, 0x84, 0x19, 0xbb, 0xba, 0x17, 0x98, 0x9f, 0xc2, 0x96, 0xa8, 0xd0, 0x48, 0xb2, 0xa1, 0x5b, 0x58, 0x22, 0x9b, 0xb1, 0x6b, 0xf8, 0xa1, 0xf5, 0x19, 0xb0, 0x44, 0x83, 0x46, 0x7a, 0x1b, 0x3a,
0xa4, 0x6d, 0x02, 0x8c, 0xf4, 0x19, 0x85, 0xe8, 0x09, 0xfb, 0x4a, 0x94, 0x54, 0x01, 0xa6, 0xfe, 0x45, 0xda, 0x26, 0xc0, 0x48, 0x9f, 0x61, 0x84, 0x9e, 0x6c, 0x5f, 0x89, 0x92, 0x2a, 0xc0, 0xb4,
0x0b, 0x97, 0x9d, 0x24, 0x9d, 0xb3, 0xec, 0x59, 0x1a, 0x67, 0x21, 0x8b, 0xf3, 0x22, 0x3b, 0x26, 0x7f, 0xe5, 0xb2, 0x93, 0xa6, 0x73, 0x16, 0x9e, 0x65, 0x71, 0x16, 0xf2, 0x38, 0x2f, 0xc2, 0x31,
0xe3, 0x7c, 0x49, 0xce, 0xf9, 0x7f, 0x2a, 0xc2, 0xd2, 0x43, 0x57, 0x37, 0x62, 0x31, 0xfc, 0xec, 0xd9, 0xce, 0x97, 0xe4, 0x3b, 0xff, 0xcf, 0x45, 0x58, 0x7a, 0xe8, 0x19, 0x66, 0x22, 0x86, 0x3f,
0x39, 0xff, 0x5d, 0xa8, 0x30, 0x8d, 0xee, 0x96, 0x28, 0xae, 0x9b, 0x22, 0x2e, 0xae, 0xed, 0x31, 0xff, 0x9d, 0x7f, 0x1b, 0x2a, 0x4c, 0xa3, 0x3b, 0x25, 0x8a, 0xeb, 0x65, 0x11, 0x17, 0xd7, 0xf6,
0x85, 0x7b, 0x14, 0xa0, 0xf1, 0x41, 0xe8, 0x26, 0xb4, 0x3c, 0x3c, 0xb2, 0xcc, 0x81, 0xde, 0x77, 0x84, 0xc2, 0x3d, 0x0a, 0xd0, 0xf9, 0x20, 0xf4, 0x32, 0xb4, 0x7c, 0x3c, 0xb4, 0xad, 0xbe, 0xd1,
0xc6, 0xf6, 0x3e, 0xf6, 0xba, 0xe5, 0x65, 0x65, 0xa5, 0xac, 0x35, 0x39, 0x74, 0x87, 0x02, 0xd1, 0x73, 0x47, 0xce, 0x3e, 0xf6, 0x3b, 0xe5, 0x65, 0x65, 0xa5, 0xac, 0x37, 0x39, 0x74, 0x87, 0x02,
0x8f, 0xa1, 0x79, 0x60, 0x62, 0xcb, 0xe8, 0x53, 0x93, 0xb0, 0xbd, 0xd5, 0xad, 0x2c, 0x17, 0x57, 0xd1, 0x8f, 0xa0, 0x79, 0x60, 0x61, 0xdb, 0xec, 0x51, 0x93, 0xb0, 0xbd, 0xd5, 0xa9, 0x2c, 0x17,
0xea, 0x1b, 0x6f, 0xae, 0x65, 0xad, 0xd1, 0x9a, 0x94, 0x23, 0x6b, 0xf7, 0xc9, 0xf0, 0x6d, 0x36, 0x57, 0xea, 0x1b, 0x6f, 0xae, 0xe5, 0xad, 0xd1, 0x9a, 0x74, 0x47, 0xd6, 0xee, 0x93, 0xe1, 0xdb,
0xfa, 0x1d, 0x27, 0xf0, 0x4e, 0xb5, 0xc6, 0x41, 0x02, 0x84, 0xba, 0x30, 0xc7, 0xd9, 0xdb, 0x9d, 0x6c, 0xf4, 0xbb, 0x6e, 0xe8, 0x9f, 0xea, 0x8d, 0x83, 0x14, 0x08, 0x75, 0x60, 0x8e, 0x6f, 0x6f,
0x5b, 0x56, 0x56, 0xaa, 0x5a, 0xf8, 0x89, 0x5e, 0x81, 0xb6, 0x87, 0x7d, 0x77, 0xec, 0x0d, 0x70, 0x67, 0x6e, 0x59, 0x59, 0xa9, 0xea, 0xd1, 0x27, 0x7a, 0x05, 0xda, 0x3e, 0x0e, 0xbc, 0x91, 0xdf,
0x7f, 0xe8, 0xb9, 0xe3, 0x91, 0xdf, 0xad, 0x2e, 0x17, 0x57, 0x6a, 0x5a, 0x2b, 0x04, 0x3f, 0xa0, 0xc7, 0xbd, 0x81, 0xef, 0x8d, 0x86, 0x41, 0xa7, 0xba, 0x5c, 0x5c, 0xa9, 0xe9, 0xad, 0x08, 0xfc,
0xd0, 0xde, 0xdb, 0x30, 0x9f, 0xc1, 0x82, 0x3a, 0x50, 0x3c, 0xc2, 0xa7, 0x74, 0x23, 0x8a, 0x1a, 0x80, 0x42, 0xbb, 0xef, 0xc0, 0x7c, 0x0e, 0x0b, 0x52, 0xa1, 0x78, 0x84, 0x4f, 0x29, 0x23, 0x8a,
0xf9, 0x89, 0x16, 0xa1, 0x7c, 0xac, 0x5b, 0x63, 0xcc, 0x59, 0xcd, 0x3e, 0xbe, 0x53, 0xb8, 0xa3, 0x3a, 0xf9, 0x89, 0x16, 0xa1, 0x7c, 0x6c, 0xd8, 0x23, 0xcc, 0xb7, 0x9a, 0x7d, 0x7c, 0xa7, 0x70,
0xa8, 0xbf, 0xaf, 0x40, 0x57, 0xc3, 0x16, 0xd6, 0x7d, 0xfc, 0x79, 0x6e, 0xe9, 0x65, 0xa8, 0x38, 0x47, 0xd1, 0xfe, 0x40, 0x81, 0x8e, 0x8e, 0x6d, 0x6c, 0x04, 0xf8, 0xf3, 0x64, 0xe9, 0x65, 0xa8,
0xae, 0x81, 0xb7, 0xb7, 0xe8, 0x96, 0x16, 0x35, 0xfe, 0xa5, 0x7e, 0xa2, 0xc0, 0xe2, 0x03, 0x1c, 0xb8, 0x9e, 0x89, 0xb7, 0xb7, 0x28, 0x4b, 0x8b, 0x3a, 0xff, 0xd2, 0x3e, 0x55, 0x60, 0xf1, 0x01,
0x10, 0x35, 0x30, 0xfd, 0xc0, 0x1c, 0x44, 0x7a, 0xfe, 0x5d, 0x28, 0x7a, 0xf8, 0x29, 0xa7, 0xec, 0x0e, 0x89, 0x1a, 0x58, 0x41, 0x68, 0xf5, 0x63, 0x3d, 0x7f, 0x1b, 0x8a, 0x3e, 0x7e, 0xca, 0x29,
0x55, 0x91, 0xb2, 0xc8, 0xfc, 0xcb, 0x46, 0x6a, 0x64, 0x1c, 0x7a, 0x11, 0x1a, 0x86, 0x6d, 0xf5, 0x7b, 0x55, 0xa4, 0x2c, 0x36, 0xff, 0xb2, 0x91, 0x3a, 0x19, 0x87, 0x5e, 0x84, 0x86, 0xe9, 0xd8,
0x07, 0x87, 0xba, 0xe3, 0x60, 0x8b, 0x29, 0x52, 0x4d, 0xab, 0x1b, 0xb6, 0xb5, 0xc9, 0x41, 0xe8, 0xbd, 0xfe, 0xa1, 0xe1, 0xba, 0xd8, 0x66, 0x8a, 0x54, 0xd3, 0xeb, 0xa6, 0x63, 0x6f, 0x72, 0x10,
0x3a, 0x80, 0x8f, 0x87, 0x36, 0x76, 0x82, 0xd8, 0x26, 0x27, 0x20, 0x68, 0x15, 0xe6, 0x0f, 0x3c, 0xba, 0x0e, 0x10, 0xe0, 0x81, 0x83, 0xdd, 0x30, 0xb1, 0xc9, 0x29, 0x08, 0x5a, 0x85, 0xf9, 0x03,
0xd7, 0xee, 0xfb, 0x87, 0xba, 0x67, 0xf4, 0x2d, 0xac, 0x1b, 0xd8, 0xa3, 0xd4, 0x57, 0xb5, 0x36, 0xdf, 0x73, 0x7a, 0xc1, 0xa1, 0xe1, 0x9b, 0x3d, 0x1b, 0x1b, 0x26, 0xf6, 0x29, 0xf5, 0x55, 0xbd,
0x69, 0xd8, 0x23, 0xf0, 0x87, 0x14, 0x8c, 0x6e, 0x43, 0xd9, 0x1f, 0xb8, 0x23, 0x4c, 0x25, 0xad, 0x4d, 0x1a, 0xf6, 0x08, 0xfc, 0x21, 0x05, 0xa3, 0xdb, 0x50, 0x0e, 0xfa, 0xde, 0x10, 0x53, 0x49,
0xb5, 0x71, 0x4d, 0x26, 0x43, 0x5b, 0x7a, 0xa0, 0xef, 0x91, 0x4e, 0x1a, 0xeb, 0xab, 0xfe, 0x75, 0x6b, 0x6d, 0x5c, 0x93, 0xc9, 0xd0, 0x96, 0x11, 0x1a, 0x7b, 0xa4, 0x93, 0xce, 0xfa, 0x6a, 0x7f,
0x89, 0xa9, 0xda, 0x17, 0xdc, 0xc8, 0x25, 0xd4, 0xb1, 0xfc, 0x7c, 0xd4, 0xb1, 0x92, 0x4b, 0x1d, 0x53, 0x62, 0xaa, 0xf6, 0x05, 0x37, 0x72, 0x29, 0x75, 0x2c, 0x3f, 0x1f, 0x75, 0xac, 0x4c, 0xa5,
0xe7, 0xce, 0x56, 0xc7, 0x0c, 0xd7, 0xce, 0xa3, 0x8e, 0xd5, 0xa9, 0xea, 0x58, 0x93, 0xa9, 0x23, 0x8e, 0x73, 0x67, 0xab, 0x63, 0x6e, 0xd7, 0xce, 0xa3, 0x8e, 0xd5, 0x89, 0xea, 0x58, 0x93, 0xa9,
0x7a, 0x07, 0xda, 0x2c, 0x80, 0x30, 0x9d, 0x03, 0xb7, 0x6f, 0x99, 0x7e, 0xd0, 0x05, 0x4a, 0xe6, 0x23, 0x7a, 0x17, 0xda, 0x2c, 0x80, 0xb0, 0xdc, 0x03, 0xaf, 0x67, 0x5b, 0x41, 0xd8, 0x01, 0x4a,
0xb5, 0xb4, 0x84, 0x1a, 0xf8, 0xd9, 0x1a, 0x43, 0xec, 0x1c, 0xb8, 0x5a, 0xd3, 0x0c, 0x7f, 0x3e, 0xe6, 0xb5, 0xac, 0x84, 0x9a, 0xf8, 0xd9, 0x1a, 0x43, 0xec, 0x1e, 0x78, 0x7a, 0xd3, 0x8a, 0x7e,
0x34, 0xfd, 0x60, 0x76, 0xad, 0xfe, 0xdb, 0x58, 0xab, 0xbf, 0xe8, 0xd2, 0x13, 0x6b, 0x7e, 0x59, 0x3e, 0xb4, 0x82, 0x70, 0x76, 0xad, 0xfe, 0xbb, 0x44, 0xab, 0xbf, 0xe8, 0xd2, 0x93, 0x68, 0x7e,
0xd0, 0xfc, 0x3f, 0x51, 0xe0, 0x2b, 0x0f, 0x70, 0x10, 0x91, 0x4f, 0x14, 0x19, 0x7f, 0x41, 0xdd, 0x59, 0xd0, 0xfc, 0x3f, 0x55, 0xe0, 0x2b, 0x0f, 0x70, 0x18, 0x93, 0x4f, 0x14, 0x19, 0x7f, 0x41,
0xfc, 0x9f, 0x2b, 0xd0, 0x93, 0xd1, 0x3a, 0x8b, 0xab, 0xff, 0x10, 0x2e, 0x47, 0x38, 0xfa, 0x06, 0xdd, 0xfc, 0x5f, 0x28, 0xd0, 0x95, 0xd1, 0x3a, 0x8b, 0xab, 0xff, 0x08, 0x2e, 0xc7, 0x38, 0x7a,
0xf6, 0x07, 0x9e, 0x39, 0xa2, 0xdb, 0x48, 0x6d, 0x55, 0x7d, 0xe3, 0x86, 0x4c, 0xf0, 0xd3, 0x14, 0x26, 0x0e, 0xfa, 0xbe, 0x35, 0xa4, 0x6c, 0xa4, 0xb6, 0xaa, 0xbe, 0x71, 0x43, 0x26, 0xf8, 0x59,
0x2c, 0x45, 0x53, 0x6c, 0x25, 0x66, 0x50, 0x7f, 0xaa, 0xc0, 0x12, 0xb1, 0x8d, 0xdc, 0x98, 0x11, 0x0a, 0x96, 0xe2, 0x29, 0xb6, 0x52, 0x33, 0x68, 0x3f, 0x51, 0x60, 0x89, 0xd8, 0x46, 0x6e, 0xcc,
0x09, 0xbc, 0x30, 0x5f, 0x45, 0x33, 0x59, 0xc8, 0x98, 0xc9, 0x1c, 0x3c, 0xa6, 0x21, 0x76, 0x9a, 0x88, 0x04, 0x5e, 0x78, 0x5f, 0x45, 0x33, 0x59, 0xc8, 0x99, 0xc9, 0x29, 0xf6, 0x98, 0x86, 0xd8,
0x9e, 0x59, 0x78, 0xf7, 0x4d, 0x28, 0x13, 0x05, 0x0c, 0x59, 0xf5, 0x82, 0x8c, 0x55, 0x49, 0x64, 0x59, 0x7a, 0x66, 0xd9, 0xbb, 0x6f, 0x42, 0x99, 0x28, 0x60, 0xb4, 0x55, 0x2f, 0xc8, 0xb6, 0x2a,
0xac, 0xb7, 0xea, 0x30, 0x2a, 0x62, 0xbb, 0x3d, 0x83, 0xb8, 0xa5, 0x97, 0x5d, 0x90, 0x2c, 0xfb, 0x8d, 0x8c, 0xf5, 0xd6, 0x5c, 0x46, 0x45, 0x62, 0xb7, 0x67, 0x10, 0xb7, 0xec, 0xb2, 0x0b, 0x92,
0x37, 0x15, 0xb8, 0x92, 0x41, 0x38, 0xcb, 0xba, 0xdf, 0x82, 0x0a, 0xf5, 0x46, 0xe1, 0xc2, 0x5f, 0x65, 0xff, 0xa6, 0x02, 0x57, 0x72, 0x08, 0x67, 0x59, 0xf7, 0x5b, 0x50, 0xa1, 0xde, 0x28, 0x5a,
0x92, 0x2e, 0x3c, 0x81, 0x8e, 0x58, 0x1b, 0x8d, 0x8f, 0x51, 0x5d, 0xe8, 0xa4, 0xdb, 0x88, 0x9f, 0xf8, 0x4b, 0xd2, 0x85, 0xa7, 0xd0, 0x11, 0x6b, 0xa3, 0xf3, 0x31, 0x9a, 0x07, 0x6a, 0xb6, 0x8d,
0xe4, 0x3e, 0xb2, 0xef, 0xe8, 0x36, 0x63, 0x40, 0x4d, 0xab, 0x73, 0xd8, 0x8e, 0x6e, 0x63, 0xf4, 0xf8, 0x49, 0xee, 0x23, 0x7b, 0xae, 0xe1, 0xb0, 0x0d, 0xa8, 0xe9, 0x75, 0x0e, 0xdb, 0x31, 0x1c,
0x15, 0xa8, 0x12, 0x95, 0xed, 0x9b, 0x46, 0xb8, 0xfd, 0x73, 0x54, 0x85, 0x0d, 0x1f, 0x5d, 0x03, 0x8c, 0xbe, 0x02, 0x55, 0xa2, 0xb2, 0x3d, 0xcb, 0x8c, 0xd8, 0x3f, 0x47, 0x55, 0xd8, 0x0c, 0xd0,
0xa0, 0x4d, 0xba, 0x61, 0x78, 0xcc, 0x85, 0xd6, 0xb4, 0x1a, 0x81, 0xdc, 0x25, 0x00, 0xf5, 0xf7, 0x35, 0x00, 0xda, 0x64, 0x98, 0xa6, 0xcf, 0x5c, 0x68, 0x4d, 0xaf, 0x11, 0xc8, 0x5d, 0x02, 0xd0,
0x14, 0xb8, 0xbe, 0x77, 0xea, 0x0c, 0x76, 0xf0, 0xc9, 0xa6, 0x87, 0xf5, 0x00, 0xc7, 0x46, 0xfb, 0x7e, 0x5f, 0x81, 0xeb, 0x7b, 0xa7, 0x6e, 0x7f, 0x07, 0x9f, 0x6c, 0xfa, 0xd8, 0x08, 0x71, 0x62,
0x53, 0x65, 0x3c, 0x5a, 0x86, 0x7a, 0x42, 0x7f, 0xb9, 0x48, 0x26, 0x41, 0xea, 0x5f, 0x28, 0xd0, 0xb4, 0x3f, 0xd3, 0x8d, 0x47, 0xcb, 0x50, 0x4f, 0xe9, 0x2f, 0x17, 0xc9, 0x34, 0x48, 0xfb, 0x4b,
0x20, 0x5e, 0xe4, 0x7d, 0x1c, 0xe8, 0x44, 0x44, 0xd0, 0xb7, 0xa1, 0x66, 0xb9, 0xba, 0xd1, 0x0f, 0x05, 0x1a, 0xc4, 0x8b, 0x7c, 0x80, 0x43, 0x83, 0x88, 0x08, 0xfa, 0x36, 0xd4, 0x6c, 0xcf, 0x30,
0x4e, 0x47, 0x8c, 0x9a, 0x56, 0x9a, 0x9a, 0xd8, 0xf5, 0x3c, 0x3a, 0x1d, 0x61, 0xad, 0x6a, 0xf1, 0x7b, 0xe1, 0xe9, 0x90, 0x51, 0xd3, 0xca, 0x52, 0x93, 0xb8, 0x9e, 0x47, 0xa7, 0x43, 0xac, 0x57,
0x5f, 0xb9, 0x28, 0x4a, 0x5b, 0x99, 0xa2, 0xc4, 0x52, 0xbe, 0x00, 0x75, 0x1b, 0x07, 0x9e, 0x39, 0x6d, 0xfe, 0x6b, 0x2a, 0x8a, 0xb2, 0x56, 0xa6, 0x28, 0xb1, 0x94, 0x2f, 0x40, 0xdd, 0xc1, 0xa1,
0x60, 0x44, 0x94, 0xe8, 0x56, 0x00, 0x03, 0x11, 0x44, 0xea, 0x4f, 0x2b, 0x70, 0xf9, 0x07, 0x7a, 0x6f, 0xf5, 0x19, 0x11, 0x25, 0xca, 0x0a, 0x60, 0x20, 0x82, 0x48, 0xfb, 0x49, 0x05, 0x2e, 0x7f,
0x30, 0x38, 0xdc, 0xb2, 0xc3, 0x28, 0xe6, 0xe2, 0x7c, 0x8c, 0xed, 0x72, 0x21, 0x69, 0x97, 0x9f, 0xdf, 0x08, 0xfb, 0x87, 0x5b, 0x4e, 0x14, 0xc5, 0x5c, 0x7c, 0x1f, 0x13, 0xbb, 0x5c, 0x48, 0xdb,
0x9b, 0xdd, 0x8f, 0x74, 0xb4, 0x2c, 0xd3, 0x51, 0x92, 0x98, 0xaf, 0x3d, 0xe1, 0x62, 0x96, 0xd0, 0xe5, 0xe7, 0x66, 0xf7, 0x63, 0x1d, 0x2d, 0xcb, 0x74, 0x94, 0x24, 0xe6, 0x6b, 0x4f, 0xb8, 0x98,
0xd1, 0x44, 0xb0, 0x51, 0xb9, 0x48, 0xb0, 0xb1, 0x09, 0x4d, 0xfc, 0x6c, 0x60, 0x8d, 0x89, 0xbc, 0xa5, 0x74, 0x34, 0x15, 0x6c, 0x54, 0x2e, 0x12, 0x6c, 0x6c, 0x42, 0x13, 0x3f, 0xeb, 0xdb, 0x23,
0x52, 0xec, 0x2c, 0x8a, 0xb8, 0x2e, 0xc1, 0x9e, 0x34, 0x10, 0x0d, 0x3e, 0x68, 0x9b, 0xd3, 0xc0, 0x22, 0xaf, 0x14, 0x3b, 0x8b, 0x22, 0xae, 0x4b, 0xb0, 0xa7, 0x0d, 0x44, 0x83, 0x0f, 0xda, 0xe6,
0x64, 0xc1, 0xc6, 0x81, 0x4e, 0x43, 0x85, 0xfa, 0xc6, 0xf2, 0x24, 0x59, 0x08, 0x05, 0x88, 0xc9, 0x34, 0x30, 0x59, 0x70, 0x70, 0x68, 0xd0, 0x50, 0xa1, 0xbe, 0xb1, 0x3c, 0x4e, 0x16, 0x22, 0x01,
0x03, 0xf9, 0x42, 0x57, 0xa1, 0xc6, 0x43, 0x9b, 0xed, 0xad, 0x6e, 0x8d, 0xb2, 0x2f, 0x06, 0x20, 0x62, 0xf2, 0x40, 0xbe, 0xd0, 0x55, 0xa8, 0xf1, 0xd0, 0x66, 0x7b, 0xab, 0x53, 0xa3, 0xdb, 0x97,
0x1d, 0x9a, 0xdc, 0x7a, 0x72, 0x0a, 0x59, 0x00, 0xf1, 0x96, 0x0c, 0x81, 0x7c, 0xb3, 0x93, 0x94, 0x00, 0x90, 0x01, 0x4d, 0x6e, 0x3d, 0x39, 0x85, 0x2c, 0x80, 0x78, 0x4b, 0x86, 0x40, 0xce, 0xec,
0xfb, 0x3c, 0xd0, 0xf1, 0x13, 0x20, 0x92, 0xf9, 0xbb, 0x07, 0x07, 0x96, 0xe9, 0xe0, 0x1d, 0xb6, 0x34, 0xe5, 0x01, 0x0f, 0x74, 0x82, 0x14, 0x88, 0x64, 0xfe, 0xde, 0xc1, 0x81, 0x6d, 0xb9, 0x78,
0xc3, 0x75, 0x4a, 0x84, 0x08, 0x24, 0xe1, 0xd0, 0x31, 0xf6, 0x7c, 0xd3, 0x75, 0xba, 0x0d, 0xda, 0x87, 0x71, 0xb8, 0x4e, 0x89, 0x10, 0x81, 0x24, 0x1c, 0x3a, 0xc6, 0x7e, 0x60, 0x79, 0x6e, 0xa7,
0x1e, 0x7e, 0xca, 0xa2, 0x9c, 0xe6, 0x05, 0xa2, 0x9c, 0x3e, 0xcc, 0x67, 0x28, 0x95, 0x44, 0x39, 0x41, 0xdb, 0xa3, 0x4f, 0x59, 0x94, 0xd3, 0xbc, 0x40, 0x94, 0xd3, 0x83, 0xf9, 0x1c, 0xa5, 0x92,
0xdf, 0x48, 0x46, 0x39, 0xd3, 0xb7, 0x2a, 0x11, 0x05, 0xfd, 0x4c, 0x81, 0xa5, 0xc7, 0x8e, 0x3f, 0x28, 0xe7, 0x1b, 0xe9, 0x28, 0x67, 0x32, 0xab, 0x52, 0x51, 0xd0, 0x4f, 0x15, 0x58, 0x7a, 0xec,
0xde, 0x8f, 0x58, 0xf4, 0xf9, 0xa8, 0x43, 0xda, 0x88, 0x96, 0x32, 0x46, 0x54, 0xfd, 0x87, 0x32, 0x06, 0xa3, 0xfd, 0x78, 0x8b, 0x3e, 0x1f, 0x75, 0xc8, 0x1a, 0xd1, 0x52, 0xce, 0x88, 0x6a, 0xff,
0xb4, 0xf9, 0x2a, 0x88, 0xd4, 0x50, 0x93, 0x73, 0x15, 0x6a, 0x91, 0x1f, 0xe5, 0x0c, 0x89, 0x01, 0x58, 0x86, 0x36, 0x5f, 0x05, 0x91, 0x1a, 0x6a, 0x72, 0xae, 0x42, 0x2d, 0xf6, 0xa3, 0x7c, 0x43,
0x69, 0x1b, 0x56, 0xc8, 0xd8, 0xb0, 0x5c, 0xa4, 0x85, 0x51, 0x51, 0x29, 0x11, 0x15, 0x5d, 0x03, 0x12, 0x40, 0xd6, 0x86, 0x15, 0x72, 0x36, 0x6c, 0x2a, 0xd2, 0xa2, 0xa8, 0xa8, 0x94, 0x8a, 0x8a,
0x38, 0xb0, 0xc6, 0xfe, 0x61, 0x3f, 0x30, 0x6d, 0xcc, 0xa3, 0xb2, 0x1a, 0x85, 0x3c, 0x32, 0x6d, 0xae, 0x01, 0x1c, 0xd8, 0xa3, 0xe0, 0xb0, 0x17, 0x5a, 0x0e, 0xe6, 0x51, 0x59, 0x8d, 0x42, 0x1e,
0x8c, 0xee, 0x42, 0x63, 0xdf, 0x74, 0x2c, 0x77, 0xd8, 0x1f, 0xe9, 0xc1, 0xa1, 0xcf, 0xd3, 0x62, 0x59, 0x0e, 0x46, 0x77, 0xa1, 0xb1, 0x6f, 0xb9, 0xb6, 0x37, 0xe8, 0x0d, 0x8d, 0xf0, 0x30, 0xe0,
0xd9, 0xb6, 0xd0, 0x18, 0xf6, 0x1e, 0xed, 0xab, 0xd5, 0xd9, 0x98, 0x5d, 0x32, 0x04, 0x5d, 0x87, 0x69, 0xb1, 0x8c, 0x2d, 0x34, 0x86, 0xbd, 0x47, 0xfb, 0xea, 0x75, 0x36, 0x66, 0x97, 0x0c, 0x41,
0xba, 0x33, 0xb6, 0xfb, 0xee, 0x41, 0xdf, 0x73, 0x4f, 0x7c, 0x9a, 0xfc, 0x16, 0xb5, 0x9a, 0x33, 0xd7, 0xa1, 0xee, 0x8e, 0x9c, 0x9e, 0x77, 0xd0, 0xf3, 0xbd, 0x93, 0x80, 0x26, 0xbf, 0x45, 0xbd,
0xb6, 0x3f, 0x38, 0xd0, 0xdc, 0x13, 0xe2, 0xc7, 0x6a, 0xc4, 0xa3, 0xf9, 0x96, 0x3b, 0x64, 0x89, 0xe6, 0x8e, 0x9c, 0x0f, 0x0f, 0x74, 0xef, 0x84, 0xf8, 0xb1, 0x1a, 0xf1, 0x68, 0x81, 0xed, 0x0d,
0xef, 0xf4, 0xf9, 0xe3, 0x01, 0x64, 0xb4, 0x81, 0xad, 0x40, 0xa7, 0xa3, 0x6b, 0xf9, 0x46, 0x47, 0x58, 0xe2, 0x3b, 0x79, 0xfe, 0x64, 0x00, 0x19, 0x6d, 0x62, 0x3b, 0x34, 0xe8, 0xe8, 0xda, 0x74,
0x03, 0xd0, 0xcb, 0xd0, 0x1a, 0xb8, 0xf6, 0x48, 0xa7, 0x1c, 0xba, 0xef, 0xb9, 0x36, 0x55, 0xc0, 0xa3, 0xe3, 0x01, 0xe8, 0x26, 0xb4, 0xfa, 0x9e, 0x33, 0x34, 0xe8, 0x0e, 0xdd, 0xf7, 0x3d, 0x87,
0xa2, 0x96, 0x82, 0xa2, 0x4d, 0xa8, 0xc7, 0x4a, 0xe0, 0x77, 0xeb, 0x14, 0x8f, 0x2a, 0xd3, 0xd2, 0x2a, 0x60, 0x51, 0xcf, 0x40, 0xd1, 0x26, 0xd4, 0x13, 0x25, 0x08, 0x3a, 0x75, 0x8a, 0x47, 0x93,
0x44, 0x28, 0x4f, 0x04, 0x14, 0x22, 0x2d, 0xf0, 0x89, 0x64, 0x84, 0xca, 0xee, 0x9b, 0x1f, 0x63, 0x69, 0x69, 0x2a, 0x94, 0x27, 0x02, 0x0a, 0xb1, 0x16, 0x04, 0x44, 0x32, 0x22, 0x65, 0x0f, 0xac,
0xae, 0x68, 0x75, 0x0e, 0xdb, 0x33, 0x3f, 0xc6, 0x24, 0x3d, 0x32, 0x1d, 0x1f, 0x7b, 0x41, 0x98, 0x4f, 0x30, 0x57, 0xb4, 0x3a, 0x87, 0xed, 0x59, 0x9f, 0x60, 0x92, 0x1e, 0x59, 0x6e, 0x80, 0xfd,
0xac, 0x76, 0x9b, 0x54, 0x7c, 0x9a, 0x0c, 0xca, 0x05, 0x1b, 0x6d, 0x41, 0xcb, 0x0f, 0x74, 0x2f, 0x30, 0x4a, 0x56, 0x3b, 0x4d, 0x2a, 0x3e, 0x4d, 0x06, 0xe5, 0x82, 0x8d, 0xb6, 0xa0, 0x15, 0x84,
0xe8, 0x8f, 0x5c, 0x9f, 0x0a, 0x40, 0xb7, 0x45, 0x65, 0x3b, 0xa5, 0x92, 0xb6, 0x3f, 0x24, 0x82, 0x86, 0x1f, 0xf6, 0x86, 0x5e, 0x40, 0x05, 0xa0, 0xd3, 0xa2, 0xb2, 0x9d, 0x51, 0x49, 0x27, 0x18,
0xbd, 0xcb, 0x3b, 0x69, 0x4d, 0x3a, 0x28, 0xfc, 0x24, 0xb3, 0x50, 0x4e, 0xc4, 0xb3, 0xb4, 0x73, 0x10, 0xc1, 0xde, 0xe5, 0x9d, 0xf4, 0x26, 0x1d, 0x14, 0x7d, 0x92, 0x59, 0xe8, 0x4e, 0x24, 0xb3,
0xcd, 0x42, 0x07, 0x85, 0x9f, 0xea, 0xff, 0x14, 0xa0, 0x25, 0x2e, 0x9a, 0x18, 0x13, 0x96, 0x6b, 0xb4, 0xa7, 0x9a, 0x85, 0x0e, 0x8a, 0x3e, 0xb5, 0xff, 0x29, 0x40, 0x4b, 0x5c, 0x34, 0x31, 0x26,
0x85, 0x92, 0x1c, 0x7e, 0x12, 0x16, 0x60, 0x47, 0xdf, 0xb7, 0x30, 0x4b, 0xec, 0xa8, 0x20, 0x57, 0x2c, 0xd7, 0x8a, 0x24, 0x39, 0xfa, 0x24, 0x5b, 0x80, 0x5d, 0x63, 0xdf, 0xc6, 0x2c, 0xb1, 0xa3,
0xb5, 0x3a, 0x83, 0xd1, 0x09, 0x88, 0x40, 0x32, 0x56, 0x53, 0xed, 0x29, 0xd2, 0xe5, 0xd7, 0x28, 0x82, 0x5c, 0xd5, 0xeb, 0x0c, 0x46, 0x27, 0x20, 0x02, 0xc9, 0xb6, 0x9a, 0x6a, 0x4f, 0x91, 0x2e,
0x84, 0x06, 0x20, 0x5d, 0x98, 0x0b, 0x73, 0x42, 0x26, 0xc6, 0xe1, 0x27, 0x69, 0xd9, 0x1f, 0x9b, 0xbf, 0x46, 0x21, 0x34, 0x00, 0xe9, 0xc0, 0x5c, 0x94, 0x13, 0x32, 0x31, 0x8e, 0x3e, 0x49, 0xcb,
0x14, 0x2b, 0x13, 0xe3, 0xf0, 0x13, 0x6d, 0x41, 0x83, 0x4d, 0x39, 0xd2, 0x3d, 0xdd, 0x0e, 0x85, 0xfe, 0xc8, 0xa2, 0x58, 0x99, 0x18, 0x47, 0x9f, 0x68, 0x0b, 0x1a, 0x6c, 0xca, 0xa1, 0xe1, 0x1b,
0xf8, 0x45, 0xa9, 0x21, 0x78, 0x0f, 0x9f, 0x3e, 0x21, 0x36, 0x65, 0x57, 0x37, 0x3d, 0x8d, 0x6d, 0x4e, 0x24, 0xc4, 0x2f, 0x4a, 0x0d, 0xc1, 0xfb, 0xf8, 0xf4, 0x09, 0xb1, 0x29, 0xbb, 0x86, 0xe5,
0xfa, 0x2e, 0x1d, 0x85, 0x56, 0xa0, 0xc3, 0x66, 0x39, 0x30, 0x2d, 0xcc, 0xd5, 0x61, 0x8e, 0x25, 0xeb, 0x8c, 0xe9, 0xbb, 0x74, 0x14, 0x5a, 0x01, 0x95, 0xcd, 0x72, 0x60, 0xd9, 0x98, 0xab, 0xc3,
0x86, 0x14, 0x7e, 0xdf, 0xb4, 0x30, 0x93, 0xf8, 0x68, 0x09, 0x74, 0x9b, 0xab, 0x4c, 0xe0, 0x29, 0x1c, 0x4b, 0x0c, 0x29, 0xfc, 0xbe, 0x65, 0x63, 0x26, 0xf1, 0xf1, 0x12, 0x28, 0x9b, 0xab, 0x4c,
0x84, 0x6e, 0xf2, 0x0d, 0x60, 0xb6, 0xb1, 0x1f, 0x5a, 0x5c, 0xe6, 0x16, 0x18, 0x8d, 0x4f, 0xb8, 0xe0, 0x29, 0x84, 0x32, 0xf9, 0x06, 0x30, 0xdb, 0xd8, 0x8b, 0x2c, 0x2e, 0x73, 0x0b, 0x8c, 0xc6,
0xd9, 0x25, 0x81, 0xd6, 0xd8, 0x66, 0x2a, 0x03, 0x6c, 0x39, 0xce, 0xd8, 0x26, 0x0a, 0xa3, 0xfe, 0x27, 0xdc, 0xec, 0x92, 0x40, 0x6b, 0xe4, 0x30, 0x95, 0x01, 0xb6, 0x1c, 0x77, 0xe4, 0x10, 0x85,
0x63, 0x09, 0x16, 0x88, 0xdd, 0xe0, 0x26, 0x64, 0x06, 0xb7, 0x7f, 0x0d, 0xc0, 0xf0, 0x83, 0xbe, 0xd1, 0xfe, 0xa9, 0x04, 0x0b, 0xc4, 0x6e, 0x70, 0x13, 0x32, 0x83, 0xdb, 0xbf, 0x06, 0x60, 0x06,
0x60, 0xeb, 0x6a, 0x86, 0x1f, 0x70, 0xa7, 0xf0, 0xed, 0xd0, 0x6b, 0x17, 0x27, 0x27, 0x21, 0x29, 0x61, 0x4f, 0xb0, 0x75, 0x35, 0x33, 0x08, 0xb9, 0x53, 0xf8, 0x76, 0xe4, 0xb5, 0x8b, 0xe3, 0x93,
0x3b, 0x96, 0xf5, 0xdc, 0x17, 0xaa, 0xda, 0xdd, 0x80, 0x26, 0xcf, 0xc0, 0x85, 0x74, 0xb1, 0xc1, 0x90, 0x8c, 0x1d, 0xcb, 0x7b, 0xee, 0x0b, 0x55, 0xed, 0x6e, 0x40, 0x93, 0x67, 0xe0, 0x42, 0xba,
0x80, 0x3b, 0x72, 0x6b, 0x5c, 0x91, 0x56, 0x0f, 0x13, 0xde, 0x7b, 0x6e, 0x36, 0xef, 0x5d, 0x4d, 0xd8, 0x60, 0xc0, 0x1d, 0xb9, 0x35, 0xae, 0x48, 0xab, 0x87, 0x29, 0xef, 0x3d, 0x37, 0x9b, 0xf7,
0x7b, 0xef, 0xfb, 0xd0, 0x16, 0x15, 0x28, 0xb4, 0x40, 0x53, 0x34, 0xa8, 0x25, 0x68, 0x90, 0x9f, 0xae, 0x66, 0xbd, 0xf7, 0x7d, 0x68, 0x8b, 0x0a, 0x14, 0x59, 0xa0, 0x09, 0x1a, 0xd4, 0x12, 0x34,
0x74, 0xbe, 0x20, 0x3a, 0xdf, 0x1b, 0xd0, 0x74, 0x30, 0x36, 0xfa, 0x81, 0xa7, 0x3b, 0xfe, 0x01, 0x28, 0x48, 0x3b, 0x5f, 0x10, 0x9d, 0xef, 0x0d, 0x68, 0xba, 0x18, 0x9b, 0xbd, 0xd0, 0x37, 0xdc,
0xf6, 0xa8, 0xf3, 0xae, 0x6a, 0x0d, 0x02, 0x7c, 0xc4, 0x61, 0xe8, 0x2d, 0x00, 0xba, 0x46, 0x56, 0xe0, 0x00, 0xfb, 0xd4, 0x79, 0x57, 0xf5, 0x06, 0x01, 0x3e, 0xe2, 0x30, 0xf4, 0x16, 0x00, 0x5d,
0x74, 0x6a, 0x4c, 0x2e, 0x3a, 0x51, 0xa1, 0xa1, 0x45, 0x27, 0xca, 0x14, 0xfa, 0x53, 0xfd, 0xe7, 0x23, 0x2b, 0x3a, 0x35, 0xc6, 0x17, 0x9d, 0xa8, 0xd0, 0xd0, 0xa2, 0x13, 0xdd, 0x14, 0xfa, 0x53,
0x02, 0x5c, 0xe6, 0xd5, 0x83, 0xd9, 0x05, 0x6a, 0x92, 0xe3, 0x0c, 0x3d, 0x4f, 0xf1, 0x8c, 0x7c, 0xfb, 0x97, 0x02, 0x5c, 0xe6, 0xd5, 0x83, 0xd9, 0x05, 0x6a, 0x9c, 0xe3, 0x8c, 0x3c, 0x4f, 0xf1,
0xbc, 0x94, 0x23, 0xb6, 0x2c, 0x4b, 0x62, 0x4b, 0x31, 0x27, 0xad, 0x64, 0x72, 0xd2, 0xa8, 0x1c, 0x8c, 0x7c, 0xbc, 0x34, 0x45, 0x6c, 0x59, 0x96, 0xc4, 0x96, 0x62, 0x4e, 0x5a, 0xc9, 0xe5, 0xa4,
0x37, 0x97, 0xbf, 0x1c, 0x87, 0x16, 0xa1, 0x4c, 0x13, 0x25, 0xba, 0xe9, 0x35, 0x8d, 0x7d, 0xe4, 0x71, 0x39, 0x6e, 0x6e, 0xfa, 0x72, 0x1c, 0x5a, 0x84, 0x32, 0x4d, 0x94, 0x28, 0xd3, 0x6b, 0x3a,
0xda, 0x0e, 0xf5, 0x77, 0x0b, 0xd0, 0xdc, 0xc3, 0xba, 0x37, 0x38, 0x0c, 0xf9, 0xf8, 0x46, 0xb2, 0xfb, 0x98, 0x8a, 0x1d, 0xda, 0xef, 0x15, 0xa0, 0xb9, 0x87, 0x0d, 0xbf, 0x7f, 0x18, 0xed, 0xe3,
0x7c, 0xf9, 0xd2, 0x84, 0xf2, 0xa5, 0x30, 0xe4, 0x4b, 0x53, 0xb7, 0x24, 0x08, 0x02, 0x37, 0xd0, 0x1b, 0xe9, 0xf2, 0xe5, 0x4b, 0x63, 0xca, 0x97, 0xc2, 0x90, 0x2f, 0x4d, 0xdd, 0x92, 0x20, 0x08,
0x23, 0x2a, 0xfb, 0xce, 0xd8, 0xe6, 0x35, 0xbd, 0x36, 0x6d, 0xe0, 0xa4, 0xee, 0x8c, 0x6d, 0xf5, 0xbd, 0xd0, 0x88, 0xa9, 0xec, 0xb9, 0x23, 0x87, 0xd7, 0xf4, 0xda, 0xb4, 0x81, 0x93, 0xba, 0x33,
0xbf, 0x14, 0x68, 0x7c, 0x9f, 0x4c, 0x13, 0x32, 0xe6, 0x4e, 0x92, 0x31, 0x2f, 0x4f, 0x60, 0x8c, 0x72, 0xb4, 0xff, 0x56, 0xa0, 0xf1, 0x3d, 0x32, 0x4d, 0xb4, 0x31, 0x77, 0xd2, 0x1b, 0x73, 0x73,
0x46, 0x72, 0x1e, 0x7c, 0x8c, 0xbf, 0x74, 0x25, 0xdd, 0xbf, 0x53, 0xa0, 0x47, 0x32, 0x5e, 0x8d, 0xcc, 0xc6, 0xe8, 0x24, 0xe7, 0xc1, 0xc7, 0xf8, 0x4b, 0x57, 0xd2, 0xfd, 0x7b, 0x05, 0xba, 0x24,
0x19, 0x8c, 0xd9, 0xb5, 0xeb, 0x06, 0x34, 0x8f, 0x85, 0xd8, 0xb2, 0x40, 0x85, 0xb3, 0x71, 0x9c, 0xe3, 0xd5, 0x99, 0xc1, 0x98, 0x5d, 0xbb, 0x6e, 0x40, 0xf3, 0x58, 0x88, 0x2d, 0x0b, 0x54, 0x38,
0xcc, 0xd0, 0x35, 0xe8, 0x84, 0x15, 0x56, 0xbe, 0xd8, 0xd0, 0x7e, 0xbf, 0x22, 0xa3, 0x3a, 0x45, 0x1b, 0xc7, 0xe9, 0x0c, 0x5d, 0x07, 0x35, 0xaa, 0xb0, 0xf2, 0xc5, 0x46, 0xf6, 0xfb, 0x15, 0x19,
0x1c, 0xb5, 0x7f, 0x6d, 0x4f, 0x04, 0xaa, 0xbf, 0xa5, 0xc0, 0x82, 0xa4, 0x23, 0xba, 0x02, 0x73, 0xd5, 0x19, 0xe2, 0xa8, 0xfd, 0x6b, 0xfb, 0x22, 0x50, 0xfb, 0x2d, 0x05, 0x16, 0x24, 0x1d, 0xd1,
0xbc, 0x1a, 0xc0, 0x1d, 0x3d, 0xd3, 0x77, 0x83, 0x6c, 0x4f, 0x5c, 0xcf, 0x32, 0x8d, 0x6c, 0xc0, 0x15, 0x98, 0xe3, 0xd5, 0x00, 0xee, 0xe8, 0x99, 0xbe, 0x9b, 0x84, 0x3d, 0x49, 0x3d, 0xcb, 0x32,
0x6a, 0x90, 0x04, 0x37, 0x4a, 0x7d, 0x8c, 0xcc, 0xfe, 0x18, 0x3e, 0xea, 0x41, 0x95, 0x9b, 0xc1, 0xf3, 0x01, 0xab, 0x49, 0x12, 0xdc, 0x38, 0xf5, 0x31, 0x73, 0xfc, 0x31, 0x03, 0xd4, 0x85, 0x2a,
0x30, 0xa7, 0x8c, 0xbe, 0xd5, 0x23, 0x40, 0x0f, 0x70, 0xec, 0x74, 0x66, 0xe1, 0x68, 0x6c, 0x6f, 0x37, 0x83, 0x51, 0x4e, 0x19, 0x7f, 0x6b, 0x47, 0x80, 0x1e, 0xe0, 0xc4, 0xe9, 0xcc, 0xb2, 0xa3,
0x62, 0x42, 0x93, 0x46, 0xc8, 0x50, 0xff, 0x5d, 0x81, 0x05, 0x01, 0xdb, 0x2c, 0x55, 0x9b, 0xd8, 0x89, 0xbd, 0x49, 0x08, 0x4d, 0x1b, 0x21, 0x53, 0xfb, 0x4f, 0x05, 0x16, 0x04, 0x6c, 0xb3, 0x54,
0x31, 0x16, 0x2e, 0xe2, 0x18, 0x85, 0xca, 0x44, 0xf1, 0x5c, 0x95, 0x89, 0xeb, 0x00, 0x11, 0xff, 0x6d, 0x12, 0xc7, 0x58, 0xb8, 0x88, 0x63, 0x14, 0x2a, 0x13, 0xc5, 0x73, 0x55, 0x26, 0xae, 0x03,
0x43, 0x8e, 0x26, 0x20, 0xea, 0xdf, 0x28, 0x70, 0xf9, 0x5d, 0xdd, 0x31, 0xdc, 0x83, 0x83, 0xd9, 0xc4, 0xfb, 0x1f, 0xed, 0x68, 0x0a, 0xa2, 0xfd, 0xad, 0x02, 0x97, 0xdf, 0x33, 0x5c, 0xd3, 0x3b,
0x45, 0x75, 0x13, 0x84, 0x2c, 0x34, 0x6f, 0x6d, 0x4e, 0x4c, 0x5d, 0x5f, 0x85, 0x79, 0x8f, 0x79, 0x38, 0x98, 0x5d, 0x54, 0x37, 0x41, 0xc8, 0x42, 0xa7, 0xad, 0xcd, 0x89, 0xa9, 0xeb, 0xab, 0x30,
0x26, 0x43, 0x94, 0xe5, 0xa2, 0xd6, 0x09, 0x1b, 0x22, 0x19, 0xfd, 0xb3, 0x02, 0x20, 0xb2, 0xea, 0xef, 0x33, 0xcf, 0x64, 0x8a, 0xb2, 0x5c, 0xd4, 0xd5, 0xa8, 0x21, 0x96, 0xd1, 0x3f, 0x2f, 0x00,
0x7b, 0xba, 0xa5, 0x3b, 0x03, 0x7c, 0x71, 0xd2, 0x6f, 0x42, 0x4b, 0x88, 0x3d, 0xa2, 0xb3, 0xf2, 0x22, 0xab, 0xbe, 0x67, 0xd8, 0x86, 0xdb, 0xc7, 0x17, 0x27, 0xfd, 0x65, 0x68, 0x09, 0xb1, 0x47,
0x64, 0xf0, 0xe1, 0xa3, 0xf7, 0xa0, 0xb5, 0xcf, 0x50, 0xf5, 0x3d, 0xac, 0xfb, 0xae, 0xc3, 0xb7, 0x7c, 0x56, 0x9e, 0x0e, 0x3e, 0x02, 0xf4, 0x3e, 0xb4, 0xf6, 0x19, 0xaa, 0x9e, 0x8f, 0x8d, 0xc0,
0x43, 0x5a, 0x86, 0x7b, 0xe4, 0x99, 0xc3, 0x21, 0xf6, 0x36, 0x5d, 0xc7, 0xe0, 0x41, 0xf4, 0x7e, 0x73, 0x39, 0x3b, 0xa4, 0x65, 0xb8, 0x47, 0xbe, 0x35, 0x18, 0x60, 0x7f, 0xd3, 0x73, 0x4d, 0x1e,
0x48, 0x26, 0x19, 0x4a, 0x94, 0x21, 0x0e, 0xc4, 0xa2, 0xcd, 0x89, 0x22, 0x31, 0xca, 0x0a, 0x1f, 0x44, 0xef, 0x47, 0x64, 0x92, 0xa1, 0x44, 0x19, 0x92, 0x40, 0x2c, 0x66, 0x4e, 0x1c, 0x89, 0xd1,
0xeb, 0x56, 0xcc, 0x88, 0xd8, 0x1b, 0x76, 0x58, 0xc3, 0xde, 0xe4, 0x2a, 0xac, 0x24, 0x30, 0x52, 0xad, 0x08, 0xb0, 0x61, 0x27, 0x1b, 0x91, 0x78, 0x43, 0x95, 0x35, 0xec, 0x8d, 0xaf, 0xc2, 0x4a,
0xff, 0x52, 0x01, 0x14, 0x65, 0xca, 0xb4, 0xb4, 0x40, 0x35, 0x3a, 0x3d, 0x54, 0x91, 0x38, 0xe5, 0x02, 0x23, 0xed, 0xaf, 0x14, 0x40, 0x71, 0xa6, 0x4c, 0x4b, 0x0b, 0x54, 0xa3, 0xb3, 0x43, 0x15,
0xab, 0x50, 0x33, 0xc2, 0x91, 0xdc, 0x04, 0xc5, 0x00, 0xea, 0x23, 0x29, 0xd1, 0x7d, 0x22, 0x79, 0x89, 0x53, 0xbe, 0x0a, 0x35, 0x33, 0x1a, 0xc9, 0x4d, 0x50, 0x02, 0xa0, 0x3e, 0x92, 0x12, 0xdd,
0xd8, 0x08, 0x33, 0x51, 0x06, 0x7c, 0x48, 0x61, 0x62, 0x5c, 0x55, 0x4a, 0xc7, 0x55, 0xc9, 0x22, 0x23, 0x92, 0x87, 0xcd, 0x28, 0x13, 0x65, 0xc0, 0x87, 0x14, 0x26, 0xc6, 0x55, 0xa5, 0x6c, 0x5c,
0x63, 0x59, 0x28, 0x32, 0xaa, 0x3f, 0x2b, 0x40, 0x87, 0xba, 0x90, 0xcd, 0xb8, 0x5a, 0x94, 0x8b, 0x95, 0x2e, 0x32, 0x96, 0x85, 0x22, 0xa3, 0xf6, 0xd3, 0x02, 0xa8, 0xd4, 0x85, 0x6c, 0x26, 0xd5,
0xe8, 0x1b, 0xd0, 0xe4, 0x77, 0x4d, 0x04, 0xc2, 0x1b, 0x4f, 0x13, 0x93, 0xa1, 0x5b, 0xb0, 0xc8, 0xa2, 0xa9, 0x88, 0xbe, 0x01, 0x4d, 0x7e, 0xd7, 0x44, 0x20, 0xbc, 0xf1, 0x34, 0x35, 0x19, 0xba,
0x3a, 0x79, 0xd8, 0x1f, 0x5b, 0x71, 0x12, 0xc6, 0xb2, 0x10, 0xf4, 0x94, 0xf9, 0x2e, 0xd2, 0x14, 0x05, 0x8b, 0xac, 0x93, 0x8f, 0x83, 0x91, 0x9d, 0x24, 0x61, 0x2c, 0x0b, 0x41, 0x4f, 0x99, 0xef,
0x8e, 0x78, 0x0c, 0x97, 0x87, 0x96, 0xbb, 0xaf, 0x5b, 0x7d, 0x71, 0x7b, 0xd8, 0x1e, 0xe6, 0x90, 0x22, 0x4d, 0xd1, 0x88, 0xc7, 0x70, 0x79, 0x60, 0x7b, 0xfb, 0x86, 0xdd, 0x13, 0xd9, 0xc3, 0x78,
0xf8, 0x45, 0x36, 0x7c, 0x2f, 0xb9, 0x87, 0x3e, 0xba, 0x07, 0x4d, 0x1f, 0xe3, 0xa3, 0x38, 0x33, 0x38, 0x85, 0xc4, 0x2f, 0xb2, 0xe1, 0x7b, 0x69, 0x1e, 0x06, 0xe8, 0x1e, 0x34, 0x03, 0x8c, 0x8f,
0x2b, 0xe7, 0xc9, 0xcc, 0x1a, 0x64, 0x4c, 0x94, 0x98, 0xfd, 0x81, 0x02, 0xed, 0xd4, 0x11, 0x41, 0x92, 0xcc, 0xac, 0x3c, 0x4d, 0x66, 0xd6, 0x20, 0x63, 0xe2, 0xc4, 0xec, 0x0f, 0x15, 0x68, 0x67,
0xba, 0x8e, 0xa0, 0x64, 0xeb, 0x08, 0x77, 0xa0, 0x4c, 0x2c, 0x15, 0xf3, 0x2d, 0x2d, 0x79, 0x8e, 0x8e, 0x08, 0xb2, 0x75, 0x04, 0x25, 0x5f, 0x47, 0xb8, 0x03, 0x65, 0x62, 0xa9, 0x98, 0x6f, 0x69,
0x2b, 0xce, 0xaa, 0xb1, 0x01, 0x68, 0x1d, 0x16, 0x24, 0x57, 0x11, 0xf8, 0xf6, 0xa3, 0xec, 0x4d, 0xc9, 0x73, 0x5c, 0x71, 0x56, 0x9d, 0x0d, 0x40, 0xeb, 0xb0, 0x20, 0xb9, 0x8a, 0xc0, 0xd9, 0x8f,
0x04, 0xf5, 0xe7, 0x25, 0xa8, 0x27, 0x58, 0x31, 0xa5, 0x04, 0xf2, 0x5c, 0x4a, 0xbd, 0x93, 0x8e, 0xf2, 0x37, 0x11, 0xb4, 0x9f, 0x95, 0xa0, 0x9e, 0xda, 0x8a, 0x09, 0x25, 0x90, 0xe7, 0x52, 0xea,
0x9e, 0x89, 0xc8, 0xd9, 0xd8, 0x66, 0x09, 0x1b, 0xcf, 0x1e, 0x6d, 0x6c, 0xd3, 0x74, 0x2d, 0x99, 0x1d, 0x77, 0xf4, 0x4c, 0x44, 0xce, 0xc1, 0x0e, 0x4b, 0xd8, 0x78, 0xf6, 0xe8, 0x60, 0x87, 0xa6,
0x89, 0x55, 0x84, 0x4c, 0x2c, 0x95, 0xab, 0xce, 0x9d, 0x91, 0xab, 0x56, 0xc5, 0x5c, 0x55, 0x50, 0x6b, 0xe9, 0x4c, 0xac, 0x22, 0x64, 0x62, 0x99, 0x5c, 0x75, 0xee, 0x8c, 0x5c, 0xb5, 0x2a, 0xe6,
0xa1, 0x5a, 0x5a, 0x85, 0xf2, 0x56, 0x25, 0x6e, 0xc1, 0xc2, 0x80, 0x95, 0xd2, 0xef, 0x9d, 0x6e, 0xaa, 0x82, 0x0a, 0xd5, 0xb2, 0x2a, 0x34, 0x6d, 0x55, 0xe2, 0x16, 0x2c, 0xf4, 0x59, 0x29, 0xfd,
0x46, 0x4d, 0x3c, 0x28, 0x95, 0x35, 0xa1, 0xfb, 0x71, 0xbd, 0x91, 0xed, 0x32, 0xcb, 0x16, 0xe4, 0xde, 0xe9, 0x66, 0xdc, 0xc4, 0x83, 0x52, 0x59, 0x13, 0xba, 0x9f, 0xd4, 0x1b, 0x19, 0x97, 0x59,
0xa9, 0x30, 0xdf, 0x1b, 0xb6, 0xc9, 0xa1, 0x65, 0xa6, 0x5f, 0xe9, 0x7a, 0x48, 0xf3, 0x42, 0xf5, 0xb6, 0x20, 0x4f, 0x85, 0x39, 0x6f, 0x18, 0x93, 0x23, 0xcb, 0x4c, 0xbf, 0xb2, 0xf5, 0x90, 0xe6,
0x90, 0x17, 0xa0, 0x1e, 0x46, 0x2a, 0x44, 0xd3, 0x5b, 0xcc, 0xe8, 0x85, 0x66, 0xc0, 0xf0, 0x05, 0x85, 0xea, 0x21, 0x2f, 0x40, 0x3d, 0x8a, 0x54, 0x88, 0xa6, 0xb7, 0x98, 0xd1, 0x8b, 0xcc, 0x80,
0x3b, 0xd0, 0x16, 0x0f, 0x1b, 0xd2, 0x85, 0x84, 0x4e, 0xb6, 0x90, 0x70, 0x05, 0xe6, 0x4c, 0xbf, 0x19, 0x08, 0x76, 0xa0, 0x2d, 0x1e, 0x36, 0x64, 0x0b, 0x09, 0x6a, 0xbe, 0x90, 0x70, 0x05, 0xe6,
0x7f, 0xa0, 0x1f, 0xe1, 0xee, 0x3c, 0x6d, 0xad, 0x98, 0xfe, 0x7d, 0xfd, 0x08, 0xab, 0xff, 0x5a, 0xac, 0xa0, 0x77, 0x60, 0x1c, 0xe1, 0xce, 0x3c, 0x6d, 0xad, 0x58, 0xc1, 0x7d, 0xe3, 0x08, 0x6b,
0x84, 0x56, 0xec, 0x60, 0x73, 0x5b, 0x90, 0x3c, 0xd7, 0x71, 0x76, 0xa0, 0x13, 0xc7, 0x3d, 0x94, 0xff, 0x56, 0x84, 0x56, 0xe2, 0x60, 0xa7, 0xb6, 0x20, 0xd3, 0x5c, 0xc7, 0xd9, 0x01, 0x35, 0x89,
0xc3, 0x67, 0x26, 0xcf, 0xe9, 0x13, 0xbc, 0xf6, 0x28, 0xa5, 0xaf, 0x82, 0xbb, 0x2f, 0x9d, 0xcb, 0x7b, 0xe8, 0x0e, 0x9f, 0x99, 0x3c, 0x67, 0x4f, 0xf0, 0xda, 0xc3, 0x8c, 0xbe, 0x0a, 0xee, 0xbe,
0xdd, 0xcf, 0x78, 0x50, 0x7f, 0x1b, 0x96, 0x22, 0xdf, 0x2b, 0x2c, 0x9b, 0x25, 0x58, 0x8b, 0x61, 0x74, 0x2e, 0x77, 0x3f, 0xe3, 0x41, 0xfd, 0x6d, 0x58, 0x8a, 0x7d, 0xaf, 0xb0, 0x6c, 0x96, 0x60,
0xe3, 0x6e, 0x72, 0xf9, 0x13, 0x4c, 0xc0, 0xdc, 0x24, 0x13, 0x90, 0x16, 0x81, 0x6a, 0x46, 0x04, 0x2d, 0x46, 0x8d, 0xbb, 0xe9, 0xe5, 0x8f, 0x31, 0x01, 0x73, 0xe3, 0x4c, 0x40, 0x56, 0x04, 0xaa,
0xb2, 0xf7, 0x05, 0x6a, 0x92, 0xfb, 0x02, 0xea, 0x63, 0x58, 0xa0, 0xb5, 0x5f, 0x7f, 0xe0, 0x99, 0x39, 0x11, 0xc8, 0xdf, 0x17, 0xa8, 0x49, 0xee, 0x0b, 0x68, 0x8f, 0x61, 0x81, 0xd6, 0x7e, 0x83,
0xfb, 0x38, 0x4a, 0x01, 0xf2, 0x6c, 0x6b, 0x0f, 0xaa, 0xa9, 0x2c, 0x22, 0xfa, 0x56, 0x7f, 0xa2, 0xbe, 0x6f, 0xed, 0xe3, 0x38, 0x05, 0x98, 0x86, 0xad, 0x5d, 0xa8, 0x66, 0xb2, 0x88, 0xf8, 0x5b,
0xc0, 0xe5, 0xec, 0xbc, 0x54, 0x62, 0x62, 0x43, 0xa2, 0x08, 0x86, 0xe4, 0x97, 0x61, 0x21, 0x11, 0xfb, 0xb1, 0x02, 0x97, 0xf3, 0xf3, 0x52, 0x89, 0x49, 0x0c, 0x89, 0x22, 0x18, 0x92, 0x5f, 0x86,
0x51, 0x0a, 0x33, 0x4f, 0x88, 0xc0, 0x25, 0x84, 0x6b, 0x28, 0x9e, 0x23, 0x84, 0xa9, 0x3f, 0x57, 0x85, 0x54, 0x44, 0x29, 0xcc, 0x3c, 0x26, 0x02, 0x97, 0x10, 0xae, 0xa3, 0x64, 0x8e, 0x08, 0xa6,
0xa2, 0x12, 0x3a, 0x81, 0x0d, 0xe9, 0xf9, 0x04, 0xf1, 0x6b, 0xae, 0x63, 0x99, 0x4e, 0x54, 0x29, 0xfd, 0x4c, 0x89, 0x4b, 0xe8, 0x04, 0x36, 0xa0, 0xe7, 0x13, 0xc4, 0xaf, 0x79, 0xae, 0x6d, 0xb9,
0xe1, 0x6b, 0x64, 0x40, 0x5e, 0x29, 0x79, 0x17, 0xda, 0xbc, 0x53, 0xe4, 0x9e, 0x72, 0x06, 0x64, 0x71, 0xa5, 0x84, 0xaf, 0x91, 0x01, 0x79, 0xa5, 0xe4, 0x3d, 0x68, 0xf3, 0x4e, 0xb1, 0x7b, 0x9a,
0x2d, 0x36, 0x2e, 0x72, 0x4c, 0x37, 0xa1, 0xc5, 0x0f, 0x0e, 0x42, 0x7c, 0x45, 0xd9, 0x71, 0xc2, 0x32, 0x20, 0x6b, 0xb1, 0x71, 0xb1, 0x63, 0x7a, 0x19, 0x5a, 0xfc, 0xe0, 0x20, 0xc2, 0x57, 0x94,
0xf7, 0xa0, 0x13, 0x76, 0x3b, 0xaf, 0x43, 0x6c, 0xf3, 0x81, 0x51, 0x60, 0xf7, 0x1b, 0x0a, 0x74, 0x1d, 0x27, 0x7c, 0x17, 0xd4, 0xa8, 0xdb, 0x79, 0x1d, 0x62, 0x9b, 0x0f, 0x8c, 0x03, 0xbb, 0xdf,
0x45, 0xf7, 0x98, 0x58, 0xfe, 0xf9, 0xc3, 0xbb, 0x37, 0xc5, 0xe3, 0xe2, 0x9b, 0x67, 0xd0, 0x13, 0x50, 0xa0, 0x23, 0xba, 0xc7, 0xd4, 0xf2, 0xcf, 0x1f, 0xde, 0xbd, 0x29, 0x1e, 0x17, 0xbf, 0x7c,
0xe3, 0x09, 0x0f, 0x8d, 0x7f, 0xa7, 0x40, 0xcf, 0xfe, 0x49, 0xaa, 0xb7, 0x65, 0xfa, 0x81, 0x67, 0x06, 0x3d, 0x09, 0x9e, 0xe8, 0xd0, 0xf8, 0x77, 0x0a, 0xf4, 0xec, 0x9f, 0xa4, 0x7a, 0x5b, 0x56,
0xee, 0x8f, 0x67, 0x3b, 0xc0, 0xd4, 0xa1, 0x3e, 0x38, 0xc4, 0x83, 0xa3, 0x91, 0x6b, 0xc6, 0xbb, 0x10, 0xfa, 0xd6, 0xfe, 0x68, 0xb6, 0x03, 0x4c, 0x03, 0xea, 0xfd, 0x43, 0xdc, 0x3f, 0x1a, 0x7a,
0xf2, 0xb6, 0x8c, 0xa6, 0xc9, 0x68, 0xd7, 0x36, 0xe3, 0x19, 0xd8, 0x09, 0x50, 0x72, 0xce, 0xde, 0x56, 0xc2, 0x95, 0x77, 0x64, 0x34, 0x8d, 0x47, 0xbb, 0xb6, 0x99, 0xcc, 0xc0, 0x4e, 0x80, 0xd2,
0x0f, 0xa1, 0x93, 0xee, 0x90, 0x3c, 0x78, 0xa9, 0xb1, 0x83, 0x97, 0xdb, 0xe2, 0xc1, 0xcb, 0x94, 0x73, 0x76, 0x7f, 0x00, 0x6a, 0xb6, 0x43, 0xfa, 0xe0, 0xa5, 0xc6, 0x0e, 0x5e, 0x6e, 0x8b, 0x07,
0x48, 0x23, 0x71, 0xee, 0xf2, 0x57, 0x05, 0xf8, 0xaa, 0x94, 0xb6, 0x59, 0xb2, 0xa4, 0x49, 0x75, 0x2f, 0x13, 0x22, 0x8d, 0xd4, 0xb9, 0xcb, 0x5f, 0x17, 0xe0, 0xab, 0x52, 0xda, 0x66, 0xc9, 0x92,
0xa4, 0x7b, 0x50, 0x4d, 0x25, 0xb5, 0x2f, 0x9f, 0xb1, 0x7f, 0xbc, 0x96, 0xca, 0x6a, 0x7a, 0x7e, 0xc6, 0xd5, 0x91, 0xee, 0x41, 0x35, 0x93, 0xd4, 0xde, 0x3c, 0x83, 0x7f, 0xbc, 0x96, 0xca, 0x6a,
0x1c, 0x5b, 0xc5, 0x0a, 0x5f, 0x9a, 0x3c, 0x07, 0xd7, 0x3b, 0x61, 0x8e, 0x70, 0x1c, 0xba, 0x0b, 0x7a, 0x41, 0x12, 0x5b, 0x25, 0x0a, 0x5f, 0x1a, 0x3f, 0x07, 0xd7, 0x3b, 0x61, 0x8e, 0x68, 0x1c,
0x0d, 0x56, 0x30, 0xe8, 0x1f, 0x9b, 0xf8, 0x24, 0x3c, 0xd6, 0xbc, 0x2e, 0x35, 0xcd, 0xb4, 0xdf, 0xba, 0x0b, 0x0d, 0x56, 0x30, 0xe8, 0x1d, 0x5b, 0xf8, 0x24, 0x3a, 0xd6, 0xbc, 0x2e, 0x35, 0xcd,
0x13, 0x13, 0x9f, 0x68, 0x75, 0x2b, 0xfa, 0xed, 0xab, 0xff, 0x5d, 0x04, 0x88, 0xdb, 0x48, 0x76, 0xb4, 0xdf, 0x13, 0x0b, 0x9f, 0xe8, 0x75, 0x3b, 0xfe, 0x1d, 0x68, 0xbf, 0x5b, 0x02, 0x48, 0xda,
0x16, 0xeb, 0x3c, 0x57, 0xe2, 0x04, 0x84, 0xc4, 0x12, 0x62, 0xe4, 0x1a, 0x7e, 0x22, 0x2d, 0x3e, 0x48, 0x76, 0x96, 0xe8, 0x3c, 0x57, 0xe2, 0x14, 0x84, 0xc4, 0x12, 0x62, 0xe4, 0x1a, 0x7d, 0x22,
0x56, 0x30, 0x4c, 0x3f, 0xe0, 0x7c, 0x59, 0x3f, 0x9b, 0x96, 0x90, 0x45, 0x64, 0xcb, 0xb8, 0xcc, 0x3d, 0x39, 0x56, 0x30, 0xad, 0x20, 0xe4, 0xfb, 0xb2, 0x7e, 0x36, 0x2d, 0xd1, 0x16, 0x11, 0x96,
0xf8, 0x31, 0x04, 0xbd, 0x0e, 0x68, 0xe8, 0xb9, 0x27, 0xa6, 0x33, 0x4c, 0xe6, 0x1b, 0x2c, 0x2d, 0x71, 0x99, 0x09, 0x12, 0x08, 0x7a, 0x1d, 0xd0, 0xc0, 0xf7, 0x4e, 0x2c, 0x77, 0x90, 0xce, 0x37,
0x99, 0xe7, 0x2d, 0x89, 0x84, 0xe3, 0x47, 0xd0, 0x49, 0x75, 0x0f, 0x59, 0x72, 0x7b, 0x0a, 0x19, 0x58, 0x5a, 0x32, 0xcf, 0x5b, 0x52, 0x09, 0xc7, 0x0f, 0x41, 0xcd, 0x74, 0x8f, 0xb6, 0xe4, 0xf6,
0x0f, 0x84, 0xb9, 0xb8, 0xf8, 0xb6, 0x45, 0x0c, 0x7e, 0xaf, 0x0f, 0x9d, 0x34, 0xbd, 0x92, 0xb3, 0x04, 0x32, 0x1e, 0x08, 0x73, 0x71, 0xf1, 0x6d, 0x8b, 0x18, 0xe8, 0x19, 0xe6, 0x23, 0xc3, 0x1f,
0xc3, 0x6f, 0x8a, 0x22, 0x7c, 0x96, 0xa5, 0x21, 0xd3, 0x24, 0x84, 0xb8, 0xa7, 0xc3, 0xa2, 0x8c, 0xe0, 0x88, 0xa3, 0x3c, 0x0e, 0x13, 0x81, 0xdd, 0x1e, 0xa8, 0xd9, 0x55, 0x49, 0x4e, 0x18, 0xbf,
0x12, 0x09, 0x92, 0x0b, 0xeb, 0xc9, 0xdb, 0x51, 0xb0, 0x4b, 0x39, 0x3c, 0xc9, 0x7f, 0x24, 0x6a, 0x29, 0x0a, 0xfa, 0x59, 0xf6, 0x88, 0x4c, 0x93, 0x12, 0xf5, 0xae, 0x01, 0x8b, 0x32, 0x7a, 0x25,
0xc1, 0x05, 0xa1, 0x16, 0xac, 0xfe, 0xbd, 0x02, 0x28, 0x2b, 0xd8, 0xa8, 0x05, 0x85, 0x68, 0x92, 0x48, 0x2e, 0xac, 0x4d, 0xef, 0xc4, 0x21, 0x31, 0xe5, 0xc3, 0x38, 0x2f, 0x93, 0xaa, 0x18, 0x17,
0xc2, 0xf6, 0x56, 0x4a, 0x90, 0x0a, 0x19, 0x41, 0xba, 0x0a, 0xb5, 0xc8, 0x9f, 0x73, 0xe3, 0x1d, 0x84, 0x8a, 0xb1, 0xf6, 0x0f, 0x0a, 0xa0, 0xbc, 0xf8, 0xa3, 0x16, 0x14, 0xe2, 0x49, 0x0a, 0xdb,
0x03, 0x92, 0x62, 0x56, 0x12, 0xc5, 0x2c, 0x41, 0x58, 0x59, 0x2c, 0x52, 0xdf, 0x82, 0x45, 0x4b, 0x5b, 0x19, 0x71, 0x2b, 0xe4, 0xc4, 0xed, 0x2a, 0xd4, 0x62, 0xaf, 0xcf, 0x4d, 0x7c, 0x02, 0x48,
0xf7, 0x83, 0x3e, 0xab, 0x85, 0x07, 0xa6, 0x8d, 0xfd, 0x40, 0xb7, 0x47, 0x34, 0x58, 0x2e, 0x69, 0x0b, 0x63, 0x49, 0x14, 0xc6, 0x14, 0x61, 0x65, 0xb1, 0x94, 0x7d, 0x0b, 0x16, 0x6d, 0x23, 0x08,
0x88, 0xb4, 0x6d, 0x91, 0xa6, 0x47, 0x61, 0x8b, 0x7a, 0x08, 0x28, 0xab, 0x5e, 0x49, 0xdc, 0x8a, 0x7b, 0xac, 0x62, 0x1e, 0x5a, 0x0e, 0x0e, 0x42, 0xc3, 0x19, 0x52, 0x56, 0x96, 0x74, 0x44, 0xda,
0x88, 0x7b, 0xda, 0x9a, 0x12, 0xb4, 0x15, 0x45, 0xa6, 0xfd, 0x51, 0x11, 0x50, 0x1c, 0xe3, 0x44, 0xb6, 0x48, 0xd3, 0xa3, 0xa8, 0x45, 0x3b, 0x04, 0x94, 0x57, 0xc2, 0x34, 0x6e, 0x45, 0xc4, 0x3d,
0xa7, 0xad, 0x79, 0x02, 0x83, 0x75, 0x58, 0xc8, 0x46, 0x40, 0x61, 0xd8, 0x87, 0x32, 0xf1, 0x8f, 0x69, 0x4d, 0x29, 0xda, 0x8a, 0xe2, 0xa6, 0xfd, 0x71, 0x11, 0x50, 0x12, 0x09, 0xc5, 0x67, 0xb2,
0x2c, 0x56, 0x29, 0xca, 0xee, 0x36, 0xbe, 0x11, 0x19, 0x44, 0x16, 0xd0, 0x5d, 0x9f, 0x58, 0xaa, 0xd3, 0x84, 0x0f, 0xeb, 0xb0, 0x90, 0x8f, 0x93, 0xa2, 0xe0, 0x10, 0xe5, 0xa2, 0x24, 0x59, 0x44,
0x17, 0x6d, 0xe2, 0x0f, 0xd3, 0x77, 0x22, 0x99, 0x86, 0xdd, 0x91, 0x1a, 0xaf, 0xcc, 0x92, 0xa7, 0x53, 0x94, 0xdd, 0x80, 0x7c, 0x23, 0x36, 0x9b, 0x2c, 0xec, 0xbb, 0x3e, 0xb6, 0xa0, 0x2f, 0x5a,
0x5e, 0x88, 0x14, 0x42, 0xcd, 0xca, 0x79, 0x42, 0xcd, 0xd9, 0x6f, 0x30, 0xfe, 0x5b, 0x01, 0xe6, 0xce, 0x1f, 0x64, 0x6f, 0x4e, 0x32, 0x3d, 0xbc, 0x23, 0x35, 0x71, 0xb9, 0x25, 0x4f, 0xbc, 0x36,
0x23, 0x46, 0x9e, 0x6b, 0x93, 0xa6, 0x1f, 0x8c, 0x7f, 0xca, 0xbb, 0xf2, 0x91, 0x7c, 0x57, 0xbe, 0x29, 0x04, 0xa4, 0x95, 0xf3, 0x04, 0xa4, 0xb3, 0xdf, 0x73, 0xfc, 0xf7, 0x02, 0xcc, 0xc7, 0x1b,
0x75, 0x66, 0xb8, 0x9f, 0x77, 0x53, 0x66, 0xe7, 0xec, 0xc7, 0x30, 0xc7, 0x0b, 0xb7, 0x19, 0x43, 0x79, 0x2e, 0x26, 0x4d, 0x3e, 0x3e, 0xff, 0x8c, 0xb9, 0xf2, 0xb1, 0x9c, 0x2b, 0xdf, 0x3a, 0x33,
0x91, 0x27, 0xa1, 0x5e, 0x84, 0x32, 0xb1, 0x4b, 0x61, 0xd5, 0x8d, 0x7d, 0x30, 0x96, 0x26, 0x6f, 0x29, 0x98, 0x96, 0x29, 0xb3, 0xef, 0xec, 0x27, 0x30, 0xc7, 0xcb, 0xbb, 0x39, 0x43, 0x31, 0x4d,
0xc8, 0x72, 0x5b, 0xd1, 0x14, 0x2e, 0xc8, 0xaa, 0xff, 0xa9, 0x00, 0xec, 0x9d, 0x3a, 0x83, 0xbb, 0xda, 0xbd, 0x08, 0x65, 0x62, 0x97, 0xa2, 0xda, 0x1c, 0xfb, 0x60, 0x5b, 0x9a, 0xbe, 0x47, 0xcb,
0x4c, 0x49, 0x6f, 0x41, 0x69, 0xda, 0x7d, 0x2a, 0xd2, 0x9b, 0xca, 0x16, 0xed, 0x99, 0x63, 0x73, 0x6d, 0x45, 0x53, 0xb8, 0x46, 0xab, 0xfd, 0x47, 0x01, 0x60, 0xef, 0xd4, 0xed, 0xdf, 0x65, 0x4a,
0x85, 0x92, 0x41, 0x31, 0x5d, 0x32, 0x98, 0x94, 0xec, 0x4f, 0x36, 0x65, 0xdf, 0x82, 0x12, 0x09, 0x7a, 0x0b, 0x4a, 0x93, 0x6e, 0x5d, 0x91, 0xde, 0x54, 0xb6, 0x68, 0xcf, 0x29, 0x98, 0x2b, 0x14,
0xf4, 0xf8, 0x75, 0xa3, 0x5c, 0x07, 0x9e, 0x74, 0x80, 0xfa, 0x49, 0x01, 0xae, 0x10, 0xea, 0x9f, 0x16, 0x8a, 0xd9, 0xc2, 0xc2, 0xb8, 0x92, 0xc0, 0x78, 0x53, 0xf6, 0x2d, 0x28, 0x91, 0x70, 0x90,
0x4f, 0x54, 0x98, 0x67, 0x6b, 0x12, 0xd6, 0xb2, 0x28, 0x5a, 0xcb, 0x3b, 0x30, 0xc7, 0xd2, 0xfd, 0x5f, 0x4a, 0x9a, 0xea, 0x58, 0x94, 0x0e, 0x40, 0x2b, 0x10, 0xb9, 0xb6, 0x6d, 0x97, 0xf9, 0x2e,
0x30, 0xbe, 0xb9, 0x3e, 0x89, 0xd7, 0x6c, 0x67, 0xb4, 0xb0, 0xfb, 0xac, 0x39, 0xa3, 0x70, 0xd8, 0x7a, 0x82, 0x5c, 0xd4, 0xb3, 0x60, 0x74, 0x13, 0x5a, 0xac, 0xa0, 0x14, 0x77, 0x64, 0xb9, 0x51,
0x5a, 0x99, 0xed, 0xb0, 0x75, 0x2e, 0x5d, 0x14, 0x4c, 0x6c, 0x5a, 0x55, 0xb4, 0xf1, 0x8f, 0xa1, 0x06, 0x9a, 0xf7, 0x8c, 0x35, 0x89, 0x67, 0xd4, 0x3e, 0x2d, 0xc0, 0x15, 0xb2, 0x6b, 0xcf, 0x27,
0xa9, 0x25, 0x05, 0x0f, 0x21, 0x28, 0x25, 0xee, 0x2f, 0xd2, 0xdf, 0x34, 0xcd, 0xd3, 0x47, 0xfa, 0x66, 0x9d, 0x46, 0x24, 0x52, 0x56, 0xba, 0x28, 0x5a, 0xe9, 0x3b, 0x30, 0xc7, 0x8a, 0x11, 0x51,
0xc0, 0x0c, 0x4e, 0x29, 0x3b, 0xcb, 0x5a, 0xf4, 0x2d, 0x97, 0x72, 0xf5, 0x7f, 0x15, 0xb8, 0x1c, 0xf4, 0x75, 0x7d, 0x1c, 0x8f, 0x99, 0x44, 0xe8, 0x51, 0xf7, 0x59, 0x33, 0x5a, 0xe1, 0x28, 0xb8,
0x1e, 0xea, 0x71, 0x1d, 0xba, 0xf8, 0x8e, 0x6e, 0xc0, 0x12, 0x57, 0x98, 0x94, 0xe6, 0xb0, 0x60, 0x32, 0xdb, 0x51, 0xf0, 0x5c, 0xb6, 0x64, 0x99, 0x12, 0x96, 0xaa, 0xe8, 0x5b, 0x1e, 0x43, 0x53,
0x6e, 0x81, 0xc1, 0xc4, 0x65, 0x6c, 0xc0, 0x52, 0xa0, 0x7b, 0x43, 0x1c, 0xa4, 0xc7, 0xb0, 0xfd, 0x4f, 0x0b, 0x3c, 0x42, 0x50, 0x4a, 0xdd, 0xae, 0xa4, 0xbf, 0x69, 0x12, 0x6a, 0x0c, 0x8d, 0xbe,
0x5e, 0x60, 0x8d, 0xe2, 0x98, 0x3c, 0x87, 0xaa, 0x2f, 0xb0, 0x0b, 0x39, 0x9c, 0xb5, 0x5c, 0x05, 0x15, 0x9e, 0xd2, 0xed, 0x2c, 0xeb, 0xf1, 0xb7, 0x5c, 0xbb, 0xb4, 0xff, 0x55, 0xe0, 0x72, 0x74,
0xc0, 0x19, 0xdb, 0x7c, 0x95, 0xea, 0x09, 0x5c, 0x65, 0x37, 0x88, 0xf7, 0x45, 0x8a, 0x66, 0xaa, 0xe4, 0xc8, 0x75, 0xf7, 0xe2, 0x1c, 0xdd, 0x80, 0x25, 0xae, 0xa8, 0x19, 0x8d, 0x65, 0xa1, 0xe6,
0xa9, 0x4b, 0xd7, 0x9d, 0xb2, 0x18, 0x7f, 0xa8, 0xc0, 0xb5, 0x09, 0x98, 0x67, 0xc9, 0x26, 0x1e, 0x02, 0x83, 0x89, 0xcb, 0xd8, 0x80, 0xa5, 0x90, 0x0a, 0x59, 0x76, 0x0c, 0xe3, 0xf7, 0x02, 0x6b,
0x4a, 0xb1, 0x4f, 0xc8, 0xfd, 0x04, 0xbc, 0xec, 0x26, 0x9b, 0x48, 0xe4, 0x27, 0x25, 0x98, 0xcf, 0x14, 0xc7, 0x4c, 0x73, 0xe4, 0xfb, 0x02, 0xbb, 0x2e, 0xc4, 0xb7, 0x96, 0xab, 0x1e, 0xb8, 0x23,
0x74, 0x3a, 0xb7, 0xcc, 0xbd, 0x06, 0x88, 0x6c, 0x42, 0xf4, 0x5a, 0x8e, 0xa6, 0xd3, 0xdc, 0x35, 0x87, 0xaf, 0x52, 0x3b, 0x81, 0xab, 0xec, 0x7e, 0xf3, 0xbe, 0x48, 0xd1, 0x4c, 0x15, 0x7f, 0xe9,
0x75, 0x9c, 0xb1, 0x1d, 0xbd, 0x94, 0x23, 0x19, 0x35, 0x32, 0x59, 0x6f, 0x56, 0x51, 0x8f, 0x76, 0xba, 0x33, 0x96, 0xea, 0x8f, 0x14, 0xb8, 0x36, 0x06, 0xf3, 0x2c, 0xb9, 0xce, 0x43, 0x29, 0xf6,
0xae, 0x34, 0xf9, 0x51, 0x44, 0x86, 0xc0, 0xb5, 0x9d, 0xb1, 0xcd, 0x8a, 0xef, 0x7c, 0x97, 0x99, 0x31, 0x99, 0xa9, 0x80, 0x97, 0xdd, 0xb3, 0x13, 0x89, 0xfc, 0xb4, 0x04, 0xf3, 0xb9, 0x4e, 0xe7,
0xbb, 0x21, 0xa8, 0x04, 0x30, 0x3a, 0x80, 0x79, 0x7a, 0x5d, 0x6b, 0x1c, 0x0c, 0x5d, 0x12, 0xd0, 0x96, 0xb9, 0xd7, 0x00, 0x11, 0x26, 0xc4, 0x6f, 0xf9, 0x68, 0xb2, 0xcf, 0x5d, 0xa2, 0xea, 0x8e,
0x53, 0xba, 0x98, 0x53, 0xfb, 0x4e, 0x6e, 0x4c, 0x1f, 0xf0, 0xd1, 0x84, 0x78, 0x1e, 0xd3, 0x3b, 0x9c, 0xf8, 0x1d, 0x1f, 0xc9, 0xf7, 0x91, 0xc5, 0x7a, 0xb3, 0x7a, 0x7f, 0xcc, 0xb9, 0xd2, 0xf8,
0x22, 0x34, 0xc4, 0x63, 0x3a, 0x03, 0xd7, 0x8e, 0xf0, 0x54, 0xce, 0x89, 0x67, 0x9b, 0x8f, 0x16, 0x27, 0x1b, 0x39, 0x02, 0xd7, 0x76, 0x46, 0x0e, 0x3b, 0x1a, 0xe0, 0x5c, 0x66, 0x6e, 0x8e, 0xa0,
0xf1, 0x24, 0xa1, 0xbd, 0x4d, 0x58, 0x92, 0x2e, 0x7d, 0x9a, 0x1b, 0x2d, 0x27, 0xf3, 0x83, 0x7b, 0x12, 0xc0, 0xe8, 0x00, 0xe6, 0xe9, 0x65, 0xb2, 0x51, 0x38, 0xf0, 0x48, 0xba, 0x41, 0xe9, 0x62,
0xb0, 0x28, 0x5b, 0xd5, 0x05, 0xe6, 0xc8, 0x50, 0x7c, 0x9e, 0x39, 0xd4, 0x3f, 0x2d, 0x40, 0x73, 0xce, 0xf4, 0x3b, 0x53, 0x63, 0xfa, 0x90, 0x8f, 0x26, 0xc4, 0xf3, 0x8c, 0xc3, 0x15, 0xa1, 0x11,
0x0b, 0x5b, 0x38, 0xc0, 0x9f, 0xee, 0x99, 0x67, 0xe6, 0x00, 0xb7, 0x98, 0x3d, 0xc0, 0xcd, 0x9c, 0x1e, 0xcb, 0xed, 0x7b, 0x4e, 0x8c, 0xa7, 0x72, 0x4e, 0x3c, 0xdb, 0x7c, 0xb4, 0x88, 0x27, 0x0d,
0x46, 0x97, 0x24, 0xa7, 0xd1, 0xd7, 0xa2, 0x43, 0x78, 0x32, 0x4b, 0x59, 0xf4, 0xd0, 0x06, 0x7a, 0xed, 0x6e, 0xc2, 0x92, 0x74, 0xe9, 0x93, 0xdc, 0x77, 0x39, 0x9d, 0x97, 0xdc, 0x83, 0x45, 0xd9,
0x13, 0x1a, 0x23, 0xcf, 0xb4, 0x75, 0xef, 0xb4, 0x7f, 0x84, 0x4f, 0x7d, 0xee, 0x34, 0xba, 0x52, 0xaa, 0x2e, 0x30, 0x47, 0x8e, 0xe2, 0xf3, 0xcc, 0xa1, 0xfd, 0x59, 0x01, 0x9a, 0x5b, 0xd8, 0xc6,
0xb7, 0xb3, 0xbd, 0xe5, 0x6b, 0x75, 0xde, 0xfb, 0x3d, 0x7c, 0x4a, 0x0f, 0xf8, 0xa3, 0x64, 0x83, 0x21, 0xfe, 0x6c, 0x4f, 0x64, 0x73, 0xc7, 0xcb, 0xc5, 0xfc, 0xf1, 0x72, 0xee, 0xac, 0xbc, 0x24,
0x5d, 0xc5, 0x2a, 0x69, 0x09, 0xc8, 0xea, 0x32, 0xd4, 0xa2, 0x1b, 0x2f, 0xa8, 0x0a, 0xa5, 0xfb, 0x39, 0x2b, 0xbf, 0x16, 0x5f, 0x11, 0x20, 0xb3, 0x94, 0xc5, 0xc8, 0xc0, 0x44, 0x6f, 0x42, 0x63,
0x63, 0xcb, 0xea, 0x5c, 0x42, 0x35, 0x28, 0xd3, 0x74, 0xa4, 0xa3, 0xac, 0xfe, 0x12, 0xd4, 0xa2, 0xe8, 0x5b, 0x8e, 0xe1, 0x9f, 0xf6, 0x8e, 0xf0, 0x69, 0xc0, 0x9d, 0x46, 0x47, 0xea, 0x76, 0xb6,
0x53, 0x7b, 0x54, 0x87, 0xb9, 0xc7, 0xce, 0x7b, 0x8e, 0x7b, 0xe2, 0x74, 0x2e, 0xa1, 0x39, 0x28, 0xb7, 0x02, 0xbd, 0xce, 0x7b, 0xbf, 0x8f, 0x4f, 0xe9, 0xf5, 0x83, 0x38, 0xc9, 0x61, 0x17, 0xc5,
0xde, 0xb5, 0xac, 0x8e, 0x82, 0x9a, 0x50, 0xdb, 0x0b, 0x3c, 0xac, 0x93, 0x3d, 0xeb, 0x14, 0x50, 0x4a, 0x7a, 0x0a, 0xb2, 0xba, 0x0c, 0xb5, 0xf8, 0x3e, 0x0e, 0xaa, 0x42, 0xe9, 0xfe, 0xc8, 0xb6,
0x0b, 0xe0, 0x5d, 0xd3, 0x0f, 0x5c, 0xcf, 0x1c, 0xe8, 0x56, 0xa7, 0xb8, 0xfa, 0x31, 0xb4, 0xc4, 0xd5, 0x4b, 0xa8, 0x06, 0x65, 0x9a, 0x06, 0xa9, 0xca, 0xea, 0x2f, 0x41, 0x2d, 0xbe, 0x53, 0x80,
0x22, 0x2e, 0x6a, 0x40, 0x75, 0xc7, 0x0d, 0xde, 0x79, 0x66, 0xfa, 0x41, 0xe7, 0x12, 0xe9, 0xbf, 0xea, 0x30, 0xf7, 0xd8, 0x7d, 0xdf, 0xf5, 0x4e, 0x5c, 0xf5, 0x12, 0x9a, 0x83, 0xe2, 0x5d, 0xdb,
0xe3, 0x06, 0xbb, 0x1e, 0xf6, 0xb1, 0x13, 0x74, 0x14, 0x04, 0x50, 0xf9, 0xc0, 0xd9, 0x32, 0xfd, 0x56, 0x15, 0xd4, 0x84, 0xda, 0x5e, 0xe8, 0x63, 0x83, 0xf0, 0x4c, 0x2d, 0xa0, 0x16, 0xc0, 0x7b,
0xa3, 0x4e, 0x01, 0x2d, 0xf0, 0xf3, 0x19, 0xdd, 0xda, 0xe6, 0x95, 0xd1, 0x4e, 0x91, 0x0c, 0x8f, 0x56, 0x10, 0x7a, 0xbe, 0xd5, 0x37, 0x6c, 0xb5, 0xb8, 0xfa, 0x09, 0xb4, 0xc4, 0x12, 0x33, 0x6a,
0xbe, 0x4a, 0xa8, 0x03, 0x8d, 0xa8, 0xcb, 0x83, 0xdd, 0xc7, 0x9d, 0x32, 0xa1, 0x9e, 0xfd, 0xac, 0x40, 0x75, 0xc7, 0x0b, 0xdf, 0x7d, 0x66, 0x05, 0xa1, 0x7a, 0x89, 0xf4, 0xdf, 0xf1, 0xc2, 0x5d,
0xac, 0x1a, 0xd0, 0x49, 0x9f, 0x2b, 0x92, 0x39, 0xd9, 0x22, 0x22, 0x50, 0xe7, 0x12, 0x59, 0x19, 0x1f, 0x07, 0xd8, 0x0d, 0x55, 0x05, 0x01, 0x54, 0x3e, 0x74, 0xb7, 0xac, 0xe0, 0x48, 0x2d, 0xa0,
0x3f, 0xd8, 0xed, 0x28, 0xa8, 0x0d, 0xf5, 0xc4, 0x31, 0x69, 0xa7, 0x40, 0x00, 0x0f, 0xbc, 0xd1, 0x05, 0x7e, 0x7a, 0x64, 0xd8, 0xdb, 0xbc, 0x6e, 0xab, 0x16, 0xc9, 0xf0, 0xf8, 0xab, 0x84, 0x54,
0x80, 0x0b, 0x14, 0x23, 0x81, 0x48, 0xe7, 0x16, 0xe1, 0x44, 0x69, 0xf5, 0x1e, 0x54, 0xc3, 0x90, 0x68, 0xc4, 0x5d, 0x1e, 0xec, 0x3e, 0x56, 0xcb, 0x84, 0x7a, 0xf6, 0xb3, 0xb2, 0x6a, 0x82, 0x9a,
0x9f, 0x74, 0xe5, 0x2c, 0x22, 0x9f, 0x9d, 0x4b, 0x68, 0x1e, 0x9a, 0xc2, 0xf3, 0xab, 0x8e, 0x82, 0x3d, 0xf5, 0x24, 0x73, 0xb2, 0x45, 0xc4, 0x20, 0xf5, 0x12, 0x59, 0x19, 0x3f, 0x76, 0x56, 0x15,
0x10, 0xb4, 0xc4, 0x07, 0x92, 0x9d, 0xc2, 0xea, 0x06, 0x40, 0x1c, 0x3a, 0x13, 0x72, 0xb6, 0x9d, 0xd4, 0x86, 0x7a, 0xea, 0x10, 0x57, 0x2d, 0x10, 0xc0, 0x03, 0x7f, 0xd8, 0xe7, 0x02, 0xc5, 0x48,
0x63, 0xdd, 0x32, 0x0d, 0x46, 0x1b, 0x69, 0x22, 0xdc, 0xa5, 0xdc, 0x61, 0x8a, 0xda, 0x29, 0xac, 0x20, 0xd2, 0xb9, 0x45, 0x76, 0xa2, 0xb4, 0x7a, 0x0f, 0xaa, 0x51, 0xaa, 0x41, 0xba, 0xf2, 0x2d,
0xae, 0x42, 0x35, 0x0c, 0x07, 0x09, 0x5c, 0xc3, 0xb6, 0x7b, 0x8c, 0xd9, 0xce, 0xec, 0x61, 0xc2, 0x22, 0x9f, 0xea, 0x25, 0x34, 0x0f, 0x4d, 0xe1, 0x71, 0x98, 0xaa, 0x20, 0x04, 0x2d, 0xf1, 0xf9,
0xca, 0x1a, 0x94, 0xef, 0xda, 0xd8, 0x31, 0x3a, 0x85, 0x8d, 0xff, 0x58, 0x00, 0x60, 0xa7, 0x82, 0xa6, 0x5a, 0x58, 0xdd, 0x00, 0x48, 0x42, 0x76, 0x42, 0xce, 0xb6, 0x7b, 0x6c, 0xd8, 0x96, 0xc9,
0xae, 0xeb, 0x19, 0xc8, 0xa2, 0xb7, 0x03, 0x36, 0x5d, 0x7b, 0xe4, 0x3a, 0xe1, 0x91, 0x85, 0x8f, 0x68, 0x23, 0x4d, 0x64, 0x77, 0xe9, 0xee, 0x30, 0x45, 0x55, 0x0b, 0xab, 0x6f, 0x43, 0x35, 0x0a,
0xd6, 0x52, 0xa9, 0x3a, 0xfb, 0xc8, 0x76, 0xe4, 0x8c, 0xe8, 0xbd, 0x24, 0xed, 0x9f, 0xea, 0xac, 0x43, 0x09, 0x5c, 0xc7, 0x8e, 0x77, 0x8c, 0x19, 0x67, 0xf6, 0x30, 0xd9, 0xca, 0x1a, 0x94, 0xef,
0x5e, 0x42, 0x36, 0xc5, 0x46, 0x92, 0xdb, 0x47, 0xe6, 0xe0, 0x28, 0x3a, 0x4a, 0x9c, 0xfc, 0x4a, 0x3a, 0xd8, 0x35, 0xd5, 0x02, 0x21, 0xe3, 0xf1, 0xd0, 0x34, 0x42, 0xcc, 0xc3, 0x31, 0xb5, 0xb8,
0x31, 0xd5, 0x35, 0xc4, 0x77, 0x43, 0x8a, 0x6f, 0x2f, 0xf0, 0x4c, 0x67, 0x18, 0xfa, 0x3f, 0xf5, 0xf1, 0x5f, 0x0b, 0x00, 0xec, 0x18, 0xd3, 0xf3, 0x7c, 0x13, 0xd9, 0xf4, 0x3a, 0xc3, 0xa6, 0xe7,
0x12, 0x7a, 0x9a, 0x7a, 0x23, 0x19, 0x22, 0xdc, 0xc8, 0xf3, 0x2c, 0xf2, 0x62, 0x28, 0x2d, 0x68, 0x0c, 0x3d, 0x37, 0x3a, 0x63, 0x09, 0xd0, 0x5a, 0xa6, 0x6a, 0xc0, 0x3e, 0xf2, 0x1d, 0xf9, 0xde,
0xa7, 0x1e, 0xa3, 0xa3, 0x55, 0xf9, 0x63, 0x13, 0xd9, 0xc3, 0xf9, 0xde, 0xab, 0xb9, 0xfa, 0x46, 0x74, 0x5f, 0x92, 0xf6, 0xcf, 0x74, 0xd6, 0x2e, 0x21, 0x87, 0x62, 0x23, 0x79, 0xf6, 0x23, 0xab,
0xd8, 0x4c, 0x68, 0x89, 0xaf, 0xa8, 0xd1, 0xd7, 0x26, 0x4d, 0x90, 0x79, 0xee, 0xd6, 0x5b, 0xcd, 0x7f, 0x14, 0x9f, 0x7d, 0x8e, 0x7f, 0x56, 0x99, 0xe9, 0x1a, 0xe1, 0xbb, 0x21, 0xc5, 0xb7, 0x17,
0xd3, 0x35, 0x42, 0xf5, 0x21, 0x93, 0xd5, 0x69, 0xa8, 0xa4, 0x2f, 0x0c, 0x7b, 0x67, 0x85, 0x1e, 0xfa, 0x96, 0x3b, 0x88, 0x5c, 0xa2, 0x76, 0x09, 0x3d, 0xcd, 0x3c, 0xea, 0x8c, 0x10, 0x6e, 0x4c,
0xea, 0x25, 0xf4, 0x63, 0x12, 0x25, 0xa4, 0x1e, 0xe5, 0xa1, 0xd7, 0xe4, 0x9e, 0x4d, 0xfe, 0x76, 0xf3, 0x8e, 0xf3, 0x62, 0x28, 0x6d, 0x68, 0x67, 0x5e, 0xcf, 0xa3, 0x55, 0xf9, 0xeb, 0x18, 0xd9,
0x6f, 0x1a, 0x86, 0x0f, 0xd3, 0x9a, 0x36, 0x99, 0xfa, 0xcc, 0x6b, 0xdf, 0xfc, 0xd4, 0x27, 0xa6, 0x4b, 0xff, 0xee, 0xab, 0x53, 0xf5, 0x8d, 0xb1, 0x59, 0xd0, 0x12, 0x9f, 0x7d, 0xa3, 0xaf, 0x8d,
0x3f, 0x8b, 0xfa, 0x73, 0x63, 0xb0, 0x58, 0xc2, 0x24, 0x79, 0x0e, 0x94, 0x16, 0xe5, 0x38, 0x5f, 0x9b, 0x20, 0xf7, 0x3e, 0xaf, 0xbb, 0x3a, 0x4d, 0xd7, 0x18, 0xd5, 0x47, 0x4c, 0x7c, 0x27, 0xa1,
0x99, 0xfc, 0x76, 0x68, 0x1a, 0xb6, 0x31, 0x55, 0xd2, 0xf4, 0x71, 0xf8, 0xeb, 0x13, 0x0a, 0xed, 0x92, 0x3e, 0x89, 0xec, 0x9e, 0x15, 0x8d, 0x68, 0x97, 0xd0, 0x8f, 0x48, 0xe0, 0x90, 0x79, 0x45,
0xf2, 0x77, 0x88, 0xbd, 0xb5, 0xbc, 0xdd, 0x93, 0xb2, 0x2c, 0x3e, 0x75, 0x93, 0x6f, 0x91, 0xf4, 0x88, 0x5e, 0x93, 0x3b, 0x3b, 0xf9, 0x63, 0xc3, 0x49, 0x18, 0x3e, 0xca, 0x2a, 0xdf, 0x78, 0xea,
0x79, 0x9e, 0x5c, 0x96, 0xe5, 0x2f, 0xe7, 0xd4, 0x4b, 0xe8, 0x91, 0x60, 0xd7, 0xd1, 0xcb, 0x93, 0x73, 0xcf, 0x93, 0xa7, 0xa7, 0x3e, 0x35, 0xfd, 0x59, 0xd4, 0x9f, 0x1b, 0x83, 0xcd, 0x72, 0x28,
0x44, 0x41, 0xbc, 0x1f, 0x33, 0x8d, 0x6f, 0xbf, 0x0a, 0x88, 0x69, 0xaa, 0x73, 0x60, 0x0e, 0xc7, 0xc9, 0xfb, 0xa5, 0xac, 0x28, 0x27, 0x29, 0xcc, 0xf8, 0xc7, 0x4e, 0x93, 0xb0, 0x8d, 0xa8, 0x92,
0x9e, 0xce, 0xc4, 0x78, 0x92, 0x71, 0xcb, 0x76, 0x0d, 0xd1, 0x7c, 0xfd, 0x1c, 0x23, 0xa2, 0x25, 0x66, 0xcf, 0xef, 0x5f, 0x1f, 0x73, 0x32, 0x20, 0x7f, 0x38, 0xd9, 0x5d, 0x9b, 0xb6, 0x7b, 0x5a,
0xf5, 0x01, 0x1e, 0xe0, 0xe0, 0x7d, 0xfa, 0xe6, 0xc9, 0x4f, 0xaf, 0x28, 0xb6, 0xdf, 0xbc, 0x43, 0x96, 0xc5, 0xb7, 0x79, 0x72, 0x16, 0x49, 0xdf, 0x13, 0xca, 0x65, 0x59, 0xfe, 0xd4, 0x4f, 0xbb,
0x88, 0xea, 0x95, 0xa9, 0xfd, 0x22, 0x04, 0xfb, 0x50, 0x7f, 0x40, 0x32, 0x28, 0x1a, 0x15, 0xfa, 0x84, 0x1e, 0x09, 0xa6, 0x1e, 0xdd, 0x1c, 0x27, 0x0a, 0xe2, 0x85, 0x9e, 0x49, 0xfb, 0xf6, 0xab,
0x68, 0xe2, 0xc8, 0xb0, 0x47, 0x88, 0x62, 0x65, 0x7a, 0xc7, 0xa4, 0xf1, 0x4c, 0x3d, 0xfb, 0x43, 0x80, 0x98, 0xa6, 0xba, 0x07, 0xd6, 0x60, 0xe4, 0x1b, 0x4c, 0x8c, 0xc7, 0x19, 0xb7, 0x7c, 0xd7,
0x13, 0x37, 0x36, 0xfb, 0x18, 0x51, 0x6e, 0x3c, 0x27, 0xbc, 0x23, 0x64, 0x2b, 0xa2, 0x87, 0x3d, 0x08, 0xcd, 0xd7, 0xcf, 0x31, 0x22, 0x5e, 0x52, 0x0f, 0xe0, 0x01, 0x0e, 0x3f, 0xa0, 0x8f, 0xb4,
0xef, 0x62, 0xdd, 0x0a, 0x0e, 0x27, 0xac, 0x28, 0xd1, 0xe3, 0xec, 0x15, 0x09, 0x1d, 0x23, 0x1c, 0x82, 0xec, 0x8a, 0x12, 0xfb, 0xcd, 0x3b, 0x44, 0xa8, 0x5e, 0x99, 0xd8, 0x2f, 0x46, 0xb0, 0x0f,
0x18, 0x16, 0x98, 0x16, 0x8a, 0xa9, 0xe7, 0xba, 0x7c, 0x8a, 0x6c, 0xcf, 0x9c, 0xa2, 0xa7, 0xc3, 0xf5, 0x07, 0x24, 0xa9, 0xa2, 0x81, 0x62, 0x80, 0xc6, 0x8e, 0x8c, 0x7a, 0x44, 0x28, 0x56, 0x26,
0xfc, 0x96, 0xe7, 0x8e, 0x44, 0x24, 0xaf, 0x4b, 0x91, 0x64, 0xfa, 0xe5, 0x44, 0xf1, 0x03, 0x68, 0x77, 0x4c, 0x1b, 0xcf, 0xcc, 0x3b, 0x45, 0x34, 0x96, 0xb1, 0xf9, 0xd7, 0x93, 0x72, 0xe3, 0x39,
0x84, 0x19, 0x3e, 0xcd, 0x49, 0xe4, 0x5c, 0x48, 0x76, 0xc9, 0x39, 0xf1, 0x47, 0xd0, 0x4e, 0x95, 0xe6, 0xe1, 0x23, 0x5b, 0x11, 0x3d, 0x9d, 0x7a, 0x0f, 0x1b, 0x76, 0x78, 0x38, 0x66, 0x45, 0xa9,
0x0e, 0xe4, 0x9b, 0x2e, 0xaf, 0x2f, 0x4c, 0x9b, 0xfd, 0x04, 0x10, 0x7d, 0xcb, 0x29, 0x3e, 0x47, 0x1e, 0x67, 0xaf, 0x48, 0xe8, 0x18, 0xe3, 0xc0, 0xb0, 0xc0, 0xb4, 0x50, 0xcc, 0x46, 0xd7, 0xe5,
0x97, 0xc7, 0x37, 0xd9, 0x8e, 0x21, 0x92, 0xf5, 0xdc, 0xfd, 0xa3, 0x9d, 0xff, 0x35, 0x58, 0x92, 0x53, 0xe4, 0x7b, 0x4e, 0x29, 0x7a, 0x06, 0xcc, 0x6f, 0xf9, 0xde, 0x50, 0x44, 0xf2, 0xba, 0x14,
0xa6, 0xe7, 0x69, 0x83, 0xc0, 0x2f, 0xc4, 0x9e, 0x51, 0x43, 0x48, 0x1b, 0x84, 0x33, 0x47, 0x84, 0x49, 0xae, 0xdf, 0x94, 0x28, 0xbe, 0x0f, 0x8d, 0x28, 0xe9, 0xa7, 0x69, 0x8a, 0x7c, 0x17, 0xd2,
0xf8, 0x37, 0x7e, 0x7b, 0x1e, 0x6a, 0x34, 0xce, 0xa3, 0xbb, 0xf5, 0x8b, 0x30, 0xef, 0xf9, 0x86, 0x5d, 0xa6, 0x9c, 0xf8, 0x63, 0x68, 0x67, 0xaa, 0x09, 0x72, 0xa6, 0xcb, 0x4b, 0x0e, 0x93, 0x66,
0x79, 0x1f, 0x41, 0x3b, 0xf5, 0xc6, 0x50, 0x2e, 0xb4, 0xf2, 0x87, 0x88, 0x39, 0xa2, 0x15, 0xf1, 0x3f, 0x01, 0x44, 0x1f, 0x9f, 0x8a, 0xef, 0xe7, 0xe5, 0xf1, 0x4d, 0xbe, 0x63, 0x84, 0x64, 0x7d,
0x79, 0x9e, 0xdc, 0x15, 0x4a, 0x9f, 0xf0, 0x4d, 0x9b, 0xfb, 0x09, 0x7b, 0xbf, 0x1b, 0xdd, 0x52, 0xea, 0xfe, 0x31, 0xe7, 0x7f, 0x0d, 0x96, 0xa4, 0x19, 0x7b, 0xd6, 0x20, 0xf0, 0x1b, 0xbc, 0x67,
0x78, 0x65, 0x62, 0xf1, 0x5e, 0xbc, 0xd8, 0xfa, 0xf9, 0x47, 0x41, 0x5f, 0xee, 0x08, 0xf4, 0x23, 0x94, 0x15, 0xb2, 0x06, 0xe1, 0xcc, 0x11, 0x11, 0xfe, 0x8d, 0xdf, 0x9e, 0x87, 0x1a, 0x8d, 0xf3,
0x68, 0xa7, 0x9e, 0x8e, 0xc8, 0x25, 0x46, 0xfe, 0xbe, 0x64, 0xda, 0xec, 0x9f, 0x61, 0xf0, 0x64, 0x28, 0xb7, 0x7e, 0x11, 0xe6, 0x3d, 0xdf, 0x30, 0xef, 0x63, 0x68, 0x67, 0x1e, 0x45, 0xca, 0x85,
0xc0, 0x82, 0xe4, 0xa6, 0x3e, 0x5a, 0x9b, 0x14, 0x88, 0xca, 0xaf, 0xf4, 0x4f, 0x5f, 0x50, 0x53, 0x56, 0xfe, 0x72, 0x72, 0x8a, 0x68, 0x45, 0x7c, 0x4f, 0x28, 0x77, 0x85, 0xd2, 0x37, 0x87, 0x93,
0x50, 0xd3, 0xb4, 0xbf, 0x89, 0x89, 0x4c, 0xff, 0x8f, 0x4d, 0xef, 0xb5, 0x7c, 0x7f, 0x7a, 0x13, 0xe6, 0x7e, 0xc2, 0x1e, 0x1c, 0xc7, 0x07, 0x9c, 0xaf, 0x8c, 0x3d, 0x47, 0x10, 0x6f, 0xe2, 0x7e,
0x2d, 0x68, 0x0f, 0x2a, 0xec, 0x41, 0x09, 0x7a, 0x51, 0x7e, 0x88, 0x91, 0x78, 0x6c, 0xd2, 0x9b, 0xfe, 0x51, 0xd0, 0x97, 0x3b, 0x02, 0xfd, 0x18, 0xda, 0x99, 0xb7, 0x2e, 0x72, 0x89, 0x91, 0x3f,
0xf6, 0x24, 0xc5, 0x1f, 0x5b, 0x01, 0xa1, 0xff, 0x57, 0xa0, 0xc5, 0x40, 0x11, 0x83, 0x9e, 0xe3, 0x88, 0x99, 0x34, 0xfb, 0xcf, 0x31, 0x78, 0x32, 0x61, 0x41, 0xf2, 0xb4, 0x00, 0xad, 0x8d, 0x0b,
0xe4, 0x7b, 0x50, 0xa6, 0xa6, 0x1d, 0x49, 0x8f, 0x0c, 0x92, 0xcf, 0x46, 0x7a, 0xd3, 0x5f, 0x8a, 0x44, 0xe5, 0x6f, 0x10, 0x26, 0x2f, 0xa8, 0x29, 0xa8, 0x69, 0xd6, 0xdf, 0x24, 0x44, 0x66, 0xff,
0xc4, 0x14, 0x37, 0xbf, 0xcf, 0xfe, 0x7e, 0x8c, 0x13, 0xfc, 0x3c, 0x27, 0xff, 0xff, 0x1d, 0x1b, 0x78, 0xa7, 0xfb, 0xda, 0x74, 0xff, 0xd2, 0x13, 0x2f, 0x68, 0x0f, 0x2a, 0xec, 0x05, 0x0c, 0x7a,
0x3f, 0xa3, 0x8f, 0x1e, 0xd2, 0xd7, 0x7a, 0xd0, 0xda, 0xf9, 0xee, 0x26, 0xf5, 0xd6, 0x73, 0xf7, 0x51, 0x7e, 0x9e, 0x92, 0x7a, 0x1d, 0xd3, 0x9d, 0xf4, 0x86, 0x26, 0x18, 0xd9, 0x21, 0xa1, 0xff,
0x8f, 0x30, 0xff, 0x08, 0x3a, 0xe9, 0xa3, 0x34, 0xf4, 0xea, 0x24, 0x4d, 0x94, 0xe1, 0x9c, 0xa2, 0x57, 0xa0, 0xc5, 0x40, 0xf1, 0x06, 0x3d, 0xc7, 0xc9, 0xf7, 0xa0, 0x4c, 0x4d, 0x3b, 0x92, 0x9e,
0x86, 0xdf, 0x83, 0x0a, 0xab, 0xa1, 0xca, 0xc5, 0x57, 0xa8, 0xaf, 0x4e, 0x99, 0xeb, 0xde, 0x37, 0x22, 0xa4, 0xdf, 0xb9, 0x74, 0x27, 0x3f, 0x6d, 0x49, 0x28, 0x6e, 0x7e, 0x8f, 0xfd, 0x5f, 0x1a,
0x3e, 0xdc, 0x18, 0x9a, 0xc1, 0xe1, 0x78, 0x9f, 0xb4, 0xac, 0xb3, 0xae, 0xaf, 0x9b, 0x2e, 0xff, 0x27, 0xf8, 0x79, 0x4e, 0xfe, 0xff, 0x3b, 0x36, 0x7e, 0x46, 0x5f, 0x69, 0x64, 0xef, 0x21, 0xa1,
0xb5, 0x1e, 0xee, 0xe5, 0x3a, 0x1d, 0xbd, 0x4e, 0x11, 0x8c, 0xf6, 0xf7, 0x2b, 0xf4, 0xf3, 0xf6, 0xb5, 0xf3, 0x5d, 0xa6, 0xea, 0xae, 0x4f, 0xdd, 0x3f, 0xc6, 0xfc, 0x43, 0x50, 0xb3, 0xa7, 0x6b,
0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x53, 0x45, 0xc6, 0xff, 0x50, 0x00, 0x00, 0xe8, 0xd5, 0x71, 0x9a, 0x28, 0xc3, 0x39, 0x41, 0x0d, 0xbf, 0x0b, 0x15, 0x56, 0x56, 0x95, 0x8b,
0xaf, 0x50, 0x72, 0x9d, 0x30, 0xd7, 0xbd, 0x6f, 0x7c, 0xb4, 0x31, 0xb0, 0xc2, 0xc3, 0xd1, 0x3e,
0x69, 0x59, 0x67, 0x5d, 0x5f, 0xb7, 0x3c, 0xfe, 0x6b, 0x3d, 0xe2, 0xe5, 0x3a, 0x1d, 0xbd, 0x4e,
0x11, 0x0c, 0xf7, 0xf7, 0x2b, 0xf4, 0xf3, 0xf6, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xf5, 0xd5,
0x38, 0x83, 0xb0, 0x51, 0x00, 0x00,
} }
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.

102
internal/querycoordv2/checkers/segment_checker.go
View File

@ -91,6 +91,7 @@ func (c *SegmentChecker) checkReplica(ctx context.Context, replica *meta.Replica
task.SetReason("lacks of segment", tasks...) task.SetReason("lacks of segment", tasks...)
ret = append(ret, tasks...) ret = append(ret, tasks...)
redundancies = c.filterSegmentInUse(replica, redundancies)
tasks = c.createSegmentReduceTasks(ctx, redundancies, replica.GetID(), querypb.DataScope_All) tasks = c.createSegmentReduceTasks(ctx, redundancies, replica.GetID(), querypb.DataScope_All)
task.SetReason("segment not exists in target", tasks...) task.SetReason("segment not exists in target", tasks...)
ret = append(ret, tasks...) ret = append(ret, tasks...)
@ -122,28 +123,43 @@ func (c *SegmentChecker) getStreamingSegmentDiff(targetMgr *meta.TargetManager,
log.Info("replica does not exist, skip it") log.Info("replica does not exist, skip it")
return return
} }
dist := c.getStreamingSegmentsDist(distMgr, replica)
distMap := typeutil.NewUniqueSet()
for _, s := range dist {
distMap.Insert(s.GetID())
}
nextTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.NextTarget) log := log.Ctx(context.TODO()).WithRateGroup("qcv2.SegmentChecker", 60, 1).With(
currentTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.CurrentTarget) zap.Int64("collectionID", collectionID),
currentTargetChannelMap := targetMgr.GetDmChannelsByCollection(collectionID, meta.CurrentTarget) zap.Int64("replicaID", replica.ID))
// get segment which exist on dist, but not on current target and next target leaders := distMgr.ChannelDistManager.GetShardLeadersByReplica(replica)
for _, segment := range dist { for leader, node := range leaders {
if !currentTargetSegmentIDs.Contain(segment.GetID()) && !nextTargetSegmentIDs.Contain(segment.GetID()) { view := distMgr.LeaderViewManager.GetLeaderShardView(node, leader)
if channel, ok := currentTargetChannelMap[segment.InsertChannel]; ok { targetVersion := targetMgr.GetCollectionTargetVersion(collectionID, meta.CurrentTarget)
timestampInSegment := segment.GetStartPosition().GetTimestamp() if view.TargetVersion != targetVersion {
timestampInTarget := channel.GetSeekPosition().GetTimestamp() // before shard delegator update it's readable version, skip release segment
// filter toRelease which seekPosition is newer than next target dmChannel log.RatedInfo(20, "before shard delegator update it's readable version, skip release segment",
if timestampInSegment < timestampInTarget { zap.String("channelName", leader),
log.Info("growing segment not exist in target, so release it", zap.Int64("nodeID", node),
zap.Int64("segmentID", segment.GetID()), zap.Int64("leaderVersion", view.TargetVersion),
) zap.Int64("currentVersion", targetVersion),
toRelease = append(toRelease, segment) )
continue
}
nextTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.NextTarget)
currentTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.CurrentTarget)
currentTargetChannelMap := targetMgr.GetDmChannelsByCollection(collectionID, meta.CurrentTarget)
// get segment which exist on leader view, but not on current target and next target
for _, segment := range view.GrowingSegments {
if !currentTargetSegmentIDs.Contain(segment.GetID()) && !nextTargetSegmentIDs.Contain(segment.GetID()) {
if channel, ok := currentTargetChannelMap[segment.InsertChannel]; ok {
timestampInSegment := segment.GetStartPosition().GetTimestamp()
timestampInTarget := channel.GetSeekPosition().GetTimestamp()
// filter toRelease which seekPosition is newer than next target dmChannel
if timestampInSegment < timestampInTarget {
log.Info("growing segment not exist in target, so release it",
zap.Int64("segmentID", segment.GetID()),
)
toRelease = append(toRelease, segment)
}
} }
} }
} }
@ -152,18 +168,6 @@ func (c *SegmentChecker) getStreamingSegmentDiff(targetMgr *meta.TargetManager,
return return
} }
func (c *SegmentChecker) getStreamingSegmentsDist(distMgr *meta.DistributionManager, replica *meta.Replica) map[int64]*meta.Segment {
segments := make(map[int64]*meta.Segment, 0)
for _, node := range replica.GetNodes() {
segmentsOnNodes := distMgr.LeaderViewManager.GetGrowingSegmentDistByCollectionAndNode(replica.CollectionID, node)
for k, v := range segmentsOnNodes {
segments[k] = v
}
}
return segments
}
// GetHistoricalSegmentDiff get historical segment diff between target and dist // GetHistoricalSegmentDiff get historical segment diff between target and dist
func (c *SegmentChecker) getHistoricalSegmentDiff( func (c *SegmentChecker) getHistoricalSegmentDiff(
targetMgr *meta.TargetManager, targetMgr *meta.TargetManager,
@ -248,16 +252,10 @@ func (c *SegmentChecker) filterExistedOnLeader(replica *meta.Replica, segments [
if !ok { if !ok {
continue continue
} }
onLeader := false
leaderViews := c.dist.LeaderViewManager.GetLeaderView(leaderID) view := c.dist.LeaderViewManager.GetLeaderShardView(leaderID, s.GetInsertChannel())
for _, view := range leaderViews { seg, ok := view.Segments[s.GetID()]
version, ok := view.Segments[s.GetID()] if ok && seg.NodeID == s.Node {
if ok && version.NodeID == s.Node {
onLeader = true
break
}
}
if onLeader {
// if this segment is serving on leader, do not remove it for search available // if this segment is serving on leader, do not remove it for search available
continue continue
} }
@ -266,6 +264,26 @@ func (c *SegmentChecker) filterExistedOnLeader(replica *meta.Replica, segments [
return filtered return filtered
} }
func (c *SegmentChecker) filterSegmentInUse(replica *meta.Replica, segments []*meta.Segment) []*meta.Segment {
filtered := make([]*meta.Segment, 0, len(segments))
for _, s := range segments {
leaderID, ok := c.dist.ChannelDistManager.GetShardLeader(replica, s.GetInsertChannel())
if !ok {
continue
}
view := c.dist.LeaderViewManager.GetLeaderShardView(leaderID, s.GetInsertChannel())
currentTargetVersion := c.targetMgr.GetCollectionTargetVersion(s.CollectionID, meta.CurrentTarget)
partition := c.meta.CollectionManager.GetPartition(s.PartitionID)
if partition != nil && view.TargetVersion != currentTargetVersion {
// leader view version hasn't been updated, segment maybe still in use
continue
}
filtered = append(filtered, s)
}
return filtered
}
func (c *SegmentChecker) createSegmentLoadTasks(ctx context.Context, segments []*datapb.SegmentInfo, replica *meta.Replica) []task.Task { func (c *SegmentChecker) createSegmentLoadTasks(ctx context.Context, segments []*datapb.SegmentInfo, replica *meta.Replica) []task.Task {
if len(segments) == 0 { if len(segments) == 0 {
return nil return nil

96
internal/querycoordv2/checkers/segment_checker_test.go
View File

@ -202,6 +202,52 @@ func (suite *SegmentCheckerTestSuite) TestReleaseRepeatedSegments() {
suite.Len(tasks, 0) suite.Len(tasks, 0)
} }
func (suite *SegmentCheckerTestSuite) TestSkipReleaseSealedSegments() {
checker := suite.checker
collectionID := int64(1)
partitionID := int64(1)
// set meta
checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(collectionID, 1))
checker.meta.CollectionManager.PutPartition(utils.CreateTestPartition(collectionID, partitionID))
checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, collectionID, []int64{1, 2}))
// set target
segments := []*datapb.SegmentInfo{}
suite.broker.EXPECT().GetRecoveryInfoV2(mock.Anything, int64(1)).Return(
nil, segments, nil)
checker.targetMgr.UpdateCollectionNextTargetWithPartitions(collectionID, partitionID)
checker.targetMgr.UpdateCollectionCurrentTarget(collectionID)
readableVersion := checker.targetMgr.GetCollectionTargetVersion(collectionID, meta.CurrentTarget)
// set dist
nodeID := int64(2)
segmentID := int64(1)
checker.dist.ChannelDistManager.Update(nodeID, utils.CreateTestChannel(collectionID, nodeID, segmentID, "test-insert-channel"))
view := utils.CreateTestLeaderView(nodeID, collectionID, "test-insert-channel", map[int64]int64{segmentID: 2}, map[int64]*meta.Segment{})
view.TargetVersion = readableVersion - 1
checker.dist.LeaderViewManager.Update(nodeID, view)
checker.dist.SegmentDistManager.Update(nodeID, utils.CreateTestSegment(collectionID, partitionID, segmentID, nodeID, 2, "test-insert-channel"))
tasks := checker.Check(context.TODO())
suite.Len(tasks, 0)
// test less version exist on leader
view = utils.CreateTestLeaderView(nodeID, collectionID, "test-insert-channel", map[int64]int64{1: 3}, map[int64]*meta.Segment{})
view.TargetVersion = readableVersion
checker.dist.LeaderViewManager.Update(2, view)
tasks = checker.Check(context.TODO())
suite.Len(tasks, 1)
suite.Len(tasks[0].Actions(), 1)
action, ok := tasks[0].Actions()[0].(*task.SegmentAction)
suite.True(ok)
suite.EqualValues(1, tasks[0].ReplicaID())
suite.Equal(task.ActionTypeReduce, action.Type())
suite.EqualValues(segmentID, action.SegmentID())
suite.EqualValues(nodeID, action.Node())
suite.Equal(tasks[0].Priority(), task.TaskPriorityNormal)
}
func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() { func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() {
checker := suite.checker checker := suite.checker
// segment3 is compacted from segment2, and node2 has growing segments 2 and 3. checker should generate // segment3 is compacted from segment2, and node2 has growing segments 2 and 3. checker should generate
@ -239,7 +285,9 @@ func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() {
dmChannel := utils.CreateTestChannel(1, 2, 1, "test-insert-channel") dmChannel := utils.CreateTestChannel(1, 2, 1, "test-insert-channel")
dmChannel.UnflushedSegmentIds = []int64{2, 3} dmChannel.UnflushedSegmentIds = []int64{2, 3}
checker.dist.ChannelDistManager.Update(2, dmChannel) checker.dist.ChannelDistManager.Update(2, dmChannel)
checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{3: 2}, growingSegments)) view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{3: 2}, growingSegments)
view.TargetVersion = checker.targetMgr.GetCollectionTargetVersion(int64(1), meta.CurrentTarget)
checker.dist.LeaderViewManager.Update(2, view)
checker.dist.SegmentDistManager.Update(2, utils.CreateTestSegment(1, 1, 3, 2, 2, "test-insert-channel")) checker.dist.SegmentDistManager.Update(2, utils.CreateTestSegment(1, 1, 3, 2, 2, "test-insert-channel"))
tasks := checker.Check(context.TODO()) tasks := checker.Check(context.TODO())
@ -266,6 +314,52 @@ func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() {
suite.Equal(tasks[1].Priority(), task.TaskPriorityNormal) suite.Equal(tasks[1].Priority(), task.TaskPriorityNormal)
} }
func (suite *SegmentCheckerTestSuite) TestSkipReleaseGrowingSegments() {
checker := suite.checker
checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1))
checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2}))
segments := []*datapb.SegmentInfo{}
channels := []*datapb.VchannelInfo{
{
CollectionID: 1,
ChannelName: "test-insert-channel",
SeekPosition: &msgpb.MsgPosition{Timestamp: 10},
},
}
suite.broker.EXPECT().GetRecoveryInfoV2(mock.Anything, int64(1)).Return(
channels, segments, nil)
checker.targetMgr.UpdateCollectionNextTargetWithPartitions(int64(1), int64(1))
checker.targetMgr.UpdateCollectionCurrentTarget(int64(1), int64(1))
growingSegments := make(map[int64]*meta.Segment)
growingSegments[2] = utils.CreateTestSegment(1, 1, 2, 2, 0, "test-insert-channel")
growingSegments[2].SegmentInfo.StartPosition = &msgpb.MsgPosition{Timestamp: 2}
dmChannel := utils.CreateTestChannel(1, 2, 1, "test-insert-channel")
dmChannel.UnflushedSegmentIds = []int64{2, 3}
checker.dist.ChannelDistManager.Update(2, dmChannel)
view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, growingSegments)
view.TargetVersion = checker.targetMgr.GetCollectionTargetVersion(int64(1), meta.CurrentTarget) - 1
checker.dist.LeaderViewManager.Update(2, view)
tasks := checker.Check(context.TODO())
suite.Len(tasks, 0)
view.TargetVersion = checker.targetMgr.GetCollectionTargetVersion(int64(1), meta.CurrentTarget)
checker.dist.LeaderViewManager.Update(2, view)
tasks = checker.Check(context.TODO())
suite.Len(tasks, 1)
suite.Len(tasks[0].Actions(), 1)
action, ok := tasks[0].Actions()[0].(*task.SegmentAction)
suite.True(ok)
suite.EqualValues(1, tasks[0].ReplicaID())
suite.Equal(task.ActionTypeReduce, action.Type())
suite.EqualValues(2, action.SegmentID())
suite.EqualValues(2, action.Node())
suite.Equal(tasks[0].Priority(), task.TaskPriorityNormal)
}
func (suite *SegmentCheckerTestSuite) TestReleaseDroppedSegments() { func (suite *SegmentCheckerTestSuite) TestReleaseDroppedSegments() {
checker := suite.checker checker := suite.checker
checker.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) checker.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel"))

1
internal/querycoordv2/dist/dist_handler.go vendored
View File

@ -201,6 +201,7 @@ func (dh *distHandler) updateLeaderView(resp *querypb.GetDataDistributionRespons
Version: version, Version: version,
Segments: lview.GetSegmentDist(), Segments: lview.GetSegmentDist(),
GrowingSegments: segments, GrowingSegments: segments,
TargetVersion: lview.TargetVersion,
} }
updates = append(updates, view) updates = append(updates, view)
} }

2
internal/querycoordv2/meta/leader_view_manager.go
View File

@ -31,6 +31,7 @@ type LeaderView struct {
Version int64 Version int64
Segments map[int64]*querypb.SegmentDist Segments map[int64]*querypb.SegmentDist
GrowingSegments map[int64]*Segment GrowingSegments map[int64]*Segment
TargetVersion int64
} }
func (view *LeaderView) Clone() *LeaderView { func (view *LeaderView) Clone() *LeaderView {
@ -51,6 +52,7 @@ func (view *LeaderView) Clone() *LeaderView {
Version: view.Version, Version: view.Version,
Segments: segments, Segments: segments,
GrowingSegments: growings, GrowingSegments: growings,
TargetVersion: view.TargetVersion,
} }
} }

8
internal/querycoordv2/meta/target.go
View File

@ -17,6 +17,8 @@
package meta package meta
import ( import (
"time"
"github.com/samber/lo" "github.com/samber/lo"
"github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/datapb"
@ -26,12 +28,14 @@ import (
type CollectionTarget struct { type CollectionTarget struct {
segments map[int64]*datapb.SegmentInfo segments map[int64]*datapb.SegmentInfo
dmChannels map[string]*DmChannel dmChannels map[string]*DmChannel
version int64
} }
func NewCollectionTarget(segments map[int64]*datapb.SegmentInfo, dmChannels map[string]*DmChannel) *CollectionTarget { func NewCollectionTarget(segments map[int64]*datapb.SegmentInfo, dmChannels map[string]*DmChannel) *CollectionTarget {
return &CollectionTarget{ return &CollectionTarget{
segments: segments, segments: segments,
dmChannels: dmChannels, dmChannels: dmChannels,
version: time.Now().UnixNano(),
} }
} }
@ -39,6 +43,10 @@ func (p *CollectionTarget) GetAllSegments() map[int64]*datapb.SegmentInfo {
return p.segments return p.segments
} }
func (p *CollectionTarget) GetTargetVersion() int64 {
return p.version
}
func (p *CollectionTarget) GetAllDmChannels() map[string]*DmChannel { func (p *CollectionTarget) GetAllDmChannels() map[string]*DmChannel {
return p.dmChannels return p.dmChannels
} }

16
internal/querycoordv2/meta/target_manager.go
View File

@ -80,7 +80,9 @@ func (mgr *TargetManager) UpdateCollectionCurrentTarget(collectionID int64, part
log.Debug("finish to update current target for collection", log.Debug("finish to update current target for collection",
zap.Int64s("segments", newTarget.GetAllSegmentIDs()), zap.Int64s("segments", newTarget.GetAllSegmentIDs()),
zap.Strings("channels", newTarget.GetAllDmChannelNames())) zap.Strings("channels", newTarget.GetAllDmChannelNames()),
zap.Int64("version", newTarget.GetTargetVersion()),
)
return true return true
} }
@ -457,6 +459,18 @@ func (mgr *TargetManager) GetHistoricalSegment(collectionID int64, id int64, sco
return collectionTarget.GetAllSegments()[id] return collectionTarget.GetAllSegments()[id]
} }
func (mgr *TargetManager) GetCollectionTargetVersion(collectionID int64, scope TargetScope) int64 {
mgr.rwMutex.RLock()
defer mgr.rwMutex.RUnlock()
targetMap := mgr.getTarget(scope)
collectionTarget := targetMap.getCollectionTarget(collectionID)
if collectionTarget == nil {
return 0
}
return collectionTarget.GetTargetVersion()
}
func (mgr *TargetManager) IsCurrentTargetExist(collectionID int64) bool { func (mgr *TargetManager) IsCurrentTargetExist(collectionID int64) bool {
newChannels := mgr.GetDmChannelsByCollection(collectionID, CurrentTarget) newChannels := mgr.GetDmChannelsByCollection(collectionID, CurrentTarget)

20
internal/querycoordv2/meta/target_manager_test.go
View File

@ -18,6 +18,7 @@ package meta
import ( import (
"testing" "testing"
"time"
"github.com/cockroachdb/errors" "github.com/cockroachdb/errors"
"github.com/samber/lo" "github.com/samber/lo"
@ -322,6 +323,25 @@ func (suite *TargetManagerSuite) assertSegments(expected []int64, actual map[int
return suite.Len(set, 0) return suite.Len(set, 0)
} }
func (suite *TargetManagerSuite) TestGetCollectionTargetVersion() {
t1 := time.Now().UnixNano()
target := NewCollectionTarget(nil, nil)
t2 := time.Now().UnixNano()
version := target.GetTargetVersion()
suite.True(t1 <= version)
suite.True(t2 >= version)
collectionID := int64(1)
t3 := time.Now().UnixNano()
suite.mgr.updateCollectionNextTarget(collectionID)
t4 := time.Now().UnixNano()
collectionVersion := suite.mgr.GetCollectionTargetVersion(collectionID, NextTarget)
suite.True(t3 <= collectionVersion)
suite.True(t4 >= collectionVersion)
}
func TestTargetManager(t *testing.T) { func TestTargetManager(t *testing.T) {
suite.Run(t, new(TargetManagerSuite)) suite.Run(t, new(TargetManagerSuite))
} }

5
internal/querycoordv2/observers/collection_observer.go
View File

@ -38,6 +38,7 @@ type CollectionObserver struct {
meta *meta.Meta meta *meta.Meta
targetMgr *meta.TargetManager targetMgr *meta.TargetManager
targetObserver *TargetObserver targetObserver *TargetObserver
leaderObserver *LeaderObserver
checkerController *checkers.CheckerController checkerController *checkers.CheckerController
partitionLoadedCount map[int64]int partitionLoadedCount map[int64]int
@ -49,6 +50,7 @@ func NewCollectionObserver(
meta *meta.Meta, meta *meta.Meta,
targetMgr *meta.TargetManager, targetMgr *meta.TargetManager,
targetObserver *TargetObserver, targetObserver *TargetObserver,
leaderObserver *LeaderObserver,
checherController *checkers.CheckerController, checherController *checkers.CheckerController,
) *CollectionObserver { ) *CollectionObserver {
return &CollectionObserver{ return &CollectionObserver{
@ -57,6 +59,7 @@ func NewCollectionObserver(
meta: meta, meta: meta,
targetMgr: targetMgr, targetMgr: targetMgr,
targetObserver: targetObserver, targetObserver: targetObserver,
leaderObserver: leaderObserver,
checkerController: checherController, checkerController: checherController,
partitionLoadedCount: make(map[int64]int), partitionLoadedCount: make(map[int64]int),
} }
@ -203,7 +206,7 @@ func (ob *CollectionObserver) observePartitionLoadStatus(partition *meta.Partiti
} }
ob.partitionLoadedCount[partition.GetPartitionID()] = loadedCount ob.partitionLoadedCount[partition.GetPartitionID()] = loadedCount
if loadPercentage == 100 && ob.targetObserver.Check(partition.GetCollectionID()) { if loadPercentage == 100 && ob.targetObserver.Check(partition.GetCollectionID()) && ob.leaderObserver.CheckTargetVersion(partition.GetCollectionID()) {
delete(ob.partitionLoadedCount, partition.GetPartitionID()) delete(ob.partitionLoadedCount, partition.GetPartitionID())
} }
collectionPercentage, err := ob.meta.CollectionManager.UpdateLoadPercent(partition.PartitionID, loadPercentage) collectionPercentage, err := ob.meta.CollectionManager.UpdateLoadPercent(partition.PartitionID, loadPercentage)

10
internal/querycoordv2/observers/collection_observer_test.go
View File

@ -26,6 +26,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
clientv3 "go.etcd.io/etcd/client/v3" clientv3 "go.etcd.io/etcd/client/v3"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/datapb"
@ -63,6 +64,7 @@ type CollectionObserverSuite struct {
meta *meta.Meta meta *meta.Meta
targetMgr *meta.TargetManager targetMgr *meta.TargetManager
targetObserver *TargetObserver targetObserver *TargetObserver
leaderObserver *LeaderObserver
checkerController *checkers.CheckerController checkerController *checkers.CheckerController
// Test object // Test object
@ -192,12 +194,19 @@ func (suite *CollectionObserverSuite) SetupTest() {
) )
suite.checkerController = &checkers.CheckerController{} suite.checkerController = &checkers.CheckerController{}
mockCluster := session.NewMockCluster(suite.T())
suite.leaderObserver = NewLeaderObserver(suite.dist, suite.meta, suite.targetMgr, suite.broker, mockCluster)
mockCluster.EXPECT().SyncDistribution(mock.Anything, mock.Anything, mock.Anything).Return(&commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil).Maybe()
// Test object // Test object
suite.ob = NewCollectionObserver( suite.ob = NewCollectionObserver(
suite.dist, suite.dist,
suite.meta, suite.meta,
suite.targetMgr, suite.targetMgr,
suite.targetObserver, suite.targetObserver,
suite.leaderObserver,
suite.checkerController, suite.checkerController,
) )
@ -205,6 +214,7 @@ func (suite *CollectionObserverSuite) SetupTest() {
suite.broker.EXPECT().GetPartitions(mock.Anything, collection).Return(suite.partitions[collection], nil).Maybe() suite.broker.EXPECT().GetPartitions(mock.Anything, collection).Return(suite.partitions[collection], nil).Maybe()
} }
suite.targetObserver.Start(context.Background()) suite.targetObserver.Start(context.Background())
suite.leaderObserver.Start(context.TODO())
suite.ob.Start(context.Background()) suite.ob.Start(context.Background())
suite.loadAll() suite.loadAll()
} }

101
internal/querycoordv2/observers/leader_observer.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/internal/querycoordv2/utils"
"github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/commonpbutil" "github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/samber/lo"
) )
const ( const (
@ -39,13 +40,14 @@ const (
// LeaderObserver is to sync the distribution with leader // LeaderObserver is to sync the distribution with leader
type LeaderObserver struct { type LeaderObserver struct {
wg sync.WaitGroup wg sync.WaitGroup
closeCh chan struct{} closeCh chan struct{}
dist *meta.DistributionManager dist *meta.DistributionManager
meta *meta.Meta meta *meta.Meta
target *meta.TargetManager target *meta.TargetManager
broker meta.Broker broker meta.Broker
cluster session.Cluster cluster session.Cluster
manualCheck chan checkRequest
stopOnce sync.Once stopOnce sync.Once
} }
@ -64,6 +66,12 @@ func (o *LeaderObserver) Start(ctx context.Context) {
case <-ctx.Done(): case <-ctx.Done():
log.Info("stop leader observer due to ctx done") log.Info("stop leader observer due to ctx done")
return return
case req := <-o.manualCheck:
log.Info("triggering manual check")
ret := o.observeCollection(ctx, req.CollectionID)
req.Notifier <- ret
log.Info("manual check done", zap.Bool("result", ret))
case <-ticker.C: case <-ticker.C:
o.observe(ctx) o.observe(ctx)
} }
@ -89,8 +97,9 @@ func (o *LeaderObserver) observeSegmentsDist(ctx context.Context) {
} }
} }
func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64) { func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64) bool {
replicas := o.meta.ReplicaManager.GetByCollection(collection) replicas := o.meta.ReplicaManager.GetByCollection(collection)
result := true
for _, replica := range replicas { for _, replica := range replicas {
leaders := o.dist.ChannelDistManager.GetShardLeadersByReplica(replica) leaders := o.dist.ChannelDistManager.GetShardLeadersByReplica(replica)
for ch, leaderID := range leaders { for ch, leaderID := range leaders {
@ -99,11 +108,55 @@ func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64
continue continue
} }
dists := o.dist.SegmentDistManager.GetByShardWithReplica(ch, replica) dists := o.dist.SegmentDistManager.GetByShardWithReplica(ch, replica)
needLoaded, needRemoved := o.findNeedLoadedSegments(leaderView, dists),
o.findNeedRemovedSegments(leaderView, dists) actions := o.findNeedLoadedSegments(leaderView, dists)
o.sync(ctx, replica.GetID(), leaderView, append(needLoaded, needRemoved...)) actions = append(actions, o.findNeedRemovedSegments(leaderView, dists)...)
updateVersionAction := o.checkNeedUpdateTargetVersion(leaderView)
if updateVersionAction != nil {
actions = append(actions, updateVersionAction)
}
success := o.sync(ctx, replica.GetID(), leaderView, actions)
if !success {
result = false
}
} }
} }
return result
}
func (ob *LeaderObserver) CheckTargetVersion(collectionID int64) bool {
notifier := make(chan bool)
ob.manualCheck <- checkRequest{
CollectionID: collectionID,
Notifier: notifier,
}
return <-notifier
}
func (o *LeaderObserver) checkNeedUpdateTargetVersion(leaderView *meta.LeaderView) *querypb.SyncAction {
targetVersion := o.target.GetCollectionTargetVersion(leaderView.CollectionID, meta.CurrentTarget)
if targetVersion <= leaderView.TargetVersion {
return nil
}
log.Info("Update readable segment version",
zap.Int64("collectionID", leaderView.CollectionID),
zap.String("channelName", leaderView.Channel),
zap.Int64("nodeID", leaderView.ID),
zap.Int64("oldVersion", leaderView.TargetVersion),
zap.Int64("newVersion", targetVersion),
)
sealedSegments := o.target.GetHistoricalSegmentsByCollection(leaderView.CollectionID, meta.CurrentTarget)
growingSegments := o.target.GetStreamingSegmentsByCollection(leaderView.CollectionID, meta.CurrentTarget)
return &querypb.SyncAction{
Type: querypb.SyncType_UpdateVersion,
GrowingInTarget: growingSegments.Collect(),
SealedInTarget: lo.Keys(sealedSegments),
TargetVersion: targetVersion,
}
} }
func (o *LeaderObserver) findNeedLoadedSegments(leaderView *meta.LeaderView, dists []*meta.Segment) []*querypb.SyncAction { func (o *LeaderObserver) findNeedLoadedSegments(leaderView *meta.LeaderView, dists []*meta.Segment) []*querypb.SyncAction {
@ -168,9 +221,9 @@ func (o *LeaderObserver) findNeedRemovedSegments(leaderView *meta.LeaderView, di
return ret return ret
} }
func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView *meta.LeaderView, diffs []*querypb.SyncAction) { func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView *meta.LeaderView, diffs []*querypb.SyncAction) bool {
if len(diffs) == 0 { if len(diffs) == 0 {
return return true
} }
log := log.With( log := log.With(
@ -182,12 +235,12 @@ func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView *
schema, err := o.broker.GetCollectionSchema(ctx, leaderView.CollectionID) schema, err := o.broker.GetCollectionSchema(ctx, leaderView.CollectionID)
if err != nil { if err != nil {
log.Error("sync distribution failed, cannot get schema of collection", zap.Error(err)) log.Error("sync distribution failed, cannot get schema of collection", zap.Error(err))
return return false
} }
partitions, err := utils.GetPartitions(o.meta.CollectionManager, leaderView.CollectionID) partitions, err := utils.GetPartitions(o.meta.CollectionManager, leaderView.CollectionID)
if err != nil { if err != nil {
log.Error("sync distribution failed, cannot get partitions of collection", zap.Error(err)) log.Error("sync distribution failed, cannot get partitions of collection", zap.Error(err))
return return false
} }
req := &querypb.SyncDistributionRequest{ req := &querypb.SyncDistributionRequest{
@ -209,12 +262,15 @@ func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView *
resp, err := o.cluster.SyncDistribution(ctx, leaderView.ID, req) resp, err := o.cluster.SyncDistribution(ctx, leaderView.ID, req)
if err != nil { if err != nil {
log.Error("failed to sync distribution", zap.Error(err)) log.Error("failed to sync distribution", zap.Error(err))
return return false
} }
if resp.ErrorCode != commonpb.ErrorCode_Success { if resp.ErrorCode != commonpb.ErrorCode_Success {
log.Error("failed to sync distribution", zap.String("reason", resp.GetReason())) log.Error("failed to sync distribution", zap.String("reason", resp.GetReason()))
return false
} }
return true
} }
func NewLeaderObserver( func NewLeaderObserver(
@ -225,11 +281,12 @@ func NewLeaderObserver(
cluster session.Cluster, cluster session.Cluster,
) *LeaderObserver { ) *LeaderObserver {
return &LeaderObserver{ return &LeaderObserver{
closeCh: make(chan struct{}), closeCh: make(chan struct{}),
dist: dist, dist: dist,
meta: meta, meta: meta,
target: targetMgr, target: targetMgr,
broker: broker, broker: broker,
cluster: cluster, cluster: cluster,
manualCheck: make(chan checkRequest, 10),
} }
} }

106
internal/querycoordv2/observers/leader_observer_test.go
View File

@ -73,6 +73,9 @@ func (suite *LeaderObserverTestSuite) SetupTest() {
suite.broker = meta.NewMockBroker(suite.T()) suite.broker = meta.NewMockBroker(suite.T())
suite.mockCluster = session.NewMockCluster(suite.T()) suite.mockCluster = session.NewMockCluster(suite.T())
// suite.mockCluster.EXPECT().SyncDistribution(mock.Anything, mock.Anything, mock.Anything).Return(&commonpb.Status{
// ErrorCode: commonpb.ErrorCode_Success,
// }, nil).Maybe()
distManager := meta.NewDistributionManager() distManager := meta.NewDistributionManager()
targetManager := meta.NewTargetManager(suite.broker, suite.meta) targetManager := meta.NewTargetManager(suite.broker, suite.meta)
suite.observer = NewLeaderObserver(distManager, suite.meta, targetManager, suite.broker, suite.mockCluster) suite.observer = NewLeaderObserver(distManager, suite.meta, targetManager, suite.broker, suite.mockCluster)
@ -120,7 +123,9 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegments() {
observer.target.UpdateCollectionCurrentTarget(1) observer.target.UpdateCollectionCurrentTarget(1)
observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 2, 1, "test-insert-channel")) observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 2, 1, "test-insert-channel"))
observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel"))
observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})) view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})
view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget)
observer.dist.LeaderViewManager.Update(2, view)
expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest { expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest {
return &querypb.SyncDistributionRequest{ return &querypb.SyncDistributionRequest{
@ -151,11 +156,10 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegments() {
called := atomic.NewBool(false) called := atomic.NewBool(false)
suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2),
mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). mock.AnythingOfType("*querypb.SyncDistributionRequest")).
Run(func(args mock.Arguments) { Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) {
inputReq := (args[2]).(*querypb.SyncDistributionRequest) assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req},
assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())})
[]*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())})
called.Store(true) called.Store(true)
}). }).
Return(&commonpb.Status{}, nil) Return(&commonpb.Status{}, nil)
@ -209,7 +213,9 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncLoadedSegments() {
observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 2, 1, "test-insert-channel"), observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 2, 1, "test-insert-channel"),
utils.CreateTestSegment(1, 1, 2, 2, 1, "test-insert-channel")) utils.CreateTestSegment(1, 1, 2, 2, 1, "test-insert-channel"))
observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel"))
observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})) view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})
view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget)
observer.dist.LeaderViewManager.Update(2, view)
expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest { expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest {
return &querypb.SyncDistributionRequest{ return &querypb.SyncDistributionRequest{
@ -238,11 +244,10 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncLoadedSegments() {
} }
} }
called := atomic.NewBool(false) called := atomic.NewBool(false)
suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")).
Run(func(args mock.Arguments) { Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) {
inputReq := (args[2]).(*querypb.SyncDistributionRequest) assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req},
assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())})
[]*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())})
called.Store(true) called.Store(true)
}). }).
Return(&commonpb.Status{}, nil) Return(&commonpb.Status{}, nil)
@ -290,6 +295,7 @@ func (suite *LeaderObserverTestSuite) TestIgnoreBalancedSegment() {
NodeID: 2, NodeID: 2,
Version: 2, Version: 2,
} }
leaderView.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget)
observer.dist.LeaderViewManager.Update(2, leaderView) observer.dist.LeaderViewManager.Update(2, leaderView)
observer.Start(context.TODO()) observer.Start(context.TODO())
@ -336,8 +342,12 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegmentsWithReplicas() {
observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel"))
observer.dist.SegmentDistManager.Update(4, utils.CreateTestSegment(1, 1, 1, 4, 2, "test-insert-channel")) observer.dist.SegmentDistManager.Update(4, utils.CreateTestSegment(1, 1, 1, 4, 2, "test-insert-channel"))
observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel"))
observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})) view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})
observer.dist.LeaderViewManager.Update(4, utils.CreateTestLeaderView(4, 1, "test-insert-channel", map[int64]int64{1: 4}, map[int64]*meta.Segment{})) view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget)
observer.dist.LeaderViewManager.Update(2, view)
view2 := utils.CreateTestLeaderView(4, 1, "test-insert-channel", map[int64]int64{1: 4}, map[int64]*meta.Segment{})
view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget)
observer.dist.LeaderViewManager.Update(4, view2)
expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest { expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest {
return &querypb.SyncDistributionRequest{ return &querypb.SyncDistributionRequest{
@ -367,11 +377,10 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegmentsWithReplicas() {
} }
called := atomic.NewBool(false) called := atomic.NewBool(false)
suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2),
mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). mock.AnythingOfType("*querypb.SyncDistributionRequest")).
Run(func(args mock.Arguments) { Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) {
inputReq := (args[2]).(*querypb.SyncDistributionRequest) assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req},
assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())})
[]*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())})
called.Store(true) called.Store(true)
}). }).
Return(&commonpb.Status{}, nil) Return(&commonpb.Status{}, nil)
@ -423,11 +432,10 @@ func (suite *LeaderObserverTestSuite) TestSyncRemovedSegments() {
} }
ch := make(chan struct{}) ch := make(chan struct{})
suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2),
mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). mock.AnythingOfType("*querypb.SyncDistributionRequest")).
Run(func(args mock.Arguments) { Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) {
inputReq := (args[2]).(*querypb.SyncDistributionRequest) assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req},
assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())})
[]*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())})
close(ch) close(ch)
}). }).
Return(&commonpb.Status{}, nil) Return(&commonpb.Status{}, nil)
@ -492,11 +500,10 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncRemovedSegments() {
} }
} }
called := atomic.NewBool(false) called := atomic.NewBool(false)
suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")).
Run(func(args mock.Arguments) { Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) {
inputReq := (args[2]).(*querypb.SyncDistributionRequest) assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req},
assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())})
[]*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())})
called.Store(true) called.Store(true)
}). }).
Return(&commonpb.Status{}, nil) Return(&commonpb.Status{}, nil)
@ -510,6 +517,47 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncRemovedSegments() {
) )
} }
func (suite *LeaderObserverTestSuite) TestSyncTargetVersion() {
collectionID := int64(1001)
observer := suite.observer
observer.meta.CollectionManager.PutCollection(utils.CreateTestCollection(collectionID, 1))
observer.meta.ReplicaManager.Put(utils.CreateTestReplica(1, collectionID, []int64{1, 2}))
nextTargetChannels := []*datapb.VchannelInfo{
{
CollectionID: collectionID,
ChannelName: "channel-1",
UnflushedSegmentIds: []int64{22, 33},
},
}
nextTargetSegments := []*datapb.SegmentInfo{
{
ID: 11,
PartitionID: 1,
InsertChannel: "channel-1",
},
}
suite.broker.EXPECT().GetRecoveryInfoV2(mock.Anything, collectionID).Return(nextTargetChannels, nextTargetSegments, nil)
suite.observer.target.UpdateCollectionNextTargetWithPartitions(collectionID, 1)
suite.observer.target.UpdateCollectionCurrentTarget(collectionID)
TargetVersion := suite.observer.target.GetCollectionTargetVersion(collectionID, meta.CurrentTarget)
view := utils.CreateTestLeaderView(1, collectionID, "test-channel", nil, nil)
view.TargetVersion = TargetVersion
action := observer.checkNeedUpdateTargetVersion(view)
suite.Nil(action)
view.TargetVersion = TargetVersion - 1
action = observer.checkNeedUpdateTargetVersion(view)
suite.NotNil(action)
suite.Equal(querypb.SyncType_UpdateVersion, action.Type)
suite.Len(action.GrowingInTarget, 2)
suite.Len(action.SealedInTarget, 1)
}
func TestLeaderObserverSuite(t *testing.T) { func TestLeaderObserverSuite(t *testing.T) {
suite.Run(t, new(LeaderObserverTestSuite)) suite.Run(t, new(LeaderObserverTestSuite))
} }

1
internal/querycoordv2/server.go
View File

@ -360,6 +360,7 @@ func (s *Server) initObserver() {
s.meta, s.meta,
s.targetMgr, s.targetMgr,
s.targetObserver, s.targetObserver,
s.leaderObserver,
s.checkerController, s.checkerController,
) )

1
internal/querycoordv2/server_test.go
View File

@ -517,6 +517,7 @@ func (suite *ServerSuite) hackServer() {
suite.server.meta, suite.server.meta,
suite.server.targetMgr, suite.server.targetMgr,
suite.server.targetObserver, suite.server.targetObserver,
suite.server.leaderObserver,
suite.server.checkerController, suite.server.checkerController,
) )

1
internal/querycoordv2/services_test.go
View File

@ -189,6 +189,7 @@ func (suite *ServiceSuite) SetupTest() {
suite.server.meta, suite.server.meta,
suite.server.targetMgr, suite.server.targetMgr,
suite.targetObserver, suite.targetObserver,
suite.server.leaderObserver,
&checkers.CheckerController{}, &checkers.CheckerController{},
) )

14
internal/querynodev2/delegator/delegator.go
View File

@ -77,7 +77,7 @@ func newLifetime() *lifetime {
type ShardDelegator interface { type ShardDelegator interface {
Collection() int64 Collection() int64
Version() int64 Version() int64
GetSegmentInfo() (sealed []SnapshotItem, growing []SegmentEntry) GetSegmentInfo(readable bool) (sealed []SnapshotItem, growing []SegmentEntry)
SyncDistribution(ctx context.Context, entries ...SegmentEntry) SyncDistribution(ctx context.Context, entries ...SegmentEntry)
Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error) Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error)
Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error) Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error)
@ -89,6 +89,8 @@ type ShardDelegator interface {
LoadGrowing(ctx context.Context, infos []*querypb.SegmentLoadInfo, version int64) error LoadGrowing(ctx context.Context, infos []*querypb.SegmentLoadInfo, version int64) error
LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error
ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest, force bool) error ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest, force bool) error
SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64)
GetTargetVersion() int64
// control // control
Serviceable() bool Serviceable() bool
@ -164,8 +166,8 @@ func (sd *shardDelegator) Version() int64 {
} }
// GetSegmentInfo returns current segment distribution snapshot. // GetSegmentInfo returns current segment distribution snapshot.
func (sd *shardDelegator) GetSegmentInfo() ([]SnapshotItem, []SegmentEntry) { func (sd *shardDelegator) GetSegmentInfo(readable bool) ([]SnapshotItem, []SegmentEntry) {
return sd.distribution.Peek() return sd.distribution.PeekSegments(readable)
} }
// SyncDistribution revises distribution. // SyncDistribution revises distribution.
@ -227,7 +229,7 @@ func (sd *shardDelegator) Search(ctx context.Context, req *querypb.SearchRequest
fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel). fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel).
Observe(float64(waitTr.ElapseSpan().Milliseconds())) Observe(float64(waitTr.ElapseSpan().Milliseconds()))
sealed, growing, version := sd.distribution.GetCurrent(req.GetReq().GetPartitionIDs()...) sealed, growing, version := sd.distribution.GetSegments(true, req.GetReq().GetPartitionIDs()...)
defer sd.distribution.FinishUsage(version) defer sd.distribution.FinishUsage(version)
existPartitions := sd.collection.GetPartitions() existPartitions := sd.collection.GetPartitions()
growing = lo.Filter(growing, func(segment SegmentEntry, _ int) bool { growing = lo.Filter(growing, func(segment SegmentEntry, _ int) bool {
@ -286,7 +288,7 @@ func (sd *shardDelegator) Query(ctx context.Context, req *querypb.QueryRequest)
fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel). fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel).
Observe(float64(waitTr.ElapseSpan().Milliseconds())) Observe(float64(waitTr.ElapseSpan().Milliseconds()))
sealed, growing, version := sd.distribution.GetCurrent(req.GetReq().GetPartitionIDs()...) sealed, growing, version := sd.distribution.GetSegments(true, req.GetReq().GetPartitionIDs()...)
defer sd.distribution.FinishUsage(version) defer sd.distribution.FinishUsage(version)
existPartitions := sd.collection.GetPartitions() existPartitions := sd.collection.GetPartitions()
growing = lo.Filter(growing, func(segment SegmentEntry, _ int) bool { growing = lo.Filter(growing, func(segment SegmentEntry, _ int) bool {
@ -340,7 +342,7 @@ func (sd *shardDelegator) GetStatistics(ctx context.Context, req *querypb.GetSta
return nil, err return nil, err
} }
sealed, growing, version := sd.distribution.GetCurrent(req.Req.GetPartitionIDs()...) sealed, growing, version := sd.distribution.GetSegments(true, req.Req.GetPartitionIDs()...)
defer sd.distribution.FinishUsage(version) defer sd.distribution.FinishUsage(version)
tasks, err := organizeSubTask(req, sealed, growing, sd.workerManager, func(req *querypb.GetStatisticsRequest, scope querypb.DataScope, segmentIDs []int64, targetID int64) *querypb.GetStatisticsRequest { tasks, err := organizeSubTask(req, sealed, growing, sd.workerManager, func(req *querypb.GetStatisticsRequest, scope querypb.DataScope, segmentIDs []int64, targetID int64) *querypb.GetStatisticsRequest {

63
internal/querynodev2/delegator/delegator_data.go
View File

@ -86,10 +86,11 @@ func (sd *shardDelegator) newGrowing(segmentID int64, insertData *InsertData) se
sd.segmentManager.Put(segments.SegmentTypeGrowing, segment) sd.segmentManager.Put(segments.SegmentTypeGrowing, segment)
sd.addGrowing(SegmentEntry{ sd.addGrowing(SegmentEntry{
NodeID: paramtable.GetNodeID(), NodeID: paramtable.GetNodeID(),
SegmentID: segmentID, SegmentID: segmentID,
PartitionID: insertData.PartitionID, PartitionID: insertData.PartitionID,
Version: 0, Version: 0,
TargetVersion: initialTargetVersion,
}) })
return segment return segment
} }
@ -173,7 +174,7 @@ func (sd *shardDelegator) ProcessDelete(deleteData []*DeleteData, ts uint64) {
offlineSegments := typeutil.NewConcurrentSet[int64]() offlineSegments := typeutil.NewConcurrentSet[int64]()
sealed, growing, version := sd.distribution.GetCurrent() sealed, growing, version := sd.distribution.GetSegments(false)
eg, ctx := errgroup.WithContext(context.Background()) eg, ctx := errgroup.WithContext(context.Background())
for _, entry := range sealed { for _, entry := range sealed {
@ -296,10 +297,11 @@ func (sd *shardDelegator) LoadGrowing(ctx context.Context, infos []*querypb.Segm
} }
sd.addGrowing(lo.Map(loaded, func(segment segments.Segment, _ int) SegmentEntry { sd.addGrowing(lo.Map(loaded, func(segment segments.Segment, _ int) SegmentEntry {
return SegmentEntry{ return SegmentEntry{
NodeID: paramtable.GetNodeID(), NodeID: paramtable.GetNodeID(),
SegmentID: segment.ID(), SegmentID: segment.ID(),
PartitionID: segment.Partition(), PartitionID: segment.Partition(),
Version: version, Version: version,
TargetVersion: sd.distribution.getTargetVersion(),
} }
})...) })...)
return nil return nil
@ -351,37 +353,14 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg
// alter distribution // alter distribution
entries := lo.Map(req.GetInfos(), func(info *querypb.SegmentLoadInfo, _ int) SegmentEntry { entries := lo.Map(req.GetInfos(), func(info *querypb.SegmentLoadInfo, _ int) SegmentEntry {
return SegmentEntry{ return SegmentEntry{
SegmentID: info.GetSegmentID(), SegmentID: info.GetSegmentID(),
PartitionID: info.GetPartitionID(), PartitionID: info.GetPartitionID(),
NodeID: req.GetDstNodeID(), NodeID: req.GetDstNodeID(),
Version: req.GetVersion(), Version: req.GetVersion(),
TargetVersion: sd.distribution.getTargetVersion(),
} }
}) })
removed, signal := sd.distribution.AddDistributions(entries...) sd.distribution.AddDistributions(entries...)
// release possible matched growing segments async
if len(removed) > 0 {
go func() {
<-signal
worker, err := sd.workerManager.GetWorker(paramtable.GetNodeID())
if err != nil {
log.Warn("failed to get local worker when try to release related growing", zap.Error(err))
return
}
err = worker.ReleaseSegments(context.Background(), &querypb.ReleaseSegmentsRequest{
Base: commonpbutil.NewMsgBase(commonpbutil.WithTargetID(paramtable.GetNodeID())),
CollectionID: sd.collectionID,
NodeID: paramtable.GetNodeID(),
Scope: querypb.DataScope_Streaming,
SegmentIDs: removed,
Shard: sd.vchannelName,
NeedTransfer: false,
})
if err != nil {
log.Warn("failed to call release segments(local)", zap.Error(err))
}
}()
}
return nil return nil
} }
@ -615,3 +594,11 @@ func (sd *shardDelegator) ReleaseSegments(ctx context.Context, req *querypb.Rele
return nil return nil
} }
func (sd *shardDelegator) SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) {
sd.distribution.SyncTargetVersion(newVersion, growingInTarget, sealedInTarget)
}
func (sd *shardDelegator) GetTargetVersion() int64 {
return sd.distribution.getTargetVersion()
}

15
internal/querynodev2/delegator/delegator_data_test.go
View File

@ -345,7 +345,7 @@ func (s *DelegatorDataSuite) TestLoadSegments() {
}) })
s.NoError(err) s.NoError(err)
sealed, _ := s.delegator.GetSegmentInfo() sealed, _ := s.delegator.GetSegmentInfo(false)
s.Require().Equal(1, len(sealed)) s.Require().Equal(1, len(sealed))
s.Equal(int64(1), sealed[0].NodeID) s.Equal(int64(1), sealed[0].NodeID)
s.ElementsMatch([]SegmentEntry{ s.ElementsMatch([]SegmentEntry{
@ -421,7 +421,7 @@ func (s *DelegatorDataSuite) TestLoadSegments() {
}) })
s.NoError(err) s.NoError(err)
sealed, _ := s.delegator.GetSegmentInfo() sealed, _ := s.delegator.GetSegmentInfo(false)
s.Require().Equal(1, len(sealed)) s.Require().Equal(1, len(sealed))
s.Equal(int64(1), sealed[0].NodeID) s.Equal(int64(1), sealed[0].NodeID)
s.ElementsMatch([]SegmentEntry{ s.ElementsMatch([]SegmentEntry{
@ -625,7 +625,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
}) })
s.Require().NoError(err) s.Require().NoError(err)
sealed, growing := s.delegator.GetSegmentInfo() sealed, growing := s.delegator.GetSegmentInfo(false)
s.Require().Equal(1, len(sealed)) s.Require().Equal(1, len(sealed))
s.Equal(int64(1), sealed[0].NodeID) s.Equal(int64(1), sealed[0].NodeID)
s.ElementsMatch([]SegmentEntry{ s.ElementsMatch([]SegmentEntry{
@ -652,7 +652,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
}, false) }, false)
s.NoError(err) s.NoError(err)
sealed, _ = s.delegator.GetSegmentInfo() sealed, _ = s.delegator.GetSegmentInfo(false)
s.Equal(0, len(sealed)) s.Equal(0, len(sealed))
err = s.delegator.ReleaseSegments(ctx, &querypb.ReleaseSegmentsRequest{ err = s.delegator.ReleaseSegments(ctx, &querypb.ReleaseSegmentsRequest{
@ -663,7 +663,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
}, false) }, false)
s.NoError(err) s.NoError(err)
_, growing = s.delegator.GetSegmentInfo() _, growing = s.delegator.GetSegmentInfo(false)
s.Equal(0, len(growing)) s.Equal(0, len(growing))
err = s.delegator.ReleaseSegments(ctx, &querypb.ReleaseSegmentsRequest{ err = s.delegator.ReleaseSegments(ctx, &querypb.ReleaseSegmentsRequest{
@ -687,6 +687,11 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
s.NoError(err) s.NoError(err)
} }
func (s *DelegatorSuite) TestSyncTargetVersion() {
s.delegator.SyncTargetVersion(int64(5), []int64{}, []int64{})
s.Equal(int64(5), s.delegator.GetTargetVersion())
}
func TestDelegatorDataSuite(t *testing.T) { func TestDelegatorDataSuite(t *testing.T) {
suite.Run(t, new(DelegatorDataSuite)) suite.Run(t, new(DelegatorDataSuite))
} }

8
internal/querynodev2/delegator/delegator_test.go
View File

@ -174,7 +174,7 @@ func (s *DelegatorSuite) TestBasicInfo() {
} }
func (s *DelegatorSuite) TestGetSegmentInfo() { func (s *DelegatorSuite) TestGetSegmentInfo() {
sealed, growing := s.delegator.GetSegmentInfo() sealed, growing := s.delegator.GetSegmentInfo(false)
s.Equal(0, len(sealed)) s.Equal(0, len(sealed))
s.Equal(0, len(growing)) s.Equal(0, len(growing))
@ -185,7 +185,7 @@ func (s *DelegatorSuite) TestGetSegmentInfo() {
Version: 2001, Version: 2001,
}) })
sealed, growing = s.delegator.GetSegmentInfo() sealed, growing = s.delegator.GetSegmentInfo(false)
s.EqualValues([]SnapshotItem{ s.EqualValues([]SnapshotItem{
{ {
NodeID: 1, NodeID: 1,
@ -241,6 +241,7 @@ func (s *DelegatorSuite) TestSearch() {
Version: 2001, Version: 2001,
}, },
) )
s.delegator.SyncTargetVersion(2001, []int64{1004}, []int64{1000, 1001, 1002, 1003})
s.Run("normal", func() { s.Run("normal", func() {
defer func() { defer func() {
s.workerManager.ExpectedCalls = nil s.workerManager.ExpectedCalls = nil
@ -493,6 +494,7 @@ func (s *DelegatorSuite) TestQuery() {
Version: 2001, Version: 2001,
}, },
) )
s.delegator.SyncTargetVersion(2001, []int64{1004}, []int64{1000, 1001, 1002, 1003})
s.Run("normal", func() { s.Run("normal", func() {
defer func() { defer func() {
s.workerManager.ExpectedCalls = nil s.workerManager.ExpectedCalls = nil
@ -712,6 +714,8 @@ func (s *DelegatorSuite) TestGetStats() {
Version: 2001, Version: 2001,
}, },
) )
s.delegator.SyncTargetVersion(2001, []int64{1004}, []int64{1000, 1001, 1002, 1003})
s.Run("normal", func() { s.Run("normal", func() {
defer func() { defer func() {
s.workerManager.ExpectedCalls = nil s.workerManager.ExpectedCalls = nil

134
internal/querynodev2/delegator/distribution.go
View File

@ -20,13 +20,17 @@ import (
"sync" "sync"
"go.uber.org/atomic" "go.uber.org/atomic"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil" "github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/samber/lo"
) )
const ( const (
// wildcardNodeID matches any nodeID, used for force distribution correction. // wildcardNodeID matches any nodeID, used for force distribution correction.
wildcardNodeID = int64(-1) wildcardNodeID = int64(-1)
initialTargetVersion = int64(0)
) )
var ( var (
@ -47,11 +51,12 @@ func getClosedCh() chan struct{} {
type distribution struct { type distribution struct {
// segments information // segments information
// map[SegmentID]=>segmentEntry // map[SegmentID]=>segmentEntry
targetVersion *atomic.Int64
growingSegments map[UniqueID]SegmentEntry growingSegments map[UniqueID]SegmentEntry
sealedSegments map[UniqueID]SegmentEntry sealedSegments map[UniqueID]SegmentEntry
// version indicator // snapshotVersion indicator
version int64 snapshotVersion int64
// quick flag for current snapshot is serviceable // quick flag for current snapshot is serviceable
serviceable *atomic.Bool serviceable *atomic.Bool
offlines typeutil.Set[int64] offlines typeutil.Set[int64]
@ -66,10 +71,11 @@ type distribution struct {
// SegmentEntry stores the segment meta information. // SegmentEntry stores the segment meta information.
type SegmentEntry struct { type SegmentEntry struct {
NodeID int64 NodeID int64
SegmentID UniqueID SegmentID UniqueID
PartitionID UniqueID PartitionID UniqueID
Version int64 Version int64
TargetVersion int64
} }
// NewDistribution creates a new distribution instance with all field initialized. // NewDistribution creates a new distribution instance with all field initialized.
@ -81,20 +87,59 @@ func NewDistribution() *distribution {
snapshots: typeutil.NewConcurrentMap[int64, *snapshot](), snapshots: typeutil.NewConcurrentMap[int64, *snapshot](),
current: atomic.NewPointer[snapshot](nil), current: atomic.NewPointer[snapshot](nil),
offlines: typeutil.NewSet[int64](), offlines: typeutil.NewSet[int64](),
targetVersion: atomic.NewInt64(initialTargetVersion),
} }
dist.genSnapshot() dist.genSnapshot()
return dist return dist
} }
// GetCurrent returns current snapshot. // GetAllSegments returns segments in current snapshot, filter readable segment when readable is true
func (d *distribution) GetCurrent(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry, version int64) { func (d *distribution) GetSegments(readable bool, partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry, version int64) {
d.mut.RLock() d.mut.RLock()
defer d.mut.RUnlock() defer d.mut.RUnlock()
current := d.current.Load() current := d.current.Load()
sealed, growing = current.Get(partitions...) sealed, growing = current.Get(partitions...)
version = current.version version = current.version
if readable {
TargetVersion := current.GetTargetVersion()
sealed, growing = d.filterReadableSegments(sealed, growing, TargetVersion)
return
}
return
}
func (d *distribution) filterReadableSegments(sealed []SnapshotItem, growing []SegmentEntry, targetVersion int64) ([]SnapshotItem, []SegmentEntry) {
filterReadable := func(entry SegmentEntry, _ int) bool {
return entry.TargetVersion == targetVersion || entry.TargetVersion == initialTargetVersion
}
growing = lo.Filter(growing, filterReadable)
sealed = lo.Map(sealed, func(item SnapshotItem, _ int) SnapshotItem {
return SnapshotItem{
NodeID: item.NodeID,
Segments: lo.Filter(item.Segments, filterReadable),
}
})
return sealed, growing
}
// PeekAllSegments returns current snapshot without increasing inuse count
// show only used by GetDataDistribution.
func (d *distribution) PeekSegments(readable bool, partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) {
current := d.current.Load()
sealed, growing = current.Peek(partitions...)
if readable {
TargetVersion := current.GetTargetVersion()
sealed, growing = d.filterReadableSegments(sealed, growing, TargetVersion)
return
}
return return
} }
@ -106,12 +151,9 @@ func (d *distribution) FinishUsage(version int64) {
} }
} }
// Peek returns current snapshot without increasing inuse count func (d *distribution) getTargetVersion() int64 {
// show only used by GetDataDistribution.
func (d *distribution) Peek(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) {
current := d.current.Load() current := d.current.Load()
sealed, growing = current.Peek(partitions...) return current.GetTargetVersion()
return sealed, growing
} }
// Serviceable returns wether current snapshot is serviceable. // Serviceable returns wether current snapshot is serviceable.
@ -120,28 +162,16 @@ func (d *distribution) Serviceable() bool {
} }
// AddDistributions add multiple segment entries. // AddDistributions add multiple segment entries.
func (d *distribution) AddDistributions(entries ...SegmentEntry) ([]int64, chan struct{}) { func (d *distribution) AddDistributions(entries ...SegmentEntry) {
d.mut.Lock() d.mut.Lock()
defer d.mut.Unlock() defer d.mut.Unlock()
// remove growing if sealed is loaded
var removed []int64
for _, entry := range entries { for _, entry := range entries {
d.sealedSegments[entry.SegmentID] = entry d.sealedSegments[entry.SegmentID] = entry
d.offlines.Remove(entry.SegmentID) d.offlines.Remove(entry.SegmentID)
_, ok := d.growingSegments[entry.SegmentID]
if ok {
removed = append(removed, entry.SegmentID)
delete(d.growingSegments, entry.SegmentID)
}
} }
ch := d.genSnapshot() d.genSnapshot()
// no offline growing, return closed ch to skip wait
if len(removed) == 0 {
return removed, getClosedCh()
}
return removed, ch
} }
// AddGrowing adds growing segment distribution. // AddGrowing adds growing segment distribution.
@ -176,6 +206,45 @@ func (d *distribution) AddOfflines(segmentIDs ...int64) {
} }
} }
// UpdateTargetVersion update readable segment version
func (d *distribution) SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) {
d.mut.Lock()
defer d.mut.Unlock()
for _, segmentID := range growingInTarget {
entry, ok := d.growingSegments[segmentID]
if !ok {
log.Error("readable growing segment lost, make it unserviceable",
zap.Int64("segmentID", segmentID))
d.serviceable.Store(false)
continue
}
entry.TargetVersion = newVersion
d.growingSegments[segmentID] = entry
}
for _, segmentID := range sealedInTarget {
entry, ok := d.sealedSegments[segmentID]
if !ok {
log.Error("readable sealed segment lost, make it unserviceable",
zap.Int64("segmentID", segmentID))
d.serviceable.Store(false)
continue
}
entry.TargetVersion = newVersion
d.sealedSegments[segmentID] = entry
}
oldValue := d.targetVersion.Load()
d.targetVersion.Store(newVersion)
d.genSnapshot()
log.Info("Update readable segment version",
zap.Int64("oldVersion", oldValue),
zap.Int64("newVersion", newVersion),
zap.Int64s("growing", growingInTarget),
zap.Int64s("sealed", sealedInTarget),
)
}
// RemoveDistributions remove segments distributions and returns the clear signal channel. // RemoveDistributions remove segments distributions and returns the clear signal channel.
func (d *distribution) RemoveDistributions(sealedSegments []SegmentEntry, growingSegments []SegmentEntry) chan struct{} { func (d *distribution) RemoveDistributions(sealedSegments []SegmentEntry, growingSegments []SegmentEntry) chan struct{} {
d.mut.Lock() d.mut.Lock()
@ -219,7 +288,6 @@ func (d *distribution) RemoveDistributions(sealedSegments []SegmentEntry, growin
// in which, user could use found nodeID=>segmentID list. // in which, user could use found nodeID=>segmentID list.
// mutex RLock is required before calling this method. // mutex RLock is required before calling this method.
func (d *distribution) genSnapshot() chan struct{} { func (d *distribution) genSnapshot() chan struct{} {
nodeSegments := make(map[int64][]SegmentEntry) nodeSegments := make(map[int64][]SegmentEntry)
for _, entry := range d.sealedSegments { for _, entry := range d.sealedSegments {
nodeSegments[entry.NodeID] = append(nodeSegments[entry.NodeID], entry) nodeSegments[entry.NodeID] = append(nodeSegments[entry.NodeID], entry)
@ -243,12 +311,12 @@ func (d *distribution) genSnapshot() chan struct{} {
// stores last snapshot // stores last snapshot
// ok to be nil // ok to be nil
last := d.current.Load() last := d.current.Load()
// increase version // update snapshot version
d.version++ d.snapshotVersion++
newSnapShot := NewSnapshot(dist, growing, last, d.version) newSnapShot := NewSnapshot(dist, growing, last, d.snapshotVersion, d.targetVersion.Load())
d.current.Store(newSnapShot) d.current.Store(newSnapShot)
// shall be a new one // shall be a new one
d.snapshots.GetOrInsert(d.version, newSnapShot) d.snapshots.GetOrInsert(d.snapshotVersion, newSnapShot)
// first snapshot, return closed chan // first snapshot, return closed chan
if last == nil { if last == nil {

72
internal/querynodev2/delegator/distribution_test.go
View File

@ -30,6 +30,7 @@ type DistributionSuite struct {
func (s *DistributionSuite) SetupTest() { func (s *DistributionSuite) SetupTest() {
s.dist = NewDistribution() s.dist = NewDistribution()
s.Equal(initialTargetVersion, s.dist.getTargetVersion())
} }
func (s *DistributionSuite) TearDownTest() { func (s *DistributionSuite) TearDownTest() {
@ -145,11 +146,10 @@ func (s *DistributionSuite) TestAddDistribution() {
s.SetupTest() s.SetupTest()
defer s.TearDownTest() defer s.TearDownTest()
s.dist.AddGrowing(tc.growing...) s.dist.AddGrowing(tc.growing...)
_, _, version := s.dist.GetCurrent() _, _, version := s.dist.GetSegments(false)
_, signal := s.dist.AddDistributions(tc.input...) s.dist.AddDistributions(tc.input...)
sealed, _ := s.dist.Peek() sealed, _ := s.dist.PeekSegments(false)
s.compareSnapshotItems(tc.expected, sealed) s.compareSnapshotItems(tc.expected, sealed)
s.Equal(tc.expectedSignalClosed, s.isClosedCh(signal))
s.dist.FinishUsage(version) s.dist.FinishUsage(version)
}) })
} }
@ -215,7 +215,7 @@ func (s *DistributionSuite) TestAddGrowing() {
defer s.TearDownTest() defer s.TearDownTest()
s.dist.AddGrowing(tc.input...) s.dist.AddGrowing(tc.input...)
_, growing, version := s.dist.GetCurrent() _, growing, version := s.dist.GetSegments(false)
defer s.dist.FinishUsage(version) defer s.dist.FinishUsage(version)
s.ElementsMatch(tc.expected, growing) s.ElementsMatch(tc.expected, growing)
@ -394,7 +394,7 @@ func (s *DistributionSuite) TestRemoveDistribution() {
var version int64 var version int64
if tc.withMockRead { if tc.withMockRead {
_, _, version = s.dist.GetCurrent() _, _, version = s.dist.GetSegments(false)
} }
ch := s.dist.RemoveDistributions(tc.removalSealed, tc.removalGrowing) ch := s.dist.RemoveDistributions(tc.removalSealed, tc.removalGrowing)
@ -418,7 +418,7 @@ func (s *DistributionSuite) TestRemoveDistribution() {
case <-ch: case <-ch:
} }
sealed, growing, version := s.dist.GetCurrent() sealed, growing, version := s.dist.GetSegments(false)
defer s.dist.FinishUsage(version) defer s.dist.FinishUsage(version)
s.compareSnapshotItems(tc.expectSealed, sealed) s.compareSnapshotItems(tc.expectSealed, sealed)
s.ElementsMatch(tc.expectGrowing, growing) s.ElementsMatch(tc.expectGrowing, growing)
@ -514,7 +514,7 @@ func (s *DistributionSuite) TestPeek() {
// peek during lock // peek during lock
s.dist.AddDistributions(tc.input...) s.dist.AddDistributions(tc.input...)
s.dist.mut.Lock() s.dist.mut.Lock()
sealed, _ := s.dist.Peek() sealed, _ := s.dist.PeekSegments(false)
s.compareSnapshotItems(tc.expected, sealed) s.compareSnapshotItems(tc.expected, sealed)
s.dist.mut.Unlock() s.dist.mut.Unlock()
}) })
@ -577,6 +577,62 @@ func (s *DistributionSuite) TestAddOfflines() {
} }
} }
func (s *DistributionSuite) Test_SyncTargetVersion() {
growing := []SegmentEntry{
{
NodeID: 1,
SegmentID: 1,
PartitionID: 1,
TargetVersion: 1,
},
{
NodeID: 1,
SegmentID: 2,
PartitionID: 1,
TargetVersion: 1,
},
{
NodeID: 1,
SegmentID: 3,
PartitionID: 1,
TargetVersion: 1,
},
}
sealed := []SegmentEntry{
{
NodeID: 1,
SegmentID: 4,
PartitionID: 1,
TargetVersion: 1,
},
{
NodeID: 1,
SegmentID: 5,
PartitionID: 1,
TargetVersion: 1,
},
{
NodeID: 1,
SegmentID: 6,
PartitionID: 1,
TargetVersion: 1,
},
}
s.dist.AddGrowing(growing...)
s.dist.AddDistributions(sealed...)
s.dist.SyncTargetVersion(2, []int64{2, 3}, []int64{6})
s1, s2, _ := s.dist.GetSegments(true)
s.Len(s1[0].Segments, 1)
s.Len(s2, 2)
s1, s2, _ = s.dist.GetSegments(false)
s.Len(s1[0].Segments, 3)
s.Len(s2, 3)
}
func TestDistributionSuite(t *testing.T) { func TestDistributionSuite(t *testing.T) {
suite.Run(t, new(DistributionSuite)) suite.Run(t, new(DistributionSuite))
} }

196
internal/querynodev2/delegator/mock_delegator.go
View File

@ -1,4 +1,4 @@
// Code generated by mockery v2.16.0. DO NOT EDIT. // Code generated by mockery v2.21.1. DO NOT EDIT.
package delegator package delegator
@ -51,6 +51,11 @@ func (_c *MockShardDelegator_Close_Call) Return() *MockShardDelegator_Close_Call
return _c return _c
} }
func (_c *MockShardDelegator_Close_Call) RunAndReturn(run func()) *MockShardDelegator_Close_Call {
_c.Call.Return(run)
return _c
}
// Collection provides a mock function with given fields: // Collection provides a mock function with given fields:
func (_m *MockShardDelegator) Collection() int64 { func (_m *MockShardDelegator) Collection() int64 {
ret := _m.Called() ret := _m.Called()
@ -87,22 +92,30 @@ func (_c *MockShardDelegator_Collection_Call) Return(_a0 int64) *MockShardDelega
return _c return _c
} }
// GetSegmentInfo provides a mock function with given fields: func (_c *MockShardDelegator_Collection_Call) RunAndReturn(run func() int64) *MockShardDelegator_Collection_Call {
func (_m *MockShardDelegator) GetSegmentInfo() ([]SnapshotItem, []SegmentEntry) { _c.Call.Return(run)
ret := _m.Called() return _c
}
// GetSegmentInfo provides a mock function with given fields: readable
func (_m *MockShardDelegator) GetSegmentInfo(readable bool) ([]SnapshotItem, []SegmentEntry) {
ret := _m.Called(readable)
var r0 []SnapshotItem var r0 []SnapshotItem
if rf, ok := ret.Get(0).(func() []SnapshotItem); ok { var r1 []SegmentEntry
r0 = rf() if rf, ok := ret.Get(0).(func(bool) ([]SnapshotItem, []SegmentEntry)); ok {
return rf(readable)
}
if rf, ok := ret.Get(0).(func(bool) []SnapshotItem); ok {
r0 = rf(readable)
} else { } else {
if ret.Get(0) != nil { if ret.Get(0) != nil {
r0 = ret.Get(0).([]SnapshotItem) r0 = ret.Get(0).([]SnapshotItem)
} }
} }
var r1 []SegmentEntry if rf, ok := ret.Get(1).(func(bool) []SegmentEntry); ok {
if rf, ok := ret.Get(1).(func() []SegmentEntry); ok { r1 = rf(readable)
r1 = rf()
} else { } else {
if ret.Get(1) != nil { if ret.Get(1) != nil {
r1 = ret.Get(1).([]SegmentEntry) r1 = ret.Get(1).([]SegmentEntry)
@ -118,13 +131,14 @@ type MockShardDelegator_GetSegmentInfo_Call struct {
} }
// GetSegmentInfo is a helper method to define mock.On call // GetSegmentInfo is a helper method to define mock.On call
func (_e *MockShardDelegator_Expecter) GetSegmentInfo() *MockShardDelegator_GetSegmentInfo_Call { // - readable bool
return &MockShardDelegator_GetSegmentInfo_Call{Call: _e.mock.On("GetSegmentInfo")} func (_e *MockShardDelegator_Expecter) GetSegmentInfo(readable interface{}) *MockShardDelegator_GetSegmentInfo_Call {
return &MockShardDelegator_GetSegmentInfo_Call{Call: _e.mock.On("GetSegmentInfo", readable)}
} }
func (_c *MockShardDelegator_GetSegmentInfo_Call) Run(run func()) *MockShardDelegator_GetSegmentInfo_Call { func (_c *MockShardDelegator_GetSegmentInfo_Call) Run(run func(readable bool)) *MockShardDelegator_GetSegmentInfo_Call {
_c.Call.Run(func(args mock.Arguments) { _c.Call.Run(func(args mock.Arguments) {
run() run(args[0].(bool))
}) })
return _c return _c
} }
@ -134,11 +148,20 @@ func (_c *MockShardDelegator_GetSegmentInfo_Call) Return(sealed []SnapshotItem,
return _c return _c
} }
func (_c *MockShardDelegator_GetSegmentInfo_Call) RunAndReturn(run func(bool) ([]SnapshotItem, []SegmentEntry)) *MockShardDelegator_GetSegmentInfo_Call {
_c.Call.Return(run)
return _c
}
// GetStatistics provides a mock function with given fields: ctx, req // GetStatistics provides a mock function with given fields: ctx, req
func (_m *MockShardDelegator) GetStatistics(ctx context.Context, req *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error) { func (_m *MockShardDelegator) GetStatistics(ctx context.Context, req *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error) {
ret := _m.Called(ctx, req) ret := _m.Called(ctx, req)
var r0 []*internalpb.GetStatisticsResponse var r0 []*internalpb.GetStatisticsResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error)); ok {
return rf(ctx, req)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetStatisticsRequest) []*internalpb.GetStatisticsResponse); ok { if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetStatisticsRequest) []*internalpb.GetStatisticsResponse); ok {
r0 = rf(ctx, req) r0 = rf(ctx, req)
} else { } else {
@ -147,7 +170,6 @@ func (_m *MockShardDelegator) GetStatistics(ctx context.Context, req *querypb.Ge
} }
} }
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *querypb.GetStatisticsRequest) error); ok { if rf, ok := ret.Get(1).(func(context.Context, *querypb.GetStatisticsRequest) error); ok {
r1 = rf(ctx, req) r1 = rf(ctx, req)
} else { } else {
@ -181,6 +203,52 @@ func (_c *MockShardDelegator_GetStatistics_Call) Return(_a0 []*internalpb.GetSta
return _c return _c
} }
func (_c *MockShardDelegator_GetStatistics_Call) RunAndReturn(run func(context.Context, *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error)) *MockShardDelegator_GetStatistics_Call {
_c.Call.Return(run)
return _c
}
// GetTargetVersion provides a mock function with given fields:
func (_m *MockShardDelegator) GetTargetVersion() int64 {
ret := _m.Called()
var r0 int64
if rf, ok := ret.Get(0).(func() int64); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(int64)
}
return r0
}
// MockShardDelegator_GetTargetVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTargetVersion'
type MockShardDelegator_GetTargetVersion_Call struct {
*mock.Call
}
// GetTargetVersion is a helper method to define mock.On call
func (_e *MockShardDelegator_Expecter) GetTargetVersion() *MockShardDelegator_GetTargetVersion_Call {
return &MockShardDelegator_GetTargetVersion_Call{Call: _e.mock.On("GetTargetVersion")}
}
func (_c *MockShardDelegator_GetTargetVersion_Call) Run(run func()) *MockShardDelegator_GetTargetVersion_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockShardDelegator_GetTargetVersion_Call) Return(_a0 int64) *MockShardDelegator_GetTargetVersion_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockShardDelegator_GetTargetVersion_Call) RunAndReturn(run func() int64) *MockShardDelegator_GetTargetVersion_Call {
_c.Call.Return(run)
return _c
}
// LoadGrowing provides a mock function with given fields: ctx, infos, version // LoadGrowing provides a mock function with given fields: ctx, infos, version
func (_m *MockShardDelegator) LoadGrowing(ctx context.Context, infos []*querypb.SegmentLoadInfo, version int64) error { func (_m *MockShardDelegator) LoadGrowing(ctx context.Context, infos []*querypb.SegmentLoadInfo, version int64) error {
ret := _m.Called(ctx, infos, version) ret := _m.Called(ctx, infos, version)
@ -220,6 +288,11 @@ func (_c *MockShardDelegator_LoadGrowing_Call) Return(_a0 error) *MockShardDeleg
return _c return _c
} }
func (_c *MockShardDelegator_LoadGrowing_Call) RunAndReturn(run func(context.Context, []*querypb.SegmentLoadInfo, int64) error) *MockShardDelegator_LoadGrowing_Call {
_c.Call.Return(run)
return _c
}
// LoadSegments provides a mock function with given fields: ctx, req // LoadSegments provides a mock function with given fields: ctx, req
func (_m *MockShardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error { func (_m *MockShardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error {
ret := _m.Called(ctx, req) ret := _m.Called(ctx, req)
@ -258,6 +331,11 @@ func (_c *MockShardDelegator_LoadSegments_Call) Return(_a0 error) *MockShardDele
return _c return _c
} }
func (_c *MockShardDelegator_LoadSegments_Call) RunAndReturn(run func(context.Context, *querypb.LoadSegmentsRequest) error) *MockShardDelegator_LoadSegments_Call {
_c.Call.Return(run)
return _c
}
// ProcessDelete provides a mock function with given fields: deleteData, ts // ProcessDelete provides a mock function with given fields: deleteData, ts
func (_m *MockShardDelegator) ProcessDelete(deleteData []*DeleteData, ts uint64) { func (_m *MockShardDelegator) ProcessDelete(deleteData []*DeleteData, ts uint64) {
_m.Called(deleteData, ts) _m.Called(deleteData, ts)
@ -287,6 +365,11 @@ func (_c *MockShardDelegator_ProcessDelete_Call) Return() *MockShardDelegator_Pr
return _c return _c
} }
func (_c *MockShardDelegator_ProcessDelete_Call) RunAndReturn(run func([]*DeleteData, uint64)) *MockShardDelegator_ProcessDelete_Call {
_c.Call.Return(run)
return _c
}
// ProcessInsert provides a mock function with given fields: insertRecords // ProcessInsert provides a mock function with given fields: insertRecords
func (_m *MockShardDelegator) ProcessInsert(insertRecords map[int64]*InsertData) { func (_m *MockShardDelegator) ProcessInsert(insertRecords map[int64]*InsertData) {
_m.Called(insertRecords) _m.Called(insertRecords)
@ -315,11 +398,20 @@ func (_c *MockShardDelegator_ProcessInsert_Call) Return() *MockShardDelegator_Pr
return _c return _c
} }
func (_c *MockShardDelegator_ProcessInsert_Call) RunAndReturn(run func(map[int64]*InsertData)) *MockShardDelegator_ProcessInsert_Call {
_c.Call.Return(run)
return _c
}
// Query provides a mock function with given fields: ctx, req // Query provides a mock function with given fields: ctx, req
func (_m *MockShardDelegator) Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error) { func (_m *MockShardDelegator) Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error) {
ret := _m.Called(ctx, req) ret := _m.Called(ctx, req)
var r0 []*internalpb.RetrieveResults var r0 []*internalpb.RetrieveResults
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error)); ok {
return rf(ctx, req)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.QueryRequest) []*internalpb.RetrieveResults); ok { if rf, ok := ret.Get(0).(func(context.Context, *querypb.QueryRequest) []*internalpb.RetrieveResults); ok {
r0 = rf(ctx, req) r0 = rf(ctx, req)
} else { } else {
@ -328,7 +420,6 @@ func (_m *MockShardDelegator) Query(ctx context.Context, req *querypb.QueryReque
} }
} }
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *querypb.QueryRequest) error); ok { if rf, ok := ret.Get(1).(func(context.Context, *querypb.QueryRequest) error); ok {
r1 = rf(ctx, req) r1 = rf(ctx, req)
} else { } else {
@ -362,6 +453,11 @@ func (_c *MockShardDelegator_Query_Call) Return(_a0 []*internalpb.RetrieveResult
return _c return _c
} }
func (_c *MockShardDelegator_Query_Call) RunAndReturn(run func(context.Context, *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error)) *MockShardDelegator_Query_Call {
_c.Call.Return(run)
return _c
}
// ReleaseSegments provides a mock function with given fields: ctx, req, force // ReleaseSegments provides a mock function with given fields: ctx, req, force
func (_m *MockShardDelegator) ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest, force bool) error { func (_m *MockShardDelegator) ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest, force bool) error {
ret := _m.Called(ctx, req, force) ret := _m.Called(ctx, req, force)
@ -401,11 +497,20 @@ func (_c *MockShardDelegator_ReleaseSegments_Call) Return(_a0 error) *MockShardD
return _c return _c
} }
func (_c *MockShardDelegator_ReleaseSegments_Call) RunAndReturn(run func(context.Context, *querypb.ReleaseSegmentsRequest, bool) error) *MockShardDelegator_ReleaseSegments_Call {
_c.Call.Return(run)
return _c
}
// Search provides a mock function with given fields: ctx, req // Search provides a mock function with given fields: ctx, req
func (_m *MockShardDelegator) Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error) { func (_m *MockShardDelegator) Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error) {
ret := _m.Called(ctx, req) ret := _m.Called(ctx, req)
var r0 []*internalpb.SearchResults var r0 []*internalpb.SearchResults
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.SearchRequest) ([]*internalpb.SearchResults, error)); ok {
return rf(ctx, req)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.SearchRequest) []*internalpb.SearchResults); ok { if rf, ok := ret.Get(0).(func(context.Context, *querypb.SearchRequest) []*internalpb.SearchResults); ok {
r0 = rf(ctx, req) r0 = rf(ctx, req)
} else { } else {
@ -414,7 +519,6 @@ func (_m *MockShardDelegator) Search(ctx context.Context, req *querypb.SearchReq
} }
} }
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *querypb.SearchRequest) error); ok { if rf, ok := ret.Get(1).(func(context.Context, *querypb.SearchRequest) error); ok {
r1 = rf(ctx, req) r1 = rf(ctx, req)
} else { } else {
@ -448,6 +552,11 @@ func (_c *MockShardDelegator_Search_Call) Return(_a0 []*internalpb.SearchResults
return _c return _c
} }
func (_c *MockShardDelegator_Search_Call) RunAndReturn(run func(context.Context, *querypb.SearchRequest) ([]*internalpb.SearchResults, error)) *MockShardDelegator_Search_Call {
_c.Call.Return(run)
return _c
}
// Serviceable provides a mock function with given fields: // Serviceable provides a mock function with given fields:
func (_m *MockShardDelegator) Serviceable() bool { func (_m *MockShardDelegator) Serviceable() bool {
ret := _m.Called() ret := _m.Called()
@ -484,6 +593,11 @@ func (_c *MockShardDelegator_Serviceable_Call) Return(_a0 bool) *MockShardDelega
return _c return _c
} }
func (_c *MockShardDelegator_Serviceable_Call) RunAndReturn(run func() bool) *MockShardDelegator_Serviceable_Call {
_c.Call.Return(run)
return _c
}
// Start provides a mock function with given fields: // Start provides a mock function with given fields:
func (_m *MockShardDelegator) Start() { func (_m *MockShardDelegator) Start() {
_m.Called() _m.Called()
@ -511,6 +625,11 @@ func (_c *MockShardDelegator_Start_Call) Return() *MockShardDelegator_Start_Call
return _c return _c
} }
func (_c *MockShardDelegator_Start_Call) RunAndReturn(run func()) *MockShardDelegator_Start_Call {
_c.Call.Return(run)
return _c
}
// SyncDistribution provides a mock function with given fields: ctx, entries // SyncDistribution provides a mock function with given fields: ctx, entries
func (_m *MockShardDelegator) SyncDistribution(ctx context.Context, entries ...SegmentEntry) { func (_m *MockShardDelegator) SyncDistribution(ctx context.Context, entries ...SegmentEntry) {
_va := make([]interface{}, len(entries)) _va := make([]interface{}, len(entries))
@ -554,6 +673,46 @@ func (_c *MockShardDelegator_SyncDistribution_Call) Return() *MockShardDelegator
return _c return _c
} }
func (_c *MockShardDelegator_SyncDistribution_Call) RunAndReturn(run func(context.Context, ...SegmentEntry)) *MockShardDelegator_SyncDistribution_Call {
_c.Call.Return(run)
return _c
}
// SyncTargetVersion provides a mock function with given fields: newVersion, growingInTarget, sealedInTarget
func (_m *MockShardDelegator) SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) {
_m.Called(newVersion, growingInTarget, sealedInTarget)
}
// MockShardDelegator_SyncTargetVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SyncTargetVersion'
type MockShardDelegator_SyncTargetVersion_Call struct {
*mock.Call
}
// SyncTargetVersion is a helper method to define mock.On call
// - newVersion int64
// - growingInTarget []int64
// - sealedInTarget []int64
func (_e *MockShardDelegator_Expecter) SyncTargetVersion(newVersion interface{}, growingInTarget interface{}, sealedInTarget interface{}) *MockShardDelegator_SyncTargetVersion_Call {
return &MockShardDelegator_SyncTargetVersion_Call{Call: _e.mock.On("SyncTargetVersion", newVersion, growingInTarget, sealedInTarget)}
}
func (_c *MockShardDelegator_SyncTargetVersion_Call) Run(run func(newVersion int64, growingInTarget []int64, sealedInTarget []int64)) *MockShardDelegator_SyncTargetVersion_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(int64), args[1].([]int64), args[2].([]int64))
})
return _c
}
func (_c *MockShardDelegator_SyncTargetVersion_Call) Return() *MockShardDelegator_SyncTargetVersion_Call {
_c.Call.Return()
return _c
}
func (_c *MockShardDelegator_SyncTargetVersion_Call) RunAndReturn(run func(int64, []int64, []int64)) *MockShardDelegator_SyncTargetVersion_Call {
_c.Call.Return(run)
return _c
}
// Version provides a mock function with given fields: // Version provides a mock function with given fields:
func (_m *MockShardDelegator) Version() int64 { func (_m *MockShardDelegator) Version() int64 {
ret := _m.Called() ret := _m.Called()
@ -590,6 +749,11 @@ func (_c *MockShardDelegator_Version_Call) Return(_a0 int64) *MockShardDelegator
return _c return _c
} }
func (_c *MockShardDelegator_Version_Call) RunAndReturn(run func() int64) *MockShardDelegator_Version_Call {
_c.Call.Return(run)
return _c
}
type mockConstructorTestingTNewMockShardDelegator interface { type mockConstructorTestingTNewMockShardDelegator interface {
mock.TestingT mock.TestingT
Cleanup(func()) Cleanup(func())

22
internal/querynodev2/delegator/snapshot.go
View File

@ -36,8 +36,9 @@ type snapshotCleanup func()
// snapshot records segment distribution with ref count. // snapshot records segment distribution with ref count.
type snapshot struct { type snapshot struct {
dist []SnapshotItem dist []SnapshotItem
growing []SegmentEntry growing []SegmentEntry
targetVersion int64
// version ID for tracking // version ID for tracking
version int64 version int64
@ -57,13 +58,14 @@ type snapshot struct {
} }
// NewSnapshot returns a prepared snapshot with channel initialized. // NewSnapshot returns a prepared snapshot with channel initialized.
func NewSnapshot(sealed []SnapshotItem, growing []SegmentEntry, last *snapshot, version int64) *snapshot { func NewSnapshot(sealed []SnapshotItem, growing []SegmentEntry, last *snapshot, version int64, targetVersion int64) *snapshot {
return &snapshot{ return &snapshot{
version: version, version: version,
growing: growing, growing: growing,
dist: sealed, dist: sealed,
last: last, last: last,
cleared: make(chan struct{}), cleared: make(chan struct{}),
targetVersion: targetVersion,
} }
} }
@ -80,6 +82,10 @@ func (s *snapshot) Get(partitions ...int64) (sealed []SnapshotItem, growing []Se
return s.filter(partitions...) return s.filter(partitions...)
} }
func (s *snapshot) GetTargetVersion() int64 {
return s.targetVersion
}
// Peek returns segment distributions without increasing inUse. // Peek returns segment distributions without increasing inUse.
func (s *snapshot) Peek(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) { func (s *snapshot) Peek(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) {
return s.filter(partitions...) return s.filter(partitions...)

6
internal/querynodev2/delegator/snapshot_test.go
View File

@ -30,8 +30,9 @@ type SnapshotSuite struct {
} }
func (s *SnapshotSuite) SetupTest() { func (s *SnapshotSuite) SetupTest() {
last := NewSnapshot(nil, nil, nil, 0) last := NewSnapshot(nil, nil, nil, 0, initialTargetVersion)
last.Expire(func() {}) last.Expire(func() {})
s.Equal(initialTargetVersion, last.GetTargetVersion())
dist := []SnapshotItem{ dist := []SnapshotItem{
{ {
@ -72,7 +73,8 @@ func (s *SnapshotSuite) SetupTest() {
}, },
} }
s.snapshot = NewSnapshot(dist, growing, last, 1) s.snapshot = NewSnapshot(dist, growing, last, 1, 1)
s.Equal(int64(1), s.snapshot.GetTargetVersion())
} }
func (s *SnapshotSuite) TearDownTest() { func (s *SnapshotSuite) TearDownTest() {

2
internal/querynodev2/handlers.go
View File

@ -271,7 +271,7 @@ func (node *QueryNode) optimizeSearchParams(ctx context.Context, req *querypb.Se
switch plan.GetNode().(type) { switch plan.GetNode().(type) {
case *planpb.PlanNode_VectorAnns: case *planpb.PlanNode_VectorAnns:
// ignore growing ones for now since they will always be brute force // ignore growing ones for now since they will always be brute force
sealed, _ := deleg.GetSegmentInfo() sealed, _ := deleg.GetSegmentInfo(true)
sealedNum := lo.Reduce(sealed, func(sum int, item delegator.SnapshotItem, _ int) int { sealedNum := lo.Reduce(sealed, func(sum int, item delegator.SnapshotItem, _ int) int {
return sum + len(item.Segments) return sum + len(item.Segments)
}, 0) }, 0)

2
internal/querynodev2/handlers_test.go
View File

@ -164,7 +164,7 @@ func (suite *OptimizeSearchParamSuite) SetupSuite() {
suite.channel = "test-channel" suite.channel = "test-channel"
suite.delegator = &delegator.MockShardDelegator{} suite.delegator = &delegator.MockShardDelegator{}
suite.delegator.EXPECT().GetSegmentInfo().Return([]delegator.SnapshotItem{{NodeID: 1, Segments: []delegator.SegmentEntry{{SegmentID: 100}}}}, []delegator.SegmentEntry{}) suite.delegator.EXPECT().GetSegmentInfo(mock.Anything).Return([]delegator.SnapshotItem{{NodeID: 1, Segments: []delegator.SegmentEntry{{SegmentID: 100}}}}, []delegator.SegmentEntry{})
} }
func (suite *OptimizeSearchParamSuite) SetupTest() { func (suite *OptimizeSearchParamSuite) SetupTest() {

6
internal/querynodev2/services.go
View File

@ -1184,7 +1184,7 @@ func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.Get
Version: value.Version(), Version: value.Version(),
}) })
sealed, growing := value.GetSegmentInfo() sealed, growing := value.GetSegmentInfo(false)
sealedSegments := make(map[int64]*querypb.SegmentDist) sealedSegments := make(map[int64]*querypb.SegmentDist)
for _, item := range sealed { for _, item := range sealed {
for _, segment := range item.Segments { for _, segment := range item.Segments {
@ -1210,6 +1210,7 @@ func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.Get
Channel: key, Channel: key,
SegmentDist: sealedSegments, SegmentDist: sealedSegments,
GrowingSegments: growingSegments, GrowingSegments: growingSegments,
TargetVersion: value.GetTargetVersion(),
}) })
return true return true
}) })
@ -1260,6 +1261,7 @@ func (node *QueryNode) SyncDistribution(ctx context.Context, req *querypb.SyncDi
log := log.With(zap.String("Action", log := log.With(zap.String("Action",
action.GetType().String()), action.GetType().String()),
zap.Int64("segmentID", action.SegmentID), zap.Int64("segmentID", action.SegmentID),
zap.Int64("TargetVersion", action.GetTargetVersion()),
) )
log.Info("sync action") log.Info("sync action")
switch action.GetType() { switch action.GetType() {
@ -1267,6 +1269,8 @@ func (node *QueryNode) SyncDistribution(ctx context.Context, req *querypb.SyncDi
removeActions = append(removeActions, action) removeActions = append(removeActions, action)
case querypb.SyncType_Set: case querypb.SyncType_Set:
addSegments[action.GetNodeID()] = append(addSegments[action.GetNodeID()], action.GetInfo()) addSegments[action.GetNodeID()] = append(addSegments[action.GetNodeID()], action.GetInfo())
case querypb.SyncType_UpdateVersion:
shardDelegator.SyncTargetVersion(action.GetTargetVersion(), action.GetGrowingInTarget(), action.GetSealedInTarget())
default: default:
return &commonpb.Status{ return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError, ErrorCode: commonpb.ErrorCode_UnexpectedError,

6
internal/querynodev2/services_test.go
View File

@ -1368,7 +1368,7 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() {
delegator, ok := suite.node.delegators.Get(suite.vchannel) delegator, ok := suite.node.delegators.Get(suite.vchannel)
suite.True(ok) suite.True(ok)
sealedSegments, _ := delegator.GetSegmentInfo() sealedSegments, _ := delegator.GetSegmentInfo(false)
suite.Len(sealedSegments[0].Segments, 3) suite.Len(sealedSegments[0].Segments, 3)
// data // data
@ -1392,7 +1392,7 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() {
status, err := suite.node.SyncDistribution(ctx, req) status, err := suite.node.SyncDistribution(ctx, req)
suite.NoError(err) suite.NoError(err)
suite.Equal(commonpb.ErrorCode_Success, status.ErrorCode) suite.Equal(commonpb.ErrorCode_Success, status.ErrorCode)
sealedSegments, _ = delegator.GetSegmentInfo() sealedSegments, _ = delegator.GetSegmentInfo(false)
suite.Len(sealedSegments[0].Segments, 3) suite.Len(sealedSegments[0].Segments, 3)
releaseAction = &querypb.SyncAction{ releaseAction = &querypb.SyncAction{
@ -1406,7 +1406,7 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() {
status, err = suite.node.SyncDistribution(ctx, req) status, err = suite.node.SyncDistribution(ctx, req)
suite.NoError(err) suite.NoError(err)
suite.Equal(commonpb.ErrorCode_Success, status.ErrorCode) suite.Equal(commonpb.ErrorCode_Success, status.ErrorCode)
sealedSegments, _ = delegator.GetSegmentInfo() sealedSegments, _ = delegator.GetSegmentInfo(false)
suite.Len(sealedSegments[0].Segments, 2) suite.Len(sealedSegments[0].Segments, 2)
} }