test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Make FieldBinlog in a segment unique for a field (#14632)

Browse Source 
See also: #14620

Signed-off-by: yangxuan <xuan.yang@zilliz.com>
This commit is contained in:
XuanYang-cn 2021-12-30 20:29:22 +08:00 committed by GitHub
parent 148dad23ac
commit f0a40f1569
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 56 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
internal/datanode/binlog_io.go
View File

@ -114,7 +114,11 @@ func (b *binlogIO) upload(
p := &cpaths{}
kvs := make(map[string]string)
var (
inPathm = make(map[UniqueID]*datapb.FieldBinlog) // FieldID > its FieldBinlog
statsPathm = make(map[UniqueID]*datapb.FieldBinlog) // FieldID > its statsBinlog
kvs = make(map[string]string)
)
for _, iData := range iDatas {
tf, ok := iData.Data[common.TimeStampField]
@ -136,9 +140,33 @@ func (b *binlogIO) upload(
kvs[k] = v
}
p.inPaths = append(p.inPaths, inpaths...)
p.statsPaths = append(p.statsPaths, statspaths...)
for fID, fieldBinlog := range inpaths {
tmpfb, ok := inPathm[fID]
if !ok {
tmpfb = fieldBinlog
} else {
tmpfb.Binlogs = append(tmpfb.Binlogs, fieldBinlog.GetBinlogs()...)
}
inPathm[fID] = tmpfb
}
for fID, fieldBinlog := range statspaths {
tmpfb, ok := statsPathm[fID]
if !ok {
tmpfb = fieldBinlog
} else {
tmpfb.Binlogs = append(tmpfb.Binlogs, fieldBinlog.GetBinlogs()...)
}
statsPathm[fID] = tmpfb
}
}
for _, bs := range inPathm {
p.inPaths = append(p.inPaths, bs)
}
for _, bs := range statsPathm {
p.statsPaths = append(p.statsPaths, bs)
}
// If there are delta logs
@ -208,16 +236,18 @@ func (b *binlogIO) genDeltaBlobs(data *DeleteData, collID, partID, segID UniqueI
}
// genInsertBlobs returns kvs, insert-paths, stats-paths
func (b *binlogIO) genInsertBlobs(data *InsertData, partID, segID UniqueID, meta *etcdpb.CollectionMeta) (map[string]string, []*datapb.FieldBinlog, []*datapb.FieldBinlog, error) {
func (b *binlogIO) genInsertBlobs(data *InsertData, partID, segID UniqueID, meta *etcdpb.CollectionMeta) (map[string]string, map[UniqueID]*datapb.FieldBinlog, map[UniqueID]*datapb.FieldBinlog, error) {
inCodec := storage.NewInsertCodec(meta)
inlogs, statslogs, err := inCodec.Serialize(partID, segID, data)
if err != nil {
return nil, nil, nil, err
}
kvs := make(map[string]string, len(inlogs)+len(statslogs))
inpaths := make([]*datapb.FieldBinlog, 0, len(inlogs))
statspaths := make([]*datapb.FieldBinlog, 0, len(statslogs))
var (
kvs = make(map[string]string, len(inlogs)+len(statslogs))
inpaths = make(map[UniqueID]*datapb.FieldBinlog)
statspaths = make(map[UniqueID]*datapb.FieldBinlog)
)
notifyGenIdx := make(chan struct{})
defer close(notifyGenIdx)
@ -237,15 +267,10 @@ func (b *binlogIO) genInsertBlobs(data *InsertData, partID, segID UniqueID, meta
fileLen := len(value)
kvs[key] = value
inpaths = append(inpaths, &datapb.FieldBinlog{
inpaths[fID] = &datapb.FieldBinlog{
FieldID: fID,
Binlogs: []*datapb.Binlog{
{
LogSize: int64(fileLen),
LogPath: key,
},
},
})
Binlogs: []*datapb.Binlog{{LogSize: int64(fileLen), LogPath: key}},
}
}
for _, blob := range statslogs {
@ -259,16 +284,10 @@ func (b *binlogIO) genInsertBlobs(data *InsertData, partID, segID UniqueID, meta
fileLen := len(value)
kvs[key] = value
statspaths = append(statspaths, &datapb.FieldBinlog{
statspaths[fID] = &datapb.FieldBinlog{
FieldID: fID,
Binlogs: []*datapb.Binlog{
{
LogSize: int64(fileLen),
LogPath: key,
},
},
})
Binlogs: []*datapb.Binlog{{LogSize: int64(fileLen), LogPath: key}},
}
}
return kvs, inpaths, statspaths, nil

15
internal/datanode/binlog_io_test.go
View File

@ -23,6 +23,7 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/kv"
memkv "github.com/milvus-io/milvus/internal/kv/mem"
"github.com/milvus-io/milvus/internal/log"
@ -52,6 +53,16 @@ func TestBinlogIOInterfaceMethods(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 11, len(p.inPaths))
assert.Equal(t, 3, len(p.statsPaths))
assert.Equal(t, 1, len(p.inPaths[0].GetBinlogs()))
assert.Equal(t, 1, len(p.statsPaths[0].GetBinlogs()))
assert.NotNil(t, p.deltaInfo)
p, err = b.upload(context.TODO(), 1, 10, []*InsertData{iData, iData}, dData, meta)
assert.NoError(t, err)
assert.Equal(t, 11, len(p.inPaths))
assert.Equal(t, 3, len(p.statsPaths))
assert.Equal(t, 2, len(p.inPaths[0].GetBinlogs()))
assert.Equal(t, 2, len(p.statsPaths[0].GetBinlogs()))
assert.NotNil(t, p.deltaInfo)
ctx, cancel := context.WithCancel(context.Background())
@ -243,8 +254,8 @@ func TestBinlogIOInnerMethods(t *testing.T) {
log.Debug("test paths",
zap.Any("kvs no.", len(kvs)),
zap.String("insert paths field0", pin[0].GetBinlogs()[0].GetLogPath()),
zap.String("stats paths field0", pstats[0].GetBinlogs()[0].GetLogPath()))
zap.String("insert paths field0", pin[common.TimeStampField].GetBinlogs()[0].GetLogPath()),
zap.String("stats paths field0", pstats[common.TimeStampField].GetBinlogs()[0].GetLogPath()))
})
t.Run("Test genInsertBlobs error", func(t *testing.T) {