test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Rename serializeDiskIndex to getIndexFileInfo (#19416)

Browse Source 
Signed-off-by: xige-16 <xi.ge@zilliz.com>

Signed-off-by: xige-16 <xi.ge@zilliz.com>
This commit is contained in:
xige-16 2022-09-27 10:32:53 +08:00 committed by GitHub
parent 911ea6c264
commit 3fb8e80147
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/indexnode/task.go
View File

@ -399,7 +399,7 @@ func (it *indexBuildTask) BuildDiskAnnIndex(ctx context.Context) error {
it.tr.Record("build index done") it.tr.Record("build index done")
indexBlobs, err := it.index.SerializeDiskIndex() fileInfos, err := it.index.GetIndexFileInfo()
if err != nil { if err != nil {
log.Ctx(ctx).Error("IndexNode index Serialize failed", zap.Error(err)) log.Ctx(ctx).Error("IndexNode index Serialize failed", zap.Error(err))
return err return err
@ -408,8 +408,12 @@ func (it *indexBuildTask) BuildDiskAnnIndex(ctx context.Context) error {
// use serialized size before encoding // use serialized size before encoding
it.serializedSize = 0 it.serializedSize = 0
for _, blob := range indexBlobs { for _, info := range fileInfos {
it.serializedSize += uint64(blob.Size) it.serializedSize += uint64(info.FileSize)
it.indexBlobs = append(it.indexBlobs, &storage.Blob{
Key: info.FileName,
Size: info.FileSize,
})
} }
// early release index for gc, and we can ensure that Delete is idempotent. // early release index for gc, and we can ensure that Delete is idempotent.
@ -419,7 +423,6 @@ func (it *indexBuildTask) BuildDiskAnnIndex(ctx context.Context) error {
encodeIndexFileDur := it.tr.Record("index codec serialize done") encodeIndexFileDur := it.tr.Record("index codec serialize done")
metrics.IndexNodeEncodeIndexFileLatency.WithLabelValues(strconv.FormatInt(Params.IndexNodeCfg.GetNodeID(), 10)).Observe(float64(encodeIndexFileDur.Milliseconds())) metrics.IndexNodeEncodeIndexFileLatency.WithLabelValues(strconv.FormatInt(Params.IndexNodeCfg.GetNodeID(), 10)).Observe(float64(encodeIndexFileDur.Milliseconds()))
it.indexBlobs = indexBlobs
return nil return nil
} }

19
internal/util/indexcgowrapper/index.go
View File

@ -25,10 +25,15 @@ import (
type Blob = storage.Blob type Blob = storage.Blob
type IndexFileInfo struct {
FileName string
FileSize int64
}
type CodecIndex interface { type CodecIndex interface {
Build(*Dataset) error Build(*Dataset) error
Serialize() ([]*Blob, error) Serialize() ([]*Blob, error)
SerializeDiskIndex() ([]*Blob, error) GetIndexFileInfo() ([]*IndexFileInfo, error)
Load([]*Blob) error Load([]*Blob) error
Delete() error Delete() error
CleanLocalData() error CleanLocalData() error
@ -233,7 +238,7 @@ func (index *CgoIndex) Serialize() ([]*Blob, error) {
return ret, nil return ret, nil
} }
func (index *CgoIndex) SerializeDiskIndex() ([]*Blob, error) { func (index *CgoIndex) GetIndexFileInfo() ([]*IndexFileInfo, error) {
var cBinarySet C.CBinarySet var cBinarySet C.CBinarySet
status := C.SerializeIndexToBinarySet(index.indexPtr, &cBinarySet) status := C.SerializeIndexToBinarySet(index.indexPtr, &cBinarySet)
@ -250,17 +255,17 @@ func (index *CgoIndex) SerializeDiskIndex() ([]*Blob, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
ret := make([]*Blob, 0) ret := make([]*IndexFileInfo, 0)
for _, key := range keys { for _, key := range keys {
size, err := GetBinarySetSize(cBinarySet, key) size, err := GetBinarySetSize(cBinarySet, key)
if err != nil { if err != nil {
return nil, err return nil, err
} }
blob := &Blob{ info := &IndexFileInfo{
Key: key, FileName: key,
Size: size, FileSize: size,
} }
ret = append(ret, blob) ret = append(ret, info)
} }
return ret, nil return ret, nil