test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 10:59:32 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: [2.4]Use fastjson lib for unmarshal delete log (#33787) (#33802)

Browse Source 
Cherry-pick from master
pr: #33878
```
goos: linux
goarch: amd64
GOMAXPROC=1
cpu: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz
BenchmarkJsonSerdeStd             343872              3568 ns/op            1335 B/op         25 allocs/op
BenchmarkJsonSerdeFastjson       5124177               234.9 ns/op            16 B/op          1 allocs/op
```

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2024-06-13 10:27:57 +08:00 committed by GitHub
parent efd1fa8b8a
commit 86f3433053
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -204,6 +204,7 @@ require (
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/valyala/fastjson v1.6.4 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect

2
go.sum
View File

@ -870,6 +870,8 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=

18
internal/storage/data_codec.go
View File

@ -26,6 +26,7 @@ import (
"strings"
"github.com/samber/lo"
"github.com/valyala/fastjson"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
@ -1059,14 +1060,18 @@ func (deleteCodec *DeleteCodec) Deserialize(blobs []*Blob) (partitionID UniqueID
}
defer rr.Release()
var p fastjson.Parser
deleteLog := &DeleteLog{}
for rr.Next() {
rec := rr.Record()
defer rec.Release()
column := rec.Column(0)
for i := 0; i < column.Len(); i++ {
deleteLog := &DeleteLog{}
strVal := column.ValueStr(i)
if err = json.Unmarshal([]byte(strVal), deleteLog); err != nil {
v, err := p.Parse(strVal)
if err != nil {
// compatible with versions that only support int64 type primary keys
// compatible with fmt.Sprintf("%d,%d", pk, ts)
// compatible error info (unmarshal err invalid character ',' after top-level value)
@ -1086,6 +1091,15 @@ func (deleteCodec *DeleteCodec) Deserialize(blobs []*Blob) (partitionID UniqueID
if err != nil {
return err
}
} else {
deleteLog.Ts = v.GetUint64("ts")
deleteLog.PkType = v.GetInt64("pkType")
switch deleteLog.PkType {
case int64(schemapb.DataType_Int64):
deleteLog.Pk = &Int64PrimaryKey{Value: v.GetInt64("pk")}
case int64(schemapb.DataType_VarChar):
deleteLog.Pk = &VarCharPrimaryKey{Value: string(v.GetStringBytes("pk"))}
}
}
result.Append(deleteLog.Pk, deleteLog.Ts)