test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 11:59:00 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Fix error when cover string to bytes in write node

Browse Source 
Signed-off-by: xige-16 <xi.ge@zilliz.com>
This commit is contained in:
xige-16 2020-09-29 10:55:48 +08:00 committed by yefu.chen
parent 14d81f19cd
commit 12e49781e6
5 changed files with 8 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/examples/simple/create_collection.cpp
View File

@ -5,7 +5,7 @@
#include <interface/ConnectionImpl.h>
#include "utils/Utils.h"
const int DIM = 128;
const int DIM = 512;
int main(int argc , char**argv) {

2
sdk/examples/simple/insert.cpp
View File

@ -22,7 +22,7 @@
#include <random>
int N = 6000000;
int DIM = 128;
int DIM = 512;
int LOOP = 2000;
int ID_START = 0;

2
sdk/examples/simple/search.cpp
View File

@ -20,7 +20,7 @@
const int TOP_K = 10;
const int LOOP = 1000;
const int DIM = 128;
const int DIM = 512;
std::default_random_engine eng(42);

38
writer/main.go
View File

@ -44,17 +44,6 @@ func main() {
const Debug = true
if Debug {
//var shouldBenchmark = false
//var start time.Time
//var LogRecords int64
//var logFlag int64
//var logString = ""
//logFile, err := os.OpenFile("writenode.benchmark", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
//defer logFile.Close()
//if err != nil {
// log.Fatalf("Prepare log file error, " + err.Error())
//}
const CountInsertMsgBaseline = 1000 * 1000
var BaselineCounter int64 = 0
@ -63,13 +52,6 @@ func main() {
break
}
msgLength := wn.MessageClient.PrepareBatchMsg()
// wait until first 100,000 rows are successfully wrote
//if wn.MsgCounter.InsertCounter >= CountMsgNum && shouldBenchmark == false {
// shouldBenchmark = true
// wn.MsgCounter.InsertCounter = 0
// wn.MsgCounter.InsertedRecordSize = 0
// start = time.Now()
//}
if wn.MsgCounter.InsertCounter/CountInsertMsgBaseline != BaselineCounter {
wn.WriteWriterLog()
@ -80,26 +62,6 @@ func main() {
wn.DoWriteNode(ctx)
fmt.Println("write node do a batch message, storage len: ", msgLength)
}
// Test insert time
// ignore if less than 1000 records per time interval
//if shouldBenchmark && wn.MsgCounter.InsertCounter > 1000 {
// LogRecords += msgCounter.InsertCounter
// timeSince := time.Since(start)
// if timeSince >= timeInterval {
// speed := wn.MsgCounter.InsertedRecordSize / timeInterval.Seconds() / MB
// logString = fmt.Sprintln("============> Insert", wn.MsgCounter.InsertCounter, "records, cost:", timeSince, "speed:", speed, "M/s", "<============")
// newFlag := LogRecords / (10000 * 100)
// if newFlag != logFlag {
// logFlag = newFlag
// fmt.Fprintln(logFile, logString)
// logString = ""
// }
// wn.MsgCounter.InsertCounter = 0
// wn.MsgCounter.InsertedRecordSize = 0
// start = time.Now()
// }
//}
}
}

13
writer/write_node/writer_node.go
View File

@ -2,6 +2,7 @@ package write_node
import (
"context"
"encoding/binary"
"encoding/json"
"fmt"
"github.com/czs007/suvlim/conf"
@ -82,23 +83,19 @@ func (wn *WriteNode) InsertBatchData(ctx context.Context, data []*msgpb.InsertOr
var suffixKeys []string
var binaryData [][]byte
var timeStamp []uint64
byteArr := make([]byte, 8)
intData := uint64(0)
binary.BigEndian.PutUint64(byteArr, intData)
for i := 0; i < len(data); i++ {
prefixKey = data[i].CollectionName + "-" + strconv.FormatUint(uint64(data[i].Uid), 10)
suffixKey = strconv.FormatUint(uint64(data[i].SegmentId), 10)
prefixKeys = append(prefixKeys, []byte(prefixKey))
suffixKeys = append(suffixKeys, suffixKey)
binaryData = append(binaryData, []byte(data[i].String()))
binaryData = append(binaryData, byteArr)
timeStamp = append(timeStamp, uint64(data[i].Timestamp))
}
//wn.WriterLog(len(data))
//wn.MsgCounter.InsertCounter += int64(len(timeStamp))
//if len(timeStamp) > 0 {
// // assume each record is same size
// wn.MsgCounter.InsertedRecordSize += float64(len(timeStamp) * len(data[0].RowsData.Blob))
//}
error := (*wn.KvStore).PutRows(ctx, prefixKeys, binaryData, suffixKeys, timeStamp)
if error != nil {
fmt.Println("Can't insert data!")