Support Benchmark

Signed-off-by: FluorineDog <guilin.gou@zilliz.com>

cmd/storage/benchmark.go

@@ -5,13 +5,13 @@ import (
 	"crypto/md5"
 	"flag"
 	"fmt"
-	minio "github.com/czs007/suvlim/storage/internal/minio"
-	tikv "github.com/czs007/suvlim/storage/internal/tikv"
-	"github.com/czs007/suvlim/storage/pkg/types"
 	"github.com/pivotal-golang/bytefmt"
 	"log"
 	"math/rand"
 	"os"
+	minio "storage/internal/minio"
+	tikv "storage/internal/tikv"
+	"storage/pkg/types"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -32,14 +32,13 @@ var logFile *os.File
 var store types.Store
 var wg sync.WaitGroup
 
-
 func runSet() {
 	for time.Now().Before(endTime) {
 		num := atomic.AddInt32(&keyNum, 1)
 		key := []byte(fmt.Sprint("key", num))
 		for ver := 1; ver <= numVersion; ver++ {
 			atomic.AddInt32(&counter, 1)
-			err := store.PutRow(context.Background(), key, valueData,"empty" ,uint64(ver))
+			err := store.Set(context.Background(), key, valueData, uint64(ver))
 			if err != nil {
 				log.Fatalf("Error setting key %s, %s", key, err.Error())
 				//atomic.AddInt32(&setCount, -1)
@@ -55,14 +54,12 @@ func runBatchSet() {
 	for time.Now().Before(endTime) {
 		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
 		keys := make([][]byte, batchOpSize)
-		versions := make([]uint64, batchOpSize)
-		batchSuffix := make([]string, batchOpSize)
 		for n := batchOpSize; n > 0; n-- {
 			keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
 		}
 		for ver := 1; ver <= numVersion; ver++ {
 			atomic.AddInt32(&counter, 1)
-			err := store.PutRows(context.Background(), keys, batchValueData, batchSuffix,versions)
+			err := store.BatchSet(context.Background(), keys, batchValueData, uint64(numVersion))
 			if err != nil {
 				log.Fatalf("Error setting batch keys %s %s", keys, err.Error())
 				//atomic.AddInt32(&batchSetCount, -1)
@@ -73,7 +70,6 @@ func runBatchSet() {
 	wg.Done()
 }
 
-
 func runGet() {
 	for time.Now().Before(endTime) {
 		num := atomic.AddInt32(&counter, 1)
@@ -81,7 +77,7 @@ func runGet() {
 		//key := []byte(fmt.Sprint("key", num))
 		num = num % totalKeyCount
 		key := totalKeys[num]
-		_, err := store.GetRow(context.Background(), key, uint64(numVersion))
+		_, err := store.Get(context.Background(), key, uint64(numVersion))
 		if err != nil {
 			log.Fatalf("Error getting key %s, %s", key, err.Error())
 			//atomic.AddInt32(&getCount, -1)
@@ -105,12 +101,8 @@ func runBatchGet() {
 		}
 		start := end - int32(batchOpSize)
 		keys := totalKeys[start:end]
-		versions := make([]uint64, batchOpSize)
-		for i, _ := range versions{
-			versions[i]= uint64(numVersion)
-		}
 		atomic.AddInt32(&counter, 1)
-		_, err := store.GetRows(context.Background(), keys, versions)
+		_, err := store.BatchGet(context.Background(), keys, uint64(numVersion))
 		if err != nil {
 			log.Fatalf("Error getting key %s, %s", keys, err.Error())
 			//atomic.AddInt32(&batchGetCount, -1)
@@ -128,7 +120,7 @@ func runDelete() {
 		//key := []byte(fmt.Sprint("key", num))
 		num = num % totalKeyCount
 		key := totalKeys[num]
-		err := store.DeleteRow(context.Background(), key, uint64(numVersion))
+		err := store.Delete(context.Background(), key, uint64(numVersion))
 		if err != nil {
 			log.Fatalf("Error getting key %s, %s", key, err.Error())
 			//atomic.AddInt32(&deleteCount, -1)
@@ -153,11 +145,7 @@ func runBatchDelete() {
 		start := end - int32(batchOpSize)
 		keys := totalKeys[start:end]
 		atomic.AddInt32(&counter, 1)
-		versions := make([]uint64, batchOpSize)
-		for i, _ := range versions{
-			versions[i]= uint64(numVersion)
-		}
-		err := store.DeleteRows(context.Background(), keys, versions)
+		err := store.BatchDelete(context.Background(), keys, uint64(numVersion))
 		if err != nil {
 			log.Fatalf("Error getting key %s, %s", keys, err.Error())
 			//atomic.AddInt32(&batchDeleteCount, -1)
@@ -171,15 +159,15 @@ func runBatchDelete() {
 func main() {
 	// Parse command line
 	myflag := flag.NewFlagSet("myflag", flag.ExitOnError)
-	myflag.IntVar(&durationSecs, "d", 30, "Duration of each test in seconds")
+	myflag.IntVar(&durationSecs, "d", 5, "Duration of each test in seconds")
 	myflag.IntVar(&threads, "t", 1, "Number of threads to run")
 	myflag.IntVar(&loops, "l", 1, "Number of times to repeat test")
 	var sizeArg string
 	var storeType string
-	myflag.StringVar(&sizeArg, "z", "2k", "Size of objects in bytes with postfix K, M, and G")
+	myflag.StringVar(&sizeArg, "z", "1K", "Size of objects in bytes with postfix K, M, and G")
 	myflag.StringVar(&storeType, "s", "tikv", "Storage type, tikv or minio")
 	myflag.IntVar(&numVersion, "v", 1, "Max versions for each key")
-	myflag.IntVar(&batchOpSize, "b", 1000, "Batch operation kv pair number")
+	myflag.IntVar(&batchOpSize, "b", 100, "Batch operation kv pair number")
 
 	if err := myflag.Parse(os.Args[1:]); err != nil {
 		os.Exit(1)
@@ -201,7 +189,7 @@ func main() {
 			log.Fatalf("Error when creating storage " + err.Error())
 		}
 	case "minio":
-		store, err = minio.NewMinioDriver(context.Background())
+		store, err = minio.NewMinioStore(context.Background())
 		if err != nil {
 			log.Fatalf("Error when creating storage " + err.Error())
 		}
@@ -240,10 +228,25 @@ func main() {
 		totalKeyCount = 0
 		totalKeys = nil
 
+		// Run the set case
+		startTime := time.Now()
+		endTime = startTime.Add(time.Second * time.Duration(durationSecs))
+		for n := 1; n <= threads; n++ {
+			wg.Add(1)
+			go runSet()
+		}
+		wg.Wait()
+
+		setTime := setFinish.Sub(startTime).Seconds()
+
+		bps := float64(uint64(counter)*valueSize) / setTime
+		fmt.Fprint(logFile, fmt.Sprintf("Loop %d: PUT time %.1f secs, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
+			loop, setTime, counter, bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter)/setTime))
+
 		// Run the batchSet case
 		// key seq start from setCount
 		counter = 0
-		startTime := time.Now()
+		startTime = time.Now()
 		endTime = startTime.Add(time.Second * time.Duration(durationSecs))
 		for n := 1; n <= threads; n++ {
 			wg.Add(1)
@@ -251,8 +254,8 @@ func main() {
 		}
 		wg.Wait()
 
-		setTime := setFinish.Sub(startTime).Seconds()
-		bps := float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
+		setTime = setFinish.Sub(startTime).Seconds()
+		bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
 		fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH PUT time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
 			loop, setTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter * int32(batchOpSize))/setTime))
 
@@ -326,6 +329,12 @@ func main() {
 		// Print line mark
 		lineMark := "\n"
 		fmt.Fprint(logFile, lineMark)
+
+		// Clear test data
+		err = store.BatchDelete(context.Background(), totalKeys, uint64(numVersion))
+		if err != nil {
+			log.Print("Clean test data error " + err.Error())
+		}
 	}
 	log.Print("Benchmark test done.")
 }

conf/config.yaml

@@ -11,12 +11,12 @@
 
 master:
     address: localhost
-    port: 53100
+    port: 6000
 
 etcd:
   address: localhost
   port: 2379
-  rootpath: by-dev/
+  rootpath: suvlim
   segthreshold: 10000
 
 timesync: 

core/CMakeLists_old.txt

@@ -0,0 +1,20 @@
+project(sulvim_core)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+cmake_minimum_required(VERSION 3.16)
+set( CMAKE_CXX_STANDARD 17 )
+set( CMAKE_CXX_STANDARD_REQUIRED on )
+set (CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+include_directories(src)
+add_subdirectory(src)
+add_subdirectory(unittest)
+
+install(
+        DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/dog_segment/
+        DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/include
+        FILES_MATCHING PATTERN "*_c.h"
+)
+
+install(FILES ${CMAKE_BINARY_DIR}/src/dog_segment/libmilvus_dog_segment.so
+        DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/lib)

core/include/segment_c.h

@@ -73,9 +73,6 @@ BuildIndex(CSegmentBase c_segment);
 bool
 IsOpened(CSegmentBase c_segment);
 
-long int
-GetMemoryUsageInBytes(CSegmentBase c_segment);
-
 //////////////////////////////////////////////////////////////////
 
 long int

core/release-build.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+
+# Compile jobs variable; Usage: $ jobs=12 ./build.sh ...
+if [[ ! ${jobs+1} ]]; then
+    jobs=$(nproc)
+fi
+
+BUILD_OUTPUT_DIR="cmake-build-release"
+BUILD_TYPE="Release"
+BUILD_UNITTEST="OFF"
+INSTALL_PREFIX=$(pwd)/milvus
+MAKE_CLEAN="OFF"
+BUILD_COVERAGE="OFF"
+DB_PATH="/tmp/milvus"
+PROFILING="OFF"
+RUN_CPPLINT="OFF"
+CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+GPU_VERSION="OFF" #defaults to CPU version
+WITH_PROMETHEUS="ON"
+CUDA_ARCH="DEFAULT"
+CUSTOM_THIRDPARTY_PATH=""
+
+while getopts "p:d:t:s:f:ulrcghzme" arg; do
+  case $arg in
+  f)
+    CUSTOM_THIRDPARTY_PATH=$OPTARG
+    ;;
+  p)
+    INSTALL_PREFIX=$OPTARG
+    ;;
+  d)
+    DB_PATH=$OPTARG
+    ;;
+  t)
+    BUILD_TYPE=$OPTARG # BUILD_TYPE
+    ;;
+  u)
+    echo "Build and run unittest cases"
+    BUILD_UNITTEST="ON"
+    ;;
+  l)
+    RUN_CPPLINT="ON"
+    ;;
+  r)
+    if [[ -d ${BUILD_OUTPUT_DIR} ]]; then
+      MAKE_CLEAN="ON"
+    fi
+    ;;
+  c)
+    BUILD_COVERAGE="ON"
+    ;;
+  z)
+    PROFILING="ON"
+    ;;
+  g)
+    GPU_VERSION="ON"
+    ;;
+  e)
+    WITH_PROMETHEUS="OFF"
+    ;;
+  s)
+    CUDA_ARCH=$OPTARG
+    ;;
+  h) # help
+    echo "
+
+parameter:
+-f: custom paths of thirdparty downloaded files(default: NULL)
+-p: install prefix(default: $(pwd)/milvus)
+-d: db data path(default: /tmp/milvus)
+-t: build type(default: Debug)
+-u: building unit test options(default: OFF)
+-l: run cpplint, clang-format and clang-tidy(default: OFF)
+-r: remove previous build directory(default: OFF)
+-c: code coverage(default: OFF)
+-z: profiling(default: OFF)
+-g: build GPU version(default: OFF)
+-e: build without prometheus(default: OFF)
+-s: build with CUDA arch(default:DEFAULT), for example '-gencode=compute_61,code=sm_61;-gencode=compute_75,code=sm_75'
+-h: help
+
+usage:
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} -f\${CUSTOM_THIRDPARTY_PATH} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h]
+                "
+    exit 0
+    ;;
+  ?)
+    echo "ERROR! unknown argument"
+    exit 1
+    ;;
+  esac
+done
+
+if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
+  mkdir ${BUILD_OUTPUT_DIR}
+fi
+
+cd ${BUILD_OUTPUT_DIR}
+
+# remove make cache since build.sh -l use default variables
+# force update the variables each time
+make rebuild_cache >/dev/null 2>&1
+
+
+if [[ ${MAKE_CLEAN} == "ON" ]]; then
+  echo "Runing make clean in ${BUILD_OUTPUT_DIR} ..."
+  make clean
+  exit 0
+fi
+
+CMAKE_CMD="cmake \
+-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DOpenBLAS_SOURCE=AUTO \
+-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DBUILD_COVERAGE=${BUILD_COVERAGE} \
+-DMILVUS_DB_PATH=${DB_PATH} \
+-DENABLE_CPU_PROFILING=${PROFILING} \
+-DMILVUS_GPU_VERSION=${GPU_VERSION} \
+-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
+-DMILVUS_CUDA_ARCH=${CUDA_ARCH} \
+-DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} \
+../"
+echo ${CMAKE_CMD}
+${CMAKE_CMD}
+
+
+if [[ ${RUN_CPPLINT} == "ON" ]]; then
+  # cpplint check
+  make lint
+  if [ $? -ne 0 ]; then
+    echo "ERROR! cpplint check failed"
+    exit 1
+  fi
+  echo "cpplint check passed!"
+
+  # clang-format check
+  make check-clang-format
+  if [ $? -ne 0 ]; then
+    echo "ERROR! clang-format check failed"
+    exit 1
+  fi
+  echo "clang-format check passed!"
+
+  # clang-tidy check
+  make check-clang-tidy
+  if [ $? -ne 0 ]; then
+      echo "ERROR! clang-tidy check failed"
+      exit 1
+  fi
+  echo "clang-tidy check passed!"
+else
+  # compile and build
+  make -j ${jobs} install || exit 1
+fi

core/src/CMakeLists_old.txt

@@ -0,0 +1,4 @@
+add_subdirectory(utils)
+add_subdirectory(dog_segment)
+#add_subdirectory(index)
+add_subdirectory(query)

core/src/dog_segment/CMakeLists.txt

@@ -1,5 +1,3 @@
-aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../pb PB_SRC_FILES)
-
 set(DOG_SEGMENT_FILES
         SegmentNaive.cpp
         IndexMeta.cpp
@@ -9,13 +7,10 @@ set(DOG_SEGMENT_FILES
         collection_c.cpp
         partition_c.cpp
         segment_c.cpp
-	${PB_SRC_FILES}
         )
 add_library(milvus_dog_segment SHARED
         ${DOG_SEGMENT_FILES}
         )
-
-
 #add_dependencies( segment sqlite mysqlpp )
 
-target_link_libraries(milvus_dog_segment tbb utils pthread knowhere log libprotobuf)
+target_link_libraries(milvus_dog_segment tbb utils pthread knowhere log)

core/src/dog_segment/Collection.cpp

@@ -1,7 +1,5 @@
 #include "Collection.h"
-#include "pb/master.pb.h"
-//using Collection = masterpb::Collection;
-#include <google/protobuf/text_format.h>
+
 namespace milvus::dog_segment {
 
 Collection::Collection(std::string &collection_name, std::string &schema):
@@ -14,32 +12,10 @@ Collection::set_index() {}
 
 void
 Collection::parse() {
-    if(schema_json_ == "") {
-        auto schema = std::make_shared<Schema>();
-        schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16);
-        schema->AddField("age", DataType::INT32);
-        schema_ = schema;
-        return;
-    }
-
-    masterpb::Collection collection;
-    auto suc = google::protobuf::TextFormat::ParseFromString(schema_json_, &collection);
-
-
-    if (!suc) {
-      std::cerr << "unmarshal failed" << std::endl;
-    }
     auto schema = std::make_shared<Schema>();
-    for (const milvus::grpc::FieldMeta & child: collection.schema().field_metas()){
-            std::cout<<"add Field, name :" << child.field_name() << ", datatype :" << child.type() << ", dim :" << int(child.dim()) << std::endl;
-            schema->AddField(std::string_view(child.field_name()), DataType {child.type()}, int(child.dim()));
-    }
-    /*
     schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16);
     schema->AddField("age", DataType::INT32);
-    */
     schema_ = schema;
-    
 }
 
 }

core/src/dog_segment/SegmentDefs.h

@@ -161,10 +161,6 @@ class Schema {
         }
     }
 
-    const std::vector<FieldMeta>& get_fields() {
-      return fields_;
-    }
-
     const FieldMeta&
     operator[](const std::string& field_name) const {
         auto offset_iter = offsets_.find(field_name);

core/src/dog_segment/SegmentNaive.cpp

@@ -19,20 +19,8 @@ TestABI() {
 std::unique_ptr<SegmentBase>
 CreateSegment(SchemaPtr schema, IndexMetaPtr remote_index_meta) {
     if (remote_index_meta == nullptr) {
-        int dim = 0;
-        std::string index_field_name;
-
-        for (auto& field: schema->get_fields()) {
-            if (field.get_data_type() == DataType::VECTOR_FLOAT) {
-                dim = field.get_dim();
-                index_field_name = field.get_name();
-            }
-        }
-
-        assert(dim != 0);
-        assert(!index_field_name.empty());
-
         auto index_meta = std::make_shared<IndexMeta>(schema);
+        auto dim = schema->operator[]("fakevec").get_dim();
         // TODO: this is merge of query conf and insert conf
         // TODO: should be splitted into multiple configs
         auto conf = milvus::knowhere::Config{
@@ -44,7 +32,7 @@ CreateSegment(SchemaPtr schema, IndexMetaPtr remote_index_meta) {
                 {milvus::knowhere::Metric::TYPE,        milvus::knowhere::Metric::L2},
                 {milvus::knowhere::meta::DEVICEID,      0},
         };
-        index_meta->AddEntry("fakeindex", index_field_name, knowhere::IndexEnum::INDEX_FAISS_IVFPQ,
+        index_meta->AddEntry("fakeindex", "fakevec", knowhere::IndexEnum::INDEX_FAISS_IVFPQ,
                              knowhere::IndexMode::MODE_CPU, conf);
         remote_index_meta = index_meta;
     }
@@ -153,11 +141,7 @@ Status
 SegmentNaive::Insert(int64_t reserved_begin, int64_t size, const int64_t *uids_raw, const Timestamp *timestamps_raw,
                      const DogDataChunk &entities_raw) {
     assert(entities_raw.count == size);
-    if (entities_raw.sizeof_per_row != schema_->get_total_sizeof()) {
-      std::string msg = "entity length = " + std::to_string(entities_raw.sizeof_per_row) +
-          ", schema length = " + std::to_string(schema_->get_total_sizeof());
-        throw std::runtime_error(msg);
-    }
+    assert(entities_raw.sizeof_per_row == schema_->get_total_sizeof());
     auto raw_data = reinterpret_cast<const char *>(entities_raw.raw_data);
     //    std::vector<char> entities(raw_data, raw_data + size * len_per_row);
 

core/src/dog_segment/segment_c.cpp

@@ -177,13 +177,6 @@ IsOpened(CSegmentBase c_segment) {
   return status == milvus::dog_segment::SegmentBase::SegmentState::Open;
 }
 
-long int
-GetMemoryUsageInBytes(CSegmentBase c_segment) {
-  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
-  auto mem_size = segment->GetMemoryUsageInBytes();
-  return mem_size;
-}
-
 //////////////////////////////////////////////////////////////////
 
 long int

core/src/dog_segment/segment_c.h

@@ -73,9 +73,6 @@ BuildIndex(CSegmentBase c_segment);
 bool
 IsOpened(CSegmentBase c_segment);
 
-long int
-GetMemoryUsageInBytes(CSegmentBase c_segment);
-
 //////////////////////////////////////////////////////////////////
 
 long int

core/src/utils/Types.h

@@ -41,7 +41,6 @@ using ResultIds = std::vector<idx_t>;
 using ResultDistances = std::vector<distance_t>;
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
-
 enum class DataType {
     NONE = 0,
     BOOL = 1,

core/thirdparty/CMakeLists.txt

@@ -45,6 +45,4 @@ endif()
 # ****************************** Thirdparty opentracing ***************************************
 if ( MILVUS_WITH_OPENTRACING )
     add_subdirectory( opentracing )
-endif()
-
-add_subdirectory( protobuf )
+endif()

core/thirdparty/protobuf/CMakeLists.txt

@@ -1,66 +0,0 @@
-# Copyright 2017 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if ( DEFINED ENV{MILVUS_GTEST_URL} )
-    set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" )
-else()
-    set( GTEST_SOURCE_URL
-	 "https://github.com/protocolbuffers/protobuf/releases/download/v3.9.0/protobuf-cpp-3.9.0.zip")
-endif()
-
-message( STATUS "Building protobuf-cpp-3.9.0 from source" )
-
-FetchContent_Declare(
-    protobuf
-    URL        ${GTEST_SOURCE_URL} 
-    URL_MD5    "9562b27cc6ac5ebd087f201f1310c885"
-    DOWNLOAD_DIR    ${MILVUS_BINARY_DIR}/3rdparty_download/download
-    SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/protobuf-src
-    BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/protobuf-build
-
-)
-
-
-if ( NOT protobuf_POPULATED )
-    FetchContent_Populate( protobuf )
-
-    add_subdirectory(${protobuf_SOURCE_DIR}/cmake 
-	    ${protobuf_BINARY_DIR} 
-	    EXCLUDE_FROM_ALL)
-
-endif()
-
-
-set(_PROTOBUF_LIBRARY_NAME "libprotobuf")
-set(gRPC_PROTOBUF_PROVIDER "module" CACHE STRING "Provider of protobuf library")
-set_property(CACHE gRPC_PROTOBUF_PROVIDER PROPERTY STRINGS "module" "package")
-
-# Building the protobuf tests require gmock what is not part of a standard protobuf checkout.
-# Disable them unless they are explicitly requested from the cmake command line (when we assume
-# gmock is downloaded to the right location inside protobuf).
-if(NOT protobuf_BUILD_TESTS)
-set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests")
-endif()
-
-# Disable building protobuf with zlib. Building protobuf with zlib breaks
-# the build if zlib is not installed on the system.
-if(NOT protobuf_WITH_ZLIB)
-set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build protobuf with zlib.")
-endif()
-
-if(NOT PROTOBUF_ROOT_DIR)
-  #set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf)
-  set(PROTOBUF_ROOT_DIR ${protobuf_SOURCE_DIR})
-endif()
-

core/thirdparty/versions.txt

@@ -3,5 +3,4 @@ GTEST_VERSION=1.8.1
 YAMLCPP_VERSION=0.6.3
 ZLIB_VERSION=v1.2.11
 OPENTRACING_VERSION=v1.5.1
-PROTOBUF_VERSION=3.9.0
 # vim: set filetype=sh:

core/unittest/test_c_api.cpp

@@ -6,440 +6,529 @@
 #include "dog_segment/collection_c.h"
 #include "dog_segment/segment_c.h"
 
+#include <chrono>
+namespace chrono = std::chrono;
 
 TEST(CApiTest, CollectionTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  DeleteCollection(collection);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    DeleteCollection(collection);
 }
 
 
 TEST(CApiTest, PartitonTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  DeleteCollection(collection);
-  DeletePartition(partition);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    DeleteCollection(collection);
+    DeletePartition(partition);
 }
 
 
 TEST(CApiTest, SegmentTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, InsertTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  std::vector<char> raw_data;
-  std::vector<uint64_t> timestamps;
-  std::vector<int64_t> uids;
-  int N = 10000;
-  std::default_random_engine e(67);
-  for (int i = 0; i < N; ++i) {
-    uids.push_back(100000 + i);
-    timestamps.push_back(0);
-    // append vec
-    float vec[16];
-    for (auto &x: vec) {
-      x = e() % 2000 * 0.001 - 1.0;
+    std::vector<char> raw_data;
+    std::vector<uint64_t> timestamps;
+    std::vector<int64_t> uids;
+    int N = 10000;
+    std::default_random_engine e(67);
+    for (int i = 0; i < N; ++i) {
+        uids.push_back(100000 + i);
+        timestamps.push_back(0);
+        // append vec
+        float vec[16];
+        for (auto& x : vec) {
+            x = e() % 2000 * 0.001 - 1.0;
+        }
+        raw_data.insert(
+            raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
+        int age = e() % 100;
+        raw_data.insert(
+            raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
     }
-    raw_data.insert(raw_data.end(), (const char *) std::begin(vec), (const char *) std::end(vec));
-    int age = e() % 100;
-    raw_data.insert(raw_data.end(), (const char *) &age, ((const char *) &age) + sizeof(age));
-  }
 
-  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
+    auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
 
-  auto offset = PreInsert(segment, N);
+    auto offset = PreInsert(segment, N);
 
-  auto res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int) line_sizeof, N);
+    auto res = Insert(segment,
+                      offset,
+                      N,
+                      uids.data(),
+                      timestamps.data(),
+                      raw_data.data(),
+                      (int)line_sizeof,
+                      N);
 
-  assert(res == 0);
+    assert(res == 0);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, DeleteTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  long delete_primary_keys[] = {100000, 100001, 100002};
-  unsigned long delete_timestamps[] = {0, 0, 0};
+    long delete_primary_keys[] = {100000, 100001, 100002};
+    unsigned long delete_timestamps[] = {0, 0, 0};
 
-  auto offset = PreDelete(segment, 3);
+    auto offset = PreDelete(segment, 3);
 
-  auto del_res = Delete(segment, offset, 3, delete_primary_keys, delete_timestamps);
-  assert(del_res == 0);
+    auto del_res = Delete(segment, offset, 3, delete_primary_keys, delete_timestamps);
+    assert(del_res == 0);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, SearchTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  std::vector<char> raw_data;
-  std::vector<uint64_t> timestamps;
-  std::vector<int64_t> uids;
-  int N = 10000;
-  std::default_random_engine e(67);
-  for (int i = 0; i < N; ++i) {
-    uids.push_back(100000 + i);
-    timestamps.push_back(0);
-    // append vec
-    float vec[16];
-    for (auto &x: vec) {
-      x = e() % 2000 * 0.001 - 1.0;
+    std::vector<char> raw_data;
+    std::vector<uint64_t> timestamps;
+    std::vector<int64_t> uids;
+    int N = 10000;
+    std::default_random_engine e(67);
+    for (int i = 0; i < N; ++i) {
+        uids.push_back(100000 + i);
+        timestamps.push_back(0);
+        // append vec
+        float vec[16];
+        for (auto& x : vec) {
+            x = e() % 2000 * 0.001 - 1.0;
+        }
+        raw_data.insert(
+            raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
+        int age = e() % 100;
+        raw_data.insert(
+            raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
     }
-    raw_data.insert(raw_data.end(), (const char *) std::begin(vec), (const char *) std::end(vec));
-    int age = e() % 100;
-    raw_data.insert(raw_data.end(), (const char *) &age, ((const char *) &age) + sizeof(age));
-  }
 
-  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
+    auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
 
-  auto offset = PreInsert(segment, N);
+    auto offset = PreInsert(segment, N);
 
-  auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int) line_sizeof, N);
-  assert(ins_res == 0);
+    auto ins_res = Insert(segment,
+                          offset,
+                          N,
+                          uids.data(),
+                          timestamps.data(),
+                          raw_data.data(),
+                          (int)line_sizeof,
+                          N);
+    assert(ins_res == 0);
 
-  long result_ids[10];
-  float result_distances[10];
+    long result_ids[10];
+    float result_distances[10];
 
-  auto query_json = std::string(R"({"field_name":"fakevec","num_queries":1,"topK":10})");
-  std::vector<float> query_raw_data(16);
-  for (int i = 0; i < 16; i++) {
-    query_raw_data[i] = e() % 2000 * 0.001 - 1.0;
-  }
+    auto query_json =
+        std::string(R"({"field_name":"fakevec","num_queries":1,"topK":10})");
+    std::vector<float> query_raw_data(16);
+    for (int i = 0; i < 16; i++) {
+        query_raw_data[i] = e() % 2000 * 0.001 - 1.0;
+    }
 
-  CQueryInfo queryInfo{1, 10, "fakevec"};
+    CQueryInfo queryInfo{1, 10, "fakevec"};
 
-  auto sea_res = Search(segment, queryInfo, 1, query_raw_data.data(), 16, result_ids, result_distances);
-  assert(sea_res == 0);
+    auto sea_res = Search(
+        segment, queryInfo, 1, query_raw_data.data(), 16, result_ids, result_distances);
+    assert(sea_res == 0);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, SearchSimpleTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  std::vector<char> raw_data;
-  std::vector<uint64_t> timestamps;
-  std::vector<int64_t> uids;
+    std::vector<char> raw_data;
+    std::vector<uint64_t> timestamps;
+    std::vector<int64_t> uids;
 
-  int N = 3;
-  int DIM = 16;
+    int N = 3;
+    int DIM = 16;
 
-  std::vector<float> vec(DIM);
-  for (int i = 0; i < DIM; i++) {
-    vec[i] = i;
-  }
+    std::vector<float> vec(DIM);
+    for (int i = 0; i < DIM; i++) {
+        vec[i] = i;
+    }
 
-  for (int i = 0; i < N; i++) {
-    uids.push_back(i);
-    timestamps.emplace_back(i);
-    // append vec
+    for (int i = 0; i < N; i++) {
+        uids.push_back(i);
+        timestamps.emplace_back(i);
+        // append vec
 
-    raw_data.insert(raw_data.end(), (const char *) &vec, ((const char *) &vec) + sizeof(float) * vec.size());
-    int age = i;
-    raw_data.insert(raw_data.end(), (const char *) &age, ((const char *) &age) + sizeof(age));
-  }
+        raw_data.insert(raw_data.end(),
+                        (const char*)&vec,
+                        ((const char*)&vec) + sizeof(float) * vec.size());
+        int age = i;
+        raw_data.insert(
+            raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
+    }
 
-  auto line_sizeof = (sizeof(int) + sizeof(float) * DIM);
+    auto line_sizeof = (sizeof(int) + sizeof(float) * DIM);
 
-  auto offset = PreInsert(segment, N);
+    auto offset = PreInsert(segment, N);
 
-  auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int) line_sizeof, N);
-  assert(ins_res == 0);
+    auto ins_res = Insert(segment,
+                          offset,
+                          N,
+                          uids.data(),
+                          timestamps.data(),
+                          raw_data.data(),
+                          (int)line_sizeof,
+                          N);
+    assert(ins_res == 0);
 
-  Close(segment);
-  BuildIndex(segment);
+    Close(segment);
+    BuildIndex(segment);
 
-  long result_ids[10];
-  float result_distances[10];
+    long result_ids[10];
+    float result_distances[10];
 
-  std::vector<float> query_raw_data(DIM);
-  for (int i = 0; i < DIM; i++) {
-    query_raw_data[i] = i;
-  }
+    std::vector<float> query_raw_data(DIM);
+    for (int i = 0; i < DIM; i++) {
+        query_raw_data[i] = i;
+    }
 
-  CQueryInfo queryInfo{1, 10, "fakevec"};
+    CQueryInfo queryInfo{1, 10, "fakevec"};
 
-  auto sea_res = Search(segment, queryInfo, 1, query_raw_data.data(), DIM, result_ids, result_distances);
-  assert(sea_res == 0);
+    auto sea_res = Search(
+        segment, queryInfo, 1, query_raw_data.data(), DIM, result_ids, result_distances);
+    assert(sea_res == 0);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, IsOpenedTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  auto is_opened = IsOpened(segment);
-  assert(is_opened);
+    auto is_opened = IsOpened(segment);
+    assert(is_opened);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, CloseTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  auto status = Close(segment);
-  assert(status == 0);
+    auto status = Close(segment);
+    assert(status == 0);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
-}
-
-
-TEST(CApiTest, GetMemoryUsageInBytesTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
-
-  std::vector<char> raw_data;
-  std::vector<uint64_t> timestamps;
-  std::vector<int64_t> uids;
-  int N = 10000;
-  std::default_random_engine e(67);
-  for (int i = 0; i < N; ++i) {
-    uids.push_back(100000 + i);
-    timestamps.push_back(0);
-    // append vec
-    float vec[16];
-    for (auto &x: vec) {
-      x = e() % 2000 * 0.001 - 1.0;
-    }
-    raw_data.insert(raw_data.end(), (const char *) std::begin(vec), (const char *) std::end(vec));
-    int age = e() % 100;
-    raw_data.insert(raw_data.end(), (const char *) &age, ((const char *) &age) + sizeof(age));
-  }
-
-  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
-
-  auto offset = PreInsert(segment, N);
-
-  auto res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int) line_sizeof, N);
-
-  assert(res == 0);
-
-  auto memory_usage_size = GetMemoryUsageInBytes(segment);
-
-  assert(memory_usage_size == 1898459);
-
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 namespace {
-    auto generate_data(int N) {
-      std::vector<char> raw_data;
-      std::vector<uint64_t> timestamps;
-      std::vector<int64_t> uids;
-      std::default_random_engine er(42);
-      std::uniform_real_distribution<> distribution(0.0, 1.0);
-      std::default_random_engine ei(42);
-      for (int i = 0; i < N; ++i) {
+auto
+generate_data(int N) {
+    std::vector<char> raw_data;
+    std::vector<uint64_t> timestamps;
+    std::vector<int64_t> uids;
+    std::default_random_engine er(42);
+    std::uniform_real_distribution<> distribution(0.0, 1.0);
+    std::default_random_engine ei(42);
+    for (int i = 0; i < N; ++i) {
         uids.push_back(10 * N + i);
         timestamps.push_back(0);
         // append vec
         float vec[16];
-        for (auto &x: vec) {
-          x = distribution(er);
+        for (auto& x : vec) {
+            x = distribution(er);
         }
-        raw_data.insert(raw_data.end(), (const char *) std::begin(vec), (const char *) std::end(vec));
+        raw_data.insert(
+            raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
         int age = ei() % 100;
-        raw_data.insert(raw_data.end(), (const char *) &age, ((const char *) &age) + sizeof(age));
-      }
-      return std::make_tuple(raw_data, timestamps, uids);
+        raw_data.insert(
+            raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
     }
+    return std::make_tuple(raw_data, timestamps, uids);
 }
+}    // namespace
 
 
 TEST(CApiTest, TestSearchWithIndex) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
 
-  int N = 1000 * 1000;
-  auto[raw_data, timestamps, uids] = generate_data(N);
-  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
-  auto offset = PreInsert(segment, N);
-  auto res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int) line_sizeof, N);
-  assert(res == 0);
+    auto beg = chrono::high_resolution_clock::now();
+    auto next = beg;
+    int N = 1000 * 1000 * 10;
+    auto [raw_data, timestamps, uids] = generate_data(N);
+    auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
 
-  auto row_count = GetRowCount(segment);
-  assert(row_count == N);
-
-  std::vector<long> result_ids(10);
-  std::vector<float> result_distances(10);
-  CQueryInfo queryInfo{1, 10, "fakevec"};
-  auto sea_res = Search(segment, queryInfo, 1, (float *) raw_data.data(), 16, result_ids.data(),
-                        result_distances.data());
-
-  ASSERT_EQ(sea_res, 0);
-  ASSERT_EQ(result_ids[0], 10 * N);
-  ASSERT_EQ(result_distances[0], 0);
-
-  auto N_del = N / 2;
-  std::vector<uint64_t> del_ts(N_del, 100);
-  auto pre_off = PreDelete(segment, N_del);
-  Delete(segment, pre_off, N_del, uids.data(), del_ts.data());
-
-  Close(segment);
-  BuildIndex(segment);
+    next = chrono::high_resolution_clock::now();
+    std::cout << "generate_data: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
 
 
-  std::vector<long> result_ids2(10);
-  std::vector<float> result_distances2(10);
+    auto offset = PreInsert(segment, N);
+    auto res = Insert(segment,
+                      offset,
+                      N,
+                      uids.data(),
+                      timestamps.data(),
+                      raw_data.data(),
+                      (int)line_sizeof,
+                      N);
+    assert(res == 0);
+    next = chrono::high_resolution_clock::now();
+    std::cout << "insert: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
 
-  sea_res = Search(segment, queryInfo, 104, (float *) raw_data.data(), 16, result_ids2.data(),
-                   result_distances2.data());
-//    sea_res = Search(segment, nullptr, 104, result_ids2.data(), result_distances2.data());
+    auto N_del = N / 100;
+    std::vector<uint64_t> del_ts(N_del, 100);
+    auto pre_off = PreDelete(segment, N_del);
+    Delete(segment, pre_off, N_del, uids.data(), del_ts.data());
 
-  std::cout << "case 1" << std::endl;
-  for (int i = 0; i < 10; ++i) {
-    std::cout << result_ids[i] << "->" << result_distances[i] << std::endl;
-  }
-  std::cout << "case 2" << std::endl;
-  for (int i = 0; i < 10; ++i) {
-    std::cout << result_ids2[i] << "->" << result_distances2[i] << std::endl;
-  }
-
-  for (auto x: result_ids2) {
-    ASSERT_GE(x, 10 * N + N_del);
-    ASSERT_LT(x, 10 * N + N);
-  }
-
-//    auto iter = 0;
-//    for(int i = 0; i < result_ids.size(); ++i) {
-//        auto uid = result_ids[i];
-//        auto dis = result_distances[i];
-//        if(uid >= 10 * N + N_del) {
-//            auto uid2 = result_ids2[iter];
-//            auto dis2 = result_distances2[iter];
-//            ASSERT_EQ(uid, uid2);
-//            ASSERT_EQ(dis, dis2);
-//            ++iter;
-//        }
-//    }
+    next = chrono::high_resolution_clock::now();
+    std::cout << "delete1: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
 
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    auto row_count = GetRowCount(segment);
+    assert(row_count == N);
+
+    std::vector<long> result_ids(10 * 16);
+    std::vector<float> result_distances(10 * 16);
+
+    CQueryInfo queryInfo{1, 10, "fakevec"};
+    auto sea_res = Search(segment,
+                          queryInfo,
+                          104,
+                          (float*)raw_data.data(),
+                          16,
+                          result_ids.data(),
+                          result_distances.data());
+
+    //    ASSERT_EQ(sea_res, 0);
+    //    ASSERT_EQ(result_ids[0], 10 * N);
+    //    ASSERT_EQ(result_distances[0], 0);
+
+    next = chrono::high_resolution_clock::now();
+    std::cout << "query1: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
+    sea_res = Search(segment,
+                     queryInfo,
+                     104,
+                     (float*)raw_data.data(),
+                     16,
+                     result_ids.data(),
+                     result_distances.data());
+
+    //    ASSERT_EQ(sea_res, 0);
+    //    ASSERT_EQ(result_ids[0], 10 * N);
+    //    ASSERT_EQ(result_distances[0], 0);
+
+    next = chrono::high_resolution_clock::now();
+    std::cout << "query2: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
+
+    // Close(segment);
+    // BuildIndex(segment);
+
+    next = chrono::high_resolution_clock::now();
+    std::cout << "build index: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
+
+
+    std::vector<int64_t> result_ids2(10);
+    std::vector<float> result_distances2(10);
+
+    sea_res = Search(segment,
+                     queryInfo,
+                     104,
+                     (float*)raw_data.data(),
+                     16,
+                     result_ids2.data(),
+                     result_distances2.data());
+
+    //    sea_res = Search(segment, nullptr, 104, result_ids2.data(),
+    //    result_distances2.data());
+
+    next = chrono::high_resolution_clock::now();
+    std::cout << "search10: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
+
+    sea_res = Search(segment,
+                     queryInfo,
+                     104,
+                     (float*)raw_data.data(),
+                     16,
+                     result_ids2.data(),
+                     result_distances2.data());
+
+    next = chrono::high_resolution_clock::now();
+    std::cout << "search11: "
+              << chrono::duration_cast<chrono::milliseconds>(next - beg).count() << "ms"
+              << std::endl;
+    beg = next;
+
+    //    std::cout << "case 1" << std::endl;
+    //    for (int i = 0; i < 10; ++i) {
+    //        std::cout << result_ids[i] << "->" << result_distances[i] << std::endl;
+    //    }
+    //    std::cout << "case 2" << std::endl;
+    //    for (int i = 0; i < 10; ++i) {
+    //        std::cout << result_ids2[i] << "->" << result_distances2[i] << std::endl;
+    //    }
+    //
+    //    for (auto x : result_ids2) {
+    //        ASSERT_GE(x, 10 * N + N_del);
+    //        ASSERT_LT(x, 10 * N + N);
+    //    }
+
+    //    auto iter = 0;
+    //    for(int i = 0; i < result_ids.size(); ++i) {
+    //        auto uid = result_ids[i];
+    //        auto dis = result_distances[i];
+    //        if(uid >= 10 * N + N_del) {
+    //            auto uid2 = result_ids2[iter];
+    //            auto dis2 = result_distances2[iter];
+    //            ASSERT_EQ(uid, uid2);
+    //            ASSERT_EQ(dis, dis2);
+    //            ++iter;
+    //        }
+    //    }
+
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 TEST(CApiTest, GetDeletedCountTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
-  long delete_primary_keys[] = {100000, 100001, 100002};
-  unsigned long delete_timestamps[] = {0, 0, 0};
+    long delete_primary_keys[] = {100000, 100001, 100002};
+    unsigned long delete_timestamps[] = {0, 0, 0};
 
-  auto offset = PreDelete(segment, 3);
+    auto offset = PreDelete(segment, 3);
 
-  auto del_res = Delete(segment, offset, 3, delete_primary_keys, delete_timestamps);
-  assert(del_res == 0);
+    auto del_res = Delete(segment, offset, 3, delete_primary_keys, delete_timestamps);
+    assert(del_res == 0);
 
-  // TODO: assert(deleted_count == len(delete_primary_keys))
-  auto deleted_count = GetDeletedCount(segment);
-  assert(deleted_count == 0);
+    // TODO: assert(deleted_count == len(delete_primary_keys))
+    auto deleted_count = GetDeletedCount(segment);
+    assert(deleted_count == 0);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }
 
 
 TEST(CApiTest, GetRowCountTest) {
-  auto collection_name = "collection0";
-  auto schema_tmp_conf = "null_schema";
-  auto collection = NewCollection(collection_name, schema_tmp_conf);
-  auto partition_name = "partition0";
-  auto partition = NewPartition(collection, partition_name);
-  auto segment = NewSegment(partition, 0);
+    auto collection_name = "collection0";
+    auto schema_tmp_conf = "null_schema";
+    auto collection = NewCollection(collection_name, schema_tmp_conf);
+    auto partition_name = "partition0";
+    auto partition = NewPartition(collection, partition_name);
+    auto segment = NewSegment(partition, 0);
 
 
-  int N = 10000;
-  auto[raw_data, timestamps, uids] = generate_data(N);
-  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
-  auto offset = PreInsert(segment, N);
-  auto res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int) line_sizeof, N);
-  assert(res == 0);
+    int N = 10000;
+    auto [raw_data, timestamps, uids] = generate_data(N);
+    auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
+    auto offset = PreInsert(segment, N);
+    auto res = Insert(segment,
+                      offset,
+                      N,
+                      uids.data(),
+                      timestamps.data(),
+                      raw_data.data(),
+                      (int)line_sizeof,
+                      N);
+    assert(res == 0);
 
-  auto row_count = GetRowCount(segment);
-  assert(row_count == N);
+    auto row_count = GetRowCount(segment);
+    assert(row_count == N);
 
-  DeleteCollection(collection);
-  DeletePartition(partition);
-  DeleteSegment(segment);
+    DeleteCollection(collection);
+    DeletePartition(partition);
+    DeleteSegment(segment);
 }

go.mod

@@ -44,7 +44,6 @@ require (
 	github.com/prometheus/client_golang v1.5.1 // indirect
 	github.com/prometheus/common v0.10.0 // indirect
 	github.com/prometheus/procfs v0.1.3 // indirect
-	github.com/rs/xid v1.2.1
 	github.com/sirupsen/logrus v1.6.0
 	github.com/stretchr/testify v1.6.1
 	github.com/tikv/client-go v0.0.0-20200824032810-95774393107b

pkg/master/common/config.go

@@ -3,7 +3,7 @@ package common
 import "time"
 
 const (
-	PULSAR_URL              = "pulsar://localhost:6650"
+	PULSAR_URL              = "pulsar://localhost:16650"
 	PULSAR_MONITER_INTERVAL = 1 * time.Second
 	PULSAR_TOPIC            = "monitor-topic"
 	ETCD_ROOT_PATH          = "by-dev"

pkg/master/id/id.go

@@ -1,43 +0,0 @@
-package id
-
-import (
-	"encoding/binary"
-
-	"github.com/rs/xid"
-
-	"github.com/czs007/suvlim/errors"
-)
-
-type ID struct {
-	xid.ID
-}
-
-func BytesToUint64(b []byte) (uint64, error) {
-	if len(b) != 12 {
-		return 0, errors.Errorf("invalid data, must 12 bytes, but %d", len(b))
-	}
-
-	return binary.BigEndian.Uint64(b), nil
-}
-
-// Uint64ToBytes converts uint64 to a byte slice.
-func Uint64ToBytes(v uint64) []byte {
-	b := make([]byte, 12)
-	binary.BigEndian.PutUint64(b, v)
-	return b
-}
-
-func New() ID {
-	return ID{
-		xid.New(),
-	}
-}
-
-func (id ID) Uint64() uint64 {
-	b := id.Bytes()
-	if len(b) != 12 {
-		return 0
-	}
-	return binary.BigEndian.Uint64(b)
-
-}

pkg/master/kv/etcd_kv.go

@@ -34,11 +34,6 @@ func NewEtcdKVBase(client *clientv3.Client, rootPath string) *EtcdKVBase {
 	}
 }
 
-func (kv *EtcdKVBase) Close(){
-	kv.client.Close()
-}
-
-
 func (kv *EtcdKVBase) LoadWithPrefix(key string) ( []string, []string) {
 	key = path.Join(kv.rootPath, key)
 	println("in loadWithPrefix,", key)

pkg/master/mock/collection.go

@@ -1,11 +1,13 @@
 package mock
 
 import (
+	"fmt"
 	"time"
 
 	pb "github.com/czs007/suvlim/pkg/master/grpc/master"
 	messagepb "github.com/czs007/suvlim/pkg/master/grpc/message"
 	"github.com/golang/protobuf/proto"
+	"github.com/google/uuid"
 	jsoniter "github.com/json-iterator/go"
 )
 
@@ -23,9 +25,9 @@ type Collection struct {
 }
 
 type FieldMeta struct {
-	FieldName string             `json:"field_name"`
-	Type      messagepb.DataType `json:"type"`
-	DIM       int64              `json:"dimension"`
+	FieldName string `json:"field_name"`
+	Type      string `json:"type"`
+	DIM       int64  `json:"dimension"`
 }
 
 func GrpcMarshal(c *Collection) *Collection {
@@ -35,16 +37,6 @@ func GrpcMarshal(c *Collection) *Collection {
 	pbSchema := &messagepb.Schema{
 		FieldMetas: []*messagepb.FieldMeta{},
 	}
-	schemaSlice := []*messagepb.FieldMeta{}
-	for _, v := range c.Schema {
-		newpbMeta := &messagepb.FieldMeta{
-			FieldName: v.FieldName,
-			Type:      v.Type,
-			Dim:       v.DIM,
-		}
-		schemaSlice = append(schemaSlice, newpbMeta)
-	}
-	pbSchema.FieldMetas = schemaSlice
 	grpcCollection := &pb.Collection{
 		Id:            c.ID,
 		Name:          c.Name,
@@ -53,24 +45,27 @@ func GrpcMarshal(c *Collection) *Collection {
 		SegmentIds:    c.SegmentIDs,
 		PartitionTags: c.PartitionTags,
 	}
-	out := proto.MarshalTextString(grpcCollection)
-	c.GrpcMarshalString = out
+	out, err := proto.Marshal(grpcCollection)
+	if err != nil {
+		fmt.Println(err)
+	}
+	c.GrpcMarshalString = string(out)
 	return c
 }
 
-func NewCollection(id uint64, name string, createTime time.Time,
-	schema []*messagepb.FieldMeta, sIds []uint64, ptags []string) Collection {
+func NewCollection(id uuid.UUID, name string, createTime time.Time,
+	schema []*messagepb.FieldMeta, sIds []uuid.UUID, ptags []string) Collection {
 
 	segementIDs := []uint64{}
 	newSchema := []FieldMeta{}
 	for _, v := range schema {
-		newSchema = append(newSchema, FieldMeta{FieldName: v.FieldName, Type: v.Type, DIM: v.Dim})
+		newSchema = append(newSchema, FieldMeta{FieldName: v.FieldName, Type: v.Type.String(), DIM: v.Dim})
 	}
 	for _, sid := range sIds {
-		segementIDs = append(segementIDs, sid)
+		segementIDs = append(segementIDs, uint64(sid.ID()))
 	}
 	return Collection{
-		ID:            id,
+		ID:            uint64(id.ID()),
 		Name:          name,
 		CreateTime:    uint64(createTime.Unix()),
 		Schema:        newSchema,

pkg/master/mock/collection_test.go

@@ -6,18 +6,11 @@ import (
 	"time"
 )
 
-var s = FieldMeta{
-	FieldName: "test-schema-1",
-	Type:      1,
-	DIM:       int64(512),
-}
-
 var C = Collection{
 	ID:            uint64(11111),
 	Name:          "test-collection",
 	CreateTime:    uint64(time.Now().Unix()),
 	SegmentIDs:    []uint64{uint64(10111)},
-	Schema:        []FieldMeta{s},
 	PartitionTags: []string{"default"},
 }
 

pkg/master/mock/segment.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"encoding/gob"
 	"time"
+
+	"github.com/google/uuid"
 )
 
 type SegmentStats struct {
@@ -43,10 +45,10 @@ type Segment struct {
 	CollectionName string `json:"collection_name"`
 }
 
-func NewSegment(id uint64, collectioID uint64, cName string, ptag string, chStart int, chEnd int, openTime time.Time, closeTime time.Time) Segment {
+func NewSegment(id uuid.UUID, collectioID uuid.UUID, cName string, ptag string, chStart int, chEnd int, openTime time.Time, closeTime time.Time) Segment {
 	return Segment{
-		SegmentID:      id,
-		CollectionID:   collectioID,
+		SegmentID:      uint64(id.ID()),
+		CollectionID:   uint64(id.ID()),
 		CollectionName: cName,
 		PartitionTag:   ptag,
 		ChannelStart:   chStart,

pkg/master/server.go

@@ -11,10 +11,10 @@ import (
 	"github.com/czs007/suvlim/pkg/master/common"
 	pb "github.com/czs007/suvlim/pkg/master/grpc/master"
 	messagepb "github.com/czs007/suvlim/pkg/master/grpc/message"
-	"github.com/czs007/suvlim/pkg/master/id"
 	"github.com/czs007/suvlim/pkg/master/informer"
 	"github.com/czs007/suvlim/pkg/master/kv"
 	"github.com/czs007/suvlim/pkg/master/mock"
+	"github.com/google/uuid"
 	"go.etcd.io/etcd/clientv3"
 	"google.golang.org/grpc"
 )
@@ -55,7 +55,6 @@ func SegmentStatsController() {
 
 func ComputeCloseTime(ss mock.SegmentStats, kvbase kv.Base) error {
 	if int(ss.MemorySize) > common.SEGMENT_THRESHOLE*0.8 {
-		currentTime := time.Now()
 		memRate := int(ss.MemoryRate)
 		if memRate == 0 {
 			memRate = 1
@@ -69,35 +68,12 @@ func ComputeCloseTime(ss mock.SegmentStats, kvbase kv.Base) error {
 		if err != nil {
 			return err
 		}
-		seg.CloseTimeStamp = uint64(currentTime.Add(time.Duration(sec) * time.Second).Unix())
+		seg.CloseTimeStamp = uint64(time.Now().Add(time.Duration(sec) * time.Second).Unix())
 		updateData, err := mock.Segment2JSON(*seg)
 		if err != nil {
 			return err
 		}
 		kvbase.Save(strconv.Itoa(int(ss.SegementID)), updateData)
-		//create new segment
-		newSegID := id.New().Uint64()
-		newSeg := mock.NewSegment(newSegID, seg.CollectionID, seg.CollectionName, "default", seg.ChannelStart, seg.ChannelEnd, currentTime, time.Unix(1<<36-1, 0))
-		newSegData, err := mock.Segment2JSON(*&newSeg)
-		if err != nil {
-			return err
-		}
-		//save to kv store
-		kvbase.Save(strconv.Itoa(int(newSegID)), newSegData)
-		// update collection data
-		c, _ := kvbase.Load(strconv.Itoa(int(seg.CollectionID)))
-		collection, err := mock.JSON2Collection(c)
-		if err != nil {
-			return err
-		}
-		segIDs := collection.SegmentIDs
-		segIDs = append(segIDs, newSegID)
-		collection.SegmentIDs = segIDs
-		cData, err := mock.Collection2JSON(*collection)
-		if err != nil {
-			return err
-		}
-		kvbase.Save(strconv.Itoa(int(seg.CollectionID)), cData)
 	}
 	return nil
 }
@@ -121,7 +97,7 @@ type GRPCMasterServer struct {
 }
 
 func (ms GRPCMasterServer) CreateCollection(ctx context.Context, in *messagepb.Mapping) (*messagepb.Status, error) {
-	//	ms.CreateRequest <- in2
+	//	ms.CreateRequest <- in
 	fmt.Println("Handle a new create collection request")
 	err := WriteCollection2Datastore(in)
 	if err != nil {
@@ -150,37 +126,27 @@ func CollectionController(ch chan *messagepb.Mapping) {
 	defer cli.Close()
 	kvbase := kv.NewEtcdKVBase(cli, common.ETCD_ROOT_PATH)
 	for collection := range ch {
-		sID := id.New().Uint64()
-		cID := id.New().Uint64()
-		s2ID := id.New().Uint64()
+		sID := uuid.New()
+		cID := uuid.New()
 		fieldMetas := []*messagepb.FieldMeta{}
 		if collection.Schema != nil {
 			fieldMetas = collection.Schema.FieldMetas
 		}
 		c := mock.NewCollection(cID, collection.CollectionName,
-			time.Now(), fieldMetas, []uint64{sID, s2ID},
+			time.Now(), fieldMetas, []uuid.UUID{sID},
 			[]string{"default"})
 		cm := mock.GrpcMarshal(&c)
-		s := mock.NewSegment(sID, cID, collection.CollectionName, "default", 0, 511, time.Now(), time.Unix(1<<36-1, 0))
-		s2 := mock.NewSegment(s2ID, cID, collection.CollectionName, "default", 512, 1023, time.Now(), time.Unix(1<<36-1, 0))
+		s := mock.NewSegment(sID, cID, collection.CollectionName, "default", 0, 100, time.Now(), time.Unix(1<<36-1, 0))
 		collectionData, _ := mock.Collection2JSON(*cm)
 		segmentData, err := mock.Segment2JSON(s)
 		if err != nil {
 			log.Fatal(err)
 		}
-		s2Data, err := mock.Segment2JSON(s2)
+		err = kvbase.Save("collection/"+cID.String(), collectionData)
 		if err != nil {
 			log.Fatal(err)
 		}
-		err = kvbase.Save("collection/"+strconv.FormatUint(cID, 10), collectionData)
-		if err != nil {
-			log.Fatal(err)
-		}
-		err = kvbase.Save("segment/"+strconv.FormatUint(sID, 10), segmentData)
-		if err != nil {
-			log.Fatal(err)
-		}
-		err = kvbase.Save("segment/"+strconv.FormatUint(s2ID, 10), s2Data)
+		err = kvbase.Save("segment/"+sID.String(), segmentData)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -194,33 +160,29 @@ func WriteCollection2Datastore(collection *messagepb.Mapping) error {
 	})
 	defer cli.Close()
 	kvbase := kv.NewEtcdKVBase(cli, common.ETCD_ROOT_PATH)
-	sID := id.New().Uint64()
-	cID := id.New().Uint64()
+	sID := uuid.New()
+	cID := uuid.New()
 	fieldMetas := []*messagepb.FieldMeta{}
 	if collection.Schema != nil {
 		fieldMetas = collection.Schema.FieldMetas
 	}
 	c := mock.NewCollection(cID, collection.CollectionName,
-		time.Now(), fieldMetas, []uint64{sID},
+		time.Now(), fieldMetas, []uuid.UUID{sID},
 		[]string{"default"})
 	cm := mock.GrpcMarshal(&c)
 	s := mock.NewSegment(sID, cID, collection.CollectionName, "default", 0, 100, time.Now(), time.Unix(1<<36-1, 0))
-	collectionData, err := mock.Collection2JSON(*cm)
-	if err != nil {
-		log.Fatal(err)
-		return err
-	}
+	collectionData, _ := mock.Collection2JSON(*cm)
 	segmentData, err := mock.Segment2JSON(s)
 	if err != nil {
 		log.Fatal(err)
 		return err
 	}
-	err = kvbase.Save("collection/"+strconv.FormatUint(cID, 10), collectionData)
+	err = kvbase.Save("collection/"+cID.String(), collectionData)
 	if err != nil {
 		log.Fatal(err)
 		return err
 	}
-	err = kvbase.Save("segment/"+strconv.FormatUint(sID, 10), segmentData)
+	err = kvbase.Save("segment/"+sID.String(), segmentData)
 	if err != nil {
 		log.Fatal(err)
 		return err

proxy/src/CMakeLists.txt

@@ -80,7 +80,7 @@ set( SERVER_LIBS server )
 
 add_executable( milvus_server   ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
                                 )
-target_include_directories(server PRIVATE ${PROJECT_BINARY_DIR}/thirdparty)
+target_include_directories(server PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar-client-cpp/pulsar-client-cpp-src/pulsar-client-cpp/include)
 
 
 target_link_libraries( milvus_server PRIVATE ${SERVER_LIBS} )

proxy/src/config/ConfigMgr.cpp

@@ -82,7 +82,7 @@ ConfigMgr::ConfigMgr() {
         {"network.port", CreateIntegerConfig("network.port", false, 0, 65535, &config.network.port.value,
                                                   19530, nullptr, nullptr)},
 
-
+        
         /* pulsar */
         {"pulsar.address", CreateStringConfig("pulsar.address", false, &config.pulsar.address.value,
                                                     "localhost", nullptr, nullptr)},
@@ -94,17 +94,6 @@ ConfigMgr::ConfigMgr() {
         {"master.port", CreateIntegerConfig("master.port", false, 0, 65535, &config.master.port.value,
                                             6000, nullptr, nullptr)},
 
-        /* etcd */
-        {"etcd.address", CreateStringConfig("etcd.address", false, &config.etcd.address.value, "localhost", nullptr,
-                                            nullptr)},
-        {"etcd.port", CreateIntegerConfig("etcd.port", false, 0, 65535, &config.etcd.port.value,
-            6000,nullptr, nullptr)},
-        {"etcd.rootpath", CreateStringConfig("etcd.rootpath", false, &config.etcd.rootpath.value, "by-dev", nullptr,
-                                             nullptr)},
-
-        /* time sync */
-        {"timesync.interval", CreateIntegerConfig("timesync.interval", false, 0, std::numeric_limits<int64_t >::max(), &config.timesync.interval.value, 10,
-                                                   nullptr, nullptr)},
 
         /* log */
         {"logs.level", CreateStringConfig("logs.level", false, &config.logs.level.value, "debug", nullptr, nullptr)},
@@ -157,9 +146,6 @@ ConfigMgr::Load(const std::string& path) {
     // auto proxy_yaml = yaml["porxy"];
     auto other_yaml = YAML::Node{};
     other_yaml["pulsar"] = yaml["pulsar"];
-    other_yaml["master"] = yaml["master"];
-    other_yaml["etcd"] = yaml["etcd"];
-    other_yaml["timesync"] = yaml["timesync"];
     Flatten(yaml["proxy"], flattened, "");
     Flatten(other_yaml, flattened, "");
     // Flatten(yaml["proxy"], flattened, "");

proxy/src/config/ServerConfig.h

@@ -82,15 +82,6 @@ struct ServerConfig {
       Integer  port{6000};
     }master;
 
-    struct Etcd{
-      String address{"localhost"};
-      Integer port{2379};
-      String rootpath{"by-dev"};
-    }etcd;
-
-    struct TimeSync{
-      Integer interval{10};
-    }timesync;
 
     struct Engine {
         Integer build_index_threshold{4096};

proxy/src/grpc/etcd.grpc.pb.cc

@@ -66,66 +66,5 @@ Watch::Service::~Service() {
 }
 
 
-static const char* KV_method_names[] = {
-  "/etcdserverpb.KV/Range",
-};
-
-std::unique_ptr< KV::Stub> KV::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) {
-  (void)options;
-  std::unique_ptr< KV::Stub> stub(new KV::Stub(channel));
-  return stub;
-}
-
-KV::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel)
-  : channel_(channel), rpcmethod_Range_(KV_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
-  {}
-
-::grpc::Status KV::Stub::Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::etcdserverpb::RangeResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Range_, context, request, response);
-}
-
-void KV::Stub::experimental_async::Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response, std::function<void(::grpc::Status)> f) {
-  ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Range_, context, request, response, std::move(f));
-}
-
-void KV::Stub::experimental_async::Range(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::etcdserverpb::RangeResponse* response, std::function<void(::grpc::Status)> f) {
-  ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Range_, context, request, response, std::move(f));
-}
-
-void KV::Stub::experimental_async::Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
-  ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Range_, context, request, response, reactor);
-}
-
-void KV::Stub::experimental_async::Range(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::etcdserverpb::RangeResponse* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
-  ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Range_, context, request, response, reactor);
-}
-
-::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>* KV::Stub::AsyncRangeRaw(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) {
-  return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::etcdserverpb::RangeResponse>::Create(channel_.get(), cq, rpcmethod_Range_, context, request, true);
-}
-
-::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>* KV::Stub::PrepareAsyncRangeRaw(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) {
-  return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::etcdserverpb::RangeResponse>::Create(channel_.get(), cq, rpcmethod_Range_, context, request, false);
-}
-
-KV::Service::Service() {
-  AddMethod(new ::grpc::internal::RpcServiceMethod(
-      KV_method_names[0],
-      ::grpc::internal::RpcMethod::NORMAL_RPC,
-      new ::grpc::internal::RpcMethodHandler< KV::Service, ::etcdserverpb::RangeRequest, ::etcdserverpb::RangeResponse>(
-          std::mem_fn(&KV::Service::Range), this)));
-}
-
-KV::Service::~Service() {
-}
-
-::grpc::Status KV::Service::Range(::grpc::ServerContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response) {
-  (void) context;
-  (void) request;
-  (void) response;
-  return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-}
-
-
 }  // namespace etcdserverpb
 

proxy/src/grpc/etcd.grpc.pb.h

@@ -229,217 +229,6 @@ class Watch final {
   typedef Service StreamedService;
 };
 
-class KV final {
- public:
-  static constexpr char const* service_full_name() {
-    return "etcdserverpb.KV";
-  }
-  class StubInterface {
-   public:
-    virtual ~StubInterface() {}
-    // Range gets the keys in the range from the key-value store.
-    virtual ::grpc::Status Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::etcdserverpb::RangeResponse* response) = 0;
-    std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::etcdserverpb::RangeResponse>> AsyncRange(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) {
-      return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::etcdserverpb::RangeResponse>>(AsyncRangeRaw(context, request, cq));
-    }
-    std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::etcdserverpb::RangeResponse>> PrepareAsyncRange(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) {
-      return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::etcdserverpb::RangeResponse>>(PrepareAsyncRangeRaw(context, request, cq));
-    }
-    class experimental_async_interface {
-     public:
-      virtual ~experimental_async_interface() {}
-      // Range gets the keys in the range from the key-value store.
-      virtual void Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response, std::function<void(::grpc::Status)>) = 0;
-      virtual void Range(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::etcdserverpb::RangeResponse* response, std::function<void(::grpc::Status)>) = 0;
-      virtual void Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0;
-      virtual void Range(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::etcdserverpb::RangeResponse* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0;
-    };
-    virtual class experimental_async_interface* experimental_async() { return nullptr; }
-  private:
-    virtual ::grpc::ClientAsyncResponseReaderInterface< ::etcdserverpb::RangeResponse>* AsyncRangeRaw(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) = 0;
-    virtual ::grpc::ClientAsyncResponseReaderInterface< ::etcdserverpb::RangeResponse>* PrepareAsyncRangeRaw(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) = 0;
-  };
-  class Stub final : public StubInterface {
-   public:
-    Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel);
-    ::grpc::Status Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::etcdserverpb::RangeResponse* response) override;
-    std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>> AsyncRange(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) {
-      return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>>(AsyncRangeRaw(context, request, cq));
-    }
-    std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>> PrepareAsyncRange(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) {
-      return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>>(PrepareAsyncRangeRaw(context, request, cq));
-    }
-    class experimental_async final :
-      public StubInterface::experimental_async_interface {
-     public:
-      void Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response, std::function<void(::grpc::Status)>) override;
-      void Range(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::etcdserverpb::RangeResponse* response, std::function<void(::grpc::Status)>) override;
-      void Range(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response, ::grpc::experimental::ClientUnaryReactor* reactor) override;
-      void Range(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::etcdserverpb::RangeResponse* response, ::grpc::experimental::ClientUnaryReactor* reactor) override;
-     private:
-      friend class Stub;
-      explicit experimental_async(Stub* stub): stub_(stub) { }
-      Stub* stub() { return stub_; }
-      Stub* stub_;
-    };
-    class experimental_async_interface* experimental_async() override { return &async_stub_; }
-
-   private:
-    std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    class experimental_async async_stub_{this};
-    ::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>* AsyncRangeRaw(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) override;
-    ::grpc::ClientAsyncResponseReader< ::etcdserverpb::RangeResponse>* PrepareAsyncRangeRaw(::grpc::ClientContext* context, const ::etcdserverpb::RangeRequest& request, ::grpc::CompletionQueue* cq) override;
-    const ::grpc::internal::RpcMethod rpcmethod_Range_;
-  };
-  static std::unique_ptr<Stub> NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions());
-
-  class Service : public ::grpc::Service {
-   public:
-    Service();
-    virtual ~Service();
-    // Range gets the keys in the range from the key-value store.
-    virtual ::grpc::Status Range(::grpc::ServerContext* context, const ::etcdserverpb::RangeRequest* request, ::etcdserverpb::RangeResponse* response);
-  };
-  template <class BaseClass>
-  class WithAsyncMethod_Range : public BaseClass {
-   private:
-    void BaseClassMustBeDerivedFromService(const Service* /*service*/) {}
-   public:
-    WithAsyncMethod_Range() {
-      ::grpc::Service::MarkMethodAsync(0);
-    }
-    ~WithAsyncMethod_Range() override {
-      BaseClassMustBeDerivedFromService(this);
-    }
-    // disable synchronous version of this method
-    ::grpc::Status Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/) override {
-      abort();
-      return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-    }
-    void RequestRange(::grpc::ServerContext* context, ::etcdserverpb::RangeRequest* request, ::grpc::ServerAsyncResponseWriter< ::etcdserverpb::RangeResponse>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) {
-      ::grpc::Service::RequestAsyncUnary(0, context, request, response, new_call_cq, notification_cq, tag);
-    }
-  };
-  typedef WithAsyncMethod_Range<Service > AsyncService;
-  template <class BaseClass>
-  class ExperimentalWithCallbackMethod_Range : public BaseClass {
-   private:
-    void BaseClassMustBeDerivedFromService(const Service* /*service*/) {}
-   public:
-    ExperimentalWithCallbackMethod_Range() {
-      ::grpc::Service::experimental().MarkMethodCallback(0,
-        new ::grpc_impl::internal::CallbackUnaryHandler< ::etcdserverpb::RangeRequest, ::etcdserverpb::RangeResponse>(
-          [this](::grpc::ServerContext* context,
-                 const ::etcdserverpb::RangeRequest* request,
-                 ::etcdserverpb::RangeResponse* response,
-                 ::grpc::experimental::ServerCallbackRpcController* controller) {
-                   return this->Range(context, request, response, controller);
-                 }));
-    }
-    void SetMessageAllocatorFor_Range(
-        ::grpc::experimental::MessageAllocator< ::etcdserverpb::RangeRequest, ::etcdserverpb::RangeResponse>* allocator) {
-      static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::etcdserverpb::RangeRequest, ::etcdserverpb::RangeResponse>*>(
-          ::grpc::Service::experimental().GetHandler(0))
-              ->SetMessageAllocator(allocator);
-    }
-    ~ExperimentalWithCallbackMethod_Range() override {
-      BaseClassMustBeDerivedFromService(this);
-    }
-    // disable synchronous version of this method
-    ::grpc::Status Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/) override {
-      abort();
-      return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-    }
-    virtual void Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); }
-  };
-  typedef ExperimentalWithCallbackMethod_Range<Service > ExperimentalCallbackService;
-  template <class BaseClass>
-  class WithGenericMethod_Range : public BaseClass {
-   private:
-    void BaseClassMustBeDerivedFromService(const Service* /*service*/) {}
-   public:
-    WithGenericMethod_Range() {
-      ::grpc::Service::MarkMethodGeneric(0);
-    }
-    ~WithGenericMethod_Range() override {
-      BaseClassMustBeDerivedFromService(this);
-    }
-    // disable synchronous version of this method
-    ::grpc::Status Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/) override {
-      abort();
-      return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-    }
-  };
-  template <class BaseClass>
-  class WithRawMethod_Range : public BaseClass {
-   private:
-    void BaseClassMustBeDerivedFromService(const Service* /*service*/) {}
-   public:
-    WithRawMethod_Range() {
-      ::grpc::Service::MarkMethodRaw(0);
-    }
-    ~WithRawMethod_Range() override {
-      BaseClassMustBeDerivedFromService(this);
-    }
-    // disable synchronous version of this method
-    ::grpc::Status Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/) override {
-      abort();
-      return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-    }
-    void RequestRange(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) {
-      ::grpc::Service::RequestAsyncUnary(0, context, request, response, new_call_cq, notification_cq, tag);
-    }
-  };
-  template <class BaseClass>
-  class ExperimentalWithRawCallbackMethod_Range : public BaseClass {
-   private:
-    void BaseClassMustBeDerivedFromService(const Service* /*service*/) {}
-   public:
-    ExperimentalWithRawCallbackMethod_Range() {
-      ::grpc::Service::experimental().MarkMethodRawCallback(0,
-        new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>(
-          [this](::grpc::ServerContext* context,
-                 const ::grpc::ByteBuffer* request,
-                 ::grpc::ByteBuffer* response,
-                 ::grpc::experimental::ServerCallbackRpcController* controller) {
-                   this->Range(context, request, response, controller);
-                 }));
-    }
-    ~ExperimentalWithRawCallbackMethod_Range() override {
-      BaseClassMustBeDerivedFromService(this);
-    }
-    // disable synchronous version of this method
-    ::grpc::Status Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/) override {
-      abort();
-      return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-    }
-    virtual void Range(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); }
-  };
-  template <class BaseClass>
-  class WithStreamedUnaryMethod_Range : public BaseClass {
-   private:
-    void BaseClassMustBeDerivedFromService(const Service* /*service*/) {}
-   public:
-    WithStreamedUnaryMethod_Range() {
-      ::grpc::Service::MarkMethodStreamed(0,
-        new ::grpc::internal::StreamedUnaryHandler< ::etcdserverpb::RangeRequest, ::etcdserverpb::RangeResponse>(std::bind(&WithStreamedUnaryMethod_Range<BaseClass>::StreamedRange, this, std::placeholders::_1, std::placeholders::_2)));
-    }
-    ~WithStreamedUnaryMethod_Range() override {
-      BaseClassMustBeDerivedFromService(this);
-    }
-    // disable regular version of this method
-    ::grpc::Status Range(::grpc::ServerContext* /*context*/, const ::etcdserverpb::RangeRequest* /*request*/, ::etcdserverpb::RangeResponse* /*response*/) override {
-      abort();
-      return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
-    }
-    // replace default version of method with streamed unary
-    virtual ::grpc::Status StreamedRange(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::etcdserverpb::RangeRequest,::etcdserverpb::RangeResponse>* server_unary_streamer) = 0;
-  };
-  typedef WithStreamedUnaryMethod_Range<Service > StreamedUnaryService;
-  typedef Service SplitStreamedService;
-  typedef WithStreamedUnaryMethod_Range<Service > StreamedService;
-};
-
 }  // namespace etcdserverpb
 
 

proxy/src/grpc/master.pb.cc

@@ -101,7 +101,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_master_2eproto::offsets[] PROT
   PROTOBUF_FIELD_OFFSET(::masterpb::Segment, channel_end_),
   PROTOBUF_FIELD_OFFSET(::masterpb::Segment, open_timestamp_),
   PROTOBUF_FIELD_OFFSET(::masterpb::Segment, close_timestamp_),
-  PROTOBUF_FIELD_OFFSET(::masterpb::Segment, collection_name_),
   ~0u,  // no _has_bits_
   PROTOBUF_FIELD_OFFSET(::masterpb::SegmentStat, _internal_metadata_),
   ~0u,  // no _extensions_
@@ -114,7 +113,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_master_2eproto::offsets[] PROT
 static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   { 0, -1, sizeof(::masterpb::Collection)},
   { 11, -1, sizeof(::masterpb::Segment)},
-  { 24, -1, sizeof(::masterpb::SegmentStat)},
+  { 23, -1, sizeof(::masterpb::SegmentStat)},
 };
 
 static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@@ -128,16 +127,16 @@ const char descriptor_table_protodef_master_2eproto[] PROTOBUF_SECTION_VARIABLE(
   "\215\001\n\nCollection\022\n\n\002id\030\001 \001(\004\022\014\n\004name\030\002 \001(\t"
   "\022#\n\006schema\030\003 \001(\0132\023.milvus.grpc.Schema\022\023\n"
   "\013create_time\030\004 \001(\004\022\023\n\013segment_ids\030\005 \003(\004\022"
-  "\026\n\016partition_tags\030\006 \003(\t\"\301\001\n\007Segment\022\022\n\ns"
+  "\026\n\016partition_tags\030\006 \003(\t\"\250\001\n\007Segment\022\022\n\ns"
   "egment_id\030\001 \001(\004\022\025\n\rcollection_id\030\002 \001(\004\022\025"
   "\n\rpartition_tag\030\003 \001(\t\022\025\n\rchannel_start\030\004"
   " \001(\005\022\023\n\013channel_end\030\005 \001(\005\022\026\n\016open_timest"
-  "amp\030\006 \001(\004\022\027\n\017close_timestamp\030\007 \001(\004\022\027\n\017co"
-  "llection_name\030\010 \001(\t\"K\n\013SegmentStat\022\022\n\nse"
-  "gment_id\030\001 \001(\004\022\023\n\013memory_size\030\002 \001(\004\022\023\n\013m"
-  "emory_rate\030\003 \001(\0022I\n\006Master\022\?\n\020CreateColl"
-  "ection\022\024.milvus.grpc.Mapping\032\023.milvus.gr"
-  "pc.Status\"\000B\010Z\006masterb\006proto3"
+  "amp\030\006 \001(\004\022\027\n\017close_timestamp\030\007 \001(\004\"K\n\013Se"
+  "gmentStat\022\022\n\nsegment_id\030\001 \001(\004\022\023\n\013memory_"
+  "size\030\002 \001(\004\022\023\n\013memory_rate\030\003 \001(\0022I\n\006Maste"
+  "r\022\?\n\020CreateCollection\022\024.milvus.grpc.Mapp"
+  "ing\032\023.milvus.grpc.Status\"\000B\010Z\006masterb\006pr"
+  "oto3"
   ;
 static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_master_2eproto_deps[1] = {
   &::descriptor_table_message_2eproto,
@@ -150,7 +149,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mas
 static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_master_2eproto_once;
 static bool descriptor_table_master_2eproto_initialized = false;
 const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_master_2eproto = {
-  &descriptor_table_master_2eproto_initialized, descriptor_table_protodef_master_2eproto, "master.proto", 549,
+  &descriptor_table_master_2eproto_initialized, descriptor_table_protodef_master_2eproto, "master.proto", 524,
   &descriptor_table_master_2eproto_once, descriptor_table_master_2eproto_sccs, descriptor_table_master_2eproto_deps, 3, 1,
   schemas, file_default_instances, TableStruct_master_2eproto::offsets,
   file_level_metadata_master_2eproto, 3, file_level_enum_descriptors_master_2eproto, file_level_service_descriptors_master_2eproto,
@@ -735,10 +734,6 @@ Segment::Segment(const Segment& from)
   if (!from.partition_tag().empty()) {
     partition_tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_tag_);
   }
-  collection_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-  if (!from.collection_name().empty()) {
-    collection_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.collection_name_);
-  }
   ::memcpy(&segment_id_, &from.segment_id_,
     static_cast<size_t>(reinterpret_cast<char*>(&close_timestamp_) -
     reinterpret_cast<char*>(&segment_id_)) + sizeof(close_timestamp_));
@@ -748,7 +743,6 @@ Segment::Segment(const Segment& from)
 void Segment::SharedCtor() {
   ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Segment_master_2eproto.base);
   partition_tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-  collection_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
   ::memset(&segment_id_, 0, static_cast<size_t>(
       reinterpret_cast<char*>(&close_timestamp_) -
       reinterpret_cast<char*>(&segment_id_)) + sizeof(close_timestamp_));
@@ -761,7 +755,6 @@ Segment::~Segment() {
 
 void Segment::SharedDtor() {
   partition_tag_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-  collection_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
 }
 
 void Segment::SetCachedSize(int size) const {
@@ -780,7 +773,6 @@ void Segment::Clear() {
   (void) cached_has_bits;
 
   partition_tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-  collection_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
   ::memset(&segment_id_, 0, static_cast<size_t>(
       reinterpret_cast<char*>(&close_timestamp_) -
       reinterpret_cast<char*>(&segment_id_)) + sizeof(close_timestamp_));
@@ -844,13 +836,6 @@ const char* Segment::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::in
           CHK_(ptr);
         } else goto handle_unusual;
         continue;
-      // string collection_name = 8;
-      case 8:
-        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 66)) {
-          ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_collection_name(), ptr, ctx, "masterpb.Segment.collection_name");
-          CHK_(ptr);
-        } else goto handle_unusual;
-        continue;
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
@@ -974,21 +959,6 @@ bool Segment::MergePartialFromCodedStream(
         break;
       }
 
-      // string collection_name = 8;
-      case 8: {
-        if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (66 & 0xFF)) {
-          DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString(
-                input, this->mutable_collection_name()));
-          DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
-            this->collection_name().data(), static_cast<int>(this->collection_name().length()),
-            ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE,
-            "masterpb.Segment.collection_name"));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
       default: {
       handle_unusual:
         if (tag == 0) {
@@ -1056,16 +1026,6 @@ void Segment::SerializeWithCachedSizes(
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt64(7, this->close_timestamp(), output);
   }
 
-  // string collection_name = 8;
-  if (this->collection_name().size() > 0) {
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
-      this->collection_name().data(), static_cast<int>(this->collection_name().length()),
-      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
-      "masterpb.Segment.collection_name");
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased(
-      8, this->collection_name(), output);
-  }
-
   if (_internal_metadata_.have_unknown_fields()) {
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
         _internal_metadata_.unknown_fields(), output);
@@ -1120,17 +1080,6 @@ void Segment::SerializeWithCachedSizes(
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteUInt64ToArray(7, this->close_timestamp(), target);
   }
 
-  // string collection_name = 8;
-  if (this->collection_name().size() > 0) {
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
-      this->collection_name().data(), static_cast<int>(this->collection_name().length()),
-      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
-      "masterpb.Segment.collection_name");
-    target =
-      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray(
-        8, this->collection_name(), target);
-  }
-
   if (_internal_metadata_.have_unknown_fields()) {
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
         _internal_metadata_.unknown_fields(), target);
@@ -1159,13 +1108,6 @@ size_t Segment::ByteSizeLong() const {
         this->partition_tag());
   }
 
-  // string collection_name = 8;
-  if (this->collection_name().size() > 0) {
-    total_size += 1 +
-      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
-        this->collection_name());
-  }
-
   // uint64 segment_id = 1;
   if (this->segment_id() != 0) {
     total_size += 1 +
@@ -1239,10 +1181,6 @@ void Segment::MergeFrom(const Segment& from) {
 
     partition_tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_tag_);
   }
-  if (from.collection_name().size() > 0) {
-
-    collection_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.collection_name_);
-  }
   if (from.segment_id() != 0) {
     set_segment_id(from.segment_id());
   }
@@ -1286,8 +1224,6 @@ void Segment::InternalSwap(Segment* other) {
   _internal_metadata_.Swap(&other->_internal_metadata_);
   partition_tag_.Swap(&other->partition_tag_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
     GetArenaNoVirtual());
-  collection_name_.Swap(&other->collection_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
-    GetArenaNoVirtual());
   swap(segment_id_, other->segment_id_);
   swap(collection_id_, other->collection_id_);
   swap(channel_start_, other->channel_start_);

proxy/src/grpc/master.pb.h

@@ -383,7 +383,6 @@ class Segment :
 
   enum : int {
     kPartitionTagFieldNumber = 3,
-    kCollectionNameFieldNumber = 8,
     kSegmentIdFieldNumber = 1,
     kCollectionIdFieldNumber = 2,
     kChannelStartFieldNumber = 4,
@@ -402,17 +401,6 @@ class Segment :
   std::string* release_partition_tag();
   void set_allocated_partition_tag(std::string* partition_tag);
 
-  // string collection_name = 8;
-  void clear_collection_name();
-  const std::string& collection_name() const;
-  void set_collection_name(const std::string& value);
-  void set_collection_name(std::string&& value);
-  void set_collection_name(const char* value);
-  void set_collection_name(const char* value, size_t size);
-  std::string* mutable_collection_name();
-  std::string* release_collection_name();
-  void set_allocated_collection_name(std::string* collection_name);
-
   // uint64 segment_id = 1;
   void clear_segment_id();
   ::PROTOBUF_NAMESPACE_ID::uint64 segment_id() const;
@@ -449,7 +437,6 @@ class Segment :
 
   ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
   ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_tag_;
-  ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr collection_name_;
   ::PROTOBUF_NAMESPACE_ID::uint64 segment_id_;
   ::PROTOBUF_NAMESPACE_ID::uint64 collection_id_;
   ::PROTOBUF_NAMESPACE_ID::int32 channel_start_;
@@ -973,57 +960,6 @@ inline void Segment::set_close_timestamp(::PROTOBUF_NAMESPACE_ID::uint64 value)
   // @@protoc_insertion_point(field_set:masterpb.Segment.close_timestamp)
 }
 
-// string collection_name = 8;
-inline void Segment::clear_collection_name() {
-  collection_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-}
-inline const std::string& Segment::collection_name() const {
-  // @@protoc_insertion_point(field_get:masterpb.Segment.collection_name)
-  return collection_name_.GetNoArena();
-}
-inline void Segment::set_collection_name(const std::string& value) {
-  
-  collection_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
-  // @@protoc_insertion_point(field_set:masterpb.Segment.collection_name)
-}
-inline void Segment::set_collection_name(std::string&& value) {
-  
-  collection_name_.SetNoArena(
-    &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
-  // @@protoc_insertion_point(field_set_rvalue:masterpb.Segment.collection_name)
-}
-inline void Segment::set_collection_name(const char* value) {
-  GOOGLE_DCHECK(value != nullptr);
-  
-  collection_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
-  // @@protoc_insertion_point(field_set_char:masterpb.Segment.collection_name)
-}
-inline void Segment::set_collection_name(const char* value, size_t size) {
-  
-  collection_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
-      ::std::string(reinterpret_cast<const char*>(value), size));
-  // @@protoc_insertion_point(field_set_pointer:masterpb.Segment.collection_name)
-}
-inline std::string* Segment::mutable_collection_name() {
-  
-  // @@protoc_insertion_point(field_mutable:masterpb.Segment.collection_name)
-  return collection_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-}
-inline std::string* Segment::release_collection_name() {
-  // @@protoc_insertion_point(field_release:masterpb.Segment.collection_name)
-  
-  return collection_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
-}
-inline void Segment::set_allocated_collection_name(std::string* collection_name) {
-  if (collection_name != nullptr) {
-    
-  } else {
-    
-  }
-  collection_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), collection_name);
-  // @@protoc_insertion_point(field_set_allocated:masterpb.Segment.collection_name)
-}
-
 // -------------------------------------------------------------------
 
 // SegmentStat

proxy/src/grpc/proto/etcd.proto

@@ -11,90 +11,6 @@ service Watch {
   }
 }
 
-service KV {
-  // Range gets the keys in the range from the key-value store.
-  rpc Range(RangeRequest) returns (RangeResponse) {
-  }
-}
-
-message RangeRequest {
-  enum SortOrder {
-	NONE = 0; // default, no sorting
-	ASCEND = 1; // lowest target value first
-	DESCEND = 2; // highest target value first
-  }
-  enum SortTarget {
-	KEY = 0;
-	VERSION = 1;
-	CREATE = 2;
-	MOD = 3;
-	VALUE = 4;
-  }
-
-  // key is the first key for the range. If range_end is not given, the request only looks up key.
-  bytes key = 1;
-  // range_end is the upper bound on the requested range [key, range_end).
-  // If range_end is '\0', the range is all keys >= key.
-  // If range_end is key plus one (e.g., "aa"+1 == "ab", "a\xff"+1 == "b"),
-  // then the range request gets all keys prefixed with key.
-  // If both key and range_end are '\0', then the range request returns all keys.
-  bytes range_end = 2;
-  // limit is a limit on the number of keys returned for the request. When limit is set to 0,
-  // it is treated as no limit.
-  int64 limit = 3;
-  // revision is the point-in-time of the key-value store to use for the range.
-  // If revision is less or equal to zero, the range is over the newest key-value store.
-  // If the revision has been compacted, ErrCompacted is returned as a response.
-  int64 revision = 4;
-
-  // sort_order is the order for returned sorted results.
-  SortOrder sort_order = 5;
-
-  // sort_target is the key-value field to use for sorting.
-  SortTarget sort_target = 6;
-
-  // serializable sets the range request to use serializable member-local reads.
-  // Range requests are linearizable by default; linearizable requests have higher
-  // latency and lower throughput than serializable requests but reflect the current
-  // consensus of the cluster. For better performance, in exchange for possible stale reads,
-  // a serializable range request is served locally without needing to reach consensus
-  // with other nodes in the cluster.
-  bool serializable = 7;
-
-  // keys_only when set returns only the keys and not the values.
-  bool keys_only = 8;
-
-  // count_only when set returns only the count of the keys in the range.
-  bool count_only = 9;
-
-  // min_mod_revision is the lower bound for returned key mod revisions; all keys with
-  // lesser mod revisions will be filtered away.
-  int64 min_mod_revision = 10;
-
-  // max_mod_revision is the upper bound for returned key mod revisions; all keys with
-  // greater mod revisions will be filtered away.
-  int64 max_mod_revision = 11;
-
-  // min_create_revision is the lower bound for returned key create revisions; all keys with
-  // lesser create revisions will be filtered away.
-  int64 min_create_revision = 12;
-
-  // max_create_revision is the upper bound for returned key create revisions; all keys with
-  // greater create revisions will be filtered away.
-  int64 max_create_revision = 13;
-}
-
-message RangeResponse {
-  ResponseHeader header = 1;
-  // kvs is the list of key-value pairs matched by the range request.
-  // kvs is empty when count is requested.
-  repeated KeyValue kvs = 2;
-  // more indicates if there are more keys to return in the requested range.
-  bool more = 3;
-  // count is set to the number of keys within the range when requested.
-  int64 count = 4;
-}
-
 message WatchRequest {
   // request_union is a request to either create a new watcher or cancel an existing watcher.
   oneof request_union {

proxy/src/message_client/ClientV2.cpp

@@ -2,7 +2,6 @@
 #include "pulsar/Result.h"
 #include "PartitionPolicy.h"
 #include "utils/CommonUtil.h"
-#include "M3_hash.h"
 #include "config/ServerConfig.h"
 #include <omp.h>
 #include <numeric>
@@ -12,10 +11,7 @@ namespace milvus::message_client {
 
 std::map<int64_t, std::vector<std::shared_ptr<grpc::QueryResult>>> total_results;
 
-MsgClientV2::MsgClientV2(int64_t client_id,
-                         const std::string &service_url,
-                         const uint32_t mut_parallelism,
-                         const pulsar::ClientConfiguration &config)
+MsgClientV2::MsgClientV2(int64_t client_id, const std::string &service_url, const uint32_t mut_parallelism, const pulsar::ClientConfiguration &config)
     : client_id_(client_id), service_url_(service_url), mut_parallelism_(mut_parallelism) {}
 
 Status MsgClientV2::Init(const std::string &insert_delete,
@@ -52,13 +48,13 @@ Status MsgClientV2::Init(const std::string &insert_delete,
 }
 
 int64_t GetQueryNodeNum() {
-  return config.query_node_num();
+    return config.query_node_num();
 }
 
 Status
-Aggregation(std::vector<std::shared_ptr<grpc::QueryResult>> results, milvus::grpc::QueryResult *result) {
+Aggregation(std::vector<std::shared_ptr<grpc::QueryResult>> results, milvus::grpc::QueryResult* result) {
   if (results.empty()) {
-    return Status(DB_ERROR, "The result is null!");
+      return  Status(DB_ERROR, "The result is null!");
   }
 
   std::vector<float> all_scores;
@@ -68,101 +64,99 @@ Aggregation(std::vector<std::shared_ptr<grpc::QueryResult>> results, milvus::grp
   std::vector<grpc::RowData> all_row_data;
   std::vector<grpc::KeyValuePair> all_kv_pairs;
 
-  grpc::Status status;
-  int row_num = 0;
+    grpc::Status status;
+    int row_num = 0;
 
-  for (auto &result_per_node : results) {
-    if (result_per_node->status().error_code() != grpc::ErrorCode::SUCCESS) {
+  for (auto & result_per_node : results) {
+      if (result_per_node->status().error_code() != grpc::ErrorCode::SUCCESS){
 //    if (one_node_res->status().error_code() != grpc::ErrorCode::SUCCESS ||
 //        one_node_res->entities().status().error_code() != grpc::ErrorCode::SUCCESS) {
-      return Status(DB_ERROR, "QueryNode return wrong status!");
-    }
-    for (int j = 0; j < result_per_node->distances_size(); j++) {
-      all_scores.push_back(result_per_node->scores()[j]);
-      all_distance.push_back(result_per_node->distances()[j]);
+          return Status(DB_ERROR, "QueryNode return wrong status!");
+      }
+      for (int j = 0; j < result_per_node->distances_size(); j++) {
+          all_scores.push_back(result_per_node->scores()[j]);
+          all_distance.push_back(result_per_node->distances()[j]);
 //          all_kv_pairs.push_back(result_per_node->extra_params()[j]);
-    }
-    for (int k = 0; k < result_per_node->entities().ids_size(); ++k) {
-      all_entities_ids.push_back(result_per_node->entities().ids(k));
+      }
+      for (int k = 0; k < result_per_node->entities().ids_size(); ++k) {
+          all_entities_ids.push_back(result_per_node->entities().ids(k));
 //          all_valid_row.push_back(result_per_node->entities().valid_row(k));
 //          all_row_data.push_back(result_per_node->entities().rows_data(k));
-    }
-    if (result_per_node->row_num() > row_num) {
-      row_num = result_per_node->row_num();
-    }
-    status = result_per_node->status();
+      }
+      if (result_per_node->row_num() > row_num){
+          row_num = result_per_node->row_num();
+      }
+      status = result_per_node->status();
   }
 
-  std::vector<int> index(all_distance.size());
+    std::vector<int> index(all_distance.size());
 
-  iota(index.begin(), index.end(), 0);
+    iota(index.begin(), index.end(), 0);
 
-  std::stable_sort(index.begin(), index.end(),
-                   [&all_distance](size_t i1, size_t i2) { return all_distance[i1] > all_distance[i2]; });
+    std::stable_sort(index.begin(), index.end(),
+                     [&all_distance](size_t i1, size_t i2) {return all_distance[i1] > all_distance[i2];});
 
-  grpc::Entities result_entities;
+    grpc::Entities result_entities;
 
-  for (int m = 0; m < result->row_num(); ++m) {
-    result->add_scores(all_scores[index[m]]);
-    result->add_distances(all_distance[index[m]]);
+    for (int m = 0; m < result->row_num(); ++m) {
+        result->add_scores(all_scores[index[m]]);
+        result->add_distances(all_distance[index[m]]);
 //        result->add_extra_params();
 //        result->mutable_extra_params(m)->CopyFrom(all_kv_pairs[index[m]]);
 
-    result_entities.add_ids(all_entities_ids[index[m]]);
+        result_entities.add_ids(all_entities_ids[index[m]]);
 //        result_entities.add_valid_row(all_valid_row[index[m]]);
 //        result_entities.add_rows_data();
 //        result_entities.mutable_rows_data(m)->CopyFrom(all_row_data[index[m]]);
-  }
+    }
 
-  result_entities.mutable_status()->CopyFrom(status);
+    result_entities.mutable_status()->CopyFrom(status);
 
-  result->set_row_num(row_num);
-  result->mutable_entities()->CopyFrom(result_entities);
-  result->set_query_id(results[0]->query_id());
+    result->set_row_num(row_num);
+    result->mutable_entities()->CopyFrom(result_entities);
+    result->set_query_id(results[0]->query_id());
 //  result->set_client_id(results[0]->client_id());
 
   return Status::OK();
 }
 
-Status MsgClientV2::GetQueryResult(int64_t query_id, milvus::grpc::QueryResult *result) {
+Status MsgClientV2::GetQueryResult(int64_t query_id, milvus::grpc::QueryResult* result) {
 
-  int64_t query_node_num = GetQueryNodeNum();
+    int64_t query_node_num = GetQueryNodeNum();
 
-  auto t1 = std::chrono::high_resolution_clock::now();
+    auto t1 = std::chrono::high_resolution_clock::now();
 
-  while (true) {
-    auto received_result = total_results[query_id];
-    if (received_result.size() == query_node_num) {
-      break;
+    while (true) {
+        auto received_result = total_results[query_id];
+        if (received_result.size() == query_node_num) {
+            break;
+        }
+        Message msg;
+        consumer_->receive(msg);
+
+        grpc::QueryResult search_res_msg;
+        auto status = search_res_msg.ParseFromString(msg.getDataAsString());
+        if (status) {
+            auto message = std::make_shared<grpc::QueryResult>(search_res_msg);
+            total_results[message->query_id()].push_back(message);
+            consumer_->acknowledge(msg);
+        } else {
+            return Status(DB_ERROR, "can't parse message which from pulsar!");
+        }
     }
-    Message msg;
-    consumer_->receive(msg);
+    auto status = Aggregation(total_results[query_id], result);
 
-    grpc::QueryResult search_res_msg;
-    auto status = search_res_msg.ParseFromString(msg.getDataAsString());
-    if (status) {
-      auto message = std::make_shared<grpc::QueryResult>(search_res_msg);
-      total_results[message->query_id()].push_back(message);
-      consumer_->acknowledge(msg);
-    } else {
-      return Status(DB_ERROR, "can't parse message which from pulsar!");
-    }
-  }
-  auto status = Aggregation(total_results[query_id], result);
-
-  return status;
+    return status;
 }
 
-Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
-                                   uint64_t timestamp,
-                                   const std::function<uint64_t(const std::string &collection_name,
-                                                                uint64_t channel_id,
-                                                                uint64_t timestamp)> &segment_id) {
+Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request, uint64_t timestamp) {
   // may have retry policy?
   auto row_count = request.rows_data_size();
-  auto stats = std::vector<Status>(ParallelNum);
+  // TODO: Get the segment from master
+  int64_t segment = 0;
+  auto stats = std::vector<pulsar::Result>(ParallelNum);
 
-#pragma omp parallel for default(none), shared(row_count, request, timestamp, stats, segment_id), num_threads(ParallelNum)
+#pragma omp parallel for default(none), shared(row_count, request, timestamp, segment, stats), num_threads(ParallelNum)
   for (auto i = 0; i < row_count; i++) {
     milvus::grpc::InsertOrDeleteMsg mut_msg;
     int this_thread = omp_get_thread_num();
@@ -172,48 +166,34 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
     mut_msg.set_timestamp(timestamp);
     mut_msg.set_collection_name(request.collection_name());
     mut_msg.set_partition_tag(request.partition_tag());
-    uint64_t uid = request.entity_id_array(i);
-    auto channel_id = makeHash(&uid, sizeof(uint64_t));
-    try {
-      mut_msg.set_segment_id(segment_id(request.collection_name(), channel_id, timestamp));
-      printf("%ld \n", mut_msg.segment_id());
-      mut_msg.mutable_rows_data()->CopyFrom(request.rows_data(i));
-      mut_msg.mutable_extra_params()->CopyFrom(request.extra_params());
+    mut_msg.set_segment_id(segment);
+    mut_msg.mutable_rows_data()->CopyFrom(request.rows_data(i));
+    mut_msg.mutable_extra_params()->CopyFrom(request.extra_params());
 
-      auto result = paralle_mut_producers_[this_thread]->send(mut_msg);
-      if (result != pulsar::ResultOk) {
-        stats[this_thread] = Status(DB_ERROR, pulsar::strResult(result));
-      }
-    }
-    catch (const std::exception &e) {
-      stats[this_thread] = Status(DB_ERROR, "Meta error");
+    auto result = paralle_mut_producers_[this_thread]->send(mut_msg);
+    if (result != pulsar::ResultOk) {
+      stats[this_thread] = result;
     }
   }
   for (auto &stat : stats) {
-    if (!stat.ok()) {
-      return stat;
+    if (stat != pulsar::ResultOk) {
+      return Status(DB_ERROR, pulsar::strResult(stat));
     }
   }
   return Status::OK();
 }
 
-Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request,
-                                   uint64_t timestamp,
-                                   const std::function<uint64_t(const std::string &collection_name,
-                                                                uint64_t channel_id,
-                                                                uint64_t timestamp)> &segment_id) {
+Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request, uint64_t timestamp) {
   auto stats = std::vector<pulsar::Result>(ParallelNum);
-#pragma omp parallel for default(none), shared( request, timestamp, stats, segment_id), num_threads(ParallelNum)
+#pragma omp parallel for default(none), shared( request, timestamp, stats), num_threads(ParallelNum)
   for (auto i = 0; i < request.id_array_size(); i++) {
     milvus::grpc::InsertOrDeleteMsg mut_msg;
     mut_msg.set_op(milvus::grpc::OpType::DELETE);
+    mut_msg.set_uid(GetUniqueQId());
     mut_msg.set_client_id(client_id_);
     mut_msg.set_uid(request.id_array(i));
     mut_msg.set_collection_name(request.collection_name());
     mut_msg.set_timestamp(timestamp);
-    uint64_t uid = request.id_array(i);
-    auto channel_id = makeHash(&uid, sizeof(uint64_t));
-    mut_msg.set_segment_id(segment_id(request.collection_name(), channel_id, timestamp));
 
     int this_thread = omp_get_thread_num();
     auto result = paralle_mut_producers_[this_thread]->send(mut_msg);
@@ -230,51 +210,51 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request,
 }
 
 Status MsgClientV2::SendQueryMessage(const milvus::grpc::SearchParam &request, uint64_t timestamp, int64_t &query_id) {
-  milvus::grpc::SearchMsg search_msg;
+    milvus::grpc::SearchMsg search_msg;
 
-  query_id = GetUniqueQId();
-  search_msg.set_collection_name(request.collection_name());
-  search_msg.set_uid(query_id);
-  //TODO: get client id from master
-  search_msg.set_client_id(1);
-  search_msg.set_timestamp(timestamp);
-  search_msg.set_dsl(request.dsl());
+    query_id = GetUniqueQId();
+    search_msg.set_collection_name(request.collection_name());
+    search_msg.set_uid(query_id);
+    //TODO: get client id from master
+    search_msg.set_client_id(1);
+    search_msg.set_timestamp(timestamp);
+    search_msg.set_dsl(request.dsl());
 
-  milvus::grpc::VectorRowRecord vector_row_recode;
-  std::vector<float> vectors_records;
-  std::string binary_data;
-  for (int i = 0; i < request.vector_param_size(); ++i) {
-    search_msg.add_json(request.vector_param(i).json());
-    for (int j = 0; j < request.vector_param(i).row_record().records_size(); ++j) {
-      for (int k = 0; k < request.vector_param(i).row_record().records(j).float_data_size(); ++k) {
-        vector_row_recode.add_float_data(request.vector_param(i).row_record().records(j).float_data(k));
-      }
-      binary_data.append(request.vector_param(i).row_record().records(j).binary_data());
+    milvus::grpc::VectorRowRecord vector_row_recode;
+    std::vector<float> vectors_records;
+    std::string binary_data;
+    for (int i = 0; i < request.vector_param_size(); ++i) {
+        search_msg.add_json(request.vector_param(i).json());
+        for (int j = 0; j < request.vector_param(i).row_record().records_size(); ++j) {
+            for (int k = 0; k < request.vector_param(i).row_record().records(j).float_data_size(); ++k) {
+                vector_row_recode.add_float_data(request.vector_param(i).row_record().records(j).float_data(k));
+            }
+            binary_data.append(request.vector_param(i).row_record().records(j).binary_data());
+        }
     }
-  }
-  vector_row_recode.set_binary_data(binary_data);
+    vector_row_recode.set_binary_data(binary_data);
 
-  search_msg.mutable_records()->CopyFrom(vector_row_recode);
+    search_msg.mutable_records()->CopyFrom(vector_row_recode);
 
-  for (int m = 0; m < request.partition_tag_size(); ++m) {
-    search_msg.add_partition_tag(request.partition_tag(m));
-  }
+    for (int m = 0; m < request.partition_tag_size(); ++m) {
+        search_msg.add_partition_tag(request.partition_tag(m));
+    }
 
-  for (int l = 0; l < request.extra_params_size(); ++l) {
-    search_msg.mutable_extra_params(l)->CopyFrom(request.extra_params(l));
-  }
+    for (int l = 0; l < request.extra_params_size(); ++l) {
+        search_msg.mutable_extra_params(l)->CopyFrom(request.extra_params(l));
+    }
 
-  auto result = search_producer_->send(search_msg);
-  if (result != pulsar::Result::ResultOk) {
-    return Status(DB_ERROR, pulsar::strResult(result));
-  }
+    auto result = search_producer_->send(search_msg);
+    if (result != pulsar::Result::ResultOk) {
+        return Status(DB_ERROR, pulsar::strResult(result));
+    }
 
-  return Status::OK();
+    return Status::OK();
 }
 
 MsgClientV2::~MsgClientV2() {
   // insert_delete_producer_->close();
-  for (auto &producer: paralle_mut_producers_) {
+  for (auto& producer: paralle_mut_producers_){
     producer->close();
   }
   search_producer_->close();

proxy/src/message_client/ClientV2.h

@@ -24,13 +24,9 @@ class MsgClientV2 {
               const std::string &search_result);
 
   // unpackage batch insert or delete request, and delivery message to pulsar per row
-  Status SendMutMessage(const milvus::grpc::InsertParam &request, uint64_t timestamp, const std::function<uint64_t (const std::string &collection_name,
-                                                                                                              uint64_t channel_id,
-                                                                                                              uint64_t timestam)>&);
+  Status SendMutMessage(const milvus::grpc::InsertParam &request, uint64_t timestamp);
 
-  Status SendMutMessage(const milvus::grpc::DeleteByIDParam &request, uint64_t timestamp, const std::function<uint64_t(const std::string &collection_name,
-                                                                                                                 uint64_t channel_id,
-                                                                                                                 uint64_t timestam) >&);
+  Status SendMutMessage(const milvus::grpc::DeleteByIDParam &request, uint64_t timestamp);
 
   //
   Status SendQueryMessage(const milvus::grpc::SearchParam &request, uint64_t timestamp, int64_t &query_id);

proxy/src/meta/CMakeLists.txt

@@ -2,11 +2,8 @@ include_directories(${PROJECT_BINARY_DIR}/thirdparty/grpc/grpc-src/third_party/p
 include_directories(${PROJECT_BINARY_DIR}/thirdparty/grpc/grpc-src/include)
 
 add_subdirectory( etcd_watcher )
-aux_source_directory( ./master  master_src)
-aux_source_directory(./etcd_client etcd_src)
-add_library(meta
-        ${master_src}
-        ${etcd_src}
+aux_source_directory( ./master master_src)
+add_library(meta ${master_src}
         ./etcd_watcher/Watcher.cpp
         ${PROJECT_SOURCE_DIR}/src/grpc/etcd.pb.cc
         ${PROJECT_SOURCE_DIR}/src/grpc/etcd.grpc.pb.cc

proxy/src/meta/etcd_client/Etcd_client.cpp

@@ -1,23 +0,0 @@
-#include "Etcd_client.h"
-#include "grpc++/grpc++.h"
-
-namespace milvus{
-namespace master{
-
-EtcdClient::EtcdClient(const std::string &addr) {
-  auto channel = grpc::CreateChannel(addr, grpc::InsecureChannelCredentials());
-  stub_ = etcdserverpb::KV::NewStub(channel);
-}
-
-Status
-EtcdClient::Range(const etcdserverpb::RangeRequest& request, etcdserverpb::RangeResponse& response){
-  ::grpc::ClientContext context;
-  auto status = stub_->Range(&context, request, &response);
-  if (!status.ok()){
-    return Status(DB_ERROR, status.error_message());
-  }
-  return Status::OK();
-}
-
-}
-}

proxy/src/meta/etcd_client/Etcd_client.h

@@ -1,18 +0,0 @@
-#include "grpc/etcd.grpc.pb.h"
-#include "utils/Status.h"
-
-namespace milvus {
-namespace master {
-
-class EtcdClient {
- public:
-  explicit EtcdClient(const std::string &addr);
-  Status
-  Range(const etcdserverpb::RangeRequest& request, etcdserverpb::RangeResponse& response);
-
- private:
-  std::unique_ptr<etcdserverpb::KV::Stub> stub_;
-};
-
-}
-}

proxy/src/meta/etcd_watcher/Watcher.h

@@ -30,10 +30,10 @@ class AsyncWatchAction {
   void CancelWatch();
  private:
   // Status status;
-  ::grpc::ClientContext context_;
-  ::grpc::CompletionQueue cq_;
+  grpc::ClientContext context_;
+  grpc::CompletionQueue cq_;
   etcdserverpb::WatchResponse reply_;
-  std::unique_ptr<::grpc::ClientAsyncReaderWriter<etcdserverpb::WatchRequest, etcdserverpb::WatchResponse>> stream_;
+  std::unique_ptr<grpc::ClientAsyncReaderWriter<etcdserverpb::WatchRequest, etcdserverpb::WatchResponse>> stream_;
   std::atomic<bool> cancled_ = false;
 };
 }

proxy/src/meta/master/GrpcClient.cpp

@@ -20,7 +20,7 @@ Status GrpcClient::CreateCollection(const milvus::grpc::Mapping &mapping) {
   ::grpc::Status grpc_status = stub_->CreateCollection(&context, mapping, &response);
 
   if (!grpc_status.ok()) {
-    std::cerr << "CreateHybridCollection gRPC failed!" << grpc_status.error_message() << std::endl;
+    std::cerr << "CreateHybridCollection gRPC failed!" << std::endl;
     return Status(grpc_status.error_code(), grpc_status.error_message());
   }
 

proxy/src/server/MetaWrapper.cpp

@@ -1,159 +1,20 @@
 #include "MetaWrapper.h"
 #include "config/ServerConfig.h"
-#include "nlohmann/json.hpp"
-#include <mutex>
-#include <google/protobuf/text_format.h>
-
-using Collection = masterpb::Collection;
-using Schema = milvus::grpc::Schema;
-using SegmentInfo = masterpb::Segment;
-using JSON = nlohmann::json;
-
-namespace milvus {
+namespace milvus{
 namespace server {
 
-namespace {
-void ParseSegmentInfo(const std::string &json_str, SegmentInfo &segment_info) {
-  auto json = JSON::parse(json_str);
-  segment_info.set_segment_id(json["segment_id"].get<uint64_t>());
-  segment_info.set_partition_tag(json["partition_tag"].get<std::string>());
-  segment_info.set_channel_start(json["channel_start"].get<int32_t>());
-  segment_info.set_channel_end(json["channel_end"].get<int32_t>());
-  segment_info.set_open_timestamp(json["open_timestamp"].get<uint64_t>());
-  segment_info.set_close_timestamp(json["close_timestamp"].get<uint64_t>());
-  segment_info.set_collection_id(json["collection_id"].get<uint64_t>());
-  segment_info.set_collection_name(json["collection_name"].get<std::string>());
-}
-
-void ParseCollectionSchema(const std::string &json_str, Collection &collection) {
-  auto json = JSON::parse(json_str);
-  auto proto_str = json["grpc_marshal_string"].get<std::string>();
-  auto suc = google::protobuf::TextFormat::ParseFromString(proto_str, &collection);
-  if (!suc) {
-    std::cerr << "unmarshal failed" << std::endl;
-  }
-}
-}
-
-bool MetaWrapper::IsCollectionMetaKey(const std::string &key) {
-  return key.rfind(collection_path_, 0) == 0;
-}
-
-bool MetaWrapper::IsSegmentMetaKey(const std::string &key) {
-  return key.rfind(segment_path_, 0) == 0;
-}
-
-MetaWrapper &MetaWrapper::GetInstance() {
+MetaWrapper& MetaWrapper::GetInstance() {
   static MetaWrapper wrapper;
   return wrapper;
 }
 
 Status MetaWrapper::Init() {
-  etcd_root_path_ = config.etcd.rootpath();
-  segment_path_ = etcd_root_path_ + "segment/";
-  collection_path_ = etcd_root_path_ + "collection/";
-
-  auto master_addr = config.master.address() + ":" + std::to_string(config.master.port());
-  master_client_ = std::make_shared<milvus::master::GrpcClient>(master_addr);
-
-  auto etcd_addr = config.etcd.address() + ":" + std::to_string(config.etcd.port());
-  etcd_client_ = std::make_shared<milvus::master::EtcdClient>(etcd_addr);
-
-  // init etcd watcher
-  auto f = [&](const etcdserverpb::WatchResponse &res) {
-    UpdateMeta(res);
-  };
-  watcher_ = std::make_shared<milvus::master::Watcher>(etcd_addr, segment_path_, f, true);
-
-  SyncMeta();
+  auto addr = config.master.address() + ":" + std::to_string(config.master.port());
+  client_ = std::make_shared<milvus::master::GrpcClient>(addr);
 }
 
 std::shared_ptr<milvus::master::GrpcClient> MetaWrapper::MetaClient() {
-  return master_client_;
-}
-
-void MetaWrapper::UpdateMeta(const etcdserverpb::WatchResponse &res) {
-  for (auto &event: res.events()) {
-    auto &event_key = event.kv().key();
-    auto &event_value = event.kv().value();
-
-    if (event.type() == etcdserverpb::Event_EventType::Event_EventType_PUT) {
-      if (event_key.rfind(segment_path_, 0) == 0) {
-        // segment info
-        SegmentInfo segment_info;
-        ParseSegmentInfo(event_value, segment_info);
-        std::unique_lock lock(mutex_);
-        segment_infos_[segment_info.segment_id()] = segment_info;
-        lock.unlock();
-      } else {
-        // table scheme
-        Collection collection;
-        ParseCollectionSchema(event_value, collection);
-        std::unique_lock lock(mutex_);
-        schemas_[collection.name()] = collection;
-        lock.unlock();
-      }
-    }
-    // TODO: Delete event type
-  }
-}
-
-uint64_t MetaWrapper::AskSegmentId(const std::string &collection_name, uint64_t channel_id, uint64_t timestamp) {
-  // TODO: may using some multi index data structure to speed up search
-  // index timestamp: no-unique, seems close timestamp is enough
-  // index collection_name: no-unique
-  // index channel_id: must satisfy channel_start <= channel_id < channel_end
-  std::shared_lock lock(mutex_);
-  for (auto &item: segment_infos_) {
-    auto &segment_info = item.second;
-    uint64_t open_ts = segment_info.open_timestamp();
-    uint64_t close_ts = segment_info.close_timestamp();
-    if (channel_id >= segment_info.channel_start() && channel_id < segment_info.channel_end()
-        && timestamp >= open_ts << 18 && timestamp < close_ts << 18
-        && segment_info.collection_name() == collection_name) {
-      return segment_info.segment_id();
-    }
-  }
-  throw std::runtime_error("Can't find eligible segment");
-}
-
-const Schema &MetaWrapper::AskCollectionSchema(const std::string &collection_name) {
-  std::shared_lock lock(mutex_);
-  if (schemas_.find(collection_name) != schemas_.end()) {
-    return schemas_[collection_name].schema();
-  }
-  throw std::runtime_error("Collection " + collection_name + " not existed");
-}
-
-Status MetaWrapper::SyncMeta() {
-  ::etcdserverpb::RangeRequest request;
-  request.set_key(etcd_root_path_);
-  std::string range_end(etcd_root_path_);
-  int ascii = (int) range_end[range_end.length() - 1];
-  range_end.back() = ascii + 1;
-  request.set_range_end(range_end);
-
-  ::etcdserverpb::RangeResponse response;
-  auto status = etcd_client_->Range(request, response);
-  if (status.ok()) {
-    for (auto &kv : response.kvs()) {
-      if (IsCollectionMetaKey(kv.key())) {
-        Collection collection;
-        ParseCollectionSchema(kv.value(), collection);
-        std::unique_lock lock(mutex_);
-        schemas_[collection.name()] = collection;
-        lock.unlock();
-      } else {
-        assert(IsSegmentMetaKey(kv.key()));
-        SegmentInfo segment_info;
-        ParseSegmentInfo(kv.value(), segment_info);
-        std::unique_lock lock(mutex_);
-        segment_infos_[segment_info.segment_id()] = segment_info;
-        lock.unlock();
-      }
-    }
-  }
-  return status;
+  return client_;
 }
 
 }

proxy/src/server/MetaWrapper.h

@@ -1,19 +1,12 @@
 #include "utils/Status.h"
 #include "meta/master/GrpcClient.h"
-#include "grpc/message.pb.h"
-#include "grpc/master.pb.h"
-#include "meta/etcd_watcher/Watcher.h"
-#include "meta/etcd_client/Etcd_client.h"
-#include "config/ServerConfig.h"
 
-#include <shared_mutex>
-
-namespace milvus {
-namespace server {
+namespace milvus{
+namespace server{
 
 class MetaWrapper {
  public:
-  static MetaWrapper &
+  static MetaWrapper&
   GetInstance();
 
   Status
@@ -22,34 +15,10 @@ class MetaWrapper {
   std::shared_ptr<milvus::master::GrpcClient>
   MetaClient();
 
-  uint64_t
-  AskSegmentId(const std::string &collection_name, uint64_t channel_id, uint64_t timestamp);
-
-  const milvus::grpc::Schema &
-  AskCollectionSchema(const std::string &collection_name);
-
-  Status
-  SyncMeta();
-
  private:
-  bool IsCollectionMetaKey(const std::string &key);
-
-  bool IsSegmentMetaKey(const std::string &key);
-
-  void UpdateMeta(const etcdserverpb::WatchResponse &res);
-
- private:
-  std::shared_ptr<milvus::master::GrpcClient> master_client_;
-  std::shared_ptr<milvus::master::EtcdClient> etcd_client_;
-  std::unordered_map<std::string, masterpb::Collection> schemas_;
-  std::unordered_map<uint64_t, masterpb::Segment> segment_infos_;
-  std::shared_ptr<milvus::master::Watcher> watcher_;
-  std::shared_mutex mutex_;
-
-  std::string etcd_root_path_;
-  std::string segment_path_;
-  std::string collection_path_;
+  std::shared_ptr<milvus::master::GrpcClient> client_;
 };
 
+
 }
 }

proxy/src/server/delivery/ReqHandler.cpp

@@ -43,8 +43,9 @@ namespace milvus {
 namespace server {
 
 Status
-ReqHandler::CreateCollection(const ContextPtr& context, const ::milvus::grpc::Mapping *request) {
-    BaseReqPtr req_ptr = CreateCollectionReq::Create(context, request);
+ReqHandler::CreateCollection(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
+                             milvus::json& json_param) {
+    BaseReqPtr req_ptr = CreateCollectionReq::Create(context, collection_name, fields, json_param);
     ReqScheduler::ExecReq(req_ptr);
     return req_ptr->status();
 }
@@ -71,8 +72,9 @@ ReqHandler::ListCollections(const ContextPtr& context, std::vector<std::string>&
 }
 
 Status
-ReqHandler::GetCollectionInfo(const ContextPtr& context, const ::milvus::grpc::CollectionName *request, ::milvus::grpc::Mapping& response) {
-    BaseReqPtr req_ptr = GetCollectionInfoReq::Create(context, request, response);
+ReqHandler::GetCollectionInfo(const ContextPtr& context, const std::string& collection_name,
+                              CollectionSchema& collection_schema) {
+    BaseReqPtr req_ptr = GetCollectionInfoReq::Create(context, collection_name, collection_schema);
     ReqScheduler::ExecReq(req_ptr);
     return req_ptr->status();
 }

proxy/src/server/delivery/ReqHandler.h

@@ -31,7 +31,8 @@ class ReqHandler {
     ReqHandler() = default;
 
     Status
-    CreateCollection(const ContextPtr& context, const ::milvus::grpc::Mapping *request);
+    CreateCollection(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
+                     milvus::json& json_params);
 
     Status
     DropCollection(const ContextPtr& context, const std::string& collection_name);
@@ -43,7 +44,8 @@ class ReqHandler {
     ListCollections(const ContextPtr& context, std::vector<std::string>& collections);
 
     Status
-    GetCollectionInfo(const ContextPtr& context, const ::milvus::grpc::CollectionName *request, ::milvus::grpc::Mapping& respons);
+    GetCollectionInfo(const ContextPtr& context, const std::string& collection_name,
+                      CollectionSchema& collection_schema);
 
     Status
     GetCollectionStats(const ContextPtr& context, const std::string& collection_name, std::string& collection_stats);

proxy/src/server/delivery/ReqScheduler.cpp

@@ -160,10 +160,13 @@ ReqScheduler::PutToQueue(const BaseReqPtr& req_ptr) {
 
 int64_t ReqScheduler::GetLatestDeliveredReqTime() {
   std::lock_guard lock(time_syc_mtx_);
-  if (!sending_){
-    latest_req_time_ = TSOracle::GetInstance().GetTimeStamp();
+  if (sending_){
+    return latest_req_time_;
   }
-  return latest_req_time_;
+  auto ts =  TSOracle::GetInstance().GetTimeStamp();
+  latest_req_time_ = ts;
+  assert(ts != 0);
+  return ts;
 }
 
 void ReqScheduler::UpdateLatestDeliveredReqTime(int64_t time) {

proxy/src/server/delivery/request/CreateCollectionReq.cpp

@@ -13,30 +13,32 @@
 // #include "db/Utils.h"
 #include "server/ValidationUtil.h"
 #include "utils/Log.h"
-#include "server/MetaWrapper.h"
+#include "utils/TimeRecorder.h"
 
 #include <set>
 
 namespace milvus {
 namespace server {
 
-CreateCollectionReq::CreateCollectionReq(const ContextPtr &context, const ::milvus::grpc::Mapping *request)
+CreateCollectionReq::CreateCollectionReq(const ContextPtr& context, const std::string& collection_name,
+                                         FieldsType& fields, milvus::json& extra_params)
     : BaseReq(context, ReqType::kCreateCollection),
-      request_(request) {
+      collection_name_(collection_name),
+      fields_(fields),
+      extra_params_(extra_params) {
 }
 
 BaseReqPtr
-CreateCollectionReq::Create(const ContextPtr &context, const ::milvus::grpc::Mapping *request) {
-    return std::shared_ptr<BaseReq>(new CreateCollectionReq(context, request));
+CreateCollectionReq::Create(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
+                            milvus::json& extra_params) {
+    return std::shared_ptr<BaseReq>(new CreateCollectionReq(context, collection_name, fields, extra_params));
 }
 
 Status
 CreateCollectionReq::OnExecute() {
-    auto status = MetaWrapper::GetInstance().MetaClient()->CreateCollection(*request_);
-    if (status.ok()){
-      status = MetaWrapper::GetInstance().SyncMeta();
-    }
-    return status;
+
+
+    return Status::OK();
 }
 
 }  // namespace server

proxy/src/server/delivery/request/CreateCollectionReq.h

@@ -22,18 +22,21 @@ namespace server {
 
 class CreateCollectionReq : public BaseReq {
  public:
-  static BaseReqPtr
-  Create(const ContextPtr &context, const ::milvus::grpc::Mapping *request);
+    static BaseReqPtr
+    Create(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
+           milvus::json& extra_params);
 
  protected:
-  CreateCollectionReq(const ContextPtr &context, const ::milvus::grpc::Mapping *request);
+    CreateCollectionReq(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
+                        milvus::json& extra_params);
 
-  Status
-  OnExecute() override;
+    Status
+    OnExecute() override;
 
  private:
-  const std::string collection_name_;
-  const ::milvus::grpc::Mapping *request_;
+    const std::string collection_name_;
+    std::unordered_map<std::string, FieldSchema> fields_;
+    milvus::json extra_params_;
 };
 
 }  // namespace server

proxy/src/server/delivery/request/DeleteEntityByIDReq.cpp

@@ -18,7 +18,6 @@
 #include "server/delivery/request/DeleteEntityByIDReq.h"
 #include "src/server/delivery/ReqScheduler.h"
 #include "server/MessageWrapper.h"
-#include "server/MetaWrapper.h"
 
 #include <memory>
 #include <string>
@@ -44,12 +43,7 @@ DeleteEntityByIDReq::Create(const ContextPtr& context, const ::milvus::grpc::Del
 Status
 DeleteEntityByIDReq::OnExecute() {
   auto &msg_client = MessageWrapper::GetInstance().MessageClient();
-  auto segment_id = [](const std::string &collection_name,
-                       uint64_t channel_id,
-                       uint64_t timestamp) {
-    return MetaWrapper::GetInstance().AskSegmentId(collection_name, channel_id, timestamp);
-  };
-  Status status = msg_client->SendMutMessage(*request_, timestamp_, segment_id);
+  Status status = msg_client->SendMutMessage(*request_, timestamp_);
   return status;
 }
 

proxy/src/server/delivery/request/GetCollectionInfoReq.cpp

@@ -16,36 +16,30 @@
 // #include "server/web_impl/Constants.h"
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"
-#include "server/MetaWrapper.h"
 
 #include <utility>
 
 namespace milvus {
 namespace server {
 
-GetCollectionInfoReq::GetCollectionInfoReq(const ContextPtr& context, const ::milvus::grpc::CollectionName *request, ::milvus::grpc::Mapping& response)
+GetCollectionInfoReq::GetCollectionInfoReq(const ContextPtr& context, const std::string& collection_name,
+                                           CollectionSchema& collection_schema)
     : BaseReq(context, ReqType::kGetCollectionInfo),
-      collection_name_(request->collection_name()),
-      collection_schema_(response) {
+      collection_name_(collection_name),
+      collection_schema_(collection_schema) {
 }
 
 BaseReqPtr
-GetCollectionInfoReq::Create(const ContextPtr& context, const ::milvus::grpc::CollectionName *request, ::milvus::grpc::Mapping& response) {
-    return std::shared_ptr<BaseReq>(new GetCollectionInfoReq(context, request, response));
+GetCollectionInfoReq::Create(const ContextPtr& context, const std::string& collection_name,
+                             CollectionSchema& collection_schema) {
+    return std::shared_ptr<BaseReq>(new GetCollectionInfoReq(context, collection_name, collection_schema));
 }
 
 Status
 GetCollectionInfoReq::OnExecute() {
-  try {
-    auto schema = MetaWrapper::GetInstance().AskCollectionSchema(collection_name_);
-    collection_schema_.mutable_schema()->CopyFrom(schema);
-    collection_schema_.set_collection_name(collection_schema_.collection_name());
-    return Status::OK();
-  }
-  catch (const std::exception& e){
-    return Status{DB_ERROR, e.what()};
-  }
 
+
+    return Status::OK();
 }
 
 }  // namespace server

proxy/src/server/delivery/request/GetCollectionInfoReq.h

@@ -21,18 +21,19 @@ namespace server {
 
 class GetCollectionInfoReq : public BaseReq {
  public:
-  static BaseReqPtr
-  Create(const ContextPtr& context, const ::milvus::grpc::CollectionName *request, ::milvus::grpc::Mapping& response);
+    static BaseReqPtr
+    Create(const ContextPtr& context, const std::string& collection_name, CollectionSchema& collection_schema);
 
  protected:
-  GetCollectionInfoReq(const ContextPtr& context, const ::milvus::grpc::CollectionName *request, ::milvus::grpc::Mapping& response);
+    GetCollectionInfoReq(const ContextPtr& context, const std::string& collection_name,
+                         CollectionSchema& collection_schema);
 
-  Status
-  OnExecute() override;
+    Status
+    OnExecute() override;
 
  private:
-  const std::string collection_name_;
-  ::milvus::grpc::Mapping &collection_schema_;
+    const std::string collection_name_;
+    CollectionSchema& collection_schema_;
 };
 
 }  // namespace server

proxy/src/server/delivery/request/InsertReq.cpp

@@ -16,7 +16,6 @@
 #include "utils/TimeRecorder.h"
 #include "server/delivery/ReqScheduler.h"
 #include "server/MessageWrapper.h"
-#include "server/MetaWrapper.h"
 
 #include <memory>
 #include <string>
@@ -45,13 +44,7 @@ Status
 InsertReq::OnExecute() {
   LOG_SERVER_INFO_ << LogOut("[%s][%ld] ", "insert", 0) << "Execute InsertReq.";
   auto &msg_client = MessageWrapper::GetInstance().MessageClient();
-  auto segment_id = [](const std::string &collection_name,
-                       uint64_t channel_id,
-                       uint64_t timestamp) {
-      return MetaWrapper::GetInstance().AskSegmentId(collection_name, channel_id, timestamp);
-  };
-  Status status;
-  status = msg_client->SendMutMessage(*insert_param_, timestamp_, segment_id);
+  Status status = msg_client->SendMutMessage(*insert_param_, timestamp_);
   return status;
 }
 

proxy/src/server/delivery/request/InsertReq.h

@@ -36,7 +36,7 @@ class InsertReq : public BaseReq {
   OnPostExecute() override ;
 
  private:
-  const grpc::InsertParam *insert_param_;
+  const ::milvus::grpc::InsertParam *insert_param_;
 };
 
 }  // namespace server

proxy/src/server/grpc_impl/GrpcRequestHandler.cpp

@@ -341,7 +341,7 @@ GrpcRequestHandler::CreateCollection(::grpc::ServerContext *context, const ::mil
   CHECK_NULLPTR_RETURN(request);
   LOG_SERVER_INFO_ << LogOut("Request [%s] %s begin.", GetContext(context)->ReqID().c_str(), __func__);
 
-  Status status = req_handler_.CreateCollection(GetContext(context), request);
+  Status status =  MetaWrapper::GetInstance().MetaClient()->CreateCollection(*request);
 
   LOG_SERVER_INFO_ << LogOut("Request [%s] %s end.", GetContext(context)->ReqID().c_str(), __func__);
   SET_RESPONSE(response, status, context)
@@ -468,9 +468,6 @@ GrpcRequestHandler::DescribeCollection(::grpc::ServerContext *context, const ::m
                                        ::milvus::grpc::Mapping *response) {
   LOG_SERVER_INFO_ << LogOut("Request [%s] %s begin.", GetContext(context)->ReqID().c_str(), __func__);
   CHECK_NULLPTR_RETURN(request);
-
-  Status status = req_handler_.GetCollectionInfo(GetContext(context), request, *response);
-  SET_RESPONSE(response->mutable_status(), status, context)
   return ::grpc::Status::OK;
 }
 
@@ -700,17 +697,6 @@ GrpcRequestHandler::Insert(::grpc::ServerContext *context, const ::milvus::grpc:
     return ::grpc::Status::OK;
   }
 
-  // check if collection exist, using `HasCollection` after.
-  try {
-    MetaWrapper::GetInstance().AskCollectionSchema(request->collection_name());
-  }
-  catch (const std::exception& e){
-    // means collection not exit
-    SET_RESPONSE(response->mutable_status(), Status(SERVER_COLLECTION_NOT_EXIST, "Collection not exist " + request->collection_name()), context)
-    return ::grpc::Status::OK;
-  }
-
-
   // generate uid for entities
   //if (request->entity_id_array_size() == 0) {
   //    auto ids = std::vector<int64_t >(request->rows_data_size());
@@ -872,8 +858,8 @@ GrpcRequestHandler::Search(::grpc::ServerContext *context, const ::milvus::grpc:
 
   //TODO: check if the request is legal
 
-  BaseReqPtr req_ptr = SearchReq::Create(GetContext(context), request, response);
-  ReqScheduler::ExecReq(req_ptr);
+    BaseReqPtr req_ptr = SearchReq::Create(GetContext(context), request, response);
+    ReqScheduler::ExecReq(req_ptr);
 
   return ::grpc::Status::OK;
 }

proxy/src/server/grpc_impl/GrpcServer.cpp

@@ -105,7 +105,7 @@ GrpcServer::StartService() {
   int client_id = 0;
   std::string pulsar_server_addr
       (std::string{"pulsar://"} + config.pulsar.address() + ":" + std::to_string(config.pulsar.port()));
-  timesync::TimeSync syc(client_id,GetMessageTimeSyncTime, config.timesync.interval(), pulsar_server_addr, "TimeSync");
+  timesync::TimeSync syc(client_id,GetMessageTimeSyncTime, 400, pulsar_server_addr, "TimeSync");
 
 
     // Add gRPC interceptor

proxy/thirdparty/grpc/CMakeLists.txt

@@ -64,7 +64,7 @@ add_custom_target(generate_suvlim_pb_grpc ALL DEPENDS protoc grpc_cpp_plugin)
 add_custom_command(TARGET generate_suvlim_pb_grpc
 	POST_BUILD
 	COMMAND echo "${PROTOC_EXCUTABLE}"
-#	COMMAND bash "${PROTO_GEN_SCRIPTS_DIR}/generate_go.sh" -p  "${PROTOC_EXCUTABLE}"
+	COMMAND bash "${PROTO_GEN_SCRIPTS_DIR}/generate_go.sh" -p  "${PROTOC_EXCUTABLE}"
 	COMMAND bash "${PROTO_GEN_SCRIPTS_DIR}/generate_cpp.sh" -p "${PROTOC_EXCUTABLE}" -g "${GRPC_CPP_PLUGIN_EXCUTABLE}"
 	COMMAND ${PROTOC_EXCUTABLE} -I "${PROTO_PATH}/proto" --grpc_out "${PROTO_PATH}" --cpp_out "${PROTO_PATH}"
 		--plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}"

proxy/thirdparty/pulsar/CMakeLists.txt

@@ -20,6 +20,20 @@ endif ()
 
 
 message(STATUS "Building pulsar-client-cpp-${PULSAR_CLIENT_CPP_VERSION} from source")
+
+#include(ExternalProject)
+#ExternalProject_Add(
+#        pulsar
+#        URL ${PULSAR_URL}
+#        PREFIX ${pulsar_ROOT}
+#        CONFIGURE_COMMAND cd ${pulsar_ROOT}/src/pulsar/pulsar-client-cpp && cmake -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${pulsar_ROOT}/build .
+#        BUILD_COMMAND  cd ${pulsar_ROOT}/src/pulsar/pulsar-client-cpp && cmake .
+#        BUILD_IN_SOURCE true
+#        INSTALL_COMMAND cd ${pulsar_ROOT}/src/pulsar/pulsar-client-cpp && make install
+#)
+
+
+
 FetchContent_Declare(
         pulsar
         URL ${PULSAR_URL}

reader/main.go

@@ -23,4 +23,4 @@ func main2() {
 	wg.Add(1)
 	reader.StartQueryNode2()
 	wg.Wait()
-}
+}

reader/read_node/collection.go

@@ -14,10 +14,10 @@ package reader
 import "C"
 
 type Collection struct {
-	CollectionPtr  C.CCollection
+	CollectionPtr C.CCollection
 	CollectionName string
-	CollectionID   uint64
-	Partitions     []*Partition
+	CollectionID uint64
+	Partitions []*Partition
 }
 
 func (c *Collection) NewPartition(partitionName string) *Partition {

reader/read_node/index_test.go

@@ -3,17 +3,16 @@ package reader
 import (
 	"encoding/binary"
 	"fmt"
-	"math"
-	"testing"
-
 	msgPb "github.com/czs007/suvlim/pkg/master/grpc/message"
 	"github.com/stretchr/testify/assert"
+	"math"
+	"testing"
 )
 
 func TestIndex_BuildIndex(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 

reader/read_node/meta.go

@@ -3,36 +3,36 @@ package reader
 import (
 	"context"
 	"fmt"
+	"github.com/czs007/suvlim/pkg/master/mock"
 	"reflect"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
-
 	"github.com/czs007/suvlim/conf"
 	"github.com/czs007/suvlim/pkg/master/kv"
-	"github.com/czs007/suvlim/pkg/master/mock"
 	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/mvcc/mvccpb"
 )
 
 const (
 	CollectonPrefix = "/collection/"
-	SegmentPrefix   = "/segment/"
+	SegmentPrefix = "/segment/"
 )
 
+
 func GetCollectionObjId(key string) string {
-	prefix := conf.Config.Etcd.Rootpath + CollectonPrefix
+	prefix  := conf.Config.Etcd.Rootpath + CollectonPrefix
 	return strings.TrimPrefix(key, prefix)
 }
 
 func GetSegmentObjId(key string) string {
-	prefix := conf.Config.Etcd.Rootpath + SegmentPrefix
+	prefix  := conf.Config.Etcd.Rootpath + SegmentPrefix
 	return strings.TrimPrefix(key, prefix)
 }
 
 func isCollectionObj(key string) bool {
-	prefix := conf.Config.Etcd.Rootpath + CollectonPrefix
+	prefix  := conf.Config.Etcd.Rootpath + CollectonPrefix
 	prefix = strings.TrimSpace(prefix)
 	println("prefix is :$", prefix)
 	index := strings.Index(key, prefix)
@@ -41,31 +41,31 @@ func isCollectionObj(key string) bool {
 }
 
 func isSegmentObj(key string) bool {
-	prefix := conf.Config.Etcd.Rootpath + SegmentPrefix
+	prefix  := conf.Config.Etcd.Rootpath + SegmentPrefix
 	prefix = strings.TrimSpace(prefix)
 	index := strings.Index(key, prefix)
 	return index == 0
 }
 
-func printCollectionStruct(obj *mock.Collection) {
+func printCollectionStruct(obj *mock.Collection){
 	v := reflect.ValueOf(obj)
 	v = reflect.Indirect(v)
 	typeOfS := v.Type()
 
-	for i := 0; i < v.NumField(); i++ {
-		if typeOfS.Field(i).Name == "GrpcMarshalString" {
+	for i := 0; i< v.NumField(); i++ {
+		if typeOfS.Field(i).Name == "GrpcMarshalString"{
 			continue
 		}
 		fmt.Printf("Field: %s\tValue: %v\n", typeOfS.Field(i).Name, v.Field(i).Interface())
 	}
 }
 
-func printSegmentStruct(obj *mock.Segment) {
+func printSegmentStruct(obj *mock.Segment){
 	v := reflect.ValueOf(obj)
 	v = reflect.Indirect(v)
 	typeOfS := v.Type()
 
-	for i := 0; i < v.NumField(); i++ {
+	for i := 0; i< v.NumField(); i++ {
 		fmt.Printf("Field: %s\tValue: %v\n", typeOfS.Field(i).Name, v.Field(i).Interface())
 	}
 }
@@ -78,10 +78,6 @@ func (node *QueryNode) processCollectionCreate(id string, value string) {
 		println(err.Error())
 	}
 	printCollectionStruct(collection)
-	newCollection := node.NewCollection(collection.ID, collection.Name, collection.GrpcMarshalString)
-	for _, partitionTag := range collection.PartitionTags {
-		newCollection.NewPartition(partitionTag)
-	}
 }
 
 func (node *QueryNode) processSegmentCreate(id string, value string) {
@@ -92,25 +88,17 @@ func (node *QueryNode) processSegmentCreate(id string, value string) {
 		println(err.Error())
 	}
 	printSegmentStruct(segment)
-	collection := node.GetCollectionByID(segment.CollectionID)
-	if collection != nil {
-		partition := collection.GetPartitionByName(segment.PartitionTag)
-		if partition != nil {
-			partition.NewSegment(int64(segment.SegmentID)) // todo change all to uint64
-		}
-	}
-	// segment.CollectionName
 }
 
 func (node *QueryNode) processCreate(key string, msg string) {
 	println("process create", key, ":", msg)
-	if isCollectionObj(key) {
+	if isCollectionObj(key){
 		objID := GetCollectionObjId(key)
 		node.processCollectionCreate(objID, msg)
-	} else if isSegmentObj(key) {
+	}else if isSegmentObj(key){
 		objID := GetSegmentObjId(key)
 		node.processSegmentCreate(objID, msg)
-	} else {
+	}else {
 		println("can not process create msg:", key)
 	}
 }
@@ -124,10 +112,6 @@ func (node *QueryNode) processSegmentModify(id string, value string) {
 		println(err.Error())
 	}
 	printSegmentStruct(segment)
-	seg, err := node.GetSegmentBySegmentID(int64(segment.SegmentID)) // todo change  to uint64
-	if seg != nil {
-		seg.SegmentCloseTime = segment.CloseTimeStamp
-	}
 }
 
 func (node *QueryNode) processCollectionModify(id string, value string) {
@@ -140,37 +124,37 @@ func (node *QueryNode) processCollectionModify(id string, value string) {
 	printCollectionStruct(collection)
 }
 
-func (node *QueryNode) processModify(key string, msg string) {
+func (node *QueryNode) processModify(key string, msg string){
 	println("process modify")
-	if isCollectionObj(key) {
+	if isCollectionObj(key){
 		objID := GetCollectionObjId(key)
 		node.processCollectionModify(objID, msg)
-	} else if isSegmentObj(key) {
+	}else if isSegmentObj(key){
 		objID := GetSegmentObjId(key)
 		node.processSegmentModify(objID, msg)
-	} else {
+	}else {
 		println("can not process modify msg:", key)
 	}
 }
 
-func (node *QueryNode) processSegmentDelete(id string) {
+
+func (node *QueryNode) processSegmentDelete(id string){
 	println("Delete segment: ", id)
 
 }
-
-func (node *QueryNode) processCollectionDelete(id string) {
+func (node *QueryNode) processCollectionDelete(id string){
 	println("Delete collection: ", id)
 }
 
-func (node *QueryNode) processDelete(key string) {
+func (node *QueryNode) processDelete(key string){
 	println("process delete")
-	if isCollectionObj(key) {
+	if isCollectionObj(key){
 		objID := GetCollectionObjId(key)
 		node.processCollectionDelete(objID)
-	} else if isSegmentObj(key) {
+	}else if isSegmentObj(key){
 		objID := GetSegmentObjId(key)
 		node.processSegmentDelete(objID)
-	} else {
+	}else {
 		println("can not process delete msg:", key)
 	}
 }
@@ -180,8 +164,6 @@ func (node *QueryNode) processResp(resp clientv3.WatchResponse) error {
 	if err != nil {
 		return err
 	}
-	println("processResp!!!!!\n")
-
 	for _, ev := range resp.Events {
 		if ev.IsCreate() {
 			key := string(ev.Kv.Key)
@@ -203,7 +185,7 @@ func (node *QueryNode) processResp(resp clientv3.WatchResponse) error {
 
 func (node *QueryNode) loadCollections() error {
 	keys, values := node.kvBase.LoadWithPrefix(CollectonPrefix)
-	for i := range keys {
+	for i:= range keys{
 		objID := GetCollectionObjId(keys[i])
 		node.processCollectionCreate(objID, values[i])
 	}
@@ -211,7 +193,7 @@ func (node *QueryNode) loadCollections() error {
 }
 func (node *QueryNode) loadSegments() error {
 	keys, values := node.kvBase.LoadWithPrefix(SegmentPrefix)
-	for i := range keys {
+	for i:= range keys{
 		objID := GetSegmentObjId(keys[i])
 		node.processSegmentCreate(objID, values[i])
 	}
@@ -228,7 +210,7 @@ func (node *QueryNode) InitFromMeta() error {
 		Endpoints:   []string{etcdAddr},
 		DialTimeout: 5 * time.Second,
 	})
-	//defer cli.Close()
+	defer cli.Close()
 	node.kvBase = kv.NewEtcdKVBase(cli, conf.Config.Etcd.Rootpath)
 	node.loadCollections()
 	node.loadSegments()
@@ -248,4 +230,4 @@ func (node *QueryNode) RunMetaService(ctx context.Context, wg *sync.WaitGroup) {
 			node.processResp(resp)
 		}
 	}
-}
+}

reader/read_node/partition.go

@@ -14,8 +14,8 @@ package reader
 import "C"
 
 type Partition struct {
-	PartitionPtr   C.CPartition
-	PartitionName  string
+	PartitionPtr C.CPartition
+	PartitionName string
 	OpenedSegments []*Segment
 	ClosedSegments []*Segment
 }

reader/read_node/query_node.go

@@ -102,7 +102,6 @@ func NewQueryNode(queryNodeId uint64, timeSync uint64) *QueryNode {
 
 func (node *QueryNode) Close() {
 	node.messageClient.Close()
-	node.kvBase.Close()
 }
 
 func CreateQueryNode(queryNodeId uint64, timeSync uint64, mc *message_client.MessageClient) *QueryNode {
@@ -157,12 +156,12 @@ func (node *QueryNode) QueryNodeDataInit() {
 	node.insertData = insertData
 }
 
-func (node *QueryNode) NewCollection(collectionID uint64, collectionName string, schemaConfig string) *Collection {
+func (node *QueryNode) NewCollection(collectionName string, schemaConfig string) *Collection {
 	cName := C.CString(collectionName)
 	cSchema := C.CString(schemaConfig)
 	collection := C.NewCollection(cName, cSchema)
 
-	var newCollection = &Collection{CollectionPtr: collection, CollectionName: collectionName, CollectionID: collectionID}
+	var newCollection = &Collection{CollectionPtr: collection, CollectionName: collectionName}
 	node.Collections = append(node.Collections, newCollection)
 
 	return newCollection
@@ -185,7 +184,7 @@ func (node *QueryNode) PrepareBatchMsg() []int {
 func (node *QueryNode) InitQueryNodeCollection() {
 	// TODO: remove hard code, add collection creation request
 	// TODO: error handle
-	var newCollection = node.NewCollection(0, "collection1", "fakeSchema")
+	var newCollection = node.NewCollection("collection1", "fakeSchema")
 	var newPartition = newCollection.NewPartition("partition1")
 	// TODO: add segment id
 	var segment = newPartition.NewSegment(0)

reader/read_node/result_test.go

@@ -1,27 +1,26 @@
 package reader
 
 import (
-	"testing"
-
 	masterPb "github.com/czs007/suvlim/pkg/master/grpc/master"
 	msgPb "github.com/czs007/suvlim/pkg/master/grpc/message"
+	"testing"
 )
 
 func TestResult_PublishSearchResult(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment
 
 	// TODO: start pulsar server
 	const N = 10
-	var entityIDs = msgPb.Entities{
+	var entityIDs = msgPb.Entities {
 		Ids: make([]int64, N),
 	}
-	var result = msgPb.QueryResult{
-		Entities:  &entityIDs,
+	var result = msgPb.QueryResult {
+		Entities: &entityIDs,
 		Distances: make([]float32, N),
 	}
 	for i := 0; i < N; i++ {
@@ -34,7 +33,7 @@ func TestResult_PublishSearchResult(t *testing.T) {
 func TestResult_PublishFailedSearchResult(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment
@@ -46,7 +45,7 @@ func TestResult_PublishFailedSearchResult(t *testing.T) {
 func TestResult_PublicStatistic(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment

reader/read_node/segment.go

@@ -77,18 +77,12 @@ func (s *Segment) Close() error {
 	}
 
 	// Build index after closing segment
-	// go s.buildIndex()
+	s.buildIndex()
 	return nil
 }
 
 func (s *Segment) GetMemSize() uint64 {
-	/*C.GetMemoryUsageInBytes
-	long int
-	GetMemoryUsageInBytes(CSegmentBase c_segment);
-	*/
-	var memoryUsageInBytes = C.GetMemoryUsageInBytes(s.SegmentPtr)
-
-	return uint64(memoryUsageInBytes)
+	return 100000
 }
 
 ////////////////////////////////////////////////////////////////////////////

reader/read_node/segment_service_test.go

@@ -7,7 +7,7 @@ import (
 func TestSegmentManagement_SegmentsManagement(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment
@@ -19,7 +19,7 @@ func TestSegmentManagement_SegmentsManagement(t *testing.T) {
 func TestSegmentManagement_SegmentService(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment
@@ -31,7 +31,7 @@ func TestSegmentManagement_SegmentService(t *testing.T) {
 func TestSegmentManagement_SegmentStatistic(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment
@@ -43,7 +43,7 @@ func TestSegmentManagement_SegmentStatistic(t *testing.T) {
 func TestSegmentManagement_SegmentStatisticService(t *testing.T) {
 	// Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment

reader/read_node/segment_test.go

@@ -3,17 +3,16 @@ package reader
 import (
 	"encoding/binary"
 	"fmt"
-	"math"
-	"testing"
-
 	msgPb "github.com/czs007/suvlim/pkg/master/grpc/message"
 	"github.com/stretchr/testify/assert"
+	"math"
+	"testing"
 )
 
 func TestSegment_ConstructorAndDestructor(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -26,7 +25,7 @@ func TestSegment_ConstructorAndDestructor(t *testing.T) {
 func TestSegment_SegmentInsert(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -71,7 +70,7 @@ func TestSegment_SegmentInsert(t *testing.T) {
 func TestSegment_SegmentDelete(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -96,7 +95,7 @@ func TestSegment_SegmentDelete(t *testing.T) {
 func TestSegment_SegmentSearch(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -122,7 +121,7 @@ func TestSegment_SegmentSearch(t *testing.T) {
 	var records [][]byte
 	for i := 0; i < N; i++ {
 		ids = append(ids, int64(i))
-		timestamps = append(timestamps, uint64(i+1))
+		timestamps = append(timestamps, uint64(i + 1))
 		records = append(records, rawData)
 	}
 
@@ -137,10 +136,10 @@ func TestSegment_SegmentSearch(t *testing.T) {
 	// 6. Do search
 	var queryJson = "{\"field_name\":\"fakevec\",\"num_queries\":1,\"topK\":10}"
 	var queryRawData = make([]float32, 0)
-	for i := 0; i < 16; i++ {
+	for i := 0; i < 16; i ++ {
 		queryRawData = append(queryRawData, float32(i))
 	}
-	var vectorRecord = msgPb.VectorRowRecord{
+	var vectorRecord = msgPb.VectorRowRecord {
 		FloatData: queryRawData,
 	}
 	var searchRes, searchErr = segment.SegmentSearch(queryJson, timestamps[N/2], &vectorRecord)
@@ -156,7 +155,7 @@ func TestSegment_SegmentSearch(t *testing.T) {
 func TestSegment_SegmentPreInsert(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -173,7 +172,7 @@ func TestSegment_SegmentPreInsert(t *testing.T) {
 func TestSegment_SegmentPreDelete(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -192,7 +191,7 @@ func TestSegment_SegmentPreDelete(t *testing.T) {
 func TestSegment_GetStatus(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -209,7 +208,7 @@ func TestSegment_GetStatus(t *testing.T) {
 func TestSegment_Close(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -226,7 +225,7 @@ func TestSegment_Close(t *testing.T) {
 func TestSegment_GetRowCount(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -275,7 +274,7 @@ func TestSegment_GetRowCount(t *testing.T) {
 func TestSegment_GetDeletedCount(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 
@@ -301,100 +300,3 @@ func TestSegment_GetDeletedCount(t *testing.T) {
 	collection.DeletePartition(partition)
 	node.DeleteCollection(collection)
 }
-
-func TestSegment_GetMemSize(t *testing.T) {
-	// 1. Construct node, collection, partition and segment
-	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
-	var partition = collection.NewPartition("partition0")
-	var segment = partition.NewSegment(0)
-
-	// 2. Create ids and timestamps
-	ids := []int64{1, 2, 3}
-	timestamps := []uint64{0, 0, 0}
-
-	// 3. Create records, use schema below:
-	// schema_tmp->AddField("fakeVec", DataType::VECTOR_FLOAT, 16);
-	// schema_tmp->AddField("age", DataType::INT32);
-	const DIM = 16
-	const N = 3
-	var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
-	var rawData []byte
-	for _, ele := range vec {
-		buf := make([]byte, 4)
-		binary.LittleEndian.PutUint32(buf, math.Float32bits(ele))
-		rawData = append(rawData, buf...)
-	}
-	bs := make([]byte, 4)
-	binary.LittleEndian.PutUint32(bs, 1)
-	rawData = append(rawData, bs...)
-	var records [][]byte
-	for i := 0; i < N; i++ {
-		records = append(records, rawData)
-	}
-
-	// 4. Do PreInsert
-	var offset = segment.SegmentPreInsert(N)
-	assert.GreaterOrEqual(t, offset, int64(0))
-
-	// 5. Do Insert
-	var err = segment.SegmentInsert(offset, &ids, &timestamps, &records)
-	assert.NoError(t, err)
-
-	// 6. Get memory usage in bytes
-	var memSize = segment.GetMemSize()
-	assert.Equal(t, memSize, uint64(1048714))
-
-	// 7. Destruct node, collection, and segment
-	partition.DeleteSegment(segment)
-	collection.DeletePartition(partition)
-	node.DeleteCollection(collection)
-}
-
-func TestSegment_RealSchemaTest(t *testing.T) {
-	// 1. Construct node, collection, partition and segment
-	// var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n  field_metas: \u003c\n    field_name: \"field_1\"\n    type: INT64\n  \u003e\n  field_metas: \u003c\n    field_name: \"field_2\"\n    type: FLOAT\n  \u003e\n  field_metas: \u003c\n    field_name: \"field_3\"\n    type: INT32\n  \u003e\n  field_metas: \u003c\n    field_name: \"field_vec\"\n    type: VECTOR_FLOAT\n  \u003e\n\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n"
-	// var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n  field_metas: \u003c\n    field_name: \"age\"\n    type: INT32\n  \u003e\n  field_metas: \u003c\n    field_name: \"fakevec\"\n    type: VECTOR_FLOAT\n  \u003e\n\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n"
-	var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n  field_metas: \u003c\n    field_name: \"age\"\n    type: INT32\n    dim: 1\n  \u003e\n  field_metas: \u003c\n    field_name: \"field_1\"\n    type: VECTOR_FLOAT\n    dim: 16\n  \u003e\n\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n"
-	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", schemaString)
-	var partition = collection.NewPartition("partition0")
-	var segment = partition.NewSegment(0)
-
-	// 2. Create ids and timestamps
-	ids := []int64{1, 2, 3}
-	timestamps := []uint64{0, 0, 0}
-
-	// 3. Create records, use schema below:
-	// schema_tmp->AddField("fakeVec", DataType::VECTOR_FLOAT, 16);
-	// schema_tmp->AddField("age", DataType::INT32);
-	const DIM = 16
-	const N = 3
-	var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
-	var rawData []byte
-	for _, ele := range vec {
-		buf := make([]byte, 4)
-		binary.LittleEndian.PutUint32(buf, math.Float32bits(ele))
-		rawData = append(rawData, buf...)
-	}
-	bs := make([]byte, 4)
-	binary.LittleEndian.PutUint32(bs, 1)
-	rawData = append(rawData, bs...)
-	var records [][]byte
-	for i := 0; i < N; i++ {
-		records = append(records, rawData)
-	}
-
-	// 4. Do PreInsert
-	var offset = segment.SegmentPreInsert(N)
-	assert.GreaterOrEqual(t, offset, int64(0))
-
-	// 5. Do Insert
-	var err = segment.SegmentInsert(offset, &ids, &timestamps, &records)
-	assert.NoError(t, err)
-
-	// 6. Destruct node, collection, and segment
-	partition.DeleteSegment(segment)
-	collection.DeletePartition(partition)
-	node.DeleteCollection(collection)
-}

reader/read_node/util_functions.go

@@ -29,16 +29,6 @@ func (node *QueryNode) GetKey2Segments() (*[]int64, *[]uint64, *[]int64) {
 	return &entityIDs, &timestamps, &segmentIDs
 }
 
-func (node *QueryNode) GetCollectionByID(collectionID uint64) *Collection {
-	for _, collection := range node.Collections {
-		if collection.CollectionID == collectionID {
-			return collection
-		}
-	}
-
-	return nil
-}
-
 func (node *QueryNode) GetCollectionByCollectionName(collectionName string) (*Collection, error) {
 	for _, collection := range node.Collections {
 		if collection.CollectionName == collectionName {
@@ -58,13 +48,3 @@ func (node *QueryNode) GetSegmentBySegmentID(segmentID int64) (*Segment, error)
 
 	return targetSegment, nil
 }
-
-func (c *Collection) GetPartitionByName(partitionName string) (partition *Partition) {
-	for _, partition := range c.Partitions {
-		if partition.PartitionName == partitionName {
-			return partition
-		}
-	}
-	return nil
-	// TODO: remove from c.Partitions
-}

reader/read_node/util_functions_test.go

@@ -1,9 +1,8 @@
 package reader
 
 import (
-	"testing"
-
 	"github.com/stretchr/testify/assert"
+	"testing"
 )
 
 func TestUtilFunctions_GetKey2Segments(t *testing.T) {
@@ -13,21 +12,18 @@ func TestUtilFunctions_GetKey2Segments(t *testing.T) {
 func TestUtilFunctions_GetCollectionByCollectionName(t *testing.T) {
 	// 1. Construct node, and collections
 	node := NewQueryNode(0, 0)
-	var _ = node.NewCollection(0, "collection0", "")
+	var _ = node.NewCollection("collection0", "fake schema")
 
 	// 2. Get collection by collectionName
 	var c0, err = node.GetCollectionByCollectionName("collection0")
 	assert.NoError(t, err)
 	assert.Equal(t, c0.CollectionName, "collection0")
-	c0 = node.GetCollectionByID(0)
-	assert.NotNil(t, c0)
-	assert.Equal(t, c0.CollectionID, 0)
 }
 
 func TestUtilFunctions_GetSegmentBySegmentID(t *testing.T) {
 	// 1. Construct node, collection, partition and segment
 	node := NewQueryNode(0, 0)
-	var collection = node.NewCollection(0, "collection0", "")
+	var collection = node.NewCollection("collection0", "fake schema")
 	var partition = collection.NewPartition("partition0")
 	var segment = partition.NewSegment(0)
 	node.SegmentsMap[0] = segment

sdk/CMakeLists.txt

@@ -66,7 +66,6 @@ endif ()
 
 aux_source_directory(interface interface_files)
 aux_source_directory(grpc grpc_client_files)
-aux_source_directory(utils utils_files)
 
 set(grpc_service_files
         grpc-gen/message.grpc.pb.cc
@@ -85,7 +84,6 @@ add_library(milvus_sdk SHARED
         ${interface_files}
         ${grpc_client_files}
         ${grpc_service_files}
-	${utils_files}
         )
 
 target_link_libraries(milvus_sdk

sdk/examples/CMakeLists.txt

@@ -12,3 +12,14 @@
 #-------------------------------------------------------------------------------
 
 add_subdirectory(simple)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/utils UTIL_SRC_FILES)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/common COMMON_SRC_FILES)
+
+file( GLOB APP_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp )
+foreach( sourcefile ${APP_SOURCES} )
+    file(RELATIVE_PATH filename ${CMAKE_CURRENT_SOURCE_DIR} ${sourcefile})     
+    string( REPLACE ".cpp" "" program ${filename} )
+    add_executable( ${program} ${sourcefile} ${COMMON_SRC_FILES} ${UTIL_SRC_FILES})
+    target_link_libraries( ${program} milvus_sdk pthread )
+    install(TARGETS ${program} DESTINATION bin)
+endforeach( sourcefile ${APP_SOURCES} )

sdk/examples/simple/CMakeLists.txt

@@ -1,23 +1,17 @@
-#-------------------------------------------------------------------------------
-# Copyright (C) 2019-2020 Zilliz. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied. See the License for the specific language governing permissions and limitations under the License.
-#-------------------------------------------------------------------------------
+add_executable(search search.cpp)
+target_link_libraries(search milvus_sdk pthread)
+install(TARGETS search DESTINATION test)
 
-aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../common COMMON_SRC_FILES)
 
-file( GLOB APP_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp )
-foreach( sourcefile ${APP_SOURCES} )
-    file(RELATIVE_PATH filename ${CMAKE_CURRENT_SOURCE_DIR} ${sourcefile})     
-    string( REPLACE ".cpp" "" program ${filename} )
-    add_executable( ${program} ${sourcefile} ${COMMON_SRC_FILES})
-    target_link_libraries( ${program} milvus_sdk pthread )
-    install(TARGETS ${program} DESTINATION bin)
-endforeach( sourcefile ${APP_SOURCES} )
+
+
+add_executable(insert insert.cpp)
+target_link_libraries(insert milvus_sdk pthread)
+install(TARGETS insert DESTINATION test)
+
+
+
+
+add_executable(delete delete.cpp)
+target_link_libraries(delete milvus_sdk pthread)
+install(TARGETS delete DESTINATION test)

sdk/examples/simple/CreateCollection.cpp

@@ -1,52 +0,0 @@
-
-#include <Status.h>
-#include <Field.h>
-#include <MilvusApi.h>
-#include <interface/ConnectionImpl.h>
-#include "utils/Utils.h"
-
-int main(int argc , char**argv) {
-  
-  TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv);
-  if (!parameters.is_valid){
-    return 0;
-  }
-  auto client = milvus::ConnectionImpl();
-  milvus::ConnectParam connect_param;
-  connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_;
-  connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ;
-  client.Connect(connect_param);
-
-  milvus::Status stat;
-  const std::string collectin_name = "collection0";
-
-  // Create
-  milvus::FieldPtr field_ptr1 = std::make_shared<milvus::Field>();
-  milvus::FieldPtr field_ptr2 = std::make_shared<milvus::Field>();
-  milvus::FieldPtr field_ptr3 = std::make_shared<milvus::Field>();
-  milvus::FieldPtr field_ptr4 = std::make_shared<milvus::Field>();
-
-  field_ptr1->field_name = "field_1";
-  field_ptr1->field_type = milvus::DataType::INT64;
-
-  field_ptr2->field_name = "field_2";
-  field_ptr2->field_type = milvus::DataType::FLOAT;
-
-  field_ptr3->field_name = "field_3";
-  field_ptr3->field_type = milvus::DataType::INT32;
-
-  field_ptr4->field_name = "field_vec";
-  field_ptr4->field_type = milvus::DataType::VECTOR_FLOAT;
-
-  milvus::Mapping mapping = {collectin_name, {field_ptr1, field_ptr2, field_ptr3, field_ptr4}};
-
-  stat = client.CreateCollection(mapping, "test_extra_params");
-
-
-  // Get Collection info
-  milvus::Mapping map;
-  client.GetCollectionInfo(collectin_name, map);
-  for (auto &f : map.fields) {
-    std::cout << f->field_name << ":" << int(f->field_type) << ":" << f->dim << "DIM" << std::endl;
-  }
-}

sdk/examples/simple/delete.cpp

@@ -13,19 +13,16 @@
 #include <libgen.h>
 #include <cstring>
 #include <string>
+
 #include "interface/ConnectionImpl.h"
-#include "utils/Utils.h"
+#include "ip.h"
+
 int
 main(int argc, char *argv[]) {
-  TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv);
-    if (!parameters.is_valid){
-    return 0;
-  }
   auto client = milvus::ConnectionImpl();
   milvus::ConnectParam connect_param;
-  connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_;
-  connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ;
-
+  connect_param.ip_address = IP;
+  connect_param.port = "19530";
   client.Connect(connect_param);
 
   std::vector<int64_t> delete_ids;

sdk/examples/simple/insert.cpp

@@ -15,19 +15,17 @@
 #include <string>
 #include <iostream>
 
-#include "utils/Utils.h"
+#include "examples/utils/Utils.h"
 
 #include "grpc/ClientProxy.h"
 #include "interface/ConnectionImpl.h"
-#include "utils/TimeRecorder.h"
+#include "ip.h"
 
 const int N = 100;
 const int DIM = 16;
-const int LOOP = 10;
 
 const milvus::FieldValue GetData() {
   milvus::FieldValue value_map;
-
   std::vector<int32_t> int32_data;
   for (int i = 0; i < N; i++) {
     int32_data.push_back(i);
@@ -48,29 +46,15 @@ const milvus::FieldValue GetData() {
 
 int
 main(int argc, char* argv[]) {
-  TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv);
-    if (!parameters.is_valid){
-    return 0;
-   }
-
-    auto client = milvus::ConnectionImpl();
-    milvus::ConnectParam connect_param;
-    connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_;
-    connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ;
-    client.Connect(connect_param);
-    std::vector<int64_t> ids_array;
-    auto data = GetData();
-    for (int64_t i = 0; i < N; i++) {
-        ids_array.push_back(i);
-    }
-
-    milvus_sdk::TimeRecorder insert("insert");
-    for (int j = 0; j < LOOP; ++j) {
-        auto status = client.Insert("collection1", "tag01", data, ids_array);
-        if (!status.ok()){
-            return -1;
-        }
-    }
-
-    return 0;
+  auto client = milvus::ConnectionImpl();
+  milvus::ConnectParam connect_param;
+  connect_param.ip_address = IP;
+  connect_param.port = "19530";
+  client.Connect(connect_param);
+  std::vector <int64_t> ids_array;
+  auto data = GetData();
+  for (int64_t i = 0; i < N; i++) {
+    ids_array.push_back(i);
+  }
+  auto status = client.Insert("collection1", "tag01", data, ids_array);
 }

sdk/examples/simple/ip.h

@@ -0,0 +1,16 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+
+
+const std::string IP = "localhost";
+//const std::string IP = "192.168.2.9";
+//const std::string IP = "192.168.2.28";

sdk/examples/simple/search.cpp

@@ -15,21 +15,15 @@
 #include "include/MilvusApi.h"
 #include "grpc/ClientProxy.h"
 #include "interface/ConnectionImpl.h"
-#include "utils/TimeRecorder.h"
-#include "utils/Utils.h"
+#include "ip.h"
 
 const int TOP_K = 10;
 
 int main(int argc , char**argv) {
-  TestParameters parameters = milvus_sdk::Utils::ParseTestParameters(argc, argv);
-  if (!parameters.is_valid){
-    return 0;
-   }
-
   auto client = milvus::ConnectionImpl();
   milvus::ConnectParam connect_param;
-  connect_param.ip_address = parameters.address_.empty() ? "127.0.0.1":parameters.address_;
-  connect_param.port = parameters.port_.empty() ? "19530":parameters.port_ ;
+  connect_param.ip_address = IP;
+  connect_param.port = "19530";
   client.Connect(connect_param);
   std::vector<int64_t> ids_array;
   std::vector<std::string> partition_list;
@@ -62,9 +56,13 @@ int main(int argc , char**argv) {
 
   milvus::TopKQueryResult result;
 
-  milvus_sdk::TimeRecorder test_search("search");
+
+  auto t1 = std::chrono::high_resolution_clock::now();
   auto status = client.Search("collection1", partition_list, "dsl", vectorParam, result);
 
-    return 0;
-}
+  auto t2 = std::chrono::high_resolution_clock::now();
+  auto duration = std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1).count();
 
+  std::cout << "Query run time: " << duration/1000.0 << "ms" << std::endl;
+  return 0;
+}

sdk/examples/utils/TimeRecorder.cpp

@@ -9,7 +9,7 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
-#include "TimeRecorder.h"
+#include "examples/utils/TimeRecorder.h"
 
 #include <iostream>
 

sdk/examples/utils/Utils.cpp

@@ -9,7 +9,7 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
-#include "Utils.h"
+#include "examples/utils/Utils.h"
 
 #include <time.h>
 #include <unistd.h>
@@ -25,7 +25,7 @@
 #include <cstring>
 #include <string>
 
-#include "TimeRecorder.h"
+#include "examples/utils/TimeRecorder.h"
 
 namespace {
 

sdk/grpc/ClientProxy.cpp

@@ -407,24 +407,32 @@ ClientProxy::Disconnect() {
 Status
 ClientProxy::CreateCollection(const Mapping& mapping, const std::string& extra_params) {
     try {
-        ::milvus::grpc::Mapping grpc_mapping;
-        grpc_mapping.set_collection_name(mapping.collection_name);
-        for (auto& field : mapping.fields) {
-            auto grpc_field = grpc_mapping.mutable_schema()->add_field_metas();
-            grpc_field->set_field_name(field->field_name);
-            grpc_field->set_type((::milvus::grpc::DataType)field->field_type);
-            grpc_field->set_dim(field->dim);
+//        ::milvus::grpc::Mapping grpc_mapping;
+//        grpc_mapping.set_collection_name(mapping.collection_name);
+//        for (auto& field : mapping.fields) {
+//            auto grpc_field = grpc_mapping.add_fields();
+//            grpc_field->set_name(field->field_name);
+//            grpc_field->set_type((::milvus::grpc::DataType)field->field_type);
+//            JSON json_index_param = JSON::parse(field->index_params);
+//            for (auto& json_param : json_index_param.items()) {
+//                auto grpc_index_param = grpc_field->add_index_params();
+//                grpc_index_param->set_key(json_param.key());
+//                grpc_index_param->set_value(json_param.value());
+//            }
+//
+//            auto grpc_extra_param = grpc_field->add_extra_params();
+//            grpc_extra_param->set_key(EXTRA_PARAM_KEY);
+//            grpc_extra_param->set_value(field->extra_params);
+//        }
+//        auto grpc_param = grpc_mapping.add_extra_params();
+//        grpc_param->set_key(EXTRA_PARAM_KEY);
+//        grpc_param->set_value(extra_params);
 
-        }
-        auto grpc_param = grpc_mapping.add_extra_params();
-        grpc_param->set_key(EXTRA_PARAM_KEY);
-        grpc_param->set_value(extra_params);
-
-        return client_ptr_->CreateCollection(grpc_mapping);
+//        return client_ptr_->CreateCollection(grpc_mapping);
+          return Status::OK();
     } catch (std::exception& ex) {
         return Status(StatusCode::UnknownError, "Failed to create collection: " + std::string(ex.what()));
     }
-    return Status::OK();
 }
 
 Status

sdk/grpc/GrpcClient.cpp

@@ -37,19 +37,19 @@ GrpcClient::~GrpcClient() = default;
 
 Status
 GrpcClient::CreateCollection(const milvus::grpc::Mapping& mapping) {
-    ClientContext context;
-    ::milvus::grpc::Status response;
-    ::grpc::Status grpc_status = stub_->CreateCollection(&context, mapping, &response);
-
-    if (!grpc_status.ok()) {
-        std::cerr << "CreateHybridCollection gRPC failed!" << std::endl;
-        return Status(StatusCode::RPCFailed, grpc_status.error_message());
-    }
-
-    if (response.error_code() != grpc::SUCCESS) {
-        std::cerr << response.reason() << std::endl;
-        return Status(StatusCode::ServerFailed, response.reason());
-    }
+//    ClientContext context;
+//    ::milvus::grpc::Status response;
+//    ::grpc::Status grpc_status = stub_->CreateCollection(&context, mapping, &response);
+//
+//    if (!grpc_status.ok()) {
+//        std::cerr << "CreateHybridCollection gRPC failed!" << std::endl;
+//        return Status(StatusCode::RPCFailed, grpc_status.error_message());
+//    }
+//
+//    if (response.error_code() != grpc::SUCCESS) {
+//        std::cerr << response.reason() << std::endl;
+//        return Status(StatusCode::ServerFailed, response.reason());
+//    }
     return Status::OK();
 }