Add segment status and management

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

core/include/segment_c.h

@@ -2,6 +2,7 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include "partition_c.h"
 
 typedef void* CSegmentBase;
@@ -12,6 +13,8 @@ NewSegment(CPartition partition, unsigned long segment_id);
 void
 DeleteSegment(CSegmentBase segment);
 
+//////////////////////////////////////////////////////////////////
+
 int
 Insert(CSegmentBase c_segment,
            signed long int size,
@@ -34,6 +37,40 @@ Search(CSegmentBase c_segment,
            long int* result_ids,
            float* result_distances);
 
+//////////////////////////////////////////////////////////////////
+
+int
+Close(CSegmentBase c_segment);
+
+bool
+IsOpened(CSegmentBase c_segment);
+
+//////////////////////////////////////////////////////////////////
+
+long int
+GetRowCount(CSegmentBase c_segment);
+
+long int
+GetDeletedCount(CSegmentBase c_segment);
+
+unsigned long
+GetTimeBegin(CSegmentBase c_segment);
+
+void
+SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin);
+
+unsigned long
+GetTimeEnd(CSegmentBase c_segment);
+
+void
+SetTimeEnd(CSegmentBase c_segment, unsigned long time_end);
+
+unsigned long
+GetSegmentId(CSegmentBase c_segment);
+
+void
+SetSegmentId(CSegmentBase c_segment, unsigned long segment_id);
+
 #ifdef __cplusplus
 }
 #endif

core/src/dog_segment/SegmentBase.h

@@ -92,7 +92,7 @@ class SegmentBase {
     uint64_t get_segment_id() {
         return segment_id_;
     }
-    uint64_t set_segment_id(uint64_t segment_id) {
+    void set_segment_id(uint64_t segment_id) {
         this->segment_id_ = segment_id;
     }
 

core/src/dog_segment/segment_c.cpp

@@ -28,6 +28,7 @@ DeleteSegment(CSegmentBase segment) {
   delete s;
 }
 
+//////////////////////////////////////////////////////////////////
 
 int
 Insert(CSegmentBase c_segment,
@@ -79,3 +80,80 @@ Search(CSegmentBase c_segment,
 
   return res.code();
 }
+
+//////////////////////////////////////////////////////////////////
+
+int
+Close(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto status = segment->Close();
+  return status.code();
+}
+
+
+bool
+IsOpened(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto status = segment->get_state();
+  return status == milvus::dog_segment::SegmentBase::SegmentState::Open;
+}
+
+//////////////////////////////////////////////////////////////////
+
+long int
+GetRowCount(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto row_count = segment->get_row_count();
+  return row_count;
+}
+
+
+long int
+GetDeletedCount(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto deleted_count = segment->get_deleted_count();
+  return deleted_count;
+}
+
+
+unsigned long
+GetTimeBegin(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto time_begin = segment->get_time_begin();
+  return time_begin;
+}
+
+
+void
+SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  segment->set_time_begin(time_begin);
+}
+
+
+unsigned long
+GetTimeEnd(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto time_end = segment->get_time_end();
+  return time_end;
+}
+
+void
+SetTimeEnd(CSegmentBase c_segment, unsigned long time_end) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  segment->set_time_end(time_end);
+}
+
+
+unsigned long
+GetSegmentId(CSegmentBase c_segment) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  auto segment_id = segment->get_segment_id();
+  return segment_id;
+}
+
+void
+SetSegmentId(CSegmentBase c_segment, unsigned long segment_id) {
+  auto segment = (milvus::dog_segment::SegmentBase*)c_segment;
+  segment->set_segment_id(segment_id);
+}

core/src/dog_segment/segment_c.h

@@ -2,6 +2,7 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include "partition_c.h"
 
 typedef void* CSegmentBase;
@@ -12,6 +13,8 @@ NewSegment(CPartition partition, unsigned long segment_id);
 void
 DeleteSegment(CSegmentBase segment);
 
+//////////////////////////////////////////////////////////////////
+
 int
 Insert(CSegmentBase c_segment,
            signed long int size,
@@ -34,6 +37,40 @@ Search(CSegmentBase c_segment,
            long int* result_ids,
            float* result_distances);
 
+//////////////////////////////////////////////////////////////////
+
+int
+Close(CSegmentBase c_segment);
+
+bool
+IsOpened(CSegmentBase c_segment);
+
+//////////////////////////////////////////////////////////////////
+
+long int
+GetRowCount(CSegmentBase c_segment);
+
+long int
+GetDeletedCount(CSegmentBase c_segment);
+
+unsigned long
+GetTimeBegin(CSegmentBase c_segment);
+
+void
+SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin);
+
+unsigned long
+GetTimeEnd(CSegmentBase c_segment);
+
+void
+SetTimeEnd(CSegmentBase c_segment, unsigned long time_end);
+
+unsigned long
+GetSegmentId(CSegmentBase c_segment);
+
+void
+SetSegmentId(CSegmentBase c_segment, unsigned long segment_id);
+
 #ifdef __cplusplus
 }
 #endif

core/unittest/test_c_api.cpp

@@ -138,3 +138,143 @@ TEST(CApiTest, SearchTest) {
   DeletePartition(partition);
   DeleteSegment(segment);
 }
+
+
+TEST(CApiTest, IsOpenedTest) {
+  auto collection_name = "collection0";
+  auto schema_tmp_conf = "null_schema";
+  auto collection = NewCollection(collection_name, schema_tmp_conf);
+  auto partition_name = "partition0";
+  auto partition = NewPartition(collection, partition_name);
+  auto segment = NewSegment(partition, 0);
+
+  auto is_opened = IsOpened(segment);
+  assert(is_opened);
+
+  DeleteCollection(collection);
+  DeletePartition(partition);
+  DeleteSegment(segment);
+}
+
+
+TEST(CApiTest, CloseTest) {
+  auto collection_name = "collection0";
+  auto schema_tmp_conf = "null_schema";
+  auto collection = NewCollection(collection_name, schema_tmp_conf);
+  auto partition_name = "partition0";
+  auto partition = NewPartition(collection, partition_name);
+  auto segment = NewSegment(partition, 0);
+
+  auto status = Close(segment);
+  assert(status == 0);
+
+  DeleteCollection(collection);
+  DeletePartition(partition);
+  DeleteSegment(segment);
+}
+
+
+TEST(CApiTest, GetRowCountTest) {
+  auto collection_name = "collection0";
+  auto schema_tmp_conf = "null_schema";
+  auto collection = NewCollection(collection_name, schema_tmp_conf);
+  auto partition_name = "partition0";
+  auto partition = NewPartition(collection, partition_name);
+  auto segment = NewSegment(partition, 0);
+
+  std::vector<char> raw_data;
+  std::vector<uint64_t> timestamps;
+  std::vector<uint64_t> uids;
+  int N = 10000;
+  std::default_random_engine e(67);
+  for(int i = 0; i < N; ++i) {
+    uids.push_back(100000 + i);
+    timestamps.push_back(0);
+    // append vec
+    float vec[16];
+    for(auto &x: vec) {
+      x = e() % 2000 * 0.001 - 1.0;
+    }
+    raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
+    int age = e() % 100;
+    raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
+  }
+
+  auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
+
+  auto res = Insert(segment, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
+  assert(res == 0);
+
+  auto row_count = GetRowCount(segment);
+  assert(row_count == N);
+
+  DeleteCollection(collection);
+  DeletePartition(partition);
+  DeleteSegment(segment);
+}
+
+TEST(CApiTest, GetDeletedCountTest) {
+  auto collection_name = "collection0";
+  auto schema_tmp_conf = "null_schema";
+  auto collection = NewCollection(collection_name, schema_tmp_conf);
+  auto partition_name = "partition0";
+  auto partition = NewPartition(collection, partition_name);
+  auto segment = NewSegment(partition, 0);
+
+  unsigned long delete_primary_keys[] = {100000, 100001, 100002};
+  unsigned long delete_timestamps[] = {0, 0, 0};
+
+  auto del_res = Delete(segment, 1, delete_primary_keys, delete_timestamps);
+  assert(del_res == 0);
+
+  // TODO: assert(deleted_count == len(delete_primary_keys))
+  auto deleted_count = GetDeletedCount(segment);
+  assert(deleted_count == 0);
+
+  DeleteCollection(collection);
+  DeletePartition(partition);
+  DeleteSegment(segment);
+}
+
+TEST(CApiTest, TimeGetterAndSetterTest) {
+  auto collection_name = "collection0";
+  auto schema_tmp_conf = "null_schema";
+  auto collection = NewCollection(collection_name, schema_tmp_conf);
+  auto partition_name = "partition0";
+  auto partition = NewPartition(collection, partition_name);
+  auto segment = NewSegment(partition, 0);
+
+  uint64_t TIME_BEGIN = 100;
+  uint64_t TIME_END = 200;
+
+  SetTimeBegin(segment, TIME_BEGIN);
+  auto time_begin = GetTimeBegin(segment);
+  assert(time_begin == TIME_BEGIN);
+
+  SetTimeEnd(segment, TIME_END);
+  auto time_end = GetTimeEnd(segment);
+  assert(time_end == TIME_END);
+
+  DeleteCollection(collection);
+  DeletePartition(partition);
+  DeleteSegment(segment);
+}
+
+
+TEST(CApiTest, SegmentIDTest) {
+  auto collection_name = "collection0";
+  auto schema_tmp_conf = "null_schema";
+  auto collection = NewCollection(collection_name, schema_tmp_conf);
+  auto partition_name = "partition0";
+  auto partition = NewPartition(collection, partition_name);
+  auto segment = NewSegment(partition, 0);
+
+  uint64_t SEGMENT_ID = 1;
+  SetSegmentId(segment, SEGMENT_ID);
+  auto segment_id = GetSegmentId(segment);
+  assert(segment_id == SEGMENT_ID);
+
+  DeleteCollection(collection);
+  DeletePartition(partition);
+  DeleteSegment(segment);
+}

go.sum

@@ -44,6 +44,8 @@ github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af h1:
 github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af/go.mod h1:QdYxU2iG99VVU6cvoBRkCgkazfJSL9WwPZ20PZR6aUk=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -51,6 +53,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa v0.0.1 h1:r3ncXbtIiad9owWu22r8ryYogBEV9NbJykk8k6K+u0w=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ=
@@ -59,10 +62,13 @@ github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/Qk
 github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 h1:FE783w8WFh+Rvg+7bZ5g8p7gP4SeVS4AoNwkvazlsBg=
 github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -71,6 +77,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
@@ -80,6 +87,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -87,11 +95,16 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -145,6 +158,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -153,27 +167,34 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
 github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.8.1 h1:VNUuLKyFcJ5IektwBKcZU4J5GJKEt+Odb8dl1d61BGQ=
 github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -183,6 +204,7 @@ github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -190,6 +212,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY=
 github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
@@ -210,6 +235,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
 github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -237,25 +264,33 @@ github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn
 github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
 github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:2JCf+JCLBs7IUZzYdIrSDN+GWYacKOdToIAt5zcga54=
 github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:43j3yLP9UiXa0z95/W3hN7yTjoxsQoOll5rrGBgBcnE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/protocolbuffers/protobuf v3.13.0+incompatible h1:omZA3Tuq+U2kJ2uMuqMR9c1VO5qLEgZ19m9878fXNtg=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -266,6 +301,8 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s
 github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -278,6 +315,7 @@ github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrX
 github.com/tikv/pd v1.1.0-beta h1:Ke5jqQ7P1zS9ZbbEW1ZPFmMTQPTzrQEMsCumjVkRh1Y=
 github.com/tikv/pd v2.1.19+incompatible h1:rqjHqO7t/STke/R2Yz6+lQj6NPA8u7G2Otwqup4K+P8=
 github.com/tikv/pd v2.1.19+incompatible/go.mod h1:v6C/D7ONC49SgjI4jbGnooSizvijaO/bdIm62DVR4tI=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs=
@@ -286,6 +324,7 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMV
 github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
 github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
 github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -293,7 +332,9 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.5.0-alpha.5 h1:VOolFSo3XgsmnYDLozjvZ6JL6AAwIDu1Yx1y+4EYLDo=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY=
 go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -313,6 +354,7 @@ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKY
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM=
 go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
@@ -361,6 +403,7 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -373,6 +416,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -405,6 +449,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -414,6 +459,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -439,6 +485,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -557,6 +604,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -585,6 +633,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -616,5 +665,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

proto/suvlim.proto

@@ -0,0 +1,715 @@
+syntax = "proto3";
+
+package milvus.grpc;
+
+enum ErrorCode {
+    SUCCESS = 0;
+    UNEXPECTED_ERROR = 1;
+    CONNECT_FAILED = 2;
+    PERMISSION_DENIED = 3;
+    COLLECTION_NOT_EXISTS = 4;
+    ILLEGAL_ARGUMENT = 5;
+    ILLEGAL_DIMENSION = 7;
+    ILLEGAL_INDEX_TYPE = 8;
+    ILLEGAL_COLLECTION_NAME = 9;
+    ILLEGAL_TOPK = 10;
+    ILLEGAL_ROWRECORD = 11;
+    ILLEGAL_VECTOR_ID = 12;
+    ILLEGAL_SEARCH_RESULT = 13;
+    FILE_NOT_FOUND = 14;
+    META_FAILED = 15;
+    CACHE_FAILED = 16;
+    CANNOT_CREATE_FOLDER = 17;
+    CANNOT_CREATE_FILE = 18;
+    CANNOT_DELETE_FOLDER = 19;
+    CANNOT_DELETE_FILE = 20;
+    BUILD_INDEX_ERROR = 21;
+    ILLEGAL_NLIST = 22;
+    ILLEGAL_METRIC_TYPE = 23;
+    OUT_OF_MEMORY = 24;
+}
+
+message Status {
+    ErrorCode error_code = 1;
+    string reason = 2;
+}
+
+/**
+ * @brief Field data type
+ */
+enum DataType {
+    NONE = 0;
+    BOOL = 1;
+    INT8 = 2;
+    INT16 = 3;
+    INT32 = 4;
+    INT64 = 5;
+
+    FLOAT = 10;
+    DOUBLE = 11;
+
+    STRING = 20;
+
+    VECTOR_BINARY = 100;
+    VECTOR_FLOAT = 101;
+}
+
+/**
+ * @brief General usage
+ */
+message KeyValuePair {
+    string key = 1;
+    string value = 2;
+}
+
+/**
+ * @brief Collection name
+ */
+message CollectionName {
+    string collection_name = 1;
+}
+
+/**
+ * @brief Collection name list
+ */
+message CollectionNameList {
+    Status status = 1;
+    repeated string collection_names = 2;
+}
+
+/**
+ * @brief Field name
+ */
+message FieldName {
+    string collection_name = 1;
+    string field_name = 2;
+}
+
+/**
+ * @brief Collection mapping
+ * @extra_params: key-value pair for extra parameters of the collection
+ *    typically usage:
+ *        extra_params["params"] = {segment_row_count: 1000000, auto_id: true}
+ *    Note:
+ *        the segment_row_count specify segment row count limit for merging
+ *        the auto_id = true means entity id is auto-generated by milvus
+ */
+message Mapping {
+    Status status = 1;
+    string collection_name = 2;
+    repeated FieldParam fields = 3;
+    repeated KeyValuePair extra_params = 4;
+}
+
+/**
+ * @brief Collection mapping list
+ */
+message MappingList {
+    Status status = 1;
+    repeated Mapping mapping_list = 2;
+}
+
+/**
+ * @brief Parameters of partition
+ */
+message PartitionParam {
+    string collection_name = 1;
+    string tag = 2;
+}
+
+/**
+ * @brief Partition list
+ */
+message PartitionList {
+    Status status = 1;
+    repeated string partition_tag_array = 2;
+}
+
+/**
+ * @brief Vector row record
+ */
+message VectorRowRecord {
+    repeated float float_data = 1;             //float vector data
+    bytes binary_data = 2;                      //binary vector data
+}
+
+message EntityIds {
+    Status status = 1;
+    repeated int64 entity_id_array = 2;
+}
+
+message VectorRecord {
+    repeated VectorRowRecord records = 1;
+}
+
+message VectorParam {
+    string json = 1;
+    VectorRecord row_record = 2;
+}
+
+//////////////////////////row schema and data///////////////////////////////////
+/**
+ * @brief schema
+ */
+message FieldMeta {
+    string field_name = 1;
+    DataType type = 2;
+    int64 dim = 3;
+}
+
+message Schema {
+    repeated FieldMeta field_metas = 1;
+}
+
+message RowData {
+    bytes blob = 1;
+}
+
+//////////////////////suvlim-proxy///////////////////////////////////
+message InsertParam {
+    string collection_name = 1;
+    Schema schema = 2;
+    repeated RowData rows_data = 3;
+    repeated int64 entity_id_array = 4;            //optional
+    string partition_tag = 5;
+    repeated KeyValuePair extra_params = 6;
+}
+
+message SearchParam {
+    string collection_name = 1;
+    repeated VectorParam vector_param = 2;
+    string dsl = 3;          //optional
+    repeated string partition_tag = 4; //why
+    repeated KeyValuePair extra_params = 5;
+}
+
+message SearchInSegmentParam {
+    repeated string file_id_array = 1;
+    SearchParam search_param = 2;
+}
+
+message Entities {
+    Status status = 1;
+    repeated int64 ids = 2;
+    repeated bool valid_row = 3;
+    repeated RowData rows_data = 4;
+}
+
+
+///////////////////////////milvus-server///////////////////////////
+/**
+ * @brief Query result
+ */
+message QueryResult {
+    Status status = 1;
+    Entities entities = 2;
+    int64 row_num = 3;
+    repeated float scores = 4;
+    repeated float distances = 5;
+    repeated KeyValuePair extra_params = 6;
+}
+
+/**
+ * @brief Server string Reply
+ */
+message StringReply {
+    Status status = 1;
+    string string_reply = 2;
+}
+
+/**
+ * @brief Server bool Reply
+ */
+message BoolReply {
+    Status status = 1;
+    bool bool_reply = 2;
+}
+
+/**
+ * @brief Return collection row count
+ */
+message CollectionRowCount {
+    Status status = 1;
+    int64 collection_row_count = 2;
+}
+
+/**
+ * @brief Server command parameters
+ */
+message Command {
+    string cmd = 1;
+}
+
+/**
+ * @brief Index params
+ * @collection_name: target collection
+ * @field_name: target field
+ * @index_name: a name for index provided by user, unique within this field
+ * @extra_params: index parameters in json format
+ *    for vector field:
+ *        extra_params["index_type"] = one of the values: FLAT, IVF_LAT, IVF_SQ8, NSGMIX, IVFSQ8H,
+ *                                                        PQ, HNSW, HNSW_SQ8NM, ANNOY
+ *        extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO
+ *                                                         SUBSTRUCTURE, SUPERSTRUCTURE
+ *        extra_params["params"] = extra parameters for index, for example ivflat: {nlist: 2048}
+ *    for structured field:
+ *        extra_params["index_type"] = one of the values: SORTED
+ */
+message IndexParam {
+    Status status = 1;
+    string collection_name = 2;
+    string field_name = 3;
+    string index_name = 4;
+    repeated KeyValuePair extra_params = 5;
+}
+
+/**
+ * @brief Parameters for flush action
+ */
+message FlushParam {
+    repeated string collection_name_array = 1;
+}
+
+/**
+ * @brief Parameters for flush action
+ */
+message CompactParam {
+    string collection_name = 1;
+    double threshold = 2;
+}
+
+/**
+ * @brief Parameters for deleting entities by id
+ */
+message DeleteByIDParam {
+    string collection_name = 1;
+    repeated int64 id_array = 2;
+}
+
+/**
+ * @brief Return collection stats
+ * @json_info: collection stats in json format, typically, the format is like:
+ *     {
+ *        row_count: xxx,
+ *        data_size: xxx,
+ *        partitions: [
+ *          {
+ *            tag: xxx,
+ *            id: xxx,
+ *            row_count: xxx,
+ *            data_size: xxx,
+ *            segments: [
+ *              {
+ *                 id: xxx,
+ *                 row_count: xxx,
+ *                 data_size: xxx,
+ *                 files: [
+ *                    {
+ *                       field: xxx,
+ *                       name: xxx,
+ *                       index_type: xxx,
+ *                       path: xxx,
+ *                       data_size: xxx,
+ *                    }
+ *                 ]
+ *              }
+ *            ]
+ *          }
+ *        ]
+ *     }
+ */
+message CollectionInfo {
+    Status status = 1;
+    string json_info = 2;
+}
+
+/**
+ * @brief Parameters for returning entities id of a segment
+ */
+message GetEntityIDsParam {
+    string collection_name = 1;
+    int64 segment_id = 2;
+}
+
+/**
+ * @brief Entities identity
+ */
+message EntityIdentity {
+    string collection_name = 1;
+    repeated int64 id_array = 2;
+    repeated string field_names = 3;
+}
+
+/********************************************SearchPB interface***************************************************/
+/**
+ * @brief Vector field parameters
+ */
+message VectorFieldParam {
+    int64 dimension = 1;
+}
+
+/**
+ * @brief Field type
+ */
+message FieldType {
+    oneof value {
+        DataType data_type = 1;
+        VectorFieldParam vector_param = 2;
+    }
+}
+
+/**
+ * @brief Field parameters
+ */
+message FieldParam {
+    uint64 id = 1;
+    string name = 2;
+    DataType type = 3;
+    repeated KeyValuePair index_params = 4;
+    repeated KeyValuePair extra_params = 5;
+}
+
+/**
+ * @brief Vector field record
+ */
+message VectorFieldRecord {
+    repeated VectorRowRecord value = 1;
+}
+
+///////////////////////////////////////////////////////////////////
+
+message TermQuery {
+    string field_name = 1;
+    repeated int64 int_value = 2;
+    repeated double double_value = 3;
+    int64 value_num = 4;
+    float boost = 5;
+    repeated KeyValuePair extra_params = 6;
+}
+
+enum CompareOperator {
+    LT = 0;
+    LTE = 1;
+    EQ = 2;
+    GT = 3;
+    GTE = 4;
+    NE = 5;
+}
+
+message CompareExpr {
+    CompareOperator operator = 1;
+    string operand = 2;
+}
+
+message RangeQuery {
+    string field_name = 1;
+    repeated CompareExpr operand = 2;
+    float boost = 3;
+    repeated KeyValuePair extra_params = 4;
+}
+
+message VectorQuery {
+    string field_name = 1;
+    float query_boost = 2;
+    repeated VectorRowRecord records = 3;
+    int64 topk = 4;
+    repeated KeyValuePair extra_params = 5;
+}
+
+enum Occur {
+    INVALID = 0;
+    MUST = 1;
+    SHOULD = 2;
+    MUST_NOT = 3;
+}
+
+message BooleanQuery {
+    Occur occur = 1;
+    repeated GeneralQuery general_query = 2;
+}
+
+message GeneralQuery {
+    oneof query {
+        BooleanQuery boolean_query = 1;
+        TermQuery term_query = 2;
+        RangeQuery range_query = 3;
+        VectorQuery vector_query = 4;
+    }
+}
+
+message SearchParamPB {
+    string collection_name = 1;
+    repeated string partition_tag_array = 2;
+    GeneralQuery general_query = 3;
+    repeated KeyValuePair extra_params = 4;
+}
+
+service MilvusService {
+    /**
+     * @brief This method is used to create collection
+     *
+     * @param CollectionSchema, use to provide collection information to be created.
+     *
+     * @return Status
+     */
+    rpc CreateCollection(Mapping) returns (Status){}
+
+    /**
+     * @brief This method is used to test collection existence.
+     *
+     * @param CollectionName, collection name is going to be tested.
+     *
+     * @return BoolReply
+     */
+    rpc HasCollection(CollectionName) returns (BoolReply) {}
+
+    /**
+     * @brief This method is used to get collection schema.
+     *
+     * @param CollectionName, target collection name.
+     *
+     * @return CollectionSchema
+     */
+    rpc DescribeCollection(CollectionName) returns (Mapping) {}
+
+    /**
+     * @brief This method is used to get collection schema.
+     *
+     * @param CollectionName, target collection name.
+     *
+     * @return CollectionRowCount
+     */
+    rpc CountCollection(CollectionName) returns (CollectionRowCount) {}
+
+    /**
+     * @brief This method is used to list all collections.
+     *
+     * @param Command, dummy parameter.
+     *
+     * @return CollectionNameList
+     */
+    rpc ShowCollections(Command) returns (CollectionNameList) {}
+
+    /**
+     * @brief This method is used to get collection detail information.
+     *
+     * @param CollectionName, target collection name.
+     *
+     * @return CollectionInfo
+     */
+    rpc ShowCollectionInfo(CollectionName) returns (CollectionInfo) {}
+
+    /**
+     * @brief This method is used to delete collection.
+     *
+     * @param CollectionName, collection name is going to be deleted.
+     *
+     * @return Status
+     */
+    rpc DropCollection(CollectionName) returns (Status) {}
+
+    /**
+     * @brief This method is used to build index by collection in sync mode.
+     *
+     * @param IndexParam, index paramters.
+     *
+     * @return Status
+     */
+    rpc CreateIndex(IndexParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to describe index
+     *
+     * @param IndexParam, target index.
+     *
+     * @return IndexParam
+     */
+    rpc DescribeIndex(IndexParam) returns (IndexParam) {}
+
+    /**
+     * @brief This method is used to drop index
+     *
+     * @param IndexParam, target field. if the IndexParam.field_name is empty, will drop all index of the collection
+     *
+     * @return Status
+     */
+    rpc DropIndex(IndexParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to create partition
+     *
+     * @param PartitionParam, partition parameters.
+     *
+     * @return Status
+     */
+    rpc CreatePartition(PartitionParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to test partition existence.
+     *
+     * @param PartitionParam, target partition.
+     *
+     * @return BoolReply
+     */
+    rpc HasPartition(PartitionParam) returns (BoolReply) {}
+
+    /**
+     * @brief This method is used to show partition information
+     *
+     * @param CollectionName, target collection name.
+     *
+     * @return PartitionList
+     */
+    rpc ShowPartitions(CollectionName) returns (PartitionList) {}
+
+    /**
+     * @brief This method is used to drop partition
+     *
+     * @param PartitionParam, target partition.
+     *
+     * @return Status
+     */
+    rpc DropPartition(PartitionParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to add vector array to collection.
+     *
+     * @param InsertParam, insert parameters.
+     *
+     * @return VectorIds
+     */
+    rpc Insert(InsertParam) returns (EntityIds) {}
+
+    /**
+     * @brief This method is used to get entities data by id array.
+     *
+     * @param EntitiesIdentity, target entity id array.
+     *
+     * @return EntitiesData
+     */
+    rpc GetEntityByID(EntityIdentity) returns (Entities) {}
+
+    /**
+     * @brief This method is used to get vector ids from a segment
+     *
+     * @param GetVectorIDsParam, target collection and segment
+     *
+     * @return VectorIds
+     */
+    rpc GetEntityIDs(GetEntityIDsParam) returns (EntityIds) {}
+
+    /**
+     * @brief This method is used to query vector in collection.
+     *
+     * @param SearchParam, search parameters.
+     *
+     * @return KQueryResult
+     */
+    rpc Search(SearchParam) returns (QueryResult) {}
+
+    /**
+     * @brief This method is used to query vector in specified files.
+     *
+     * @param SearchInSegmentParam, target segments to search.
+     *
+     * @return TopKQueryResult
+     */
+    rpc SearchInSegment(SearchInSegmentParam) returns (QueryResult) {}
+
+    /**
+     * @brief This method is used to give the server status.
+     *
+     * @param Command, command string
+     *
+     * @return StringReply
+     */
+    rpc Cmd(Command) returns (StringReply) {}
+
+    /**
+     * @brief This method is used to delete vector by id
+     *
+     * @param DeleteByIDParam, delete parameters.
+     *
+     * @return status
+     */
+    rpc DeleteByID(DeleteByIDParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to preload collection
+     *
+     * @param CollectionName, target collection name.
+     *
+     * @return Status
+     */
+    rpc PreloadCollection(CollectionName) returns (Status) {}
+
+    /**
+     * @brief This method is used to flush buffer into storage.
+     *
+     * @param FlushParam, flush parameters
+     *
+     * @return Status
+     */
+    rpc Flush(FlushParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to compact collection
+     *
+     * @param CompactParam, compact parameters
+     *
+     * @return Status
+     */
+    rpc Compact(CompactParam) returns (Status) {}
+
+    /********************************New Interface********************************************/
+
+    rpc SearchPB(SearchParamPB) returns (QueryResult) {}
+}
+
+////////////////////pulsar//////////////////////////////////////
+enum OpType {
+    INSERT = 0;
+    DELETE = 1;
+    SEARCH = 2;
+    SEARCH_RESULT = 3;
+}
+
+message InsertOrDeleteMsg {
+    string collection_name = 1;
+    RowData rows_data = 2;
+    int64 uid = 3;            //optional
+    string partition_tag = 4;
+    int64 timestamp =5;
+    int64 segment_id = 6;
+    int64 channel_id = 7;
+    OpType op = 8;
+    int64 client_id = 9;
+    repeated KeyValuePair extra_params = 10;
+}
+
+message SearchMsg {
+    string collection_name = 1;
+    VectorRowRecord records = 2;
+    string partition_tag = 3;
+    int64 uid = 4;
+    int64 timestamp =5;
+    int64 client_id = 6;
+    repeated KeyValuePair extra_params = 7;
+}
+
+message TimeSyncMsg{
+	int64 ClientId = 1;
+	int64 Timestamp = 2;
+	OpType MsgType = 3;
+}
+
+message SegmentRecord {
+    int64 uid = 1;
+    repeated int64 segment_id = 2;
+}
+
+message Key2SegMsg {
+	int64 client_id = 1;
+	SegmentRecord records = 2;
+	OpType msg_type = 3;
+}

pulsar/client-go/go_client.go

@@ -3,8 +3,8 @@ package client_go
 import (
 	"context"
 	"github.com/apache/pulsar/pulsar-client-go/pulsar"
+	"github.com/czs007/suvlim/pulsar/client-go/schema"
 	"log"
-	"suvlim/pulsar/client-go/schema"
 )
 
 var (

pulsar/client-go/query_node.go

@@ -2,7 +2,7 @@ package client_go
 
 import (
 	"fmt"
-	"suvlim/pulsar/client-go/schema"
+	"github.com/czs007/suvlim/pulsar/client-go/schema"
 	"sync"
 	"time"
 )
@@ -34,7 +34,7 @@ func (qn *QueryNode)ReceiveMessage() {
 	qn.mc.ReceiveMessage()
 }
 
-func main() {
+func queryNodeMain() {
 
 	mc := MessageClient{}
 	topics := []string{"insert", "delete"}

pulsar/client-go/storage_node.go

@@ -2,7 +2,7 @@ package client_go
 
 import (
 	"fmt"
-	"suvlim/pulsar/client-go/schema"
+	"github.com/czs007/suvlim/pulsar/client-go/schema"
 	"sync"
 	"time"
 )
@@ -27,7 +27,7 @@ func (wn *WriteNode) doWriteNode(wg sync.WaitGroup) {
 func (wn *WriteNode) PrepareBatchMsg() {
 	wn.mc.PrepareBatchMsg(JobType(1))
 }
-func main() {
+func storageNodeMain() {
 
 	mc := MessageClient{}
 	topics := []string{"insert", "delete"}

reader/partition.go

@@ -16,14 +16,15 @@ import "C"
 type Partition struct {
 	PartitionPtr C.CPartition
 	PartitionName string
-	Segments []*Segment
+	OpenedSegments []*Segment
+	ClosedSegments []*Segment
 }
 
 func (p *Partition) NewSegment(segmentId uint64) *Segment {
 	segmentPtr := C.NewSegment(p.PartitionPtr, C.ulong(segmentId))
 
 	var newSegment = &Segment{SegmentPtr: segmentPtr, SegmentId: segmentId}
-	p.Segments = append(p.Segments, newSegment)
+	p.OpenedSegments = append(p.OpenedSegments, newSegment)
 	return newSegment
 }
 

reader/query_node.go

@@ -125,17 +125,15 @@ func (node *QueryNode) GetTargetSegment(collectionName *string, partitionTag *st
 		return nil, errors.New("cannot found target partition")
 	}
 
-	for _, segment := range targetPartition.Segments {
+	for _, segment := range targetPartition.OpenedSegments {
-		var segmentStatus = segment.GetStatus()
+		// TODO: add other conditions
-		if segmentStatus == 0 {
+		return segment, nil
-			return segment, nil
-		}
 	}
 
 	return nil, errors.New("cannot found target segment")
 }
 
-func (node *QueryNode) GetTimeSync() uint64 {
+func (node *QueryNode) GetTSOTime() uint64 {
 	// TODO: Add time sync
 	return 0
 }
@@ -152,17 +150,22 @@ func (node *QueryNode) InitQueryNodeCollection() {
 }
 
 func (node *QueryNode) SegmentsManagement() {
-	var timeSync = node.GetTimeSync()
+	var timeNow = node.GetTSOTime()
 	for _, collection := range node.Collections {
 		for _, partition := range collection.Partitions {
-			for _, segment := range partition.Segments {
+			for _, oldSegment := range partition.OpenedSegments {
 				// TODO: check segment status
-				if timeSync >= segment.SegmentCloseTime {
+				if timeNow >= oldSegment.SegmentCloseTime {
-					segment.Close()
+					// start new segment and add it into partition.OpenedSegments
 					// TODO: add atomic segment id
 					var newSegment = partition.NewSegment(0)
-					newSegment.SegmentCloseTime = timeSync + SegmentLifetime
+					newSegment.SegmentCloseTime = timeNow + SegmentLifetime
-					partition.Segments = append(partition.Segments, newSegment)
+					partition.OpenedSegments = append(partition.OpenedSegments, newSegment)
+
+					// close old segment and move it into partition.ClosedSegments
+					// TODO: check status
+					var _ = oldSegment.Close()
+					partition.ClosedSegments = append(partition.ClosedSegments, oldSegment)
 				}
 			}
 		}
@@ -179,7 +182,7 @@ func (node *QueryNode) SegmentService() {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitGroup) schema.Status {
-	var timeSync = node.GetTimeSync()
+	var timeNow = node.GetTSOTime()
 	var collectionName = insertMessages[0].CollectionName
 	var partitionTag = insertMessages[0].PartitionTag
 	var clientId = insertMessages[0].ClientId
@@ -190,7 +193,7 @@ func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitG
 	var vectorRecords [][]*schema.FieldValue
 
 	for i, msg := range node.buffer.InsertBuffer {
-		if msg.Timestamp <= timeSync {
+		if msg.Timestamp <= timeNow {
 			entityIds = append(entityIds, msg.EntityId)
 			timestamps = append(timestamps, msg.Timestamp)
 			vectorRecords = append(vectorRecords, msg.Fields)
@@ -207,7 +210,7 @@ func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitG
 	}
 
 	for _, msg := range insertMessages {
-		if msg.Timestamp <= timeSync {
+		if msg.Timestamp <= timeNow {
 			entityIds = append(entityIds, msg.EntityId)
 			timestamps = append(timestamps, msg.Timestamp)
 			vectorRecords = append(vectorRecords, msg.Fields)
@@ -232,7 +235,7 @@ func (node *QueryNode) Insert(insertMessages []*schema.InsertMsg, wg *sync.WaitG
 }
 
 func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitGroup) schema.Status {
-	var timeSync = node.GetTimeSync()
+	var timeNow = node.GetTSOTime()
 	var clientId = deleteMessages[0].ClientId
 
 	// TODO: prevent Memory copy
@@ -240,7 +243,7 @@ func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitG
 	var timestamps []uint64
 
 	for i, msg := range node.buffer.DeleteBuffer {
-		if msg.Timestamp <= timeSync {
+		if msg.Timestamp <= timeNow {
 			entityIds = append(entityIds, msg.EntityId)
 			timestamps = append(timestamps, msg.Timestamp)
 			node.buffer.validDeleteBuffer[i] = false
@@ -256,7 +259,7 @@ func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitG
 	}
 
 	for _, msg := range deleteMessages {
-		if msg.Timestamp <= timeSync {
+		if msg.Timestamp <= timeNow {
 			entityIds = append(entityIds, msg.EntityId)
 			timestamps = append(timestamps, msg.Timestamp)
 		} else {
@@ -281,7 +284,7 @@ func (node *QueryNode) Delete(deleteMessages []*schema.DeleteMsg, wg *sync.WaitG
 }
 
 func (node *QueryNode) Search(searchMessages []*schema.SearchMsg, wg *sync.WaitGroup) schema.Status {
-	var timeSync = node.GetTimeSync()
+	var timeNow = node.GetTSOTime()
 	var collectionName = searchMessages[0].CollectionName
 	var partitionTag = searchMessages[0].PartitionTag
 	var clientId = searchMessages[0].ClientId
@@ -292,7 +295,7 @@ func (node *QueryNode) Search(searchMessages []*schema.SearchMsg, wg *sync.WaitG
 	var timestamps []uint64
 
 	for i, msg := range node.buffer.SearchBuffer {
-		if msg.Timestamp <= timeSync {
+		if msg.Timestamp <= timeNow {
 			records = append(records, *msg.VectorParam.RowRecord)
 			timestamps = append(timestamps, msg.Timestamp)
 			node.buffer.validSearchBuffer[i] = false
@@ -308,7 +311,7 @@ func (node *QueryNode) Search(searchMessages []*schema.SearchMsg, wg *sync.WaitG
 	}
 
 	for _, msg := range searchMessages {
-		if msg.Timestamp <= timeSync {
+		if msg.Timestamp <= timeNow {
 			records = append(records, *msg.VectorParam.RowRecord)
 			timestamps = append(timestamps, msg.Timestamp)
 		} else {

reader/segment.go

@@ -21,54 +21,113 @@ import (
 
 const SegmentLifetime = 20000
 
+const (
+	SegmentOpened = 0
+	SegmentClosed = 1
+)
+
 type Segment struct {
 	SegmentPtr C.CSegmentBase
 	SegmentId	uint64
 	SegmentCloseTime uint64
 }
 
-func (s *Segment) GetRowCount() int64 {
+func (s *Segment) GetStatus() int {
-	// TODO: C type to go type
+	/*C.IsOpened
-	//return C.GetRowCount(s)
+	bool
-	return 0
+	IsOpened(CSegmentBase c_segment);
+	*/
+	var isOpened = C.IsOpened(s.SegmentPtr)
+	if isOpened {
+		return SegmentOpened
+	} else {
+		return SegmentClosed
+	}
 }
 
-func (s *Segment) GetStatus() int {
+func (s *Segment) GetSegmentID() uint64 {
-	// TODO: C type to go type
+	/*C.GetSegmentId
-	//return C.GetStatus(s)
+	unsigned long
-	return 0
+	GetSegmentId(CSegmentBase c_segment);
+	*/
+	var segmentID = C.GetSegmentId(s.SegmentPtr)
+	return uint64(segmentID)
+}
+
+func (s *Segment) SetSegmentID(segmentID uint64) {
+	/*C.SetSegmentId
+	void
+	SetSegmentId(CSegmentBase c_segment, unsigned long segment_id);
+	*/
+	C.SetSegmentId(s.SegmentPtr, C.ulong(segmentID))
 }
 
 func (s *Segment) GetMaxTimestamp() uint64 {
-	// TODO: C type to go type
+	/*C.GetTimeEnd
-	//return C.GetMaxTimestamp(s)
+	unsigned long
-	return 0
+	GetTimeEnd(CSegmentBase c_segment);
+	*/
+	var maxTimestamp = C.GetTimeEnd(s.SegmentPtr)
+	return uint64(maxTimestamp)
+}
+
+func (s *Segment) SetMaxTimestamp(maxTimestamp uint64) {
+	/*C.SetTimeEnd
+	void
+	SetTimeEnd(CSegmentBase c_segment, unsigned long time_end);
+	*/
+	C.SetTimeEnd(s.SegmentPtr, C.ulong(maxTimestamp))
 }
 
 func (s *Segment) GetMinTimestamp() uint64 {
-	// TODO: C type to go type
+	/*C.GetTimeBegin
-	//return C.GetMinTimestamp(s)
+	unsigned long
-	return 0
+	GetTimeBegin(CSegmentBase c_segment);
+	*/
+	var minTimestamp = C.GetTimeBegin(s.SegmentPtr)
+	return uint64(minTimestamp)
 }
 
-func (s *Segment) GetDeletedCount() uint64 {
+func (s *Segment) SetMinTimestamp(minTimestamp uint64) {
-	// TODO: C type to go type
+	/*C.SetTimeBegin
-	//return C.GetDeletedCount(s)
+	void
-	return 0
+	SetTimeBegin(CSegmentBase c_segment, unsigned long time_begin);
+	*/
+	C.SetTimeBegin(s.SegmentPtr, C.ulong(minTimestamp))
 }
 
-func (s *Segment) Close() {
+func (s *Segment) GetRowCount() int64 {
-	// TODO: C type to go type
+	/*C.GetRowCount
-	//C.CloseSegment(s)
+	long int
+	GetRowCount(CSegmentBase c_segment);
+	*/
+	var rowCount = C.GetRowCount(s.SegmentPtr)
+	return int64(rowCount)
+}
+
+func (s *Segment) GetDeletedCount() int64 {
+	/*C.GetDeletedCount
+	long int
+	GetDeletedCount(CSegmentBase c_segment);
+	*/
+	var deletedCount = C.GetDeletedCount(s.SegmentPtr)
+	return int64(deletedCount)
+}
+
+func (s *Segment) Close() error {
+	/*C.Close
+	int
+	Close(CSegmentBase c_segment);
+	*/
+	var status = C.Close(s.SegmentPtr)
+	if status != 0 {
+		return errors.New("Close segment failed, error code = " + strconv.Itoa(int(status)))
+	}
+	return nil
 }
 
 ////////////////////////////////////////////////////////////////////////////
 func SegmentInsert(segment *Segment, entityIds *[]uint64, timestamps *[]uint64, dataChunk [][]*schema.FieldValue) (ResultEntityIds, error) {
-	// TODO: remove hard code schema
-	// auto schema_tmp = std::make_shared<Schema>();
-	// schema_tmp->AddField("fakeVec", DataType::VECTOR_FLOAT, 16);
-	// schema_tmp->AddField("age", DataType::INT32);
-
 	/*C.Insert
 	int
 	Insert(CSegmentBase c_segment,
@@ -80,6 +139,10 @@ func SegmentInsert(segment *Segment, entityIds *[]uint64, timestamps *[]uint64,
 	           signed long int count);
 	*/
 
+	// TODO: remove hard code schema
+	// auto schema_tmp = std::make_shared<Schema>();
+	// schema_tmp->AddField("fakeVec", DataType::VECTOR_FLOAT, 16);
+	// schema_tmp->AddField("age", DataType::INT32);
 	// TODO: remove hard code & fake dataChunk
 	const DIM = 4
 	const N = 3

reader/segment_test.go

@@ -71,3 +71,111 @@ func TestSegmentSearch(t *testing.T) {
 	collection.DeletePartition(partition)
 	node.DeleteCollection(collection)
 }
+
+func TestSegment_GetStatus(t *testing.T) {
+	node := NewQueryNode(0, 0)
+	var collection = node.NewCollection("collection0", "fake schema")
+	var partition = collection.NewPartition("partition0")
+	var segment = partition.NewSegment(0)
+
+	var status = segment.GetStatus()
+	assert.Equal(t, status, SegmentOpened)
+
+	partition.DeleteSegment(segment)
+	collection.DeletePartition(partition)
+	node.DeleteCollection(collection)
+}
+
+func TestSegment_Close(t *testing.T) {
+	node := NewQueryNode(0, 0)
+	var collection = node.NewCollection("collection0", "fake schema")
+	var partition = collection.NewPartition("partition0")
+	var segment = partition.NewSegment(0)
+
+	var err = segment.Close()
+	assert.NoError(t, err)
+
+	partition.DeleteSegment(segment)
+	collection.DeletePartition(partition)
+	node.DeleteCollection(collection)
+}
+
+func TestSegment_GetRowCount(t *testing.T) {
+	node := NewQueryNode(0, 0)
+	var collection = node.NewCollection("collection0", "fake schema")
+	var partition = collection.NewPartition("partition0")
+	var segment = partition.NewSegment(0)
+
+	ids :=[] uint64{1, 2, 3}
+	timestamps :=[] uint64 {0, 0, 0}
+
+	var _, err = SegmentInsert(segment, &ids, &timestamps, nil)
+	assert.NoError(t, err)
+
+	var rowCount = segment.GetRowCount()
+	assert.Equal(t, rowCount, int64(len(ids)))
+
+	partition.DeleteSegment(segment)
+	collection.DeletePartition(partition)
+	node.DeleteCollection(collection)
+}
+
+func TestSegment_GetDeletedCount(t *testing.T) {
+	node := NewQueryNode(0, 0)
+	var collection = node.NewCollection("collection0", "fake schema")
+	var partition = collection.NewPartition("partition0")
+	var segment = partition.NewSegment(0)
+
+	ids :=[] uint64{1, 2, 3}
+	timestamps :=[] uint64 {0, 0, 0}
+
+	var _, err = SegmentDelete(segment, &ids, &timestamps)
+	assert.NoError(t, err)
+
+	var deletedCount = segment.GetDeletedCount()
+	// TODO: assert.Equal(t, deletedCount, len(ids))
+	assert.Equal(t, deletedCount, int64(0))
+
+	partition.DeleteSegment(segment)
+	collection.DeletePartition(partition)
+	node.DeleteCollection(collection)
+}
+
+func TestSegment_TimestampGetterAndSetter(t *testing.T) {
+	node := NewQueryNode(0, 0)
+	var collection = node.NewCollection("collection0", "fake schema")
+	var partition = collection.NewPartition("partition0")
+	var segment = partition.NewSegment(0)
+
+	const MinTimestamp = 100
+	const MaxTimestamp = 200
+
+	segment.SetMinTimestamp(MinTimestamp)
+	var minTimestamp = segment.GetMinTimestamp()
+	assert.Equal(t, minTimestamp, uint64(MinTimestamp))
+
+	segment.SetMaxTimestamp(MaxTimestamp)
+	var maxTimestamp = segment.GetMaxTimestamp()
+	assert.Equal(t, maxTimestamp, uint64(MaxTimestamp))
+
+	partition.DeleteSegment(segment)
+	collection.DeletePartition(partition)
+	node.DeleteCollection(collection)
+}
+
+func TestSegment_SegmentIDGetterAndSetter(t *testing.T) {
+	node := NewQueryNode(0, 0)
+	var collection = node.NewCollection("collection0", "fake schema")
+	var partition = collection.NewPartition("partition0")
+	var segment = partition.NewSegment(0)
+
+	const SegmentID = 1
+
+	segment.SetSegmentID(SegmentID)
+	var segmentID = segment.GetSegmentID()
+	assert.Equal(t, segmentID, uint64(SegmentID))
+
+	partition.DeleteSegment(segment)
+	collection.DeletePartition(partition)
+	node.DeleteCollection(collection)
+}