From 85a544b79b7dd207aa7b78ce98307d38ad0055fd Mon Sep 17 00:00:00 2001 From: "cai.zhang" Date: Sat, 28 Nov 2020 19:06:48 +0800 Subject: [PATCH] Add dockerfile for each component Signed-off-by: cai.zhang --- build/docker/deploy/.env | 3 + build/docker/deploy/docker-compose.yml | 45 ++ build/docker/deploy/master/DockerFile | 22 + build/docker/deploy/proxy/DockerFile | 22 + build/docker/deploy/querynode/DockerFile | 39 ++ configs/advanced/channel.yaml | 1 - configs/milvus.yaml | 2 + docs/developer_guides/chap06_master.md | 18 +- internal/allocator/allocator.go | 2 - internal/allocator/id.go | 1 - internal/allocator/segment.go | 2 - internal/allocator/timestamp.go | 5 - internal/core/src/CMakeLists.txt | 1 + internal/core/src/common/CMakeLists.txt | 8 + internal/core/src/common/Schema.cpp | 46 ++ internal/core/src/common/Schema.h | 25 + internal/core/src/pb/schema.pb.cc | 174 +++-- internal/core/src/pb/schema.pb.h | 45 +- internal/core/src/pb/service_msg.pb.cc | 457 +------------- internal/core/src/pb/service_msg.pb.h | 363 +---------- internal/core/src/query/Plan.cpp | 6 + internal/core/src/query/PlanImpl.h | 1 + internal/core/src/query/Search.cpp | 13 +- internal/core/src/segcore/CMakeLists.txt | 1 + internal/core/src/segcore/Collection.cpp | 22 +- internal/core/src/segcore/SegmentBase.h | 3 + internal/core/src/segcore/SegmentNaive.h | 5 + .../core/src/segcore/SegmentSmallIndex.cpp | 34 + internal/core/src/segcore/SegmentSmallIndex.h | 36 +- internal/core/src/utils/Types.h | 25 +- internal/core/unittest/test_query.cpp | 79 +++ internal/master/collection_task.go | 49 +- internal/master/collection_task_test.go | 4 +- internal/master/config_task.go | 79 --- internal/master/config_task_test.go | 150 ----- internal/master/config_test.yaml | 111 ---- internal/master/grpc_service.go | 37 -- internal/master/grpc_service_test.go | 3 +- internal/master/master.go | 52 +- internal/master/master_test.go | 246 -------- internal/master/param_table.go | 51 +- internal/master/param_table_test.go | 12 +- internal/master/partition_task.go | 53 +- internal/master/partition_task_test.go | 3 +- internal/master/scheduler.go | 73 +-- internal/master/scheduler_test.go | 342 ---------- internal/master/segment_manager_test.go | 3 +- internal/master/system_config.go | 117 ---- internal/master/system_config_test.go | 209 ------ internal/master/time_snyc_producer_test.go | 27 +- internal/master/time_sync_producer.go | 21 +- internal/master/timesync.go | 2 - internal/msgstream/msg.go | 593 ------------------ internal/msgstream/task.go | 263 ++++++++ .../msgstream/{msg_test.go => task_test.go} | 0 internal/msgstream/unmarshal.go | 10 - internal/proto/internal_msg.proto | 17 +- internal/proto/internalpb/internal_msg.pb.go | 281 +++------ internal/proto/master.proto | 9 - internal/proto/masterpb/master.pb.go | 110 +--- internal/proto/schema.proto | 9 +- internal/proto/schemapb/schema.pb.go | 73 ++- internal/proto/service_msg.proto | 8 - internal/proto/servicepb/service_msg.pb.go | 159 ++--- internal/proxy/grpc_service.go | 11 +- internal/proxy/paramtable.go | 7 +- internal/proxy/proxy_test.go | 4 +- internal/proxy/task.go | 57 +- internal/proxy/task_scheduler.go | 9 +- internal/proxy/timetick.go | 5 +- internal/querynode/collection_replica_test.go | 234 ++++--- internal/querynode/collection_test.go | 39 +- internal/querynode/data_sync_service_test.go | 13 +- .../querynode/flow_graph_service_time_node.go | 3 +- internal/querynode/meta_service.go | 12 +- internal/querynode/meta_service_test.go | 72 ++- internal/querynode/param_table.go | 10 +- internal/querynode/param_table_test.go | 6 + internal/querynode/partition_test.go | 13 +- internal/querynode/plan_test.go | 26 +- internal/querynode/reduce_test.go | 13 +- internal/querynode/search_service.go | 28 +- internal/querynode/search_service_test.go | 13 +- internal/querynode/segment_test.go | 130 ++-- internal/querynode/stats_service_test.go | 25 +- internal/querynode/tsafe.go | 2 +- internal/util/paramtable/paramtable.go | 13 +- 87 files changed, 1582 insertions(+), 3845 deletions(-) create mode 100644 build/docker/deploy/.env create mode 100644 build/docker/deploy/docker-compose.yml create mode 100644 build/docker/deploy/master/DockerFile create mode 100644 build/docker/deploy/proxy/DockerFile create mode 100644 build/docker/deploy/querynode/DockerFile create mode 100644 internal/core/src/common/CMakeLists.txt create mode 100644 internal/core/src/common/Schema.cpp delete mode 100644 internal/master/config_task.go delete mode 100644 internal/master/config_task_test.go delete mode 100644 internal/master/config_test.yaml delete mode 100644 internal/master/master_test.go delete mode 100644 internal/master/scheduler_test.go delete mode 100644 internal/master/system_config.go delete mode 100644 internal/master/system_config_test.go delete mode 100644 internal/msgstream/msg.go create mode 100644 internal/msgstream/task.go rename internal/msgstream/{msg_test.go => task_test.go} (100%) diff --git a/build/docker/deploy/.env b/build/docker/deploy/.env new file mode 100644 index 0000000000..2c2ec0a3df --- /dev/null +++ b/build/docker/deploy/.env @@ -0,0 +1,3 @@ +PULSAR_ADDRESS=pulsar://pulsar:6650 +ETCD_ADDRESS=etcd:2379 +MASTER_ADDRESS=master:53100 diff --git a/build/docker/deploy/docker-compose.yml b/build/docker/deploy/docker-compose.yml new file mode 100644 index 0000000000..9588637d3f --- /dev/null +++ b/build/docker/deploy/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3.5' + +services: + master: + image: master + build: + context: ../../../ + dockerfile: build/docker/deploy/master/DockerFile + environment: + PULSAR_ADDRESS: ${PULSAR_ADDRESS} + ETCD_ADDRESS: ${ETCD_ADDRESS} + MASTER_ADDRESS: ${MASTER_ADDRESS} + networks: + - milvus + ports: + - "53100:53100" + + proxy: + image: proxy + build: + context: ../../../ + dockerfile: build/docker/deploy/proxy/DockerFile + environment: + PULSAR_ADDRESS: ${PULSAR_ADDRESS} + ETCD_ADDRESS: ${ETCD_ADDRESS} + MASTER_ADDRESS: ${MASTER_ADDRESS} + ports: + - "19530:19530" + networks: + - milvus + + querynode: + image: querynode + build: + context: ../../../ + dockerfile: build/docker/deploy/querynode/DockerFile + environment: + PULSAR_ADDRESS: ${PULSAR_ADDRESS} + ETCD_ADDRESS: ${ETCD_ADDRESS} + MASTER_ADDRESS: ${MASTER_ADDRESS} + networks: + - milvus + +networks: + milvus: diff --git a/build/docker/deploy/master/DockerFile b/build/docker/deploy/master/DockerFile new file mode 100644 index 0000000000..47a585ecda --- /dev/null +++ b/build/docker/deploy/master/DockerFile @@ -0,0 +1,22 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM alpine:3.12.1 + +COPY ./bin/master /milvus-distributed/bin/master + +COPY ./configs/ /milvus-distributed/configs/ + +WORKDIR /milvus-distributed/ + +CMD ["./bin/master"] + +EXPOSE 53100 diff --git a/build/docker/deploy/proxy/DockerFile b/build/docker/deploy/proxy/DockerFile new file mode 100644 index 0000000000..8e4857b3a6 --- /dev/null +++ b/build/docker/deploy/proxy/DockerFile @@ -0,0 +1,22 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM alpine:3.12.1 + +COPY ./bin/proxy /milvus-distributed/bin/proxy + +COPY ./configs/ /milvus-distributed/configs/ + +WORKDIR /milvus-distributed/ + +CMD ["./bin/proxy"] + +EXPOSE 19530 diff --git a/build/docker/deploy/querynode/DockerFile b/build/docker/deploy/querynode/DockerFile new file mode 100644 index 0000000000..80489b81d8 --- /dev/null +++ b/build/docker/deploy/querynode/DockerFile @@ -0,0 +1,39 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas + +#FROM alpine +FROM ubuntu:bionic-20200921 + +RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfortran + +#RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories + +#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories \ +# && apk add --no-cache libtbb gfortran + +COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/ + +RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \ + ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so + +COPY ./bin/querynode /milvus-distributed/bin/querynode + +COPY ./configs/ /milvus-distributed/configs/ + +COPY ./lib/ /milvus-distributed/lib/ + +ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib + +WORKDIR /milvus-distributed/ + +CMD ["./bin/querynode"] diff --git a/configs/advanced/channel.yaml b/configs/advanced/channel.yaml index 071441e48f..13684e4bf4 100644 --- a/configs/advanced/channel.yaml +++ b/configs/advanced/channel.yaml @@ -34,7 +34,6 @@ msgChannel: channelRange: insert: [0, 1] delete: [0, 1] - dataDefinition: [0,1] k2s: [0, 1] search: [0, 1] searchResult: [0, 1] \ No newline at end of file diff --git a/configs/milvus.yaml b/configs/milvus.yaml index c76cbd8ce7..b0eca24ad6 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -19,6 +19,8 @@ etcd: address: localhost port: 2379 rootPath: by-dev + metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath + kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath segThreshold: 10000 pulsar: diff --git a/docs/developer_guides/chap06_master.md b/docs/developer_guides/chap06_master.md index 22723637b4..bd415697f5 100644 --- a/docs/developer_guides/chap06_master.md +++ b/docs/developer_guides/chap06_master.md @@ -380,18 +380,13 @@ func (segMgr *SegmentManager) AssignSegmentID(segIDReq []*internalpb.SegIDReques // "/msg_stream/insert" message SysConfigRequest { - MsgType msg_type = 1; - int64 reqID = 2; - int64 proxyID = 3; - uint64 timestamp = 4; - repeated string keys = 5; - repeated string key_prefixes = 6; + repeated string keys = 1; + repeated string key_prefixes = 2; } message SysConfigResponse { - common.Status status = 1; - repeated string keys = 2; - repeated string values = 3; + repeated string keys = 1; + repeated string values = 2; } ``` @@ -399,11 +394,12 @@ message SysConfigResponse { ```go type SysConfig struct { - kv *kv.EtcdKV + etcdKV *etcd + etcdPathPrefix string } func (conf *SysConfig) InitFromFile(filePath string) (error) -func (conf *SysConfig) GetByPrefix(keyPrefix string) (keys []string, values []string, err error) +func (conf *SysConfig) GetByPrefix(keyPrefix string) ([]string, error) func (conf *SysConfig) Get(keys []string) ([]string, error) ``` diff --git a/internal/allocator/allocator.go b/internal/allocator/allocator.go index e3105a68b5..fe4870fc3a 100644 --- a/internal/allocator/allocator.go +++ b/internal/allocator/allocator.go @@ -3,7 +3,6 @@ package allocator import ( "context" "errors" - "fmt" "log" "sync" "time" @@ -215,7 +214,6 @@ func (ta *Allocator) sync(timeout bool) { if !timeout { ta.tChan.Reset() } - fmt.Println("synced") } func (ta *Allocator) finishSyncRequest() { diff --git a/internal/allocator/id.go b/internal/allocator/id.go index 742fc60658..c4f4cbb80f 100644 --- a/internal/allocator/id.go +++ b/internal/allocator/id.go @@ -77,7 +77,6 @@ func (ia *IDAllocator) processFunc(req request) error { idRequest := req.(*idRequest) idRequest.id = ia.idStart ia.idStart++ - fmt.Println("process ID") return nil } diff --git a/internal/allocator/segment.go b/internal/allocator/segment.go index 0efcfef522..419033d244 100644 --- a/internal/allocator/segment.go +++ b/internal/allocator/segment.go @@ -173,7 +173,6 @@ func (sa *SegIDAssigner) syncSegments() { sa.segReqs = sa.segReqs[0:0] fmt.Println("OOOOO", req.PerChannelReq) resp, err := sa.masterClient.AssignSegmentID(ctx, req) - log.Printf("resp: %v", resp) if resp.Status.GetErrorCode() != commonpb.ErrorCode_SUCCESS { log.Panic("GRPC AssignSegmentID Failed") @@ -249,7 +248,6 @@ func (sa *SegIDAssigner) processFunc(req request) error { } } segRequest.segInfo = resultSegInfo - fmt.Println("process segmentID") return nil } diff --git a/internal/allocator/timestamp.go b/internal/allocator/timestamp.go index b96e45fbfb..61f4065fb3 100644 --- a/internal/allocator/timestamp.go +++ b/internal/allocator/timestamp.go @@ -2,7 +2,6 @@ package allocator import ( "context" - "fmt" "log" "time" @@ -54,7 +53,6 @@ func (ta *TimestampAllocator) checkFunc(timeout bool) bool { } func (ta *TimestampAllocator) syncTs() { - fmt.Println("sync TS") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) req := &internalpb.TsoRequest{ PeerID: 1, @@ -62,7 +60,6 @@ func (ta *TimestampAllocator) syncTs() { Count: ta.countPerRPC, } resp, err := ta.masterClient.AllocTimestamp(ctx, req) - log.Printf("resp: %v", resp) cancel() if err != nil { @@ -77,7 +74,6 @@ func (ta *TimestampAllocator) processFunc(req request) error { tsoRequest := req.(*tsoRequest) tsoRequest.timestamp = ta.lastTsBegin ta.lastTsBegin++ - fmt.Println("process tso") return nil } @@ -95,7 +91,6 @@ func (ta *TimestampAllocator) Alloc(count uint32) ([]Timestamp, error) { } req.count = count ta.reqs <- req - fmt.Println("YYYYY ", len(ta.reqs)) req.Wait() if !req.IsValid() { diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index c4ea463cf9..cad6777cb6 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -29,3 +29,4 @@ add_subdirectory( pb ) add_subdirectory( segcore ) add_subdirectory( cache ) add_subdirectory( query ) +add_subdirectory( common ) diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt new file mode 100644 index 0000000000..f52a79e5b4 --- /dev/null +++ b/internal/core/src/common/CMakeLists.txt @@ -0,0 +1,8 @@ +set(COMMON_SRC + Schema.cpp +) + +add_library(milvus_common + ${COMMON_SRC} +) +target_link_libraries(milvus_common milvus_proto) diff --git a/internal/core/src/common/Schema.cpp b/internal/core/src/common/Schema.cpp new file mode 100644 index 0000000000..baa1a785d3 --- /dev/null +++ b/internal/core/src/common/Schema.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include "common/Schema.h" +#include + +namespace milvus { +std::shared_ptr +Schema::ParseFrom(const milvus::proto::schema::CollectionSchema& schema_proto) { + auto schema = std::make_shared(); + schema->set_auto_id(schema_proto.autoid()); + for (const milvus::proto::schema::FieldSchema& child : schema_proto.fields()) { + const auto& type_params = child.type_params(); + int64_t dim = -1; + auto data_type = DataType(child.data_type()); + for (const auto& type_param : type_params) { + if (type_param.key() == "dim") { + dim = strtoll(type_param.value().c_str(), nullptr, 10); + } + } + + if (field_is_vector(data_type)) { + AssertInfo(dim != -1, "dim not found"); + } else { + AssertInfo(dim == 1 || dim == -1, "Invalid dim field. Should be 1 or not exists"); + dim = 1; + } + + if (child.is_primary_key()) { + AssertInfo(!schema->primary_key_offset_opt_.has_value(), "repetitive primary key"); + schema->primary_key_offset_opt_ = schema->size(); + } + + schema->AddField(child.name(), data_type, dim); + } + return schema; +} +} // namespace milvus diff --git a/internal/core/src/common/Schema.h b/internal/core/src/common/Schema.h index fafb97619d..05710c7944 100644 --- a/internal/core/src/common/Schema.h +++ b/internal/core/src/common/Schema.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include namespace milvus { @@ -37,6 +39,11 @@ class Schema { total_sizeof_ += field_sizeof; } + void + set_auto_id(bool is_auto_id) { + is_auto_id_ = is_auto_id; + } + auto begin() { return fields_.begin(); @@ -46,6 +53,13 @@ class Schema { end() { return fields_.end(); } + + public: + bool + get_is_auto_id() const { + return is_auto_id_; + } + auto begin() const { return fields_.begin(); @@ -100,6 +114,15 @@ class Schema { return (*this)[offset]; } + std::optional + get_primary_key_offset() const { + return primary_key_offset_opt_; + } + + public: + static std::shared_ptr + ParseFrom(const milvus::proto::schema::CollectionSchema& schema_proto); + private: // this is where data holds std::vector fields_; @@ -109,6 +132,8 @@ class Schema { std::unordered_map offsets_; std::vector sizeof_infos_; int total_sizeof_ = 0; + bool is_auto_id_ = true; + std::optional primary_key_offset_opt_; }; using SchemaPtr = std::shared_ptr; diff --git a/internal/core/src/pb/schema.pb.cc b/internal/core/src/pb/schema.pb.cc index 1c93b5ffe4..71d00e0556 100644 --- a/internal/core/src/pb/schema.pb.cc +++ b/internal/core/src/pb/schema.pb.cc @@ -72,6 +72,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_schema_2eproto::offsets[] PROT ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldSchema, name_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldSchema, is_primary_key_), PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldSchema, description_), PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldSchema, data_type_), PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldSchema, type_params_), @@ -88,7 +89,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_schema_2eproto::offsets[] PROT }; static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::proto::schema::FieldSchema)}, - { 10, -1, sizeof(::milvus::proto::schema::CollectionSchema)}, + { 11, -1, sizeof(::milvus::proto::schema::CollectionSchema)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -98,21 +99,21 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_schema_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014schema.proto\022\023milvus.proto.schema\032\014com" - "mon.proto\"\323\001\n\013FieldSchema\022\014\n\004name\030\001 \001(\t\022" - "\023\n\013description\030\002 \001(\t\0220\n\tdata_type\030\003 \001(\0162" - "\035.milvus.proto.schema.DataType\0226\n\013type_p" - "arams\030\004 \003(\0132!.milvus.proto.common.KeyVal" - "uePair\0227\n\014index_params\030\005 \003(\0132!.milvus.pr" - "oto.common.KeyValuePair\"w\n\020CollectionSch" - "ema\022\014\n\004name\030\001 \001(\t\022\023\n\013description\030\002 \001(\t\022\016" - "\n\006autoID\030\003 \001(\010\0220\n\006fields\030\004 \003(\0132 .milvus." - "proto.schema.FieldSchema*\221\001\n\010DataType\022\010\n" - "\004NONE\020\000\022\010\n\004BOOL\020\001\022\010\n\004INT8\020\002\022\t\n\005INT16\020\003\022\t" - "\n\005INT32\020\004\022\t\n\005INT64\020\005\022\t\n\005FLOAT\020\n\022\n\n\006DOUBL" - "E\020\013\022\n\n\006STRING\020\024\022\021\n\rVECTOR_BINARY\020d\022\020\n\014VE" - "CTOR_FLOAT\020eBBZ@github.com/zilliztech/mi" - "lvus-distributed/internal/proto/schemapb" - "b\006proto3" + "mon.proto\"\353\001\n\013FieldSchema\022\014\n\004name\030\001 \001(\t\022" + "\026\n\016is_primary_key\030\002 \001(\010\022\023\n\013description\030\003" + " \001(\t\0220\n\tdata_type\030\004 \001(\0162\035.milvus.proto.s" + "chema.DataType\0226\n\013type_params\030\005 \003(\0132!.mi" + "lvus.proto.common.KeyValuePair\0227\n\014index_" + "params\030\006 \003(\0132!.milvus.proto.common.KeyVa" + "luePair\"w\n\020CollectionSchema\022\014\n\004name\030\001 \001(" + "\t\022\023\n\013description\030\002 \001(\t\022\016\n\006autoID\030\003 \001(\010\0220" + "\n\006fields\030\004 \003(\0132 .milvus.proto.schema.Fie" + "ldSchema*\221\001\n\010DataType\022\010\n\004NONE\020\000\022\010\n\004BOOL\020" + "\001\022\010\n\004INT8\020\002\022\t\n\005INT16\020\003\022\t\n\005INT32\020\004\022\t\n\005INT" + "64\020\005\022\t\n\005FLOAT\020\n\022\n\n\006DOUBLE\020\013\022\n\n\006STRING\020\024\022" + "\021\n\rVECTOR_BINARY\020d\022\020\n\014VECTOR_FLOAT\020eBBZ@" + "github.com/zilliztech/milvus-distributed" + "/internal/proto/schemapbb\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_schema_2eproto_deps[1] = { &::descriptor_table_common_2eproto, @@ -124,7 +125,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_sch static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_schema_2eproto_once; static bool descriptor_table_schema_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_schema_2eproto = { - &descriptor_table_schema_2eproto_initialized, descriptor_table_protodef_schema_2eproto, "schema.proto", 608, + &descriptor_table_schema_2eproto_initialized, descriptor_table_protodef_schema_2eproto, "schema.proto", 632, &descriptor_table_schema_2eproto_once, descriptor_table_schema_2eproto_sccs, descriptor_table_schema_2eproto_deps, 2, 1, schemas, file_default_instances, TableStruct_schema_2eproto::offsets, file_level_metadata_schema_2eproto, 2, file_level_enum_descriptors_schema_2eproto, file_level_service_descriptors_schema_2eproto, @@ -192,7 +193,9 @@ FieldSchema::FieldSchema(const FieldSchema& from) if (!from.description().empty()) { description_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.description_); } - data_type_ = from.data_type_; + ::memcpy(&is_primary_key_, &from.is_primary_key_, + static_cast(reinterpret_cast(&data_type_) - + reinterpret_cast(&is_primary_key_)) + sizeof(data_type_)); // @@protoc_insertion_point(copy_constructor:milvus.proto.schema.FieldSchema) } @@ -200,7 +203,9 @@ void FieldSchema::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_FieldSchema_schema_2eproto.base); name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); description_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - data_type_ = 0; + ::memset(&is_primary_key_, 0, static_cast( + reinterpret_cast(&data_type_) - + reinterpret_cast(&is_primary_key_)) + sizeof(data_type_)); } FieldSchema::~FieldSchema() { @@ -232,7 +237,9 @@ void FieldSchema::Clear() { index_params_.Clear(); name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); description_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - data_type_ = 0; + ::memset(&is_primary_key_, 0, static_cast( + reinterpret_cast(&data_type_) - + reinterpret_cast(&is_primary_key_)) + sizeof(data_type_)); _internal_metadata_.Clear(); } @@ -251,43 +258,50 @@ const char* FieldSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; - // string description = 2; + // bool is_primary_key = 2; case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + is_primary_key_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // string description = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_description(), ptr, ctx, "milvus.proto.schema.FieldSchema.description"); CHK_(ptr); } else goto handle_unusual; continue; - // .milvus.proto.schema.DataType data_type = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + // .milvus.proto.schema.DataType data_type = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { ::PROTOBUF_NAMESPACE_ID::uint64 val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); set_data_type(static_cast<::milvus::proto::schema::DataType>(val)); } else goto handle_unusual; continue; - // repeated .milvus.proto.common.KeyValuePair type_params = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) { + // repeated .milvus.proto.common.KeyValuePair type_params = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(add_type_params(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; - } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 34); + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); } else goto handle_unusual; continue; - // repeated .milvus.proto.common.KeyValuePair index_params = 5; - case 5: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { + // repeated .milvus.proto.common.KeyValuePair index_params = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) { ptr -= 1; do { ptr += 1; ptr = ctx->ParseMessage(add_index_params(), ptr); CHK_(ptr); if (!ctx->DataAvailable(ptr)) break; - } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 50); } else goto handle_unusual; continue; default: { @@ -335,9 +349,22 @@ bool FieldSchema::MergePartialFromCodedStream( break; } - // string description = 2; + // bool is_primary_key = 2; case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + bool, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_BOOL>( + input, &is_primary_key_))); + } else { + goto handle_unusual; + } + break; + } + + // string description = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( input, this->mutable_description())); DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( @@ -350,9 +377,9 @@ bool FieldSchema::MergePartialFromCodedStream( break; } - // .milvus.proto.schema.DataType data_type = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + // .milvus.proto.schema.DataType data_type = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { int value = 0; DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< int, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_ENUM>( @@ -364,9 +391,9 @@ bool FieldSchema::MergePartialFromCodedStream( break; } - // repeated .milvus.proto.common.KeyValuePair type_params = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (34 & 0xFF)) { + // repeated .milvus.proto.common.KeyValuePair type_params = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( input, add_type_params())); } else { @@ -375,9 +402,9 @@ bool FieldSchema::MergePartialFromCodedStream( break; } - // repeated .milvus.proto.common.KeyValuePair index_params = 5; - case 5: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { + // repeated .milvus.proto.common.KeyValuePair index_params = 6; + case 6: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( input, add_index_params())); } else { @@ -423,36 +450,41 @@ void FieldSchema::SerializeWithCachedSizes( 1, this->name(), output); } - // string description = 2; + // bool is_primary_key = 2; + if (this->is_primary_key() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBool(2, this->is_primary_key(), output); + } + + // string description = 3; if (this->description().size() > 0) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->description().data(), static_cast(this->description().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.proto.schema.FieldSchema.description"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->description(), output); + 3, this->description(), output); } - // .milvus.proto.schema.DataType data_type = 3; + // .milvus.proto.schema.DataType data_type = 4; if (this->data_type() != 0) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnum( - 3, this->data_type(), output); + 4, this->data_type(), output); } - // repeated .milvus.proto.common.KeyValuePair type_params = 4; + // repeated .milvus.proto.common.KeyValuePair type_params = 5; for (unsigned int i = 0, n = static_cast(this->type_params_size()); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 4, + 5, this->type_params(static_cast(i)), output); } - // repeated .milvus.proto.common.KeyValuePair index_params = 5; + // repeated .milvus.proto.common.KeyValuePair index_params = 6; for (unsigned int i = 0, n = static_cast(this->index_params_size()); i < n; i++) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 5, + 6, this->index_params(static_cast(i)), output); } @@ -481,7 +513,12 @@ void FieldSchema::SerializeWithCachedSizes( 1, this->name(), target); } - // string description = 2; + // bool is_primary_key = 2; + if (this->is_primary_key() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(2, this->is_primary_key(), target); + } + + // string description = 3; if (this->description().size() > 0) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->description().data(), static_cast(this->description().length()), @@ -489,29 +526,29 @@ void FieldSchema::SerializeWithCachedSizes( "milvus.proto.schema.FieldSchema.description"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( - 2, this->description(), target); + 3, this->description(), target); } - // .milvus.proto.schema.DataType data_type = 3; + // .milvus.proto.schema.DataType data_type = 4; if (this->data_type() != 0) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray( - 3, this->data_type(), target); + 4, this->data_type(), target); } - // repeated .milvus.proto.common.KeyValuePair type_params = 4; + // repeated .milvus.proto.common.KeyValuePair type_params = 5; for (unsigned int i = 0, n = static_cast(this->type_params_size()); i < n; i++) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 4, this->type_params(static_cast(i)), target); + 5, this->type_params(static_cast(i)), target); } - // repeated .milvus.proto.common.KeyValuePair index_params = 5; + // repeated .milvus.proto.common.KeyValuePair index_params = 6; for (unsigned int i = 0, n = static_cast(this->index_params_size()); i < n; i++) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 5, this->index_params(static_cast(i)), target); + 6, this->index_params(static_cast(i)), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -535,7 +572,7 @@ size_t FieldSchema::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // repeated .milvus.proto.common.KeyValuePair type_params = 4; + // repeated .milvus.proto.common.KeyValuePair type_params = 5; { unsigned int count = static_cast(this->type_params_size()); total_size += 1UL * count; @@ -546,7 +583,7 @@ size_t FieldSchema::ByteSizeLong() const { } } - // repeated .milvus.proto.common.KeyValuePair index_params = 5; + // repeated .milvus.proto.common.KeyValuePair index_params = 6; { unsigned int count = static_cast(this->index_params_size()); total_size += 1UL * count; @@ -564,14 +601,19 @@ size_t FieldSchema::ByteSizeLong() const { this->name()); } - // string description = 2; + // string description = 3; if (this->description().size() > 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->description()); } - // .milvus.proto.schema.DataType data_type = 3; + // bool is_primary_key = 2; + if (this->is_primary_key() != 0) { + total_size += 1 + 1; + } + + // .milvus.proto.schema.DataType data_type = 4; if (this->data_type() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->data_type()); @@ -614,6 +656,9 @@ void FieldSchema::MergeFrom(const FieldSchema& from) { description_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.description_); } + if (from.is_primary_key() != 0) { + set_is_primary_key(from.is_primary_key()); + } if (from.data_type() != 0) { set_data_type(from.data_type()); } @@ -646,6 +691,7 @@ void FieldSchema::InternalSwap(FieldSchema* other) { GetArenaNoVirtual()); description_.Swap(&other->description_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); + swap(is_primary_key_, other->is_primary_key_); swap(data_type_, other->data_type_); } diff --git a/internal/core/src/pb/schema.pb.h b/internal/core/src/pb/schema.pb.h index ee5c3d66c5..d5154a7c61 100644 --- a/internal/core/src/pb/schema.pb.h +++ b/internal/core/src/pb/schema.pb.h @@ -225,13 +225,14 @@ class FieldSchema : // accessors ------------------------------------------------------- enum : int { - kTypeParamsFieldNumber = 4, - kIndexParamsFieldNumber = 5, + kTypeParamsFieldNumber = 5, + kIndexParamsFieldNumber = 6, kNameFieldNumber = 1, - kDescriptionFieldNumber = 2, - kDataTypeFieldNumber = 3, + kDescriptionFieldNumber = 3, + kIsPrimaryKeyFieldNumber = 2, + kDataTypeFieldNumber = 4, }; - // repeated .milvus.proto.common.KeyValuePair type_params = 4; + // repeated .milvus.proto.common.KeyValuePair type_params = 5; int type_params_size() const; void clear_type_params(); ::milvus::proto::common::KeyValuePair* mutable_type_params(int index); @@ -242,7 +243,7 @@ class FieldSchema : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >& type_params() const; - // repeated .milvus.proto.common.KeyValuePair index_params = 5; + // repeated .milvus.proto.common.KeyValuePair index_params = 6; int index_params_size() const; void clear_index_params(); ::milvus::proto::common::KeyValuePair* mutable_index_params(int index); @@ -264,7 +265,7 @@ class FieldSchema : std::string* release_name(); void set_allocated_name(std::string* name); - // string description = 2; + // string description = 3; void clear_description(); const std::string& description() const; void set_description(const std::string& value); @@ -275,7 +276,12 @@ class FieldSchema : std::string* release_description(); void set_allocated_description(std::string* description); - // .milvus.proto.schema.DataType data_type = 3; + // bool is_primary_key = 2; + void clear_is_primary_key(); + bool is_primary_key() const; + void set_is_primary_key(bool value); + + // .milvus.proto.schema.DataType data_type = 4; void clear_data_type(); ::milvus::proto::schema::DataType data_type() const; void set_data_type(::milvus::proto::schema::DataType value); @@ -289,6 +295,7 @@ class FieldSchema : ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair > index_params_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr description_; + bool is_primary_key_; int data_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_schema_2eproto; @@ -525,7 +532,21 @@ inline void FieldSchema::set_allocated_name(std::string* name) { // @@protoc_insertion_point(field_set_allocated:milvus.proto.schema.FieldSchema.name) } -// string description = 2; +// bool is_primary_key = 2; +inline void FieldSchema::clear_is_primary_key() { + is_primary_key_ = false; +} +inline bool FieldSchema::is_primary_key() const { + // @@protoc_insertion_point(field_get:milvus.proto.schema.FieldSchema.is_primary_key) + return is_primary_key_; +} +inline void FieldSchema::set_is_primary_key(bool value) { + + is_primary_key_ = value; + // @@protoc_insertion_point(field_set:milvus.proto.schema.FieldSchema.is_primary_key) +} + +// string description = 3; inline void FieldSchema::clear_description() { description_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } @@ -576,7 +597,7 @@ inline void FieldSchema::set_allocated_description(std::string* description) { // @@protoc_insertion_point(field_set_allocated:milvus.proto.schema.FieldSchema.description) } -// .milvus.proto.schema.DataType data_type = 3; +// .milvus.proto.schema.DataType data_type = 4; inline void FieldSchema::clear_data_type() { data_type_ = 0; } @@ -590,7 +611,7 @@ inline void FieldSchema::set_data_type(::milvus::proto::schema::DataType value) // @@protoc_insertion_point(field_set:milvus.proto.schema.FieldSchema.data_type) } -// repeated .milvus.proto.common.KeyValuePair type_params = 4; +// repeated .milvus.proto.common.KeyValuePair type_params = 5; inline int FieldSchema::type_params_size() const { return type_params_.size(); } @@ -617,7 +638,7 @@ FieldSchema::type_params() const { return type_params_; } -// repeated .milvus.proto.common.KeyValuePair index_params = 5; +// repeated .milvus.proto.common.KeyValuePair index_params = 6; inline int FieldSchema::index_params_size() const { return index_params_.size(); } diff --git a/internal/core/src/pb/service_msg.pb.cc b/internal/core/src/pb/service_msg.pb.cc index 0d01d784c7..070902e4da 100644 --- a/internal/core/src/pb/service_msg.pb.cc +++ b/internal/core/src/pb/service_msg.pb.cc @@ -76,10 +76,6 @@ class PartitionDescriptionDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _PartitionDescription_default_instance_; -class SysConfigResponseDefaultTypeInternal { - public: - ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; -} _SysConfigResponse_default_instance_; class HitsDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -315,22 +311,7 @@ static void InitDefaultsscc_info_StringResponse_service_5fmsg_2eproto() { {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_StringResponse_service_5fmsg_2eproto}, { &scc_info_Status_common_2eproto.base,}}; -static void InitDefaultsscc_info_SysConfigResponse_service_5fmsg_2eproto() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - - { - void* ptr = &::milvus::proto::service::_SysConfigResponse_default_instance_; - new (ptr) ::milvus::proto::service::SysConfigResponse(); - ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); - } - ::milvus::proto::service::SysConfigResponse::InitAsDefaultInstance(); -} - -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_SysConfigResponse_service_5fmsg_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_SysConfigResponse_service_5fmsg_2eproto}, { - &scc_info_Status_common_2eproto.base,}}; - -static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_service_5fmsg_2eproto[16]; +static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_service_5fmsg_2eproto[15]; static const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* file_level_enum_descriptors_service_5fmsg_2eproto[1]; static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_service_5fmsg_2eproto = nullptr; @@ -433,14 +414,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_service_5fmsg_2eproto::offsets PROTOBUF_FIELD_OFFSET(::milvus::proto::service::PartitionDescription, name_), PROTOBUF_FIELD_OFFSET(::milvus::proto::service::PartitionDescription, statistics_), ~0u, // no _has_bits_ - PROTOBUF_FIELD_OFFSET(::milvus::proto::service::SysConfigResponse, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - PROTOBUF_FIELD_OFFSET(::milvus::proto::service::SysConfigResponse, status_), - PROTOBUF_FIELD_OFFSET(::milvus::proto::service::SysConfigResponse, keys_), - PROTOBUF_FIELD_OFFSET(::milvus::proto::service::SysConfigResponse, values_), - ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::proto::service::Hits, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -470,9 +443,8 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 73, -1, sizeof(::milvus::proto::service::IntegerRangeResponse)}, { 81, -1, sizeof(::milvus::proto::service::CollectionDescription)}, { 89, -1, sizeof(::milvus::proto::service::PartitionDescription)}, - { 97, -1, sizeof(::milvus::proto::service::SysConfigResponse)}, - { 105, -1, sizeof(::milvus::proto::service::Hits)}, - { 113, -1, sizeof(::milvus::proto::service::QueryResult)}, + { 97, -1, sizeof(::milvus::proto::service::Hits)}, + { 105, -1, sizeof(::milvus::proto::service::QueryResult)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -489,7 +461,6 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = reinterpret_cast(&::milvus::proto::service::_IntegerRangeResponse_default_instance_), reinterpret_cast(&::milvus::proto::service::_CollectionDescription_default_instance_), reinterpret_cast(&::milvus::proto::service::_PartitionDescription_default_instance_), - reinterpret_cast(&::milvus::proto::service::_SysConfigResponse_default_instance_), reinterpret_cast(&::milvus::proto::service::_Hits_default_instance_), reinterpret_cast(&::milvus::proto::service::_QueryResult_default_instance_), }; @@ -528,22 +499,20 @@ const char descriptor_table_protodef_service_5fmsg_2eproto[] PROTOBUF_SECTION_VA "\006status\030\001 \001(\0132\033.milvus.proto.common.Stat" "us\0221\n\004name\030\002 \001(\0132#.milvus.proto.service." "PartitionName\0225\n\nstatistics\030\003 \003(\0132!.milv" - "us.proto.common.KeyValuePair\"^\n\021SysConfi" - "gResponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto" - ".common.Status\022\014\n\004keys\030\002 \003(\t\022\016\n\006values\030\003" - " \003(\t\"5\n\004Hits\022\013\n\003IDs\030\001 \003(\003\022\020\n\010row_data\030\002 " - "\003(\014\022\016\n\006scores\030\003 \003(\002\"H\n\013QueryResult\022+\n\006st" - "atus\030\001 \001(\0132\033.milvus.proto.common.Status\022" - "\014\n\004hits\030\002 \003(\014*@\n\017PlaceholderType\022\010\n\004NONE" - "\020\000\022\021\n\rVECTOR_BINARY\020d\022\020\n\014VECTOR_FLOAT\020eB" - "CZAgithub.com/zilliztech/milvus-distribu" - "ted/internal/proto/servicepbb\006proto3" + "us.proto.common.KeyValuePair\"5\n\004Hits\022\013\n\003" + "IDs\030\001 \003(\003\022\020\n\010row_data\030\002 \003(\014\022\016\n\006scores\030\003 " + "\003(\002\"H\n\013QueryResult\022+\n\006status\030\001 \001(\0132\033.mil" + "vus.proto.common.Status\022\014\n\004hits\030\002 \003(\014*@\n" + "\017PlaceholderType\022\010\n\004NONE\020\000\022\021\n\rVECTOR_BIN" + "ARY\020d\022\020\n\014VECTOR_FLOAT\020eBCZAgithub.com/zi" + "lliztech/milvus-distributed/internal/pro" + "to/servicepbb\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_service_5fmsg_2eproto_deps[2] = { &::descriptor_table_common_2eproto, &::descriptor_table_schema_2eproto, }; -static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_service_5fmsg_2eproto_sccs[16] = { +static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_service_5fmsg_2eproto_sccs[15] = { &scc_info_BoolResponse_service_5fmsg_2eproto.base, &scc_info_CollectionDescription_service_5fmsg_2eproto.base, &scc_info_CollectionName_service_5fmsg_2eproto.base, @@ -559,15 +528,14 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_ser &scc_info_RowBatch_service_5fmsg_2eproto.base, &scc_info_StringListResponse_service_5fmsg_2eproto.base, &scc_info_StringResponse_service_5fmsg_2eproto.base, - &scc_info_SysConfigResponse_service_5fmsg_2eproto.base, }; static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_service_5fmsg_2eproto_once; static bool descriptor_table_service_5fmsg_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_service_5fmsg_2eproto = { - &descriptor_table_service_5fmsg_2eproto_initialized, descriptor_table_protodef_service_5fmsg_2eproto, "service_msg.proto", 1716, - &descriptor_table_service_5fmsg_2eproto_once, descriptor_table_service_5fmsg_2eproto_sccs, descriptor_table_service_5fmsg_2eproto_deps, 16, 2, + &descriptor_table_service_5fmsg_2eproto_initialized, descriptor_table_protodef_service_5fmsg_2eproto, "service_msg.proto", 1620, + &descriptor_table_service_5fmsg_2eproto_once, descriptor_table_service_5fmsg_2eproto_sccs, descriptor_table_service_5fmsg_2eproto_deps, 15, 2, schemas, file_default_instances, TableStruct_service_5fmsg_2eproto::offsets, - file_level_metadata_service_5fmsg_2eproto, 16, file_level_enum_descriptors_service_5fmsg_2eproto, file_level_service_descriptors_service_5fmsg_2eproto, + file_level_metadata_service_5fmsg_2eproto, 15, file_level_enum_descriptors_service_5fmsg_2eproto, file_level_service_descriptors_service_5fmsg_2eproto, }; // Force running AddDescriptors() at dynamic initialization time. @@ -5191,398 +5159,6 @@ void PartitionDescription::InternalSwap(PartitionDescription* other) { } -// =================================================================== - -void SysConfigResponse::InitAsDefaultInstance() { - ::milvus::proto::service::_SysConfigResponse_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::proto::common::Status*>( - ::milvus::proto::common::Status::internal_default_instance()); -} -class SysConfigResponse::_Internal { - public: - static const ::milvus::proto::common::Status& status(const SysConfigResponse* msg); -}; - -const ::milvus::proto::common::Status& -SysConfigResponse::_Internal::status(const SysConfigResponse* msg) { - return *msg->status_; -} -void SysConfigResponse::clear_status() { - if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { - delete status_; - } - status_ = nullptr; -} -SysConfigResponse::SysConfigResponse() - : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { - SharedCtor(); - // @@protoc_insertion_point(constructor:milvus.proto.service.SysConfigResponse) -} -SysConfigResponse::SysConfigResponse(const SysConfigResponse& from) - : ::PROTOBUF_NAMESPACE_ID::Message(), - _internal_metadata_(nullptr), - keys_(from.keys_), - values_(from.values_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - if (from.has_status()) { - status_ = new ::milvus::proto::common::Status(*from.status_); - } else { - status_ = nullptr; - } - // @@protoc_insertion_point(copy_constructor:milvus.proto.service.SysConfigResponse) -} - -void SysConfigResponse::SharedCtor() { - ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_SysConfigResponse_service_5fmsg_2eproto.base); - status_ = nullptr; -} - -SysConfigResponse::~SysConfigResponse() { - // @@protoc_insertion_point(destructor:milvus.proto.service.SysConfigResponse) - SharedDtor(); -} - -void SysConfigResponse::SharedDtor() { - if (this != internal_default_instance()) delete status_; -} - -void SysConfigResponse::SetCachedSize(int size) const { - _cached_size_.Set(size); -} -const SysConfigResponse& SysConfigResponse::default_instance() { - ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_SysConfigResponse_service_5fmsg_2eproto.base); - return *internal_default_instance(); -} - - -void SysConfigResponse::Clear() { -// @@protoc_insertion_point(message_clear_start:milvus.proto.service.SysConfigResponse) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - keys_.Clear(); - values_.Clear(); - if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { - delete status_; - } - status_ = nullptr; - _internal_metadata_.Clear(); -} - -#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -const char* SysConfigResponse::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { -#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure - while (!ctx->Done(&ptr)) { - ::PROTOBUF_NAMESPACE_ID::uint32 tag; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); - CHK_(ptr); - switch (tag >> 3) { - // .milvus.proto.common.Status status = 1; - case 1: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { - ptr = ctx->ParseMessage(mutable_status(), ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // repeated string keys = 2; - case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { - ptr -= 1; - do { - ptr += 1; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_keys(), ptr, ctx, "milvus.proto.service.SysConfigResponse.keys"); - CHK_(ptr); - if (!ctx->DataAvailable(ptr)) break; - } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); - } else goto handle_unusual; - continue; - // repeated string values = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { - ptr -= 1; - do { - ptr += 1; - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_values(), ptr, ctx, "milvus.proto.service.SysConfigResponse.values"); - CHK_(ptr); - if (!ctx->DataAvailable(ptr)) break; - } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 26); - } else goto handle_unusual; - continue; - default: { - handle_unusual: - if ((tag & 7) == 4 || tag == 0) { - ctx->SetLastTag(tag); - goto success; - } - ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); - CHK_(ptr != nullptr); - continue; - } - } // switch - } // while -success: - return ptr; -failure: - ptr = nullptr; - goto success; -#undef CHK_ -} -#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -bool SysConfigResponse::MergePartialFromCodedStream( - ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure - ::PROTOBUF_NAMESPACE_ID::uint32 tag; - // @@protoc_insertion_point(parse_start:milvus.proto.service.SysConfigResponse) - for (;;) { - ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // .milvus.proto.common.Status status = 1; - case 1: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( - input, mutable_status())); - } else { - goto handle_unusual; - } - break; - } - - // repeated string keys = 2; - case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( - input, this->add_keys())); - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->keys(this->keys_size() - 1).data(), - static_cast(this->keys(this->keys_size() - 1).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.proto.service.SysConfigResponse.keys")); - } else { - goto handle_unusual; - } - break; - } - - // repeated string values = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( - input, this->add_values())); - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->values(this->values_size() - 1).data(), - static_cast(this->values(this->values_size() - 1).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.proto.service.SysConfigResponse.values")); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:milvus.proto.service.SysConfigResponse) - return true; -failure: - // @@protoc_insertion_point(parse_failure:milvus.proto.service.SysConfigResponse) - return false; -#undef DO_ -} -#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - -void SysConfigResponse::SerializeWithCachedSizes( - ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:milvus.proto.service.SysConfigResponse) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // .milvus.proto.common.Status status = 1; - if (this->has_status()) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 1, _Internal::status(this), output); - } - - // repeated string keys = 2; - for (int i = 0, n = this->keys_size(); i < n; i++) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->keys(i).data(), static_cast(this->keys(i).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, - "milvus.proto.service.SysConfigResponse.keys"); - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( - 2, this->keys(i), output); - } - - // repeated string values = 3; - for (int i = 0, n = this->values_size(); i < n; i++) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->values(i).data(), static_cast(this->values(i).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, - "milvus.proto.service.SysConfigResponse.values"); - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( - 3, this->values(i), output); - } - - if (_internal_metadata_.have_unknown_fields()) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( - _internal_metadata_.unknown_fields(), output); - } - // @@protoc_insertion_point(serialize_end:milvus.proto.service.SysConfigResponse) -} - -::PROTOBUF_NAMESPACE_ID::uint8* SysConfigResponse::InternalSerializeWithCachedSizesToArray( - ::PROTOBUF_NAMESPACE_ID::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:milvus.proto.service.SysConfigResponse) - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // .milvus.proto.common.Status status = 1; - if (this->has_status()) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - InternalWriteMessageToArray( - 1, _Internal::status(this), target); - } - - // repeated string keys = 2; - for (int i = 0, n = this->keys_size(); i < n; i++) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->keys(i).data(), static_cast(this->keys(i).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, - "milvus.proto.service.SysConfigResponse.keys"); - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - WriteStringToArray(2, this->keys(i), target); - } - - // repeated string values = 3; - for (int i = 0, n = this->values_size(); i < n; i++) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( - this->values(i).data(), static_cast(this->values(i).length()), - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, - "milvus.proto.service.SysConfigResponse.values"); - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - WriteStringToArray(3, this->values(i), target); - } - - if (_internal_metadata_.have_unknown_fields()) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( - _internal_metadata_.unknown_fields(), target); - } - // @@protoc_insertion_point(serialize_to_array_end:milvus.proto.service.SysConfigResponse) - return target; -} - -size_t SysConfigResponse::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:milvus.proto.service.SysConfigResponse) - size_t total_size = 0; - - if (_internal_metadata_.have_unknown_fields()) { - total_size += - ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( - _internal_metadata_.unknown_fields()); - } - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - // repeated string keys = 2; - total_size += 1 * - ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->keys_size()); - for (int i = 0, n = this->keys_size(); i < n; i++) { - total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( - this->keys(i)); - } - - // repeated string values = 3; - total_size += 1 * - ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->values_size()); - for (int i = 0, n = this->values_size(); i < n; i++) { - total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( - this->values(i)); - } - - // .milvus.proto.common.Status status = 1; - if (this->has_status()) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( - *status_); - } - - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); - SetCachedSize(cached_size); - return total_size; -} - -void SysConfigResponse::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:milvus.proto.service.SysConfigResponse) - GOOGLE_DCHECK_NE(&from, this); - const SysConfigResponse* source = - ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( - &from); - if (source == nullptr) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.proto.service.SysConfigResponse) - ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.proto.service.SysConfigResponse) - MergeFrom(*source); - } -} - -void SysConfigResponse::MergeFrom(const SysConfigResponse& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.service.SysConfigResponse) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - keys_.MergeFrom(from.keys_); - values_.MergeFrom(from.values_); - if (from.has_status()) { - mutable_status()->::milvus::proto::common::Status::MergeFrom(from.status()); - } -} - -void SysConfigResponse::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:milvus.proto.service.SysConfigResponse) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void SysConfigResponse::CopyFrom(const SysConfigResponse& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.service.SysConfigResponse) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool SysConfigResponse::IsInitialized() const { - return true; -} - -void SysConfigResponse::InternalSwap(SysConfigResponse* other) { - using std::swap; - _internal_metadata_.Swap(&other->_internal_metadata_); - keys_.InternalSwap(CastToBase(&other->keys_)); - values_.InternalSwap(CastToBase(&other->values_)); - swap(status_, other->status_); -} - -::PROTOBUF_NAMESPACE_ID::Metadata SysConfigResponse::GetMetadata() const { - return GetMetadataStatic(); -} - - // =================================================================== void Hits::InitAsDefaultInstance() { @@ -6335,9 +5911,6 @@ template<> PROTOBUF_NOINLINE ::milvus::proto::service::CollectionDescription* Ar template<> PROTOBUF_NOINLINE ::milvus::proto::service::PartitionDescription* Arena::CreateMaybeMessage< ::milvus::proto::service::PartitionDescription >(Arena* arena) { return Arena::CreateInternal< ::milvus::proto::service::PartitionDescription >(arena); } -template<> PROTOBUF_NOINLINE ::milvus::proto::service::SysConfigResponse* Arena::CreateMaybeMessage< ::milvus::proto::service::SysConfigResponse >(Arena* arena) { - return Arena::CreateInternal< ::milvus::proto::service::SysConfigResponse >(arena); -} template<> PROTOBUF_NOINLINE ::milvus::proto::service::Hits* Arena::CreateMaybeMessage< ::milvus::proto::service::Hits >(Arena* arena) { return Arena::CreateInternal< ::milvus::proto::service::Hits >(arena); } diff --git a/internal/core/src/pb/service_msg.pb.h b/internal/core/src/pb/service_msg.pb.h index 9a6e7ba386..62d9101204 100644 --- a/internal/core/src/pb/service_msg.pb.h +++ b/internal/core/src/pb/service_msg.pb.h @@ -50,7 +50,7 @@ struct TableStruct_service_5fmsg_2eproto { PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] PROTOBUF_SECTION_VARIABLE(protodesc_cold); - static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[16] + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[15] PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; @@ -105,9 +105,6 @@ extern StringListResponseDefaultTypeInternal _StringListResponse_default_instanc class StringResponse; class StringResponseDefaultTypeInternal; extern StringResponseDefaultTypeInternal _StringResponse_default_instance_; -class SysConfigResponse; -class SysConfigResponseDefaultTypeInternal; -extern SysConfigResponseDefaultTypeInternal _SysConfigResponse_default_instance_; } // namespace service } // namespace proto } // namespace milvus @@ -127,7 +124,6 @@ template<> ::milvus::proto::service::QueryResult* Arena::CreateMaybeMessage<::mi template<> ::milvus::proto::service::RowBatch* Arena::CreateMaybeMessage<::milvus::proto::service::RowBatch>(Arena*); template<> ::milvus::proto::service::StringListResponse* Arena::CreateMaybeMessage<::milvus::proto::service::StringListResponse>(Arena*); template<> ::milvus::proto::service::StringResponse* Arena::CreateMaybeMessage<::milvus::proto::service::StringResponse>(Arena*); -template<> ::milvus::proto::service::SysConfigResponse* Arena::CreateMaybeMessage<::milvus::proto::service::SysConfigResponse>(Arena*); PROTOBUF_NAMESPACE_CLOSE namespace milvus { namespace proto { @@ -2158,178 +2154,6 @@ class PartitionDescription : }; // ------------------------------------------------------------------- -class SysConfigResponse : - public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.service.SysConfigResponse) */ { - public: - SysConfigResponse(); - virtual ~SysConfigResponse(); - - SysConfigResponse(const SysConfigResponse& from); - SysConfigResponse(SysConfigResponse&& from) noexcept - : SysConfigResponse() { - *this = ::std::move(from); - } - - inline SysConfigResponse& operator=(const SysConfigResponse& from) { - CopyFrom(from); - return *this; - } - inline SysConfigResponse& operator=(SysConfigResponse&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - - static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { - return GetDescriptor(); - } - static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { - return GetMetadataStatic().descriptor; - } - static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { - return GetMetadataStatic().reflection; - } - static const SysConfigResponse& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const SysConfigResponse* internal_default_instance() { - return reinterpret_cast( - &_SysConfigResponse_default_instance_); - } - static constexpr int kIndexInFileMessages = - 13; - - friend void swap(SysConfigResponse& a, SysConfigResponse& b) { - a.Swap(&b); - } - inline void Swap(SysConfigResponse* other) { - if (other == this) return; - InternalSwap(other); - } - - // implements Message ---------------------------------------------- - - inline SysConfigResponse* New() const final { - return CreateMaybeMessage(nullptr); - } - - SysConfigResponse* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { - return CreateMaybeMessage(arena); - } - void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void CopyFrom(const SysConfigResponse& from); - void MergeFrom(const SysConfigResponse& from); - PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; - bool IsInitialized() const final; - - size_t ByteSizeLong() const final; - #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; - #else - bool MergePartialFromCodedStream( - ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; - #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - void SerializeWithCachedSizes( - ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; - ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( - ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; - int GetCachedSize() const final { return _cached_size_.Get(); } - - private: - inline void SharedCtor(); - inline void SharedDtor(); - void SetCachedSize(int size) const final; - void InternalSwap(SysConfigResponse* other); - friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; - static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { - return "milvus.proto.service.SysConfigResponse"; - } - private: - inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { - return nullptr; - } - inline void* MaybeArenaPtr() const { - return nullptr; - } - public: - - ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; - private: - static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { - ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_service_5fmsg_2eproto); - return ::descriptor_table_service_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages]; - } - - public: - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - enum : int { - kKeysFieldNumber = 2, - kValuesFieldNumber = 3, - kStatusFieldNumber = 1, - }; - // repeated string keys = 2; - int keys_size() const; - void clear_keys(); - const std::string& keys(int index) const; - std::string* mutable_keys(int index); - void set_keys(int index, const std::string& value); - void set_keys(int index, std::string&& value); - void set_keys(int index, const char* value); - void set_keys(int index, const char* value, size_t size); - std::string* add_keys(); - void add_keys(const std::string& value); - void add_keys(std::string&& value); - void add_keys(const char* value); - void add_keys(const char* value, size_t size); - const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& keys() const; - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_keys(); - - // repeated string values = 3; - int values_size() const; - void clear_values(); - const std::string& values(int index) const; - std::string* mutable_values(int index); - void set_values(int index, const std::string& value); - void set_values(int index, std::string&& value); - void set_values(int index, const char* value); - void set_values(int index, const char* value, size_t size); - std::string* add_values(); - void add_values(const std::string& value); - void add_values(std::string&& value); - void add_values(const char* value); - void add_values(const char* value, size_t size); - const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& values() const; - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_values(); - - // .milvus.proto.common.Status status = 1; - bool has_status() const; - void clear_status(); - const ::milvus::proto::common::Status& status() const; - ::milvus::proto::common::Status* release_status(); - ::milvus::proto::common::Status* mutable_status(); - void set_allocated_status(::milvus::proto::common::Status* status); - - // @@protoc_insertion_point(class_scope:milvus.proto.service.SysConfigResponse) - private: - class _Internal; - - ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField keys_; - ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField values_; - ::milvus::proto::common::Status* status_; - mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; - friend struct ::TableStruct_service_5fmsg_2eproto; -}; -// ------------------------------------------------------------------- - class Hits : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.service.Hits) */ { public: @@ -2372,7 +2196,7 @@ class Hits : &_Hits_default_instance_); } static constexpr int kIndexInFileMessages = - 14; + 13; friend void swap(Hits& a, Hits& b) { a.Swap(&b); @@ -2543,7 +2367,7 @@ class QueryResult : &_QueryResult_default_instance_); } static constexpr int kIndexInFileMessages = - 15; + 14; friend void swap(QueryResult& a, QueryResult& b) { a.Swap(&b); @@ -4056,185 +3880,6 @@ PartitionDescription::statistics() const { // ------------------------------------------------------------------- -// SysConfigResponse - -// .milvus.proto.common.Status status = 1; -inline bool SysConfigResponse::has_status() const { - return this != internal_default_instance() && status_ != nullptr; -} -inline const ::milvus::proto::common::Status& SysConfigResponse::status() const { - const ::milvus::proto::common::Status* p = status_; - // @@protoc_insertion_point(field_get:milvus.proto.service.SysConfigResponse.status) - return p != nullptr ? *p : *reinterpret_cast( - &::milvus::proto::common::_Status_default_instance_); -} -inline ::milvus::proto::common::Status* SysConfigResponse::release_status() { - // @@protoc_insertion_point(field_release:milvus.proto.service.SysConfigResponse.status) - - ::milvus::proto::common::Status* temp = status_; - status_ = nullptr; - return temp; -} -inline ::milvus::proto::common::Status* SysConfigResponse::mutable_status() { - - if (status_ == nullptr) { - auto* p = CreateMaybeMessage<::milvus::proto::common::Status>(GetArenaNoVirtual()); - status_ = p; - } - // @@protoc_insertion_point(field_mutable:milvus.proto.service.SysConfigResponse.status) - return status_; -} -inline void SysConfigResponse::set_allocated_status(::milvus::proto::common::Status* status) { - ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == nullptr) { - delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(status_); - } - if (status) { - ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; - if (message_arena != submessage_arena) { - status = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( - message_arena, status, submessage_arena); - } - - } else { - - } - status_ = status; - // @@protoc_insertion_point(field_set_allocated:milvus.proto.service.SysConfigResponse.status) -} - -// repeated string keys = 2; -inline int SysConfigResponse::keys_size() const { - return keys_.size(); -} -inline void SysConfigResponse::clear_keys() { - keys_.Clear(); -} -inline const std::string& SysConfigResponse::keys(int index) const { - // @@protoc_insertion_point(field_get:milvus.proto.service.SysConfigResponse.keys) - return keys_.Get(index); -} -inline std::string* SysConfigResponse::mutable_keys(int index) { - // @@protoc_insertion_point(field_mutable:milvus.proto.service.SysConfigResponse.keys) - return keys_.Mutable(index); -} -inline void SysConfigResponse::set_keys(int index, const std::string& value) { - // @@protoc_insertion_point(field_set:milvus.proto.service.SysConfigResponse.keys) - keys_.Mutable(index)->assign(value); -} -inline void SysConfigResponse::set_keys(int index, std::string&& value) { - // @@protoc_insertion_point(field_set:milvus.proto.service.SysConfigResponse.keys) - keys_.Mutable(index)->assign(std::move(value)); -} -inline void SysConfigResponse::set_keys(int index, const char* value) { - GOOGLE_DCHECK(value != nullptr); - keys_.Mutable(index)->assign(value); - // @@protoc_insertion_point(field_set_char:milvus.proto.service.SysConfigResponse.keys) -} -inline void SysConfigResponse::set_keys(int index, const char* value, size_t size) { - keys_.Mutable(index)->assign( - reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:milvus.proto.service.SysConfigResponse.keys) -} -inline std::string* SysConfigResponse::add_keys() { - // @@protoc_insertion_point(field_add_mutable:milvus.proto.service.SysConfigResponse.keys) - return keys_.Add(); -} -inline void SysConfigResponse::add_keys(const std::string& value) { - keys_.Add()->assign(value); - // @@protoc_insertion_point(field_add:milvus.proto.service.SysConfigResponse.keys) -} -inline void SysConfigResponse::add_keys(std::string&& value) { - keys_.Add(std::move(value)); - // @@protoc_insertion_point(field_add:milvus.proto.service.SysConfigResponse.keys) -} -inline void SysConfigResponse::add_keys(const char* value) { - GOOGLE_DCHECK(value != nullptr); - keys_.Add()->assign(value); - // @@protoc_insertion_point(field_add_char:milvus.proto.service.SysConfigResponse.keys) -} -inline void SysConfigResponse::add_keys(const char* value, size_t size) { - keys_.Add()->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_add_pointer:milvus.proto.service.SysConfigResponse.keys) -} -inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& -SysConfigResponse::keys() const { - // @@protoc_insertion_point(field_list:milvus.proto.service.SysConfigResponse.keys) - return keys_; -} -inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* -SysConfigResponse::mutable_keys() { - // @@protoc_insertion_point(field_mutable_list:milvus.proto.service.SysConfigResponse.keys) - return &keys_; -} - -// repeated string values = 3; -inline int SysConfigResponse::values_size() const { - return values_.size(); -} -inline void SysConfigResponse::clear_values() { - values_.Clear(); -} -inline const std::string& SysConfigResponse::values(int index) const { - // @@protoc_insertion_point(field_get:milvus.proto.service.SysConfigResponse.values) - return values_.Get(index); -} -inline std::string* SysConfigResponse::mutable_values(int index) { - // @@protoc_insertion_point(field_mutable:milvus.proto.service.SysConfigResponse.values) - return values_.Mutable(index); -} -inline void SysConfigResponse::set_values(int index, const std::string& value) { - // @@protoc_insertion_point(field_set:milvus.proto.service.SysConfigResponse.values) - values_.Mutable(index)->assign(value); -} -inline void SysConfigResponse::set_values(int index, std::string&& value) { - // @@protoc_insertion_point(field_set:milvus.proto.service.SysConfigResponse.values) - values_.Mutable(index)->assign(std::move(value)); -} -inline void SysConfigResponse::set_values(int index, const char* value) { - GOOGLE_DCHECK(value != nullptr); - values_.Mutable(index)->assign(value); - // @@protoc_insertion_point(field_set_char:milvus.proto.service.SysConfigResponse.values) -} -inline void SysConfigResponse::set_values(int index, const char* value, size_t size) { - values_.Mutable(index)->assign( - reinterpret_cast(value), size); - // @@protoc_insertion_point(field_set_pointer:milvus.proto.service.SysConfigResponse.values) -} -inline std::string* SysConfigResponse::add_values() { - // @@protoc_insertion_point(field_add_mutable:milvus.proto.service.SysConfigResponse.values) - return values_.Add(); -} -inline void SysConfigResponse::add_values(const std::string& value) { - values_.Add()->assign(value); - // @@protoc_insertion_point(field_add:milvus.proto.service.SysConfigResponse.values) -} -inline void SysConfigResponse::add_values(std::string&& value) { - values_.Add(std::move(value)); - // @@protoc_insertion_point(field_add:milvus.proto.service.SysConfigResponse.values) -} -inline void SysConfigResponse::add_values(const char* value) { - GOOGLE_DCHECK(value != nullptr); - values_.Add()->assign(value); - // @@protoc_insertion_point(field_add_char:milvus.proto.service.SysConfigResponse.values) -} -inline void SysConfigResponse::add_values(const char* value, size_t size) { - values_.Add()->assign(reinterpret_cast(value), size); - // @@protoc_insertion_point(field_add_pointer:milvus.proto.service.SysConfigResponse.values) -} -inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& -SysConfigResponse::values() const { - // @@protoc_insertion_point(field_list:milvus.proto.service.SysConfigResponse.values) - return values_; -} -inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* -SysConfigResponse::mutable_values() { - // @@protoc_insertion_point(field_mutable_list:milvus.proto.service.SysConfigResponse.values) - return &values_; -} - -// ------------------------------------------------------------------- - // Hits // repeated int64 IDs = 1; @@ -4507,8 +4152,6 @@ QueryResult::mutable_hits() { // ------------------------------------------------------------------- -// ------------------------------------------------------------------- - // @@protoc_insertion_point(namespace_scope) diff --git a/internal/core/src/query/Plan.cpp b/internal/core/src/query/Plan.cpp index ee1b838899..862df2b6d4 100644 --- a/internal/core/src/query/Plan.cpp +++ b/internal/core/src/query/Plan.cpp @@ -133,6 +133,12 @@ CreatePlanImplNaive(const Schema& schema, const std::string& dsl_str) { PanicInfo("Unsupported DSL"); } plan->plan_node_->predicate_ = std::move(predicate); + // TODO: target_entry parser + // if schema autoid is true, + // prepend target_entries_ with row_id + // else + // with primary_key + // return plan; } diff --git a/internal/core/src/query/PlanImpl.h b/internal/core/src/query/PlanImpl.h index 3fec00ee8f..453959e530 100644 --- a/internal/core/src/query/PlanImpl.h +++ b/internal/core/src/query/PlanImpl.h @@ -35,6 +35,7 @@ struct Plan { const Schema& schema_; std::unique_ptr plan_node_; std::map tag2field_; // PlaceholderName -> FieldId + std::vector target_entries_; // TODO: add move extra info }; diff --git a/internal/core/src/query/Search.cpp b/internal/core/src/query/Search.cpp index 402e438808..d18f4dc57f 100644 --- a/internal/core/src/query/Search.cpp +++ b/internal/core/src/query/Search.cpp @@ -116,18 +116,21 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment, segcore::merge_into(num_queries, topK, final_dis.data(), final_uids.data(), buf_dis.data(), buf_uids.data()); } - // step 5: convert offset to uids + results.result_distances_ = std::move(final_dis); + results.internal_seg_offsets_ = std::move(final_uids); + results.topK_ = topK; + results.num_queries_ = num_queries; + + // TODO: deprecated code begin + final_uids = results.internal_seg_offsets_; for (auto& id : final_uids) { if (id == -1) { continue; } id = record.uids_[id]; } - results.result_ids_ = std::move(final_uids); - results.result_distances_ = std::move(final_dis); - results.topK_ = topK; - results.num_queries_ = num_queries; + // TODO: deprecated code end return Status::OK(); } diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 865b4d52a8..dd1ae1ae33 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(milvus_segcore SHARED target_link_libraries(milvus_segcore tbb utils pthread knowhere log milvus_proto dl backtrace + milvus_common milvus_query ) diff --git a/internal/core/src/segcore/Collection.cpp b/internal/core/src/segcore/Collection.cpp index 4a90c8dcb4..2edf4a0d66 100644 --- a/internal/core/src/segcore/Collection.cpp +++ b/internal/core/src/segcore/Collection.cpp @@ -139,27 +139,7 @@ Collection::parse() { } collection_name_ = collection_meta.schema().name(); - // TODO: delete print - std::cout << "create collection " << collection_meta.schema().name() << std::endl; - - auto schema = std::make_shared(); - for (const milvus::proto::schema::FieldSchema& child : collection_meta.schema().fields()) { - const auto& type_params = child.type_params(); - int64_t dim = 16; - for (const auto& type_param : type_params) { - if (type_param.key() == "dim") { - dim = strtoll(type_param.value().c_str(), nullptr, 10); - } - } - std::cout << "add Field, name :" << child.name() << ", datatype :" << child.data_type() << ", dim :" << dim - << std::endl; - schema->AddField(std::string_view(child.name()), DataType(child.data_type()), dim); - } - /* - schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16); - schema->AddField("age", DataType::INT32); - */ - schema_ = schema; + schema_ = Schema::ParseFrom(collection_meta.schema()); } } // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegmentBase.h b/internal/core/src/segcore/SegmentBase.h index 552a9733a3..6fa0950e2e 100644 --- a/internal/core/src/segcore/SegmentBase.h +++ b/internal/core/src/segcore/SegmentBase.h @@ -72,6 +72,9 @@ class SegmentBase { int num_groups, QueryResult& results) = 0; + virtual Status + FillTargetEntry(const query::Plan* Plan, QueryResult& results) = 0; + // stop receive insert requests virtual Status Close() = 0; diff --git a/internal/core/src/segcore/SegmentNaive.h b/internal/core/src/segcore/SegmentNaive.h index bc44084f4b..5f09213b5c 100644 --- a/internal/core/src/segcore/SegmentNaive.h +++ b/internal/core/src/segcore/SegmentNaive.h @@ -84,6 +84,11 @@ class SegmentNaive : public SegmentBase { Status BuildIndex(IndexMetaPtr index_meta) override; + Status + FillTargetEntry(const query::Plan* Plan, QueryResult& results) override { + PanicInfo("unimplemented"); + } + Status DropRawData(std::string_view field_name) override { // TODO: NO-OP diff --git a/internal/core/src/segcore/SegmentSmallIndex.cpp b/internal/core/src/segcore/SegmentSmallIndex.cpp index 32f90cdfc0..de52f78252 100644 --- a/internal/core/src/segcore/SegmentSmallIndex.cpp +++ b/internal/core/src/segcore/SegmentSmallIndex.cpp @@ -344,5 +344,39 @@ SegmentSmallIndex::Search(const query::Plan* plan, results = visitor.get_moved_result(*plan->plan_node_); return Status::OK(); } +Status +SegmentSmallIndex::FillTargetEntry(const query::Plan* plan, QueryResult& results) { + AssertInfo(plan, "empty plan"); + auto size = results.result_distances_.size(); + Assert(results.internal_seg_offsets_.size() == size); + Assert(results.result_offsets_.size() == size); + Assert(results.row_data_.size() == 0); + + if (plan->schema_.get_is_auto_id()) { + auto& uids = record_.uids_; + for (int64_t i = 0; i < size; ++i) { + auto seg_offset = results.internal_seg_offsets_[i]; + auto row_id = uids[seg_offset]; + std::vector blob(sizeof(row_id)); + memcpy(blob.data(), &row_id, sizeof(row_id)); + results.row_data_.emplace_back(std::move(blob)); + } + } else { + auto key_offset_opt = schema_->get_primary_key_offset(); + Assert(key_offset_opt.has_value()); + auto key_offset = key_offset_opt.value(); + auto field_meta = schema_->operator[](key_offset); + Assert(field_meta.get_data_type() == DataType::INT64); + auto uids = record_.get_scalar_entity(key_offset); + for (int64_t i = 0; i < size; ++i) { + auto seg_offset = results.internal_seg_offsets_[i]; + auto row_id = uids->operator[](seg_offset); + std::vector blob(sizeof(row_id)); + memcpy(blob.data(), &row_id, sizeof(row_id)); + results.row_data_.emplace_back(std::move(blob)); + } + } + return Status::OK(); +} } // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegmentSmallIndex.h b/internal/core/src/segcore/SegmentSmallIndex.h index 9d6a5deb43..cb4079d603 100644 --- a/internal/core/src/segcore/SegmentSmallIndex.h +++ b/internal/core/src/segcore/SegmentSmallIndex.h @@ -33,29 +33,6 @@ #include namespace milvus::segcore { -// struct ColumnBasedDataChunk { -// std::vector> entity_vecs; -// -// static ColumnBasedDataChunk -// from(const RowBasedRawData& source, const Schema& schema) { -// ColumnBasedDataChunk dest; -// auto count = source.count; -// auto raw_data = reinterpret_cast(source.raw_data); -// auto align = source.sizeof_per_row; -// for (auto& field : schema) { -// auto len = field.get_sizeof(); -// Assert(len % sizeof(float) == 0); -// std::vector new_col(len * count / sizeof(float)); -// for (int64_t i = 0; i < count; ++i) { -// memcpy(new_col.data() + i * len / sizeof(float), raw_data + i * align, len); -// } -// dest.entity_vecs.push_back(std::move(new_col)); -// // offset the raw_data -// raw_data += len / sizeof(float); -// } -// return dest; -// } -//}; class SegmentSmallIndex : public SegmentBase { public: @@ -164,20 +141,13 @@ class SegmentSmallIndex : public SegmentBase { std::shared_ptr get_deleted_bitmap(int64_t del_barrier, Timestamp query_timestamp, int64_t insert_barrier, bool force = false); - // Status - // QueryBruteForceImpl(query::QueryPtr query, Timestamp timestamp, QueryResult& results); - - // Status - // QueryBruteForceImpl(const query::QueryInfo& info, - // const float* query_data, - // int64_t num_queries, - // Timestamp timestamp, - // QueryResult& results); - template knowhere::IndexPtr BuildVecIndexImpl(const IndexMeta::Entry& entry); + Status + FillTargetEntry(const query::Plan* Plan, QueryResult& results) override; + private: SchemaPtr schema_; std::atomic state_ = SegmentState::Open; diff --git a/internal/core/src/utils/Types.h b/internal/core/src/utils/Types.h index 50013e95d9..33643f3258 100644 --- a/internal/core/src/utils/Types.h +++ b/internal/core/src/utils/Types.h @@ -141,8 +141,11 @@ struct QueryResult { QueryResult() = default; QueryResult(uint64_t num_queries, uint64_t topK) : topK_(topK), num_queries_(num_queries) { auto count = get_row_count(); - result_ids_.resize(count); result_distances_.resize(count); + internal_seg_offsets_.resize(count); + + // TODO: deprecated + result_ids_.resize(count); } [[nodiscard]] uint64_t @@ -153,10 +156,24 @@ struct QueryResult { uint64_t num_queries_; uint64_t topK_; // uint64_t total_row_count_; // total_row_count_ = topK * num_queries_ - engine::ResultIds result_ids_; // top1, top2, ..; - engine::ResultDistances result_distances_; - // engine::DataChunkPtr data_chunk_; + + // vector> data_reduced; + + // vector> + + // map>> + uint64_t seg_id_; + std::vector result_distances_; + + // TODO(gexi): utilize these field + std::vector internal_seg_offsets_; + std::vector result_offsets_; + std::vector> row_data_; + + // TODO: deprecated, use row_data directly + std::vector result_ids_; }; + using QueryResultPtr = std::shared_ptr; } // namespace engine diff --git a/internal/core/unittest/test_query.cpp b/internal/core/unittest/test_query.cpp index 3ac12a99e2..556cfd7cbb 100644 --- a/internal/core/unittest/test_query.cpp +++ b/internal/core/unittest/test_query.cpp @@ -20,6 +20,7 @@ #include "query/generated/ExecPlanNodeVisitor.h" #include "query/PlanImpl.h" #include "segcore/SegmentSmallIndex.h" +#include "pb/schema.pb.h" using namespace milvus; using namespace milvus::query; @@ -322,3 +323,81 @@ TEST(Query, ExecWihtoutPredicate) { Json json{results}; std::cout << json.dump(2); } + +TEST(Query, FillSegment) { + namespace pb = milvus::proto; + pb::schema::CollectionSchema proto; + proto.set_name("col"); + proto.set_description("asdfhsalkgfhsadg"); + proto.set_autoid(true); + + { + auto field = proto.add_fields(); + field->set_name("fakevec"); + field->set_is_primary_key(false); + field->set_description("asdgfsagf"); + field->set_data_type(pb::schema::DataType::VECTOR_FLOAT); + auto param = field->add_type_params(); + param->set_key("dim"); + param->set_value("16"); + } + + { + auto field = proto.add_fields(); + field->set_name("the_key"); + field->set_is_primary_key(true); + field->set_description("asdgfsagf"); + field->set_data_type(pb::schema::DataType::INT32); + } + + auto schema = Schema::ParseFrom(proto); + auto segment = CreateSegment(schema); + int N = 100000; + auto dataset = DataGen(schema, N); + segment->PreInsert(N); + segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_); + std::string dsl = R"({ + "bool": { + "must": [ + { + "vector": { + "fakevec": { + "metric_type": "L2", + "params": { + "nprobe": 10 + }, + "query": "$0", + "topk": 5 + } + } + } + ] + } + })"; + auto plan = CreatePlan(*schema, dsl); + auto ph_proto = CreatePlaceholderGroup(10, 16, 443); + auto ph = ParsePlaceholderGroup(plan.get(), ph_proto.SerializeAsString()); + std::vector groups = {ph.get()}; + std::vector timestamps = {N * 2UL}; + QueryResult result; + segment->Search(plan.get(), groups.data(), timestamps.data(), 1, result); + + // TODO: deprecated result_ids_ + ASSERT_EQ(result.result_ids_, result.internal_seg_offsets_); + auto topk = 5; + auto num_queries = 10; + + result.result_offsets_.resize(topk * num_queries); + segment->FillTargetEntry(plan.get(), result); + auto ans = result.row_data_; + ASSERT_EQ(ans.size(), topk * num_queries); + int64_t std_index = 0; + for (auto& vec : ans) { + ASSERT_EQ(vec.size(), sizeof(int64_t)); + int64_t val; + memcpy(&val, vec.data(), sizeof(int64_t)); + auto std_val = result.result_ids_[std_index]; + ASSERT_EQ(val, std_val); + ++std_index; + } +} diff --git a/internal/master/collection_task.go b/internal/master/collection_task.go index 3e3424d035..1fb9549886 100644 --- a/internal/master/collection_task.go +++ b/internal/master/collection_task.go @@ -5,7 +5,6 @@ import ( "log" "github.com/golang/protobuf/proto" - ms "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" @@ -86,23 +85,8 @@ func (t *createCollectionTask) Execute() error { // TODO: initial partition? PartitionTags: make([]string, 0), } - err = t.mt.AddCollection(&collection) - if err != nil { - return err - } - msgPack := ms.MsgPack{} - baseMsg := ms.BaseMsg{ - BeginTimestamp: t.req.Timestamp, - EndTimestamp: t.req.Timestamp, - HashValues: []int32{0}, - } - timeTickMsg := &ms.CreateCollectionMsg{ - BaseMsg: baseMsg, - CreateCollectionRequest: *t.req, - } - msgPack.Msgs = append(msgPack.Msgs, timeTickMsg) - return t.sch.ddMsgStream.Broadcast(&msgPack) + return t.mt.AddCollection(&collection) } ////////////////////////////////////////////////////////////////////////// @@ -118,7 +102,7 @@ func (t *dropCollectionTask) Ts() (Timestamp, error) { if t.req == nil { return 0, errors.New("null request") } - return t.req.Timestamp, nil + return Timestamp(t.req.Timestamp), nil } func (t *dropCollectionTask) Execute() error { @@ -134,29 +118,7 @@ func (t *dropCollectionTask) Execute() error { collectionID := collectionMeta.ID - err = t.mt.DeleteCollection(collectionID) - if err != nil { - return err - } - - ts, err := t.Ts() - if err != nil { - return err - } - - msgPack := ms.MsgPack{} - baseMsg := ms.BaseMsg{ - BeginTimestamp: ts, - EndTimestamp: ts, - HashValues: []int32{0}, - } - timeTickMsg := &ms.DropCollectionMsg{ - BaseMsg: baseMsg, - DropCollectionRequest: *t.req, - } - msgPack.Msgs = append(msgPack.Msgs, timeTickMsg) - return t.sch.ddMsgStream.Broadcast(&msgPack) - + return t.mt.DeleteCollection(collectionID) } ////////////////////////////////////////////////////////////////////////// @@ -172,7 +134,7 @@ func (t *hasCollectionTask) Ts() (Timestamp, error) { if t.req == nil { return 0, errors.New("null request") } - return t.req.Timestamp, nil + return Timestamp(t.req.Timestamp), nil } func (t *hasCollectionTask) Execute() error { @@ -185,8 +147,8 @@ func (t *hasCollectionTask) Execute() error { if err == nil { t.hasCollection = true } - return nil + return nil } ////////////////////////////////////////////////////////////////////////// @@ -219,7 +181,6 @@ func (t *describeCollectionTask) Execute() error { t.description.Schema = collection.Schema return nil - } ////////////////////////////////////////////////////////////////////////// diff --git a/internal/master/collection_task_test.go b/internal/master/collection_task_test.go index 60ae5ff038..eb4920b00e 100644 --- a/internal/master/collection_task_test.go +++ b/internal/master/collection_task_test.go @@ -33,7 +33,8 @@ func TestMaster_CollectionTask(t *testing.T) { Port: Params.Port, EtcdAddress: Params.EtcdAddress, - EtcdRootPath: "/test/root", + MetaRootPath: "/test/root/meta", + KvRootPath: "/test/root/kv", PulsarAddress: Params.PulsarAddress, ProxyIDList: []typeutil.UniqueID{1, 2}, @@ -54,7 +55,6 @@ func TestMaster_CollectionTask(t *testing.T) { // msgChannel ProxyTimeTickChannelNames: []string{"proxy1", "proxy2"}, WriteNodeTimeTickChannelNames: []string{"write3", "write4"}, - DDChannelNames: []string{"dd1", "dd2"}, InsertChannelNames: []string{"dm0", "dm1"}, K2SChannelNames: []string{"k2s0", "k2s1"}, QueryNodeStatsChannelName: "statistic", diff --git a/internal/master/config_task.go b/internal/master/config_task.go deleted file mode 100644 index 79f4ad32b0..0000000000 --- a/internal/master/config_task.go +++ /dev/null @@ -1,79 +0,0 @@ -package master - -import ( - "log" - - "github.com/zilliztech/milvus-distributed/internal/errors" - "github.com/zilliztech/milvus-distributed/internal/kv" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" -) - -type getSysConfigsTask struct { - baseTask - configkv *kv.EtcdKV - req *internalpb.SysConfigRequest - keys []string - values []string -} - -func (t *getSysConfigsTask) Type() internalpb.MsgType { - if t.req == nil { - log.Printf("null request") - return 0 - } - return t.req.MsgType -} - -func (t *getSysConfigsTask) Ts() (Timestamp, error) { - if t.req == nil { - return 0, errors.New("null request") - } - return t.req.Timestamp, nil -} - -func (t *getSysConfigsTask) Execute() error { - if t.req == nil { - return errors.New("null request") - } - - sc := &SysConfig{kv: t.configkv} - keyMap := make(map[string]bool) - - // Load configs with prefix - for _, prefix := range t.req.KeyPrefixes { - prefixKeys, prefixVals, err := sc.GetByPrefix(prefix) - if err != nil { - return errors.Errorf("Load configs by prefix wrong: %s", err.Error()) - } - t.keys = append(t.keys, prefixKeys...) - t.values = append(t.values, prefixVals...) - } - - for _, key := range t.keys { - keyMap[key] = true - } - - // Load specific configs - if len(t.req.Keys) > 0 { - // To clean up duplicated keys - cleanKeys := []string{} - for _, key := range t.req.Keys { - if v, ok := keyMap[key]; (!ok) || (ok && !v) { - cleanKeys = append(cleanKeys, key) - keyMap[key] = true - continue - } - log.Println("[GetSysConfigs] Warning: duplicate key:", key) - } - - v, err := sc.Get(cleanKeys) - if err != nil { - return errors.Errorf("Load configs wrong: %s", err.Error()) - } - - t.keys = append(t.keys, cleanKeys...) - t.values = append(t.values, v...) - } - - return nil -} diff --git a/internal/master/config_task_test.go b/internal/master/config_task_test.go deleted file mode 100644 index 53bc676a7a..0000000000 --- a/internal/master/config_task_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package master - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" - "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" - "github.com/zilliztech/milvus-distributed/internal/util/typeutil" - "go.etcd.io/etcd/clientv3" - "google.golang.org/grpc" -) - -func TestMaster_ConfigTask(t *testing.T) { - Init() - - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{Params.EtcdAddress}}) - require.Nil(t, err) - _, err = etcdCli.Delete(ctx, "/test/root", clientv3.WithPrefix()) - require.Nil(t, err) - - Params = ParamTable{ - Address: Params.Address, - Port: Params.Port, - - EtcdAddress: Params.EtcdAddress, - EtcdRootPath: "/test/root", - PulsarAddress: Params.PulsarAddress, - - ProxyIDList: []typeutil.UniqueID{1, 2}, - WriteNodeIDList: []typeutil.UniqueID{3, 4}, - - TopicNum: 5, - QueryNodeNum: 3, - SoftTimeTickBarrierInterval: 300, - - // segment - SegmentSize: 536870912 / 1024 / 1024, - SegmentSizeFactor: 0.75, - DefaultRecordSize: 1024, - MinSegIDAssignCnt: 1048576 / 1024, - MaxSegIDAssignCnt: Params.MaxSegIDAssignCnt, - SegIDAssignExpiration: 2000, - - // msgChannel - ProxyTimeTickChannelNames: []string{"proxy1", "proxy2"}, - WriteNodeTimeTickChannelNames: []string{"write3", "write4"}, - InsertChannelNames: []string{"dm0", "dm1"}, - K2SChannelNames: []string{"k2s0", "k2s1"}, - QueryNodeStatsChannelName: "statistic", - MsgChannelSubName: Params.MsgChannelSubName, - } - - svr, err := CreateServer(ctx) - require.Nil(t, err) - err = svr.Run(10002) - defer svr.Close() - require.Nil(t, err) - - conn, err := grpc.DialContext(ctx, "127.0.0.1:10002", grpc.WithInsecure(), grpc.WithBlock()) - require.Nil(t, err) - defer conn.Close() - - cli := masterpb.NewMasterClient(conn) - testKeys := []string{ - "/etcd/address", - "/master/port", - "/master/proxyidlist", - "/master/segmentthresholdfactor", - "/pulsar/token", - "/reader/stopflag", - "/proxy/timezone", - "/proxy/network/address", - "/proxy/storage/path", - "/storage/accesskey", - } - - testVals := []string{ - "localhost", - "53100", - "[1 2]", - "0.75", - "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY", - "-1", - "UTC+8", - "0.0.0.0", - "/var/lib/milvus", - "", - } - - sc := SysConfig{kv: svr.kvBase} - sc.InitFromFile(".") - - configRequest := &internalpb.SysConfigRequest{ - MsgType: internalpb.MsgType_kGetSysConfigs, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - Keys: testKeys, - KeyPrefixes: []string{}, - } - - response, err := cli.GetSysConfigs(ctx, configRequest) - assert.Nil(t, err) - assert.ElementsMatch(t, testKeys, response.Keys) - assert.ElementsMatch(t, testVals, response.Values) - assert.Equal(t, len(response.GetKeys()), len(response.GetValues())) - - configRequest = &internalpb.SysConfigRequest{ - MsgType: internalpb.MsgType_kGetSysConfigs, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - Keys: []string{}, - KeyPrefixes: []string{"/master"}, - } - - response, err = cli.GetSysConfigs(ctx, configRequest) - assert.Nil(t, err) - for i := range response.GetKeys() { - assert.True(t, strings.HasPrefix(response.GetKeys()[i], "/master")) - } - assert.Equal(t, len(response.GetKeys()), len(response.GetValues())) - - t.Run("Test duplicate keys and key prefix", func(t *testing.T) { - configRequest.Keys = []string{} - configRequest.KeyPrefixes = []string{"/master"} - - resp, err := cli.GetSysConfigs(ctx, configRequest) - require.Nil(t, err) - assert.Equal(t, len(resp.GetKeys()), len(resp.GetValues())) - assert.NotEqual(t, 0, len(resp.GetKeys())) - - configRequest.Keys = []string{"/master/port"} - configRequest.KeyPrefixes = []string{"/master"} - - respDup, err := cli.GetSysConfigs(ctx, configRequest) - require.Nil(t, err) - assert.Equal(t, len(respDup.GetKeys()), len(respDup.GetValues())) - assert.NotEqual(t, 0, len(respDup.GetKeys())) - assert.Equal(t, len(respDup.GetKeys()), len(resp.GetKeys())) - }) - -} diff --git a/internal/master/config_test.yaml b/internal/master/config_test.yaml deleted file mode 100644 index b2d2e4f676..0000000000 --- a/internal/master/config_test.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -master: # 21 - address: localhost - port: 53100 - pulsarmoniterinterval: 1 - pulsartopic: "monitor-topic" - - proxyidlist: [1, 2] - proxyTimeSyncChannels: ["proxy1", "proxy2"] - proxyTimeSyncSubName: "proxy-topic" - softTimeTickBarrierInterval: 500 - - writeidlist: [3, 4] - writeTimeSyncChannels: ["write3", "write4"] - writeTimeSyncSubName: "write-topic" - - dmTimeSyncChannels: ["dm5", "dm6"] - k2sTimeSyncChannels: ["k2s7", "k2s8"] - - defaultSizePerRecord: 1024 - minimumAssignSize: 1048576 - segmentThreshold: 536870912 - segmentExpireDuration: 2000 - segmentThresholdFactor: 0.75 - querynodenum: 1 - writenodenum: 1 - statsChannels: "statistic" - -etcd: # 4 - address: localhost - port: 2379 - rootpath: by-dev - segthreshold: 10000 - -timesync: # 1 - interval: 400 - -storage: # 5 - driver: TIKV - address: localhost - port: 2379 - accesskey: - secretkey: - -pulsar: # 6 - authentication: false - user: user-default - token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY - address: localhost - port: 6650 - topicnum: 128 - -reader: # 7 - clientid: 0 - stopflag: -1 - readerqueuesize: 10000 - searchchansize: 10000 - key2segchansize: 10000 - topicstart: 0 - topicend: 128 - -writer: # 8 - clientid: 0 - stopflag: -2 - readerqueuesize: 10000 - searchbyidchansize: 10000 - parallelism: 100 - topicstart: 0 - topicend: 128 - bucket: "zilliz-hz" - -proxy: # 21 - timezone: UTC+8 - proxy_id: 1 - numReaderNodes: 2 - tsoSaveInterval: 200 - timeTickInterval: 200 - - pulsarTopics: - readerTopicPrefix: "milvusReader" - numReaderTopics: 2 - deleteTopic: "milvusDeleter" - queryTopic: "milvusQuery" - resultTopic: "milvusResult" - resultGroup: "milvusResultGroup" - timeTickTopic: "milvusTimeTick" - - network: - address: 0.0.0.0 - port: 19530 - - logs: - level: debug - trace.enable: true - path: /tmp/logs - max_log_file_size: 1024MB - log_rotate_num: 0 - - storage: - path: /var/lib/milvus - auto_flush_interval: 1 diff --git a/internal/master/grpc_service.go b/internal/master/grpc_service.go index 3f4d8a31f9..f4449001c2 100644 --- a/internal/master/grpc_service.go +++ b/internal/master/grpc_service.go @@ -359,43 +359,6 @@ func (s *Master) ShowPartitions(ctx context.Context, in *internalpb.ShowPartitio return t.(*showPartitionTask).stringListResponse, nil } -func (s *Master) GetSysConfigs(ctx context.Context, in *internalpb.SysConfigRequest) (*servicepb.SysConfigResponse, error) { - var t task = &getSysConfigsTask{ - req: in, - configkv: s.kvBase, - baseTask: baseTask{ - sch: s.scheduler, - mt: s.metaTable, - cv: make(chan error), - }, - keys: []string{}, - values: []string{}, - } - - response := &servicepb.SysConfigResponse{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - }, - } - - var err = s.scheduler.Enqueue(t) - if err != nil { - response.Status.Reason = "Enqueue failed: " + err.Error() - return response, nil - } - - err = t.WaitToFinish(ctx) - if err != nil { - response.Status.Reason = "Get System Config failed: " + err.Error() - return response, nil - } - - response.Keys = t.(*getSysConfigsTask).keys - response.Values = t.(*getSysConfigsTask).values - response.Status.ErrorCode = commonpb.ErrorCode_SUCCESS - return response, nil -} - //----------------------------------------Internal GRPC Service-------------------------------- func (s *Master) AllocTimestamp(ctx context.Context, request *internalpb.TsoRequest) (*internalpb.TsoResponse, error) { diff --git a/internal/master/grpc_service_test.go b/internal/master/grpc_service_test.go index a32e23efc8..644a84999c 100644 --- a/internal/master/grpc_service_test.go +++ b/internal/master/grpc_service_test.go @@ -32,7 +32,8 @@ func TestMaster_CreateCollection(t *testing.T) { Port: Params.Port, EtcdAddress: Params.EtcdAddress, - EtcdRootPath: "/test/root", + MetaRootPath: "/test/root/meta", + KvRootPath: "/test/root/kv", PulsarAddress: Params.PulsarAddress, ProxyIDList: []typeutil.UniqueID{1, 2}, diff --git a/internal/master/master.go b/internal/master/master.go index d6c2f9f6e3..8ea4586be6 100644 --- a/internal/master/master.go +++ b/internal/master/master.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/zilliztech/milvus-distributed/internal/errors" "github.com/zilliztech/milvus-distributed/internal/kv" ms "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" @@ -81,8 +82,8 @@ func Init() { func CreateServer(ctx context.Context) (*Master, error) { //Init(etcdAddr, kvRootPath) etcdAddress := Params.EtcdAddress - metaRootPath := Params.EtcdRootPath - kvRootPath := Params.EtcdRootPath + metaRootPath := Params.MetaRootPath + kvRootPath := Params.KvRootPath pulsarAddr := Params.PulsarAddress etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}}) @@ -122,11 +123,6 @@ func CreateServer(ctx context.Context) (*Master, error) { } tsMsgProducer.SetWriteNodeTtBarrier(writeTimeTickBarrier) - pulsarDDStream := ms.NewPulsarMsgStream(ctx, 1024) //input stream - pulsarDDStream.SetPulsarClient(pulsarAddr) - pulsarDDStream.CreatePulsarProducers(Params.DDChannelNames) - tsMsgProducer.SetDDSyncStream(pulsarDDStream) - pulsarDMStream := ms.NewPulsarMsgStream(ctx, 1024) //input stream pulsarDMStream.SetPulsarClient(pulsarAddr) pulsarDMStream.CreatePulsarProducers(Params.InsertChannelNames) @@ -165,10 +161,7 @@ func CreateServer(ctx context.Context) (*Master, error) { return nil, err } - m.scheduler = NewDDRequestScheduler(ctx) - m.scheduler.SetDDMsgStream(pulsarDDStream) - m.scheduler.SetIDAllocator(func() (UniqueID, error) { return m.idAllocator.AllocOne() }) - + m.scheduler = NewDDRequestScheduler(func() (UniqueID, error) { return m.idAllocator.AllocOne() }) m.segmentMgr = NewSegmentManager(metakv, func() (UniqueID, error) { return m.idAllocator.AllocOne() }, func() (Timestamp, error) { return m.tsoAllocator.AllocOne() }, @@ -255,11 +248,6 @@ func (s *Master) startServerLoop(ctx context.Context, grpcPort int64) error { return err } - s.serverLoopWg.Add(1) - if err := s.scheduler.Start(); err != nil { - return err - } - s.serverLoopWg.Add(1) go s.grpcLoop(grpcPort) @@ -267,6 +255,9 @@ func (s *Master) startServerLoop(ctx context.Context, grpcPort int64) error { return err } + s.serverLoopWg.Add(1) + go s.tasksExecutionLoop() + s.serverLoopWg.Add(1) go s.segmentStatisticsLoop() @@ -279,8 +270,6 @@ func (s *Master) startServerLoop(ctx context.Context, grpcPort int64) error { func (s *Master) stopServerLoop() { s.timesSyncMsgProducer.Close() s.serverLoopWg.Done() - s.scheduler.Close() - s.serverLoopWg.Done() if s.grpcServer != nil { s.grpcServer.GracefulStop() @@ -348,6 +337,33 @@ func (s *Master) tsLoop() { } } +func (s *Master) tasksExecutionLoop() { + defer s.serverLoopWg.Done() + ctx, cancel := context.WithCancel(s.serverLoopCtx) + defer cancel() + + for { + select { + case task := <-s.scheduler.reqQueue: + timeStamp, err := (task).Ts() + if err != nil { + log.Println(err) + } else { + if timeStamp < s.scheduler.scheduleTimeStamp { + task.Notify(errors.Errorf("input timestamp = %d, schduler timestamp = %d", timeStamp, s.scheduler.scheduleTimeStamp)) + } else { + s.scheduler.scheduleTimeStamp = timeStamp + err = task.Execute() + task.Notify(err) + } + } + case <-ctx.Done(): + log.Print("server is closed, exit task execution loop") + return + } + } +} + func (s *Master) segmentStatisticsLoop() { defer s.serverLoopWg.Done() defer s.segmentStatusMsg.Close() diff --git a/internal/master/master_test.go b/internal/master/master_test.go deleted file mode 100644 index c3cdbbdfbd..0000000000 --- a/internal/master/master_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package master - -import ( - "context" - "log" - "math/rand" - "strconv" - "testing" - - "github.com/golang/protobuf/proto" - "github.com/stretchr/testify/assert" - ms "github.com/zilliztech/milvus-distributed/internal/msgstream" - "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" - internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" - "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" - "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" - "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" - "go.uber.org/zap" - "google.golang.org/grpc" -) - -func receiveTimeTickMsg(stream *ms.MsgStream) bool { - for { - result := (*stream).Consume() - if len(result.Msgs) > 0 { - return true - } - } -} - -func getTimeTickMsgPack(ttmsgs [][2]uint64) *ms.MsgPack { - msgPack := ms.MsgPack{} - for _, vi := range ttmsgs { - msgPack.Msgs = append(msgPack.Msgs, getTtMsg(internalPb.MsgType_kTimeTick, UniqueID(vi[0]), Timestamp(vi[1]))) - } - return &msgPack -} - -func TestMaster(t *testing.T) { - Init() - pulsarAddr := Params.PulsarAddress - - // Creates server. - ctx, cancel := context.WithCancel(context.Background()) - - svr, err := CreateServer(ctx) - if err != nil { - log.Print("create server failed", zap.Error(err)) - } - - if err := svr.Run(int64(Params.Port)); err != nil { - log.Fatal("run server failed", zap.Error(err)) - } - - proxyTimeTickStream := ms.NewPulsarMsgStream(ctx, 1024) //input stream - proxyTimeTickStream.SetPulsarClient(pulsarAddr) - proxyTimeTickStream.CreatePulsarProducers(Params.ProxyTimeTickChannelNames) - proxyTimeTickStream.Start() - - writeNodeStream := ms.NewPulsarMsgStream(ctx, 1024) //input stream - writeNodeStream.SetPulsarClient(pulsarAddr) - writeNodeStream.CreatePulsarProducers(Params.WriteNodeTimeTickChannelNames) - writeNodeStream.Start() - - ddMs := ms.NewPulsarMsgStream(ctx, 1024) - ddMs.SetPulsarClient(pulsarAddr) - ddMs.CreatePulsarConsumers(Params.DDChannelNames, "DDStream", ms.NewUnmarshalDispatcher(), 1024) - ddMs.Start() - - dMMs := ms.NewPulsarMsgStream(ctx, 1024) - dMMs.SetPulsarClient(pulsarAddr) - dMMs.CreatePulsarConsumers(Params.InsertChannelNames, "DMStream", ms.NewUnmarshalDispatcher(), 1024) - dMMs.Start() - - k2sMs := ms.NewPulsarMsgStream(ctx, 1024) - k2sMs.SetPulsarClient(pulsarAddr) - k2sMs.CreatePulsarConsumers(Params.K2SChannelNames, "K2SStream", ms.NewUnmarshalDispatcher(), 1024) - k2sMs.Start() - - ttsoftmsgs := [][2]uint64{ - {0, 10}, - } - msgSoftPackAddr := getTimeTickMsgPack(ttsoftmsgs) - - proxyTimeTickStream.Produce(msgSoftPackAddr) - var dMMsgstream ms.MsgStream = dMMs - assert.True(t, receiveTimeTickMsg(&dMMsgstream)) - var ddMsgstream ms.MsgStream = ddMs - assert.True(t, receiveTimeTickMsg(&ddMsgstream)) - - tthardmsgs := [][2]int{ - {3, 10}, - } - - msghardPackAddr := getMsgPack(tthardmsgs) - writeNodeStream.Produce(msghardPackAddr) - var k2sMsgstream ms.MsgStream = k2sMs - assert.True(t, receiveTimeTickMsg(&k2sMsgstream)) - - conn, err := grpc.DialContext(ctx, "127.0.0.1:53100", grpc.WithInsecure(), grpc.WithBlock()) - assert.Nil(t, err) - defer conn.Close() - - cli := masterpb.NewMasterClient(conn) - - sch := schemapb.CollectionSchema{ - Name: "name" + strconv.FormatUint(rand.Uint64(), 10), - Description: "test collection", - AutoID: false, - Fields: []*schemapb.FieldSchema{}, - } - - schemaBytes, err := proto.Marshal(&sch) - assert.Nil(t, err) - - createCollectionReq := internalpb.CreateCollectionRequest{ - MsgType: internalpb.MsgType_kCreateCollection, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - Schema: &commonpb.Blob{Value: schemaBytes}, - } - st, err := cli.CreateCollection(ctx, &createCollectionReq) - assert.Nil(t, err) - assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS) - - var consumeMsg ms.MsgStream = ddMs - var createCollectionMsg *ms.CreateCollectionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - createCollectionMsg = v.(*ms.CreateCollectionMsg) - } - break - } - } - assert.Equal(t, createCollectionReq.MsgType, createCollectionMsg.CreateCollectionRequest.MsgType) - assert.Equal(t, createCollectionReq.ReqID, createCollectionMsg.CreateCollectionRequest.ReqID) - assert.Equal(t, createCollectionReq.Timestamp, createCollectionMsg.CreateCollectionRequest.Timestamp) - assert.Equal(t, createCollectionReq.ProxyID, createCollectionMsg.CreateCollectionRequest.ProxyID) - assert.Equal(t, createCollectionReq.Schema.Value, createCollectionMsg.CreateCollectionRequest.Schema.Value) - - ////////////////////////////CreatePartition//////////////////////// - partitionName := "partitionName" + strconv.FormatUint(rand.Uint64(), 10) - createPartitionReq := internalpb.CreatePartitionRequest{ - MsgType: internalpb.MsgType_kCreatePartition, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - PartitionName: &servicepb.PartitionName{ - CollectionName: sch.Name, - Tag: partitionName, - }, - } - - st, err = cli.CreatePartition(ctx, &createPartitionReq) - assert.Nil(t, err) - assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS) - - var createPartitionMsg *ms.CreatePartitionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - createPartitionMsg = v.(*ms.CreatePartitionMsg) - } - break - } - } - assert.Equal(t, createPartitionReq.MsgType, createPartitionMsg.CreatePartitionRequest.MsgType) - assert.Equal(t, createPartitionReq.ReqID, createPartitionMsg.CreatePartitionRequest.ReqID) - assert.Equal(t, createPartitionReq.Timestamp, createPartitionMsg.CreatePartitionRequest.Timestamp) - assert.Equal(t, createPartitionReq.ProxyID, createPartitionMsg.CreatePartitionRequest.ProxyID) - assert.Equal(t, createPartitionReq.PartitionName.CollectionName, createPartitionMsg.CreatePartitionRequest.PartitionName.CollectionName) - assert.Equal(t, createPartitionReq.PartitionName.Tag, createPartitionMsg.CreatePartitionRequest.PartitionName.Tag) - - ////////////////////////////DropPartition//////////////////////// - dropPartitionReq := internalpb.DropPartitionRequest{ - MsgType: internalpb.MsgType_kDropPartition, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - PartitionName: &servicepb.PartitionName{ - CollectionName: sch.Name, - Tag: partitionName, - }, - } - - st, err = cli.DropPartition(ctx, &dropPartitionReq) - assert.Nil(t, err) - assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS) - - var dropPartitionMsg *ms.DropPartitionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - dropPartitionMsg = v.(*ms.DropPartitionMsg) - } - break - } - } - assert.Equal(t, dropPartitionReq.MsgType, dropPartitionMsg.DropPartitionRequest.MsgType) - assert.Equal(t, dropPartitionReq.ReqID, dropPartitionMsg.DropPartitionRequest.ReqID) - assert.Equal(t, dropPartitionReq.Timestamp, dropPartitionMsg.DropPartitionRequest.Timestamp) - assert.Equal(t, dropPartitionReq.ProxyID, dropPartitionMsg.DropPartitionRequest.ProxyID) - assert.Equal(t, dropPartitionReq.PartitionName.CollectionName, dropPartitionMsg.DropPartitionRequest.PartitionName.CollectionName) - - ////////////////////////////DropCollection//////////////////////// - dropCollectionReq := internalpb.DropCollectionRequest{ - MsgType: internalpb.MsgType_kDropCollection, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - CollectionName: &servicepb.CollectionName{CollectionName: sch.Name}, - } - - st, err = cli.DropCollection(ctx, &dropCollectionReq) - assert.Nil(t, err) - assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS) - - var dropCollectionMsg *ms.DropCollectionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - dropCollectionMsg = v.(*ms.DropCollectionMsg) - } - break - } - } - assert.Equal(t, dropCollectionReq.MsgType, dropCollectionMsg.DropCollectionRequest.MsgType) - assert.Equal(t, dropCollectionReq.ReqID, dropCollectionMsg.DropCollectionRequest.ReqID) - assert.Equal(t, dropCollectionReq.Timestamp, dropCollectionMsg.DropCollectionRequest.Timestamp) - assert.Equal(t, dropCollectionReq.ProxyID, dropCollectionMsg.DropCollectionRequest.ProxyID) - assert.Equal(t, dropCollectionReq.CollectionName.CollectionName, dropCollectionMsg.DropCollectionRequest.CollectionName.CollectionName) - - cancel() - svr.Close() -} diff --git a/internal/master/param_table.go b/internal/master/param_table.go index cba9e1a93a..873c7b6af2 100644 --- a/internal/master/param_table.go +++ b/internal/master/param_table.go @@ -17,7 +17,8 @@ type ParamTable struct { Port int EtcdAddress string - EtcdRootPath string + MetaRootPath string + KvRootPath string PulsarAddress string // nodeID @@ -39,7 +40,6 @@ type ParamTable struct { // msgChannel ProxyTimeTickChannelNames []string WriteNodeTimeTickChannelNames []string - DDChannelNames []string InsertChannelNames []string K2SChannelNames []string QueryNodeStatsChannelName string @@ -76,7 +76,8 @@ func (p *ParamTable) Init() { p.initPort() p.initEtcdAddress() - p.initEtcdRootPath() + p.initMetaRootPath() + p.initKvRootPath() p.initPulsarAddress() p.initProxyIDList() @@ -96,7 +97,6 @@ func (p *ParamTable) Init() { p.initProxyTimeTickChannelNames() p.initWriteNodeTimeTickChannelNames() p.initInsertChannelNames() - p.initDDChannelNames() p.initK2SChannelNames() p.initQueryNodeStatsChannelName() p.initMsgChannelSubName() @@ -140,12 +140,28 @@ func (p *ParamTable) initPulsarAddress() { p.PulsarAddress = addr } -func (p *ParamTable) initEtcdRootPath() { - path, err := p.Load("etcd.rootpath") +func (p *ParamTable) initMetaRootPath() { + rootPath, err := p.Load("etcd.rootPath") if err != nil { panic(err) } - p.EtcdRootPath = path + subPath, err := p.Load("etcd.metaSubPath") + if err != nil { + panic(err) + } + p.MetaRootPath = rootPath + "/" + subPath +} + +func (p *ParamTable) initKvRootPath() { + rootPath, err := p.Load("etcd.rootPath") + if err != nil { + panic(err) + } + subPath, err := p.Load("etcd.kvSubPath") + if err != nil { + panic(err) + } + p.KvRootPath = rootPath + "/" + subPath } func (p *ParamTable) initTopicNum() { @@ -366,27 +382,6 @@ func (p *ParamTable) initWriteNodeTimeTickChannelNames() { p.WriteNodeTimeTickChannelNames = channels } -func (p *ParamTable) initDDChannelNames() { - ch, err := p.Load("msgChannel.chanNamePrefix.dataDefinition") - if err != nil { - log.Fatal(err) - } - id, err := p.Load("nodeID.queryNodeIDList") - if err != nil { - log.Panicf("load query node id list error, %s", err.Error()) - } - ids := strings.Split(id, ",") - channels := make([]string, 0, len(ids)) - for _, i := range ids { - _, err := strconv.ParseInt(i, 10, 64) - if err != nil { - log.Panicf("load query node id list error, %s", err.Error()) - } - channels = append(channels, ch+"-"+i) - } - p.DDChannelNames = channels -} - func (p *ParamTable) initInsertChannelNames() { ch, err := p.Load("msgChannel.chanNamePrefix.insert") if err != nil { diff --git a/internal/master/param_table_test.go b/internal/master/param_table_test.go index 339ecfb6d4..ed83c3af02 100644 --- a/internal/master/param_table_test.go +++ b/internal/master/param_table_test.go @@ -22,10 +22,16 @@ func TestParamTable_Port(t *testing.T) { assert.Equal(t, port, 53100) } -func TestParamTable_EtcdRootPath(t *testing.T) { +func TestParamTable_MetaRootPath(t *testing.T) { Params.Init() - addr := Params.EtcdRootPath - assert.Equal(t, addr, "by-dev") + path := Params.MetaRootPath + assert.Equal(t, path, "by-dev/meta") +} + +func TestParamTable_KVRootPath(t *testing.T) { + Params.Init() + path := Params.KvRootPath + assert.Equal(t, path, "by-dev/kv") } func TestParamTable_TopicNum(t *testing.T) { diff --git a/internal/master/partition_task.go b/internal/master/partition_task.go index fae0fe7ccc..80a339b2fc 100644 --- a/internal/master/partition_task.go +++ b/internal/master/partition_task.go @@ -4,7 +4,6 @@ import ( "errors" "log" - ms "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" @@ -67,30 +66,7 @@ func (t *createPartitionTask) Execute() error { if err != nil { return err } - - ts, err := t.Ts() - if err != nil { - return err - } - - err = t.mt.AddPartition(collectionMeta.ID, partitionName.Tag) - if err != nil { - return err - } - - msgPack := ms.MsgPack{} - baseMsg := ms.BaseMsg{ - BeginTimestamp: ts, - EndTimestamp: ts, - HashValues: []int32{0}, - } - timeTickMsg := &ms.CreatePartitionMsg{ - BaseMsg: baseMsg, - CreatePartitionRequest: *t.req, - } - msgPack.Msgs = append(msgPack.Msgs, timeTickMsg) - return t.sch.ddMsgStream.Broadcast(&msgPack) - + return t.mt.AddPartition(collectionMeta.ID, partitionName.Tag) } ////////////////////////////////////////////////////////////////////////// @@ -122,29 +98,7 @@ func (t *dropPartitionTask) Execute() error { return err } - err = t.mt.DeletePartition(collectionMeta.ID, partitionName.Tag) - if err != nil { - return err - } - - ts, err := t.Ts() - if err != nil { - return err - } - - msgPack := ms.MsgPack{} - baseMsg := ms.BaseMsg{ - BeginTimestamp: ts, - EndTimestamp: ts, - HashValues: []int32{0}, - } - timeTickMsg := &ms.DropPartitionMsg{ - BaseMsg: baseMsg, - DropPartitionRequest: *t.req, - } - msgPack.Msgs = append(msgPack.Msgs, timeTickMsg) - return t.sch.ddMsgStream.Broadcast(&msgPack) - + return t.mt.DeletePartition(collectionMeta.ID, partitionName.Tag) } ////////////////////////////////////////////////////////////////////////// @@ -178,7 +132,6 @@ func (t *hasPartitionTask) Execute() error { t.hasPartition = t.mt.HasPartition(collectionMeta.ID, partitionName.Tag) return nil - } ////////////////////////////////////////////////////////////////////////// @@ -215,7 +168,6 @@ func (t *describePartitionTask) Execute() error { t.description = &description return nil - } ////////////////////////////////////////////////////////////////////////// @@ -256,5 +208,4 @@ func (t *showPartitionTask) Execute() error { t.stringListResponse = &stringListResponse return nil - } diff --git a/internal/master/partition_task_test.go b/internal/master/partition_task_test.go index 53b698e64a..c6642cad27 100644 --- a/internal/master/partition_task_test.go +++ b/internal/master/partition_task_test.go @@ -35,7 +35,8 @@ func TestMaster_Partition(t *testing.T) { Port: Params.Port, EtcdAddress: Params.EtcdAddress, - EtcdRootPath: "/test/root", + MetaRootPath: "/test/root/meta", + KvRootPath: "/test/root/kv", PulsarAddress: Params.PulsarAddress, ProxyIDList: []typeutil.UniqueID{1, 2}, diff --git a/internal/master/scheduler.go b/internal/master/scheduler.go index 562052ebb9..9d2611b5a6 100644 --- a/internal/master/scheduler.go +++ b/internal/master/scheduler.go @@ -1,36 +1,17 @@ package master -import ( - "context" - "log" - - "github.com/zilliztech/milvus-distributed/internal/errors" - ms "github.com/zilliztech/milvus-distributed/internal/msgstream" -) - -//type ddRequestScheduler interface {} - -//type ddReqFIFOScheduler struct {} - type ddRequestScheduler struct { - ctx context.Context - cancel context.CancelFunc - globalIDAllocator func() (UniqueID, error) reqQueue chan task scheduleTimeStamp Timestamp - ddMsgStream ms.MsgStream } -func NewDDRequestScheduler(ctx context.Context) *ddRequestScheduler { +func NewDDRequestScheduler(allocGlobalID func() (UniqueID, error)) *ddRequestScheduler { const channelSize = 1024 - ctx2, cancel := context.WithCancel(ctx) - rs := ddRequestScheduler{ - ctx: ctx2, - cancel: cancel, - reqQueue: make(chan task, channelSize), + globalIDAllocator: allocGlobalID, + reqQueue: make(chan task, channelSize), } return &rs } @@ -39,51 +20,3 @@ func (rs *ddRequestScheduler) Enqueue(task task) error { rs.reqQueue <- task return nil } - -func (rs *ddRequestScheduler) SetIDAllocator(allocGlobalID func() (UniqueID, error)) { - rs.globalIDAllocator = allocGlobalID -} - -func (rs *ddRequestScheduler) SetDDMsgStream(ddStream ms.MsgStream) { - rs.ddMsgStream = ddStream -} - -func (rs *ddRequestScheduler) scheduleLoop() { - for { - select { - case task := <-rs.reqQueue: - err := rs.schedule(task) - if err != nil { - log.Println(err) - } - case <-rs.ctx.Done(): - log.Print("server is closed, exit task execution loop") - return - } - } -} - -func (rs *ddRequestScheduler) schedule(t task) error { - timeStamp, err := t.Ts() - if err != nil { - log.Println(err) - return err - } - if timeStamp < rs.scheduleTimeStamp { - t.Notify(errors.Errorf("input timestamp = %d, schduler timestamp = %d", timeStamp, rs.scheduleTimeStamp)) - } else { - rs.scheduleTimeStamp = timeStamp - err = t.Execute() - t.Notify(err) - } - return nil -} - -func (rs *ddRequestScheduler) Start() error { - go rs.scheduleLoop() - return nil -} - -func (rs *ddRequestScheduler) Close() { - rs.cancel() -} diff --git a/internal/master/scheduler_test.go b/internal/master/scheduler_test.go deleted file mode 100644 index 7a25e5b849..0000000000 --- a/internal/master/scheduler_test.go +++ /dev/null @@ -1,342 +0,0 @@ -package master - -import ( - "context" - "math/rand" - "strconv" - "testing" - "time" - - "github.com/golang/protobuf/proto" - "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/kv" - ms "github.com/zilliztech/milvus-distributed/internal/msgstream" - "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" - "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" - "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" - "github.com/zilliztech/milvus-distributed/internal/util/tsoutil" - "go.etcd.io/etcd/clientv3" -) - -func TestMaster_Scheduler_Collection(t *testing.T) { - Init() - etcdAddress := Params.EtcdAddress - kvRootPath := Params.EtcdRootPath - pulsarAddr := Params.PulsarAddress - - producerChannels := []string{"ddstream"} - consumerChannels := []string{"ddstream"} - consumerSubName := "substream" - - cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}}) - assert.Nil(t, err) - etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root") - - meta, err := NewMetaTable(etcdKV) - assert.Nil(t, err) - defer meta.client.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pulsarDDStream := ms.NewPulsarMsgStream(ctx, 1024) //input stream - pulsarDDStream.SetPulsarClient(pulsarAddr) - pulsarDDStream.CreatePulsarProducers(producerChannels) - pulsarDDStream.Start() - defer pulsarDDStream.Close() - - consumeMs := ms.NewPulsarMsgStream(ctx, 1024) - consumeMs.SetPulsarClient(pulsarAddr) - consumeMs.CreatePulsarConsumers(consumerChannels, consumerSubName, ms.NewUnmarshalDispatcher(), 1024) - consumeMs.Start() - defer consumeMs.Close() - - idAllocator := NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase([]string{etcdAddress}, kvRootPath, "gid")) - err = idAllocator.Initialize() - assert.Nil(t, err) - - scheduler := NewDDRequestScheduler(ctx) - scheduler.SetDDMsgStream(pulsarDDStream) - scheduler.SetIDAllocator(func() (UniqueID, error) { return idAllocator.AllocOne() }) - scheduler.Start() - defer scheduler.Close() - - rand.Seed(time.Now().Unix()) - sch := schemapb.CollectionSchema{ - Name: "name" + strconv.FormatUint(rand.Uint64(), 10), - Description: "string", - AutoID: true, - Fields: nil, - } - - schemaBytes, err := proto.Marshal(&sch) - assert.Nil(t, err) - - ////////////////////////////CreateCollection//////////////////////// - createCollectionReq := internalpb.CreateCollectionRequest{ - MsgType: internalpb.MsgType_kCreateCollection, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - Schema: &commonpb.Blob{Value: schemaBytes}, - } - - var createCollectionTask task = &createCollectionTask{ - req: &createCollectionReq, - baseTask: baseTask{ - sch: scheduler, - mt: meta, - cv: make(chan error), - }, - } - - err = scheduler.Enqueue(createCollectionTask) - assert.Nil(t, err) - err = createCollectionTask.WaitToFinish(ctx) - assert.Nil(t, err) - - var consumeMsg ms.MsgStream = consumeMs - var createCollectionMsg *ms.CreateCollectionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - createCollectionMsg = v.(*ms.CreateCollectionMsg) - } - break - } - } - assert.Equal(t, createCollectionReq.MsgType, createCollectionMsg.CreateCollectionRequest.MsgType) - assert.Equal(t, createCollectionReq.ReqID, createCollectionMsg.CreateCollectionRequest.ReqID) - assert.Equal(t, createCollectionReq.Timestamp, createCollectionMsg.CreateCollectionRequest.Timestamp) - assert.Equal(t, createCollectionReq.ProxyID, createCollectionMsg.CreateCollectionRequest.ProxyID) - assert.Equal(t, createCollectionReq.Schema.Value, createCollectionMsg.CreateCollectionRequest.Schema.Value) - - ////////////////////////////DropCollection//////////////////////// - dropCollectionReq := internalpb.DropCollectionRequest{ - MsgType: internalpb.MsgType_kDropCollection, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - CollectionName: &servicepb.CollectionName{CollectionName: sch.Name}, - } - - var dropCollectionTask task = &dropCollectionTask{ - req: &dropCollectionReq, - baseTask: baseTask{ - sch: scheduler, - mt: meta, - cv: make(chan error), - }, - } - - err = scheduler.Enqueue(dropCollectionTask) - assert.Nil(t, err) - err = dropCollectionTask.WaitToFinish(ctx) - assert.Nil(t, err) - - var dropCollectionMsg *ms.DropCollectionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - dropCollectionMsg = v.(*ms.DropCollectionMsg) - } - break - } - } - assert.Equal(t, dropCollectionReq.MsgType, dropCollectionMsg.DropCollectionRequest.MsgType) - assert.Equal(t, dropCollectionReq.ReqID, dropCollectionMsg.DropCollectionRequest.ReqID) - assert.Equal(t, dropCollectionReq.Timestamp, dropCollectionMsg.DropCollectionRequest.Timestamp) - assert.Equal(t, dropCollectionReq.ProxyID, dropCollectionMsg.DropCollectionRequest.ProxyID) - assert.Equal(t, dropCollectionReq.CollectionName.CollectionName, dropCollectionMsg.DropCollectionRequest.CollectionName.CollectionName) - -} - -func TestMaster_Scheduler_Partition(t *testing.T) { - Init() - etcdAddress := Params.EtcdAddress - kvRootPath := Params.EtcdRootPath - pulsarAddr := Params.PulsarAddress - - producerChannels := []string{"ddstream"} - consumerChannels := []string{"ddstream"} - consumerSubName := "substream" - - cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}}) - assert.Nil(t, err) - etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root") - - meta, err := NewMetaTable(etcdKV) - assert.Nil(t, err) - defer meta.client.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pulsarDDStream := ms.NewPulsarMsgStream(ctx, 1024) //input stream - pulsarDDStream.SetPulsarClient(pulsarAddr) - pulsarDDStream.CreatePulsarProducers(producerChannels) - pulsarDDStream.Start() - defer pulsarDDStream.Close() - - consumeMs := ms.NewPulsarMsgStream(ctx, 1024) - consumeMs.SetPulsarClient(pulsarAddr) - consumeMs.CreatePulsarConsumers(consumerChannels, consumerSubName, ms.NewUnmarshalDispatcher(), 1024) - consumeMs.Start() - defer consumeMs.Close() - - idAllocator := NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase([]string{etcdAddress}, kvRootPath, "gid")) - err = idAllocator.Initialize() - assert.Nil(t, err) - - scheduler := NewDDRequestScheduler(ctx) - scheduler.SetDDMsgStream(pulsarDDStream) - scheduler.SetIDAllocator(func() (UniqueID, error) { return idAllocator.AllocOne() }) - scheduler.Start() - defer scheduler.Close() - - rand.Seed(time.Now().Unix()) - sch := schemapb.CollectionSchema{ - Name: "name" + strconv.FormatUint(rand.Uint64(), 10), - Description: "string", - AutoID: true, - Fields: nil, - } - - schemaBytes, err := proto.Marshal(&sch) - assert.Nil(t, err) - - ////////////////////////////CreateCollection//////////////////////// - createCollectionReq := internalpb.CreateCollectionRequest{ - MsgType: internalpb.MsgType_kCreateCollection, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - Schema: &commonpb.Blob{Value: schemaBytes}, - } - - var createCollectionTask task = &createCollectionTask{ - req: &createCollectionReq, - baseTask: baseTask{ - sch: scheduler, - mt: meta, - cv: make(chan error), - }, - } - - err = scheduler.Enqueue(createCollectionTask) - assert.Nil(t, err) - err = createCollectionTask.WaitToFinish(ctx) - assert.Nil(t, err) - - var consumeMsg ms.MsgStream = consumeMs - var createCollectionMsg *ms.CreateCollectionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - createCollectionMsg = v.(*ms.CreateCollectionMsg) - } - break - } - } - assert.Equal(t, createCollectionReq.MsgType, createCollectionMsg.CreateCollectionRequest.MsgType) - assert.Equal(t, createCollectionReq.ReqID, createCollectionMsg.CreateCollectionRequest.ReqID) - assert.Equal(t, createCollectionReq.Timestamp, createCollectionMsg.CreateCollectionRequest.Timestamp) - assert.Equal(t, createCollectionReq.ProxyID, createCollectionMsg.CreateCollectionRequest.ProxyID) - assert.Equal(t, createCollectionReq.Schema.Value, createCollectionMsg.CreateCollectionRequest.Schema.Value) - - ////////////////////////////CreatePartition//////////////////////// - partitionName := "partitionName" + strconv.FormatUint(rand.Uint64(), 10) - createPartitionReq := internalpb.CreatePartitionRequest{ - MsgType: internalpb.MsgType_kCreatePartition, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - PartitionName: &servicepb.PartitionName{ - CollectionName: sch.Name, - Tag: partitionName, - }, - } - - var createPartitionTask task = &createPartitionTask{ - req: &createPartitionReq, - baseTask: baseTask{ - sch: scheduler, - mt: meta, - cv: make(chan error), - }, - } - - err = scheduler.Enqueue(createPartitionTask) - assert.Nil(t, err) - err = createPartitionTask.WaitToFinish(ctx) - assert.Nil(t, err) - - var createPartitionMsg *ms.CreatePartitionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - createPartitionMsg = v.(*ms.CreatePartitionMsg) - } - break - } - } - assert.Equal(t, createPartitionReq.MsgType, createPartitionMsg.CreatePartitionRequest.MsgType) - assert.Equal(t, createPartitionReq.ReqID, createPartitionMsg.CreatePartitionRequest.ReqID) - assert.Equal(t, createPartitionReq.Timestamp, createPartitionMsg.CreatePartitionRequest.Timestamp) - assert.Equal(t, createPartitionReq.ProxyID, createPartitionMsg.CreatePartitionRequest.ProxyID) - assert.Equal(t, createPartitionReq.PartitionName.CollectionName, createPartitionMsg.CreatePartitionRequest.PartitionName.CollectionName) - assert.Equal(t, createPartitionReq.PartitionName.Tag, createPartitionMsg.CreatePartitionRequest.PartitionName.Tag) - - ////////////////////////////DropPartition//////////////////////// - dropPartitionReq := internalpb.DropPartitionRequest{ - MsgType: internalpb.MsgType_kDropPartition, - ReqID: 1, - Timestamp: 11, - ProxyID: 1, - PartitionName: &servicepb.PartitionName{ - CollectionName: sch.Name, - Tag: partitionName, - }, - } - - var dropPartitionTask task = &dropPartitionTask{ - req: &dropPartitionReq, - baseTask: baseTask{ - sch: scheduler, - mt: meta, - cv: make(chan error), - }, - } - - err = scheduler.Enqueue(dropPartitionTask) - assert.Nil(t, err) - err = dropPartitionTask.WaitToFinish(ctx) - assert.Nil(t, err) - - var dropPartitionMsg *ms.DropPartitionMsg - for { - result := consumeMsg.Consume() - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - dropPartitionMsg = v.(*ms.DropPartitionMsg) - } - break - } - } - assert.Equal(t, dropPartitionReq.MsgType, dropPartitionMsg.DropPartitionRequest.MsgType) - assert.Equal(t, dropPartitionReq.ReqID, dropPartitionMsg.DropPartitionRequest.ReqID) - assert.Equal(t, dropPartitionReq.Timestamp, dropPartitionMsg.DropPartitionRequest.Timestamp) - assert.Equal(t, dropPartitionReq.ProxyID, dropPartitionMsg.DropPartitionRequest.ProxyID) - assert.Equal(t, dropPartitionReq.PartitionName.CollectionName, dropPartitionMsg.DropPartitionRequest.PartitionName.CollectionName) - -} diff --git a/internal/master/segment_manager_test.go b/internal/master/segment_manager_test.go index 85942bdd62..0287ea955c 100644 --- a/internal/master/segment_manager_test.go +++ b/internal/master/segment_manager_test.go @@ -236,7 +236,8 @@ func startupMaster() { Port: Params.Port, EtcdAddress: Params.EtcdAddress, - EtcdRootPath: rootPath, + MetaRootPath: "/test/root/meta", + KvRootPath: "/test/root/kv", PulsarAddress: Params.PulsarAddress, ProxyIDList: []typeutil.UniqueID{1, 2}, diff --git a/internal/master/system_config.go b/internal/master/system_config.go deleted file mode 100644 index 1fadb37b19..0000000000 --- a/internal/master/system_config.go +++ /dev/null @@ -1,117 +0,0 @@ -package master - -import ( - "fmt" - "log" - "os" - "path" - "path/filepath" - "strings" - - "github.com/spf13/viper" - "github.com/zilliztech/milvus-distributed/internal/errors" - "github.com/zilliztech/milvus-distributed/internal/kv" -) - -type SysConfig struct { - kv *kv.EtcdKV -} - -// Initialize Configs from config files, and store them in Etcd. -func (conf *SysConfig) InitFromFile(filePath string) error { - memConfigs, err := conf.getConfigFiles(filePath) - if err != nil { - return errors.Errorf("[Init SysConfig] %s\n", err.Error()) - } - - for _, memConfig := range memConfigs { - if err := conf.saveToEtcd(memConfig, "config"); err != nil { - return errors.Errorf("[Init SysConfig] %s\n", err.Error()) - } - } - return nil -} - -func (conf *SysConfig) GetByPrefix(keyPrefix string) (keys []string, values []string, err error) { - realPrefix := path.Join("config", strings.ToLower(keyPrefix)) - keys, values, err = conf.kv.LoadWithPrefix(realPrefix) - for index := range keys { - keys[index] = strings.Replace(keys[index], conf.kv.GetPath("config"), "", 1) - } - if err != nil { - return nil, nil, err - } - log.Println("Loaded", len(keys), "pairs of configs with prefix", keyPrefix) - return keys, values, err -} - -// Get specific configs for keys. -func (conf *SysConfig) Get(keys []string) ([]string, error) { - var keysToLoad []string - for i := range keys { - keysToLoad = append(keysToLoad, path.Join("config", strings.ToLower(keys[i]))) - } - - values, err := conf.kv.MultiLoad(keysToLoad) - if err != nil { - return nil, err - } - - return values, nil -} - -func (conf *SysConfig) getConfigFiles(filePath string) ([]*viper.Viper, error) { - - var vipers []*viper.Viper - err := filepath.Walk(filePath, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // all names - if !info.IsDir() && filepath.Ext(path) == ".yaml" { - log.Println("Config files ", info.Name()) - - currentConf := viper.New() - currentConf.SetConfigFile(path) - if err := currentConf.ReadInConfig(); err != nil { - log.Panic("Config file error: ", err) - } - vipers = append(vipers, currentConf) - } - return nil - }) - if err != nil { - return nil, err - } - - if len(vipers) == 0 { - return nil, errors.Errorf("There are no config files in the path `%s`.\n", filePath) - } - return vipers, nil -} - -func (conf *SysConfig) saveToEtcd(memConfig *viper.Viper, secondRootPath string) error { - configMaps := map[string]string{} - - allKeys := memConfig.AllKeys() - for _, key := range allKeys { - etcdKey := strings.ReplaceAll(key, ".", "/") - - etcdKey = path.Join(secondRootPath, etcdKey) - - val := memConfig.Get(key) - if val == nil { - configMaps[etcdKey] = "" - continue - } - configMaps[etcdKey] = fmt.Sprintf("%v", val) - } - - if err := conf.kv.MultiSave(configMaps); err != nil { - return err - } - - return nil -} diff --git a/internal/master/system_config_test.go b/internal/master/system_config_test.go deleted file mode 100644 index 45ddacb01e..0000000000 --- a/internal/master/system_config_test.go +++ /dev/null @@ -1,209 +0,0 @@ -package master - -import ( - "context" - "fmt" - "log" - "path" - "strings" - "testing" - "time" - - "github.com/zilliztech/milvus-distributed/internal/kv" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.etcd.io/etcd/clientv3" -) - -func Test_SysConfig(t *testing.T) { - Init() - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - cli, err := clientv3.New(clientv3.Config{ - Endpoints: []string{Params.EtcdAddress}, - DialTimeout: 5 * time.Second, - }) - require.Nil(t, err) - _, err = cli.Delete(ctx, "/test/root", clientv3.WithPrefix()) - require.Nil(t, err) - - rootPath := "/test/root" - configKV := kv.NewEtcdKV(cli, rootPath) - defer configKV.Close() - - sc := SysConfig{kv: configKV} - require.Equal(t, rootPath, sc.kv.GetPath(".")) - - t.Run("tests on contig_test.yaml", func(t *testing.T) { - err = sc.InitFromFile(".") - require.Nil(t, err) - - testKeys := []string{ - "/etcd/address", - "/master/port", - "/master/proxyidlist", - "/master/segmentthresholdfactor", - "/pulsar/token", - "/reader/stopflag", - "/proxy/timezone", - "/proxy/network/address", - "/proxy/storage/path", - "/storage/accesskey", - } - - testVals := []string{ - "localhost", - "53100", - "[1 2]", - "0.75", - "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY", - "-1", - "UTC+8", - "0.0.0.0", - "/var/lib/milvus", - "", - } - - vals, err := sc.Get(testKeys) - assert.Nil(t, err) - for i := range testVals { - assert.Equal(t, testVals[i], vals[i]) - } - - keys, vals, err := sc.GetByPrefix("/master") - assert.Nil(t, err) - for i := range keys { - assert.True(t, strings.HasPrefix(keys[i], "/master/")) - } - assert.Equal(t, len(keys), len(vals)) - assert.Equal(t, 21, len(keys)) - - // Test get all configs - keys, vals, err = sc.GetByPrefix("/") - assert.Nil(t, err) - assert.Equal(t, len(keys), len(vals)) - assert.Equal(t, 73, len(vals)) - - // Test get configs with prefix not exist - keys, vals, err = sc.GetByPrefix("/config") - assert.Nil(t, err) - assert.Equal(t, len(keys), len(vals)) - assert.Equal(t, 0, len(keys)) - assert.Equal(t, 0, len(vals)) - - _, _, err = sc.GetByPrefix("//././../../../../../..//../") - assert.Nil(t, err) - _, _, err = sc.GetByPrefix("/master/./address") - assert.Nil(t, err) - _, _, err = sc.GetByPrefix(".") - assert.Nil(t, err) - _, _, err = sc.GetByPrefix("\\") - assert.Nil(t, err) - - }) - - t.Run("getConfigFiles", func(t *testing.T) { - filePath := "../../configs" - vipers, err := sc.getConfigFiles(filePath) - assert.Nil(t, err) - assert.NotNil(t, vipers[0]) - - filePath = "/path/not/exists" - _, err = sc.getConfigFiles(filePath) - assert.NotNil(t, err) - log.Println(err) - }) - - t.Run("Test saveToEtcd Normal", func(t *testing.T) { - _, err = cli.Delete(ctx, "/test/root/config", clientv3.WithPrefix()) - require.Nil(t, err) - - v := viper.New() - - v.Set("a.suba1", "v1") - v.Set("a.suba2", "v2") - v.Set("a.suba3.subsuba1", "v3") - v.Set("a.suba3.subsuba2", "v4") - - secondRootPath := "config" - err := sc.saveToEtcd(v, secondRootPath) - assert.Nil(t, err) - - value, err := sc.kv.Load(path.Join(secondRootPath, "a/suba1")) - assert.Nil(t, err) - assert.Equal(t, "v1", value) - value, err = sc.kv.Load(path.Join(secondRootPath, "a/suba2")) - assert.Nil(t, err) - assert.Equal(t, "v2", value) - value, err = sc.kv.Load(path.Join(secondRootPath, "a/suba3/subsuba1")) - assert.Nil(t, err) - assert.Equal(t, "v3", value) - value, err = sc.kv.Load(path.Join(secondRootPath, "a/suba3/subsuba2")) - assert.Nil(t, err) - assert.Equal(t, "v4", value) - - keys, values, err := sc.kv.LoadWithPrefix(path.Join(secondRootPath, "a")) - assert.Nil(t, err) - assert.Equal(t, 4, len(keys)) - assert.Equal(t, 4, len(values)) - assert.ElementsMatch(t, []string{ - path.Join(sc.kv.GetPath(secondRootPath), "/a/suba1"), - path.Join(sc.kv.GetPath(secondRootPath), "/a/suba2"), - path.Join(sc.kv.GetPath(secondRootPath), "/a/suba3/subsuba1"), - path.Join(sc.kv.GetPath(secondRootPath), "/a/suba3/subsuba2"), - }, keys) - assert.ElementsMatch(t, []string{"v1", "v2", "v3", "v4"}, values) - - keys = []string{ - "/a/suba1", - "/a/suba2", - "/a/suba3/subsuba1", - "/a/suba3/subsuba2", - } - values, err = sc.Get(keys) - assert.Nil(t, err) - assert.ElementsMatch(t, []string{"v1", "v2", "v3", "v4"}, values) - - keysAfter, values, err := sc.GetByPrefix("/a") - fmt.Println(keysAfter) - assert.Nil(t, err) - assert.ElementsMatch(t, []string{"v1", "v2", "v3", "v4"}, values) - assert.ElementsMatch(t, keys, keysAfter) - - }) - - t.Run("Test saveToEtcd Different value types", func(t *testing.T) { - v := viper.New() - - v.Set("string", "string") - v.Set("number", 1) - v.Set("nil", nil) - v.Set("float", 1.2) - v.Set("intslice", []int{100, 200}) - v.Set("stringslice", []string{"a", "b"}) - v.Set("stringmapstring", map[string]string{"k1": "1", "k2": "2"}) - - secondRootPath := "test_save_to_etcd_different_value_types" - err := sc.saveToEtcd(v, secondRootPath) - require.Nil(t, err) - - keys, values, err := sc.kv.LoadWithPrefix(path.Join("/", secondRootPath)) - assert.Nil(t, err) - assert.Equal(t, 7, len(keys)) - assert.Equal(t, 7, len(values)) - - assert.ElementsMatch(t, []string{ - path.Join(sc.kv.GetPath(secondRootPath), "nil"), - path.Join(sc.kv.GetPath(secondRootPath), "string"), - path.Join(sc.kv.GetPath(secondRootPath), "number"), - path.Join(sc.kv.GetPath(secondRootPath), "float"), - path.Join(sc.kv.GetPath(secondRootPath), "intslice"), - path.Join(sc.kv.GetPath(secondRootPath), "stringslice"), - path.Join(sc.kv.GetPath(secondRootPath), "stringmapstring"), - }, keys) - assert.ElementsMatch(t, []string{"", "string", "1", "1.2", "[100 200]", "[a b]", "map[k1:1 k2:2]"}, values) - }) -} diff --git a/internal/master/time_snyc_producer_test.go b/internal/master/time_snyc_producer_test.go index e55b1ec427..4bb98f5129 100644 --- a/internal/master/time_snyc_producer_test.go +++ b/internal/master/time_snyc_producer_test.go @@ -81,18 +81,13 @@ func receiveMsg(stream *ms.MsgStream) []uint64 { func TestStream_PulsarMsgStream_TimeTick(t *testing.T) { Init() pulsarAddress := Params.PulsarAddress + + producerChannels := []string{"proxyTtBarrier"} + consumerChannels := []string{"proxyTtBarrier"} + consumerSubName := "proxyTtBarrier" ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - - producerChannels := []string{"proxyDMTtBarrier"} - consumerChannels := []string{"proxyDMTtBarrier"} - consumerSubName := "proxyDMTtBarrier" - proxyDMTtInputStream, proxyDMTtOutputStream := initTestPulsarStream(ctx, pulsarAddress, producerChannels, consumerChannels, consumerSubName) - - producerChannels = []string{"proxyDDTtBarrier"} - consumerChannels = []string{"proxyDDTtBarrier"} - consumerSubName = "proxyDDTtBarrier" - proxyDDTtInputStream, proxyDDTtOutputStream := initTestPulsarStream(ctx, pulsarAddress, producerChannels, consumerChannels, consumerSubName) + proxyTtInputStream, proxyTtOutputStream := initTestPulsarStream(ctx, pulsarAddress, producerChannels, consumerChannels, consumerSubName) producerChannels = []string{"writeNodeBarrier"} consumerChannels = []string{"writeNodeBarrier"} @@ -102,20 +97,14 @@ func TestStream_PulsarMsgStream_TimeTick(t *testing.T) { timeSyncProducer, _ := NewTimeSyncMsgProducer(ctx) timeSyncProducer.SetProxyTtBarrier(&TestTickBarrier{ctx: ctx}) timeSyncProducer.SetWriteNodeTtBarrier(&TestTickBarrier{ctx: ctx}) - timeSyncProducer.SetDMSyncStream(*proxyDMTtInputStream) - timeSyncProducer.SetDDSyncStream(*proxyDDTtInputStream) + timeSyncProducer.SetDMSyncStream(*proxyTtInputStream) timeSyncProducer.SetK2sSyncStream(*writeNodeInputStream) - (*proxyDMTtOutputStream).Start() - (*proxyDDTtOutputStream).Start() + (*proxyTtOutputStream).Start() (*writeNodeOutputStream).Start() timeSyncProducer.Start() expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} - result1 := receiveMsg(proxyDMTtOutputStream) + result1 := receiveMsg(proxyTtOutputStream) assert.Equal(t, expected, result1) - expected = []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} - result1 = receiveMsg(proxyDDTtOutputStream) - assert.Equal(t, expected, result1) - expected = []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} result2 := receiveMsg(writeNodeOutputStream) assert.Equal(t, expected, result2) diff --git a/internal/master/time_sync_producer.go b/internal/master/time_sync_producer.go index 075de0b726..fc58198858 100644 --- a/internal/master/time_sync_producer.go +++ b/internal/master/time_sync_producer.go @@ -15,8 +15,7 @@ type timeSyncMsgProducer struct { //hardTimeTickBarrier writeNodeTtBarrier TimeTickBarrier - ddSyncStream ms.MsgStream // insert & delete - dmSyncStream ms.MsgStream + dmSyncStream ms.MsgStream // insert & delete k2sSyncStream ms.MsgStream ctx context.Context @@ -35,9 +34,6 @@ func (syncMsgProducer *timeSyncMsgProducer) SetProxyTtBarrier(proxyTtBarrier Tim func (syncMsgProducer *timeSyncMsgProducer) SetWriteNodeTtBarrier(writeNodeTtBarrier TimeTickBarrier) { syncMsgProducer.writeNodeTtBarrier = writeNodeTtBarrier } -func (syncMsgProducer *timeSyncMsgProducer) SetDDSyncStream(ddSync ms.MsgStream) { - syncMsgProducer.ddSyncStream = ddSync -} func (syncMsgProducer *timeSyncMsgProducer) SetDMSyncStream(dmSync ms.MsgStream) { syncMsgProducer.dmSyncStream = dmSync @@ -47,7 +43,7 @@ func (syncMsgProducer *timeSyncMsgProducer) SetK2sSyncStream(k2sSync ms.MsgStrea syncMsgProducer.k2sSyncStream = k2sSync } -func (syncMsgProducer *timeSyncMsgProducer) broadcastMsg(barrier TimeTickBarrier, streams []ms.MsgStream) error { +func (syncMsgProducer *timeSyncMsgProducer) broadcastMsg(barrier TimeTickBarrier, stream ms.MsgStream) error { for { select { case <-syncMsgProducer.ctx.Done(): @@ -76,9 +72,7 @@ func (syncMsgProducer *timeSyncMsgProducer) broadcastMsg(barrier TimeTickBarrier TimeTickMsg: timeTickResult, } msgPack.Msgs = append(msgPack.Msgs, timeTickMsg) - for _, stream := range streams { - err = stream.Broadcast(&msgPack) - } + err = stream.Broadcast(&msgPack) if err != nil { return err } @@ -97,17 +91,16 @@ func (syncMsgProducer *timeSyncMsgProducer) Start() error { return err } - go syncMsgProducer.broadcastMsg(syncMsgProducer.proxyTtBarrier, []ms.MsgStream{syncMsgProducer.dmSyncStream, syncMsgProducer.ddSyncStream}) - go syncMsgProducer.broadcastMsg(syncMsgProducer.writeNodeTtBarrier, []ms.MsgStream{syncMsgProducer.k2sSyncStream}) + go syncMsgProducer.broadcastMsg(syncMsgProducer.proxyTtBarrier, syncMsgProducer.dmSyncStream) + go syncMsgProducer.broadcastMsg(syncMsgProducer.writeNodeTtBarrier, syncMsgProducer.k2sSyncStream) return nil } func (syncMsgProducer *timeSyncMsgProducer) Close() { - syncMsgProducer.ddSyncStream.Close() + syncMsgProducer.proxyTtBarrier.Close() + syncMsgProducer.writeNodeTtBarrier.Close() syncMsgProducer.dmSyncStream.Close() syncMsgProducer.k2sSyncStream.Close() syncMsgProducer.cancel() - syncMsgProducer.proxyTtBarrier.Close() - syncMsgProducer.writeNodeTtBarrier.Close() } diff --git a/internal/master/timesync.go b/internal/master/timesync.go index fc502adf37..e4448cab05 100644 --- a/internal/master/timesync.go +++ b/internal/master/timesync.go @@ -75,7 +75,6 @@ func (ttBarrier *softTimeTickBarrier) Start() error { log.Printf("[softTimeTickBarrier] Warning: peerID %d not exist\n", ttmsg.PeerID) continue } - if ttmsg.Timestamp > oldT { ttBarrier.peer2LastTt[ttmsg.PeerID] = ttmsg.Timestamp @@ -85,7 +84,6 @@ func (ttBarrier *softTimeTickBarrier) Start() error { if ttBarrier.lastTt != 0 && ttBarrier.minTtInterval > ts-Timestamp(lastTt) { continue } - ttBarrier.outTt <- ts } } diff --git a/internal/msgstream/msg.go b/internal/msgstream/msg.go deleted file mode 100644 index 93b7bb4c00..0000000000 --- a/internal/msgstream/msg.go +++ /dev/null @@ -1,593 +0,0 @@ -package msgstream - -import ( - "github.com/golang/protobuf/proto" - internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" -) - -type MsgType = internalPb.MsgType - -type TsMsg interface { - BeginTs() Timestamp - EndTs() Timestamp - Type() MsgType - HashKeys() []int32 - Marshal(TsMsg) ([]byte, error) - Unmarshal([]byte) (TsMsg, error) -} - -type BaseMsg struct { - BeginTimestamp Timestamp - EndTimestamp Timestamp - HashValues []int32 -} - -func (bm *BaseMsg) BeginTs() Timestamp { - return bm.BeginTimestamp -} - -func (bm *BaseMsg) EndTs() Timestamp { - return bm.EndTimestamp -} - -func (bm *BaseMsg) HashKeys() []int32 { - return bm.HashValues -} - -/////////////////////////////////////////Insert////////////////////////////////////////// -type InsertMsg struct { - BaseMsg - internalPb.InsertRequest -} - -func (it *InsertMsg) Type() MsgType { - return it.MsgType -} - -func (it *InsertMsg) Marshal(input TsMsg) ([]byte, error) { - insertMsg := input.(*InsertMsg) - insertRequest := &insertMsg.InsertRequest - mb, err := proto.Marshal(insertRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (it *InsertMsg) Unmarshal(input []byte) (TsMsg, error) { - insertRequest := internalPb.InsertRequest{} - err := proto.Unmarshal(input, &insertRequest) - if err != nil { - return nil, err - } - insertMsg := &InsertMsg{InsertRequest: insertRequest} - for _, timestamp := range insertMsg.Timestamps { - insertMsg.BeginTimestamp = timestamp - insertMsg.EndTimestamp = timestamp - break - } - for _, timestamp := range insertMsg.Timestamps { - if timestamp > insertMsg.EndTimestamp { - insertMsg.EndTimestamp = timestamp - } - if timestamp < insertMsg.BeginTimestamp { - insertMsg.BeginTimestamp = timestamp - } - } - - return insertMsg, nil -} - -/////////////////////////////////////////Delete////////////////////////////////////////// -type DeleteMsg struct { - BaseMsg - internalPb.DeleteRequest -} - -func (dt *DeleteMsg) Type() MsgType { - return dt.MsgType -} - -func (dt *DeleteMsg) Marshal(input TsMsg) ([]byte, error) { - deleteTask := input.(*DeleteMsg) - deleteRequest := &deleteTask.DeleteRequest - mb, err := proto.Marshal(deleteRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (dt *DeleteMsg) Unmarshal(input []byte) (TsMsg, error) { - deleteRequest := internalPb.DeleteRequest{} - err := proto.Unmarshal(input, &deleteRequest) - if err != nil { - return nil, err - } - deleteMsg := &DeleteMsg{DeleteRequest: deleteRequest} - for _, timestamp := range deleteMsg.Timestamps { - deleteMsg.BeginTimestamp = timestamp - deleteMsg.EndTimestamp = timestamp - break - } - for _, timestamp := range deleteMsg.Timestamps { - if timestamp > deleteMsg.EndTimestamp { - deleteMsg.EndTimestamp = timestamp - } - if timestamp < deleteMsg.BeginTimestamp { - deleteMsg.BeginTimestamp = timestamp - } - } - - return deleteMsg, nil -} - -/////////////////////////////////////////Search////////////////////////////////////////// -type SearchMsg struct { - BaseMsg - internalPb.SearchRequest -} - -func (st *SearchMsg) Type() MsgType { - return st.MsgType -} - -func (st *SearchMsg) Marshal(input TsMsg) ([]byte, error) { - searchTask := input.(*SearchMsg) - searchRequest := &searchTask.SearchRequest - mb, err := proto.Marshal(searchRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (st *SearchMsg) Unmarshal(input []byte) (TsMsg, error) { - searchRequest := internalPb.SearchRequest{} - err := proto.Unmarshal(input, &searchRequest) - if err != nil { - return nil, err - } - searchMsg := &SearchMsg{SearchRequest: searchRequest} - searchMsg.BeginTimestamp = searchMsg.Timestamp - searchMsg.EndTimestamp = searchMsg.Timestamp - - return searchMsg, nil -} - -/////////////////////////////////////////SearchResult////////////////////////////////////////// -type SearchResultMsg struct { - BaseMsg - internalPb.SearchResult -} - -func (srt *SearchResultMsg) Type() MsgType { - return srt.MsgType -} - -func (srt *SearchResultMsg) Marshal(input TsMsg) ([]byte, error) { - searchResultTask := input.(*SearchResultMsg) - searchResultRequest := &searchResultTask.SearchResult - mb, err := proto.Marshal(searchResultRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (srt *SearchResultMsg) Unmarshal(input []byte) (TsMsg, error) { - searchResultRequest := internalPb.SearchResult{} - err := proto.Unmarshal(input, &searchResultRequest) - if err != nil { - return nil, err - } - searchResultMsg := &SearchResultMsg{SearchResult: searchResultRequest} - searchResultMsg.BeginTimestamp = searchResultMsg.Timestamp - searchResultMsg.EndTimestamp = searchResultMsg.Timestamp - - return searchResultMsg, nil -} - -/////////////////////////////////////////TimeTick////////////////////////////////////////// -type TimeTickMsg struct { - BaseMsg - internalPb.TimeTickMsg -} - -func (tst *TimeTickMsg) Type() MsgType { - return tst.MsgType -} - -func (tst *TimeTickMsg) Marshal(input TsMsg) ([]byte, error) { - timeTickTask := input.(*TimeTickMsg) - timeTick := &timeTickTask.TimeTickMsg - mb, err := proto.Marshal(timeTick) - if err != nil { - return nil, err - } - return mb, nil -} - -func (tst *TimeTickMsg) Unmarshal(input []byte) (TsMsg, error) { - timeTickMsg := internalPb.TimeTickMsg{} - err := proto.Unmarshal(input, &timeTickMsg) - if err != nil { - return nil, err - } - timeTick := &TimeTickMsg{TimeTickMsg: timeTickMsg} - timeTick.BeginTimestamp = timeTick.Timestamp - timeTick.EndTimestamp = timeTick.Timestamp - - return timeTick, nil -} - -/////////////////////////////////////////QueryNodeSegStats////////////////////////////////////////// -type QueryNodeSegStatsMsg struct { - BaseMsg - internalPb.QueryNodeSegStats -} - -func (qs *QueryNodeSegStatsMsg) Type() MsgType { - return qs.MsgType -} - -func (qs *QueryNodeSegStatsMsg) Marshal(input TsMsg) ([]byte, error) { - queryNodeSegStatsTask := input.(*QueryNodeSegStatsMsg) - queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeSegStats - mb, err := proto.Marshal(queryNodeSegStats) - if err != nil { - return nil, err - } - return mb, nil -} - -func (qs *QueryNodeSegStatsMsg) Unmarshal(input []byte) (TsMsg, error) { - queryNodeSegStats := internalPb.QueryNodeSegStats{} - err := proto.Unmarshal(input, &queryNodeSegStats) - if err != nil { - return nil, err - } - queryNodeSegStatsMsg := &QueryNodeSegStatsMsg{QueryNodeSegStats: queryNodeSegStats} - - return queryNodeSegStatsMsg, nil -} - -///////////////////////////////////////////Key2Seg////////////////////////////////////////// -//type Key2SegMsg struct { -// BaseMsg -// internalPb.Key2SegMsg -//} -// -//func (k2st *Key2SegMsg) Type() MsgType { -// return -//} - -/////////////////////////////////////////CreateCollection////////////////////////////////////////// -type CreateCollectionMsg struct { - BaseMsg - internalPb.CreateCollectionRequest -} - -func (cc *CreateCollectionMsg) Type() MsgType { - return cc.MsgType -} - -func (cc *CreateCollectionMsg) Marshal(input TsMsg) ([]byte, error) { - createCollectionMsg := input.(*CreateCollectionMsg) - createCollectionRequest := &createCollectionMsg.CreateCollectionRequest - mb, err := proto.Marshal(createCollectionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (cc *CreateCollectionMsg) Unmarshal(input []byte) (TsMsg, error) { - createCollectionRequest := internalPb.CreateCollectionRequest{} - err := proto.Unmarshal(input, &createCollectionRequest) - if err != nil { - return nil, err - } - createCollectionMsg := &CreateCollectionMsg{CreateCollectionRequest: createCollectionRequest} - createCollectionMsg.BeginTimestamp = createCollectionMsg.Timestamp - createCollectionMsg.EndTimestamp = createCollectionMsg.Timestamp - - return createCollectionMsg, nil -} - -/////////////////////////////////////////DropCollection////////////////////////////////////////// -type DropCollectionMsg struct { - BaseMsg - internalPb.DropCollectionRequest -} - -func (dc *DropCollectionMsg) Type() MsgType { - return dc.MsgType -} - -func (dc *DropCollectionMsg) Marshal(input TsMsg) ([]byte, error) { - dropCollectionMsg := input.(*DropCollectionMsg) - dropCollectionRequest := &dropCollectionMsg.DropCollectionRequest - mb, err := proto.Marshal(dropCollectionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (dc *DropCollectionMsg) Unmarshal(input []byte) (TsMsg, error) { - dropCollectionRequest := internalPb.DropCollectionRequest{} - err := proto.Unmarshal(input, &dropCollectionRequest) - if err != nil { - return nil, err - } - dropCollectionMsg := &DropCollectionMsg{DropCollectionRequest: dropCollectionRequest} - dropCollectionMsg.BeginTimestamp = dropCollectionMsg.Timestamp - dropCollectionMsg.EndTimestamp = dropCollectionMsg.Timestamp - - return dropCollectionMsg, nil -} - -/////////////////////////////////////////HasCollection////////////////////////////////////////// -type HasCollectionMsg struct { - BaseMsg - internalPb.HasCollectionRequest -} - -func (hc *HasCollectionMsg) Type() MsgType { - return hc.MsgType -} - -func (hc *HasCollectionMsg) Marshal(input TsMsg) ([]byte, error) { - hasCollectionMsg := input.(*HasCollectionMsg) - hasCollectionRequest := &hasCollectionMsg.HasCollectionRequest - mb, err := proto.Marshal(hasCollectionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (hc *HasCollectionMsg) Unmarshal(input []byte) (TsMsg, error) { - hasCollectionRequest := internalPb.HasCollectionRequest{} - err := proto.Unmarshal(input, &hasCollectionRequest) - if err != nil { - return nil, err - } - hasCollectionMsg := &HasCollectionMsg{HasCollectionRequest: hasCollectionRequest} - hasCollectionMsg.BeginTimestamp = hasCollectionMsg.Timestamp - hasCollectionMsg.EndTimestamp = hasCollectionMsg.Timestamp - - return hasCollectionMsg, nil -} - -/////////////////////////////////////////DescribeCollection////////////////////////////////////////// -type DescribeCollectionMsg struct { - BaseMsg - internalPb.DescribeCollectionRequest -} - -func (dc *DescribeCollectionMsg) Type() MsgType { - return dc.MsgType -} - -func (dc *DescribeCollectionMsg) Marshal(input TsMsg) ([]byte, error) { - describeCollectionMsg := input.(*DescribeCollectionMsg) - describeCollectionRequest := &describeCollectionMsg.DescribeCollectionRequest - mb, err := proto.Marshal(describeCollectionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (dc *DescribeCollectionMsg) Unmarshal(input []byte) (TsMsg, error) { - describeCollectionRequest := internalPb.DescribeCollectionRequest{} - err := proto.Unmarshal(input, &describeCollectionRequest) - if err != nil { - return nil, err - } - describeCollectionMsg := &DescribeCollectionMsg{DescribeCollectionRequest: describeCollectionRequest} - describeCollectionMsg.BeginTimestamp = describeCollectionMsg.Timestamp - describeCollectionMsg.EndTimestamp = describeCollectionMsg.Timestamp - - return describeCollectionMsg, nil -} - -/////////////////////////////////////////ShowCollection////////////////////////////////////////// -type ShowCollectionMsg struct { - BaseMsg - internalPb.ShowCollectionRequest -} - -func (sc *ShowCollectionMsg) Type() MsgType { - return sc.MsgType -} - -func (sc *ShowCollectionMsg) Marshal(input TsMsg) ([]byte, error) { - showCollectionMsg := input.(*ShowCollectionMsg) - showCollectionRequest := &showCollectionMsg.ShowCollectionRequest - mb, err := proto.Marshal(showCollectionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (sc *ShowCollectionMsg) Unmarshal(input []byte) (TsMsg, error) { - showCollectionRequest := internalPb.ShowCollectionRequest{} - err := proto.Unmarshal(input, &showCollectionRequest) - if err != nil { - return nil, err - } - showCollectionMsg := &ShowCollectionMsg{ShowCollectionRequest: showCollectionRequest} - showCollectionMsg.BeginTimestamp = showCollectionMsg.Timestamp - showCollectionMsg.EndTimestamp = showCollectionMsg.Timestamp - - return showCollectionMsg, nil -} - -/////////////////////////////////////////CreatePartition////////////////////////////////////////// -type CreatePartitionMsg struct { - BaseMsg - internalPb.CreatePartitionRequest -} - -func (cc *CreatePartitionMsg) Type() MsgType { - return cc.MsgType -} - -func (cc *CreatePartitionMsg) Marshal(input TsMsg) ([]byte, error) { - createPartitionMsg := input.(*CreatePartitionMsg) - createPartitionRequest := &createPartitionMsg.CreatePartitionRequest - mb, err := proto.Marshal(createPartitionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (cc *CreatePartitionMsg) Unmarshal(input []byte) (TsMsg, error) { - createPartitionRequest := internalPb.CreatePartitionRequest{} - err := proto.Unmarshal(input, &createPartitionRequest) - if err != nil { - return nil, err - } - createPartitionMsg := &CreatePartitionMsg{CreatePartitionRequest: createPartitionRequest} - createPartitionMsg.BeginTimestamp = createPartitionMsg.Timestamp - createPartitionMsg.EndTimestamp = createPartitionMsg.Timestamp - - return createPartitionMsg, nil -} - -/////////////////////////////////////////DropPartition////////////////////////////////////////// -type DropPartitionMsg struct { - BaseMsg - internalPb.DropPartitionRequest -} - -func (dc *DropPartitionMsg) Type() MsgType { - return dc.MsgType -} - -func (dc *DropPartitionMsg) Marshal(input TsMsg) ([]byte, error) { - dropPartitionMsg := input.(*DropPartitionMsg) - dropPartitionRequest := &dropPartitionMsg.DropPartitionRequest - mb, err := proto.Marshal(dropPartitionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (dc *DropPartitionMsg) Unmarshal(input []byte) (TsMsg, error) { - dropPartitionRequest := internalPb.DropPartitionRequest{} - err := proto.Unmarshal(input, &dropPartitionRequest) - if err != nil { - return nil, err - } - dropPartitionMsg := &DropPartitionMsg{DropPartitionRequest: dropPartitionRequest} - dropPartitionMsg.BeginTimestamp = dropPartitionMsg.Timestamp - dropPartitionMsg.EndTimestamp = dropPartitionMsg.Timestamp - - return dropPartitionMsg, nil -} - -/////////////////////////////////////////HasPartition////////////////////////////////////////// -type HasPartitionMsg struct { - BaseMsg - internalPb.HasPartitionRequest -} - -func (hc *HasPartitionMsg) Type() MsgType { - return hc.MsgType -} - -func (hc *HasPartitionMsg) Marshal(input TsMsg) ([]byte, error) { - hasPartitionMsg := input.(*HasPartitionMsg) - hasPartitionRequest := &hasPartitionMsg.HasPartitionRequest - mb, err := proto.Marshal(hasPartitionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (hc *HasPartitionMsg) Unmarshal(input []byte) (TsMsg, error) { - hasPartitionRequest := internalPb.HasPartitionRequest{} - err := proto.Unmarshal(input, &hasPartitionRequest) - if err != nil { - return nil, err - } - hasPartitionMsg := &HasPartitionMsg{HasPartitionRequest: hasPartitionRequest} - hasPartitionMsg.BeginTimestamp = hasPartitionMsg.Timestamp - hasPartitionMsg.EndTimestamp = hasPartitionMsg.Timestamp - - return hasPartitionMsg, nil -} - -/////////////////////////////////////////DescribePartition////////////////////////////////////////// -type DescribePartitionMsg struct { - BaseMsg - internalPb.DescribePartitionRequest -} - -func (dc *DescribePartitionMsg) Type() MsgType { - return dc.MsgType -} - -func (dc *DescribePartitionMsg) Marshal(input TsMsg) ([]byte, error) { - describePartitionMsg := input.(*DescribePartitionMsg) - describePartitionRequest := &describePartitionMsg.DescribePartitionRequest - mb, err := proto.Marshal(describePartitionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (dc *DescribePartitionMsg) Unmarshal(input []byte) (TsMsg, error) { - describePartitionRequest := internalPb.DescribePartitionRequest{} - err := proto.Unmarshal(input, &describePartitionRequest) - if err != nil { - return nil, err - } - describePartitionMsg := &DescribePartitionMsg{DescribePartitionRequest: describePartitionRequest} - describePartitionMsg.BeginTimestamp = describePartitionMsg.Timestamp - describePartitionMsg.EndTimestamp = describePartitionMsg.Timestamp - - return describePartitionMsg, nil -} - -/////////////////////////////////////////ShowPartition////////////////////////////////////////// -type ShowPartitionMsg struct { - BaseMsg - internalPb.ShowPartitionRequest -} - -func (sc *ShowPartitionMsg) Type() MsgType { - return sc.MsgType -} - -func (sc *ShowPartitionMsg) Marshal(input TsMsg) ([]byte, error) { - showPartitionMsg := input.(*ShowPartitionMsg) - showPartitionRequest := &showPartitionMsg.ShowPartitionRequest - mb, err := proto.Marshal(showPartitionRequest) - if err != nil { - return nil, err - } - return mb, nil -} - -func (sc *ShowPartitionMsg) Unmarshal(input []byte) (TsMsg, error) { - showPartitionRequest := internalPb.ShowPartitionRequest{} - err := proto.Unmarshal(input, &showPartitionRequest) - if err != nil { - return nil, err - } - showPartitionMsg := &ShowPartitionMsg{ShowPartitionRequest: showPartitionRequest} - showPartitionMsg.BeginTimestamp = showPartitionMsg.Timestamp - showPartitionMsg.EndTimestamp = showPartitionMsg.Timestamp - - return showPartitionMsg, nil -} diff --git a/internal/msgstream/task.go b/internal/msgstream/task.go new file mode 100644 index 0000000000..69fe04cd1d --- /dev/null +++ b/internal/msgstream/task.go @@ -0,0 +1,263 @@ +package msgstream + +import ( + "github.com/golang/protobuf/proto" + internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" +) + +type MsgType = internalPb.MsgType + +type TsMsg interface { + BeginTs() Timestamp + EndTs() Timestamp + Type() MsgType + HashKeys() []int32 + Marshal(TsMsg) ([]byte, error) + Unmarshal([]byte) (TsMsg, error) +} + +type BaseMsg struct { + BeginTimestamp Timestamp + EndTimestamp Timestamp + HashValues []int32 +} + +func (bm *BaseMsg) BeginTs() Timestamp { + return bm.BeginTimestamp +} + +func (bm *BaseMsg) EndTs() Timestamp { + return bm.EndTimestamp +} + +func (bm *BaseMsg) HashKeys() []int32 { + return bm.HashValues +} + +/////////////////////////////////////////Insert////////////////////////////////////////// +type InsertMsg struct { + BaseMsg + internalPb.InsertRequest +} + +func (it *InsertMsg) Type() MsgType { + return it.MsgType +} + +func (it *InsertMsg) Marshal(input TsMsg) ([]byte, error) { + insertMsg := input.(*InsertMsg) + insertRequest := &insertMsg.InsertRequest + mb, err := proto.Marshal(insertRequest) + if err != nil { + return nil, err + } + return mb, nil +} + +func (it *InsertMsg) Unmarshal(input []byte) (TsMsg, error) { + insertRequest := internalPb.InsertRequest{} + err := proto.Unmarshal(input, &insertRequest) + if err != nil { + return nil, err + } + insertMsg := &InsertMsg{InsertRequest: insertRequest} + for _, timestamp := range insertMsg.Timestamps { + insertMsg.BeginTimestamp = timestamp + insertMsg.EndTimestamp = timestamp + break + } + for _, timestamp := range insertMsg.Timestamps { + if timestamp > insertMsg.EndTimestamp { + insertMsg.EndTimestamp = timestamp + } + if timestamp < insertMsg.BeginTimestamp { + insertMsg.BeginTimestamp = timestamp + } + } + + return insertMsg, nil +} + +/////////////////////////////////////////Delete////////////////////////////////////////// +type DeleteMsg struct { + BaseMsg + internalPb.DeleteRequest +} + +func (dt *DeleteMsg) Type() MsgType { + return dt.MsgType +} + +func (dt *DeleteMsg) Marshal(input TsMsg) ([]byte, error) { + deleteTask := input.(*DeleteMsg) + deleteRequest := &deleteTask.DeleteRequest + mb, err := proto.Marshal(deleteRequest) + if err != nil { + return nil, err + } + return mb, nil +} + +func (dt *DeleteMsg) Unmarshal(input []byte) (TsMsg, error) { + deleteRequest := internalPb.DeleteRequest{} + err := proto.Unmarshal(input, &deleteRequest) + if err != nil { + return nil, err + } + deleteMsg := &DeleteMsg{DeleteRequest: deleteRequest} + for _, timestamp := range deleteMsg.Timestamps { + deleteMsg.BeginTimestamp = timestamp + deleteMsg.EndTimestamp = timestamp + break + } + for _, timestamp := range deleteMsg.Timestamps { + if timestamp > deleteMsg.EndTimestamp { + deleteMsg.EndTimestamp = timestamp + } + if timestamp < deleteMsg.BeginTimestamp { + deleteMsg.BeginTimestamp = timestamp + } + } + + return deleteMsg, nil +} + +/////////////////////////////////////////Search////////////////////////////////////////// +type SearchMsg struct { + BaseMsg + internalPb.SearchRequest +} + +func (st *SearchMsg) Type() MsgType { + return st.MsgType +} + +func (st *SearchMsg) Marshal(input TsMsg) ([]byte, error) { + searchTask := input.(*SearchMsg) + searchRequest := &searchTask.SearchRequest + mb, err := proto.Marshal(searchRequest) + if err != nil { + return nil, err + } + return mb, nil +} + +func (st *SearchMsg) Unmarshal(input []byte) (TsMsg, error) { + searchRequest := internalPb.SearchRequest{} + err := proto.Unmarshal(input, &searchRequest) + if err != nil { + return nil, err + } + searchMsg := &SearchMsg{SearchRequest: searchRequest} + searchMsg.BeginTimestamp = searchMsg.Timestamp + searchMsg.EndTimestamp = searchMsg.Timestamp + + return searchMsg, nil +} + +/////////////////////////////////////////SearchResult////////////////////////////////////////// +type SearchResultMsg struct { + BaseMsg + internalPb.SearchResult +} + +func (srt *SearchResultMsg) Type() MsgType { + return srt.MsgType +} + +func (srt *SearchResultMsg) Marshal(input TsMsg) ([]byte, error) { + searchResultTask := input.(*SearchResultMsg) + searchResultRequest := &searchResultTask.SearchResult + mb, err := proto.Marshal(searchResultRequest) + if err != nil { + return nil, err + } + return mb, nil +} + +func (srt *SearchResultMsg) Unmarshal(input []byte) (TsMsg, error) { + searchResultRequest := internalPb.SearchResult{} + err := proto.Unmarshal(input, &searchResultRequest) + if err != nil { + return nil, err + } + searchResultMsg := &SearchResultMsg{SearchResult: searchResultRequest} + searchResultMsg.BeginTimestamp = searchResultMsg.Timestamp + searchResultMsg.EndTimestamp = searchResultMsg.Timestamp + + return searchResultMsg, nil +} + +/////////////////////////////////////////TimeTick////////////////////////////////////////// +type TimeTickMsg struct { + BaseMsg + internalPb.TimeTickMsg +} + +func (tst *TimeTickMsg) Type() MsgType { + return tst.MsgType +} + +func (tst *TimeTickMsg) Marshal(input TsMsg) ([]byte, error) { + timeTickTask := input.(*TimeTickMsg) + timeTick := &timeTickTask.TimeTickMsg + mb, err := proto.Marshal(timeTick) + if err != nil { + return nil, err + } + return mb, nil +} + +func (tst *TimeTickMsg) Unmarshal(input []byte) (TsMsg, error) { + timeTickMsg := internalPb.TimeTickMsg{} + err := proto.Unmarshal(input, &timeTickMsg) + if err != nil { + return nil, err + } + timeTick := &TimeTickMsg{TimeTickMsg: timeTickMsg} + timeTick.BeginTimestamp = timeTick.Timestamp + timeTick.EndTimestamp = timeTick.Timestamp + + return timeTick, nil +} + +/////////////////////////////////////////QueryNodeSegStats////////////////////////////////////////// +type QueryNodeSegStatsMsg struct { + BaseMsg + internalPb.QueryNodeSegStats +} + +func (qs *QueryNodeSegStatsMsg) Type() MsgType { + return qs.MsgType +} + +func (qs *QueryNodeSegStatsMsg) Marshal(input TsMsg) ([]byte, error) { + queryNodeSegStatsTask := input.(*QueryNodeSegStatsMsg) + queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeSegStats + mb, err := proto.Marshal(queryNodeSegStats) + if err != nil { + return nil, err + } + return mb, nil +} + +func (qs *QueryNodeSegStatsMsg) Unmarshal(input []byte) (TsMsg, error) { + queryNodeSegStats := internalPb.QueryNodeSegStats{} + err := proto.Unmarshal(input, &queryNodeSegStats) + if err != nil { + return nil, err + } + queryNodeSegStatsMsg := &QueryNodeSegStatsMsg{QueryNodeSegStats: queryNodeSegStats} + + return queryNodeSegStatsMsg, nil +} + +///////////////////////////////////////////Key2Seg////////////////////////////////////////// +//type Key2SegMsg struct { +// BaseMsg +// internalPb.Key2SegMsg +//} +// +//func (k2st *Key2SegMsg) Type() MsgType { +// return +//} diff --git a/internal/msgstream/msg_test.go b/internal/msgstream/task_test.go similarity index 100% rename from internal/msgstream/msg_test.go rename to internal/msgstream/task_test.go diff --git a/internal/msgstream/unmarshal.go b/internal/msgstream/unmarshal.go index 27f5c8a1ad..5024b98b16 100644 --- a/internal/msgstream/unmarshal.go +++ b/internal/msgstream/unmarshal.go @@ -30,11 +30,6 @@ func (dispatcher *UnmarshalDispatcher) addDefaultMsgTemplates() { searchMsg := SearchMsg{} searchResultMsg := SearchResultMsg{} timeTickMsg := TimeTickMsg{} - createCollectionMsg := CreateCollectionMsg{} - dropCollectionMsg := DropCollectionMsg{} - createPartitionMsg := CreatePartitionMsg{} - dropPartitionMsg := DropPartitionMsg{} - queryNodeSegStatsMsg := QueryNodeSegStatsMsg{} dispatcher.tempMap = make(map[internalPb.MsgType]UnmarshalFunc) dispatcher.tempMap[internalPb.MsgType_kInsert] = insertMsg.Unmarshal @@ -43,11 +38,6 @@ func (dispatcher *UnmarshalDispatcher) addDefaultMsgTemplates() { dispatcher.tempMap[internalPb.MsgType_kSearchResult] = searchResultMsg.Unmarshal dispatcher.tempMap[internalPb.MsgType_kTimeTick] = timeTickMsg.Unmarshal dispatcher.tempMap[internalPb.MsgType_kQueryNodeSegStats] = queryNodeSegStatsMsg.Unmarshal - dispatcher.tempMap[internalPb.MsgType_kCreateCollection] = createCollectionMsg.Unmarshal - dispatcher.tempMap[internalPb.MsgType_kDropCollection] = dropCollectionMsg.Unmarshal - dispatcher.tempMap[internalPb.MsgType_kCreatePartition] = createPartitionMsg.Unmarshal - dispatcher.tempMap[internalPb.MsgType_kDropPartition] = dropPartitionMsg.Unmarshal - } func NewUnmarshalDispatcher() *UnmarshalDispatcher { diff --git a/internal/proto/internal_msg.proto b/internal/proto/internal_msg.proto index 67f5d666e0..889ad9b259 100644 --- a/internal/proto/internal_msg.proto +++ b/internal/proto/internal_msg.proto @@ -14,7 +14,6 @@ enum MsgType { kHasCollection = 102; kDescribeCollection = 103; kShowCollections = 104; - kGetSysConfigs = 105; /* Definition Requests: partition */ kCreatePartition = 200; @@ -34,7 +33,6 @@ enum MsgType { /* System Control */ kTimeTick = 1200; kQueryNodeSegStats = 1201; - } enum PeerRole { @@ -225,19 +223,6 @@ message SearchRequest { } -/** - * @brief Request of DescribePartition - */ -message SysConfigRequest { - MsgType msg_type = 1; - int64 reqID = 2; - int64 proxyID = 3; - uint64 timestamp = 4; - repeated string keys = 5; - repeated string key_prefixes = 6; -} - - message SearchResult { MsgType msg_type = 1; common.Status status = 2; @@ -281,4 +266,4 @@ message QueryNodeSegStats { MsgType msg_type = 1; int64 peerID = 2; repeated SegmentStats seg_stats = 3; -} +} \ No newline at end of file diff --git a/internal/proto/internalpb/internal_msg.pb.go b/internal/proto/internalpb/internal_msg.pb.go index 88ffd85d37..a74f020c25 100644 --- a/internal/proto/internalpb/internal_msg.pb.go +++ b/internal/proto/internalpb/internal_msg.pb.go @@ -32,7 +32,6 @@ const ( MsgType_kHasCollection MsgType = 102 MsgType_kDescribeCollection MsgType = 103 MsgType_kShowCollections MsgType = 104 - MsgType_kGetSysConfigs MsgType = 105 // Definition Requests: partition MsgType_kCreatePartition MsgType = 200 MsgType_kDropPartition MsgType = 201 @@ -57,7 +56,6 @@ var MsgType_name = map[int32]string{ 102: "kHasCollection", 103: "kDescribeCollection", 104: "kShowCollections", - 105: "kGetSysConfigs", 200: "kCreatePartition", 201: "kDropPartition", 202: "kHasPartition", @@ -78,7 +76,6 @@ var MsgType_value = map[string]int32{ "kHasCollection": 102, "kDescribeCollection": 103, "kShowCollections": 104, - "kGetSysConfigs": 105, "kCreatePartition": 200, "kDropPartition": 201, "kHasPartition": 202, @@ -1582,87 +1579,6 @@ func (m *SearchRequest) GetQuery() *commonpb.Blob { return nil } -//* -// @brief Request of DescribePartition -type SysConfigRequest struct { - MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"` - ReqID int64 `protobuf:"varint,2,opt,name=reqID,proto3" json:"reqID,omitempty"` - ProxyID int64 `protobuf:"varint,3,opt,name=proxyID,proto3" json:"proxyID,omitempty"` - Timestamp uint64 `protobuf:"varint,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Keys []string `protobuf:"bytes,5,rep,name=keys,proto3" json:"keys,omitempty"` - KeyPrefixes []string `protobuf:"bytes,6,rep,name=key_prefixes,json=keyPrefixes,proto3" json:"key_prefixes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SysConfigRequest) Reset() { *m = SysConfigRequest{} } -func (m *SysConfigRequest) String() string { return proto.CompactTextString(m) } -func (*SysConfigRequest) ProtoMessage() {} -func (*SysConfigRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{21} -} - -func (m *SysConfigRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SysConfigRequest.Unmarshal(m, b) -} -func (m *SysConfigRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SysConfigRequest.Marshal(b, m, deterministic) -} -func (m *SysConfigRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SysConfigRequest.Merge(m, src) -} -func (m *SysConfigRequest) XXX_Size() int { - return xxx_messageInfo_SysConfigRequest.Size(m) -} -func (m *SysConfigRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SysConfigRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SysConfigRequest proto.InternalMessageInfo - -func (m *SysConfigRequest) GetMsgType() MsgType { - if m != nil { - return m.MsgType - } - return MsgType_kNone -} - -func (m *SysConfigRequest) GetReqID() int64 { - if m != nil { - return m.ReqID - } - return 0 -} - -func (m *SysConfigRequest) GetProxyID() int64 { - if m != nil { - return m.ProxyID - } - return 0 -} - -func (m *SysConfigRequest) GetTimestamp() uint64 { - if m != nil { - return m.Timestamp - } - return 0 -} - -func (m *SysConfigRequest) GetKeys() []string { - if m != nil { - return m.Keys - } - return nil -} - -func (m *SysConfigRequest) GetKeyPrefixes() []string { - if m != nil { - return m.KeyPrefixes - } - return nil -} - type SearchResult struct { MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"` Status *commonpb.Status `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` @@ -1681,7 +1597,7 @@ func (m *SearchResult) Reset() { *m = SearchResult{} } func (m *SearchResult) String() string { return proto.CompactTextString(m) } func (*SearchResult) ProtoMessage() {} func (*SearchResult) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{22} + return fileDescriptor_7eb37f6b80b23116, []int{21} } func (m *SearchResult) XXX_Unmarshal(b []byte) error { @@ -1771,7 +1687,7 @@ func (m *TimeTickMsg) Reset() { *m = TimeTickMsg{} } func (m *TimeTickMsg) String() string { return proto.CompactTextString(m) } func (*TimeTickMsg) ProtoMessage() {} func (*TimeTickMsg) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{23} + return fileDescriptor_7eb37f6b80b23116, []int{22} } func (m *TimeTickMsg) XXX_Unmarshal(b []byte) error { @@ -1828,7 +1744,7 @@ func (m *Key2Seg) Reset() { *m = Key2Seg{} } func (m *Key2Seg) String() string { return proto.CompactTextString(m) } func (*Key2Seg) ProtoMessage() {} func (*Key2Seg) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{24} + return fileDescriptor_7eb37f6b80b23116, []int{23} } func (m *Key2Seg) XXX_Unmarshal(b []byte) error { @@ -1896,7 +1812,7 @@ func (m *Key2SegMsg) Reset() { *m = Key2SegMsg{} } func (m *Key2SegMsg) String() string { return proto.CompactTextString(m) } func (*Key2SegMsg) ProtoMessage() {} func (*Key2SegMsg) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{25} + return fileDescriptor_7eb37f6b80b23116, []int{24} } func (m *Key2SegMsg) XXX_Unmarshal(b []byte) error { @@ -1945,7 +1861,7 @@ func (m *SegmentStats) Reset() { *m = SegmentStats{} } func (m *SegmentStats) String() string { return proto.CompactTextString(m) } func (*SegmentStats) ProtoMessage() {} func (*SegmentStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{26} + return fileDescriptor_7eb37f6b80b23116, []int{25} } func (m *SegmentStats) XXX_Unmarshal(b []byte) error { @@ -2007,7 +1923,7 @@ func (m *QueryNodeSegStats) Reset() { *m = QueryNodeSegStats{} } func (m *QueryNodeSegStats) String() string { return proto.CompactTextString(m) } func (*QueryNodeSegStats) ProtoMessage() {} func (*QueryNodeSegStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{27} + return fileDescriptor_7eb37f6b80b23116, []int{26} } func (m *QueryNodeSegStats) XXX_Unmarshal(b []byte) error { @@ -2073,7 +1989,6 @@ func init() { proto.RegisterType((*InsertRequest)(nil), "milvus.proto.internal.InsertRequest") proto.RegisterType((*DeleteRequest)(nil), "milvus.proto.internal.DeleteRequest") proto.RegisterType((*SearchRequest)(nil), "milvus.proto.internal.SearchRequest") - proto.RegisterType((*SysConfigRequest)(nil), "milvus.proto.internal.SysConfigRequest") proto.RegisterType((*SearchResult)(nil), "milvus.proto.internal.SearchResult") proto.RegisterType((*TimeTickMsg)(nil), "milvus.proto.internal.TimeTickMsg") proto.RegisterType((*Key2Seg)(nil), "milvus.proto.internal.Key2Seg") @@ -2085,98 +2000,94 @@ func init() { func init() { proto.RegisterFile("internal_msg.proto", fileDescriptor_7eb37f6b80b23116) } var fileDescriptor_7eb37f6b80b23116 = []byte{ - // 1474 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x58, 0x4b, 0x6f, 0x1b, 0x47, - 0x12, 0xf6, 0x70, 0x28, 0x3e, 0x8a, 0x14, 0x35, 0x6a, 0x49, 0x36, 0x6d, 0x2f, 0x6c, 0x79, 0xbc, - 0x58, 0x6b, 0xbd, 0x58, 0x09, 0x2b, 0xef, 0x61, 0x7d, 0xdb, 0xb5, 0x08, 0xac, 0xb9, 0x86, 0x0c, - 0xed, 0x50, 0x48, 0x80, 0xc0, 0xc0, 0x60, 0x44, 0x96, 0x86, 0x83, 0x79, 0xaa, 0x7b, 0x28, 0x99, - 0x3a, 0xe4, 0xe4, 0x1f, 0x90, 0x1c, 0x72, 0xc8, 0x21, 0x40, 0x8e, 0x39, 0x19, 0xc9, 0xbf, 0xc8, - 0xeb, 0x14, 0x20, 0x7f, 0x22, 0x81, 0x63, 0x20, 0x71, 0xee, 0x41, 0x77, 0xcf, 0x83, 0xa3, 0xa7, - 0x01, 0x5b, 0x89, 0x00, 0xdd, 0xba, 0x6a, 0x7a, 0xba, 0xaa, 0xbe, 0xaf, 0xba, 0xba, 0xba, 0x81, - 0x38, 0x41, 0x8c, 0x34, 0xb0, 0x3c, 0xd3, 0x67, 0xf6, 0x72, 0x44, 0xc3, 0x38, 0x24, 0x0b, 0xbe, - 0xe3, 0xed, 0x8e, 0x98, 0x94, 0x96, 0xd3, 0x09, 0xd7, 0x9a, 0xfd, 0xd0, 0xf7, 0xc3, 0x40, 0xaa, - 0xaf, 0xcd, 0x32, 0xa4, 0xbb, 0x4e, 0x1f, 0xf3, 0xff, 0xf4, 0x00, 0xea, 0xdd, 0x8e, 0x81, 0x3b, - 0x23, 0x64, 0x31, 0xb9, 0x0c, 0x95, 0x08, 0x91, 0x76, 0x3b, 0x6d, 0x65, 0x51, 0x59, 0x52, 0x8d, - 0x44, 0x22, 0xf7, 0xa0, 0x4c, 0x43, 0x0f, 0xdb, 0xa5, 0x45, 0x65, 0xa9, 0xb5, 0x7a, 0x73, 0xf9, - 0x48, 0x5b, 0xcb, 0x1b, 0x88, 0xd4, 0x08, 0x3d, 0x34, 0xc4, 0x64, 0x32, 0x0f, 0x53, 0xfd, 0x70, - 0x14, 0xc4, 0x6d, 0x75, 0x51, 0x59, 0x9a, 0x36, 0xa4, 0xa0, 0xdb, 0x00, 0xdc, 0x1e, 0x8b, 0xc2, - 0x80, 0x21, 0xb9, 0x07, 0x15, 0x16, 0x5b, 0xf1, 0x88, 0x09, 0x83, 0x8d, 0xd5, 0xeb, 0xc5, 0xa5, - 0x13, 0xe7, 0x7b, 0x62, 0x8a, 0x91, 0x4c, 0x25, 0x2d, 0x28, 0x75, 0x3b, 0xc2, 0x17, 0xd5, 0x28, - 0x75, 0x3b, 0xc7, 0x18, 0x0a, 0x01, 0x36, 0x59, 0xf8, 0x3b, 0x46, 0xb6, 0x0b, 0x0d, 0x61, 0xf0, - 0x4d, 0x42, 0xfb, 0x13, 0xd4, 0x63, 0xc7, 0x47, 0x16, 0x5b, 0x7e, 0x24, 0x7c, 0x2a, 0x1b, 0xb9, - 0xe2, 0x18, 0xbb, 0xcf, 0x14, 0x68, 0xf6, 0xd0, 0xce, 0x59, 0xcc, 0xa6, 0x29, 0x13, 0xd3, 0xf8, - 0xd2, 0xfd, 0xa1, 0x15, 0x04, 0xe8, 0x25, 0xe0, 0x4d, 0x19, 0xb9, 0x82, 0x5c, 0x87, 0x7a, 0x3f, - 0xf4, 0x3c, 0x33, 0xb0, 0x7c, 0x14, 0xcb, 0xd7, 0x8d, 0x1a, 0x57, 0x3c, 0xb6, 0x7c, 0x24, 0xb7, - 0x61, 0x3a, 0xb2, 0x68, 0xec, 0xc4, 0x4e, 0x18, 0x98, 0xb1, 0x65, 0xb7, 0xcb, 0x62, 0x42, 0x33, - 0x53, 0x6e, 0x5a, 0xb6, 0xfe, 0x5c, 0x01, 0xf2, 0x1f, 0xc6, 0x1c, 0x3b, 0x28, 0x38, 0xf3, 0x56, - 0x81, 0x7f, 0x04, 0x33, 0x11, 0x52, 0x33, 0x71, 0xdb, 0xa4, 0xb8, 0xd3, 0x56, 0x17, 0xd5, 0xa5, - 0xc6, 0xea, 0xed, 0x63, 0xfe, 0x9f, 0x74, 0xc5, 0x98, 0x8e, 0x90, 0xae, 0xc9, 0x5f, 0x0d, 0xdc, - 0xd1, 0x3f, 0x51, 0x60, 0x46, 0x7c, 0x97, 0x5e, 0xfb, 0x18, 0x08, 0xe8, 0x18, 0x57, 0x25, 0xce, - 0x4a, 0xe1, 0x14, 0xe8, 0x8e, 0x64, 0xa5, 0x08, 0x68, 0xf9, 0x34, 0x40, 0xa7, 0x8e, 0x00, 0xf4, - 0xa5, 0x02, 0x73, 0x05, 0x40, 0xcf, 0x2e, 0xb1, 0xee, 0xc0, 0x0c, 0x3e, 0x8d, 0x1c, 0x8a, 0xe6, - 0x60, 0x44, 0x2d, 0xee, 0x80, 0x08, 0xa6, 0x6c, 0xb4, 0xa4, 0xba, 0x93, 0x68, 0xc9, 0x13, 0xb8, - 0x3c, 0x49, 0x80, 0x95, 0x21, 0xd7, 0x2e, 0x0b, 0x1e, 0xfe, 0x72, 0x12, 0x0f, 0x39, 0xce, 0xc6, - 0x7c, 0x4e, 0x45, 0xae, 0xd5, 0xbf, 0x57, 0xe0, 0xca, 0x1a, 0x45, 0x2b, 0xc6, 0xb5, 0xd0, 0xf3, - 0xb0, 0xcf, 0x4d, 0xa6, 0x79, 0x74, 0x1f, 0x6a, 0x3e, 0xb3, 0xcd, 0x78, 0x1c, 0xa1, 0x88, 0xbb, - 0xb5, 0x7a, 0xe3, 0x18, 0x5b, 0xeb, 0xcc, 0xde, 0x1c, 0x47, 0x68, 0x54, 0x7d, 0x39, 0xe0, 0x04, - 0x51, 0xdc, 0xc9, 0x4a, 0x86, 0x14, 0x8a, 0x88, 0xa8, 0x07, 0x11, 0x69, 0x43, 0x35, 0xa2, 0xe1, - 0xd3, 0x71, 0xb7, 0x23, 0xc8, 0x53, 0x8d, 0x54, 0x24, 0xff, 0x80, 0x0a, 0xeb, 0x0f, 0xd1, 0xb7, - 0x04, 0x69, 0x8d, 0xd5, 0xab, 0x47, 0xc2, 0xff, 0xc0, 0x0b, 0xb7, 0x8c, 0x64, 0x22, 0x67, 0x72, - 0xa1, 0x43, 0xc3, 0xe8, 0x1c, 0x47, 0xb5, 0x0e, 0x33, 0xfd, 0xcc, 0x3b, 0x99, 0xb4, 0x32, 0xbc, - 0x3f, 0x17, 0xfd, 0x49, 0x0e, 0x90, 0xe5, 0x3c, 0x14, 0x9e, 0xd0, 0x46, 0xab, 0x5f, 0x90, 0xf5, - 0x9f, 0x14, 0x98, 0x7f, 0x68, 0xb1, 0x8b, 0x13, 0xf0, 0x2f, 0x0a, 0x5c, 0xed, 0x20, 0xeb, 0x53, - 0x67, 0x0b, 0x2f, 0x4e, 0xd4, 0x9f, 0x2a, 0xb0, 0xd0, 0x1b, 0x86, 0x7b, 0xe7, 0x37, 0x62, 0xfd, - 0x85, 0x02, 0x97, 0x65, 0x4d, 0xd9, 0x48, 0x8b, 0xeb, 0xb9, 0x63, 0xe5, 0x7f, 0xd0, 0xca, 0x8f, - 0x83, 0x09, 0x52, 0x6e, 0x1f, 0x4d, 0x4a, 0x16, 0x88, 0xe0, 0x24, 0x3f, 0x49, 0x04, 0x25, 0x3f, - 0x2a, 0x30, 0xcf, 0x6b, 0xcd, 0xc5, 0x88, 0xf6, 0x07, 0x05, 0xe6, 0x1e, 0x5a, 0xec, 0x62, 0x04, - 0xfb, 0x52, 0x81, 0x76, 0x5a, 0x63, 0x2e, 0x46, 0xc4, 0xfc, 0x18, 0xe1, 0xf5, 0xe5, 0xfc, 0x46, - 0xfb, 0x96, 0x0b, 0xea, 0xcf, 0x25, 0x98, 0xee, 0x06, 0x0c, 0x69, 0x7c, 0x66, 0x91, 0xde, 0x39, - 0xec, 0xb1, 0xec, 0xf7, 0x0f, 0xf8, 0xf2, 0x5a, 0x5d, 0x3f, 0xc7, 0x8d, 0xa1, 0xcd, 0xbb, 0xb7, - 0x6e, 0x47, 0x44, 0xae, 0x1a, 0xb9, 0xa2, 0xd8, 0x38, 0x57, 0xe4, 0xd7, 0xbc, 0x71, 0x9e, 0x40, - 0xb5, 0x5a, 0x44, 0xf5, 0x06, 0x40, 0x06, 0x3e, 0x6b, 0xd7, 0x16, 0xd5, 0xa5, 0xb2, 0x31, 0xa1, - 0xe1, 0x97, 0x0a, 0x1a, 0xee, 0x75, 0x3b, 0xac, 0x5d, 0x5f, 0x54, 0xf9, 0xa5, 0x42, 0x4a, 0xe4, - 0x9f, 0x50, 0xa3, 0xe1, 0x9e, 0x39, 0xb0, 0x62, 0xab, 0x0d, 0xa2, 0x21, 0x3d, 0xa1, 0x3b, 0xab, - 0xd2, 0x70, 0xaf, 0x63, 0xc5, 0x96, 0xfe, 0xac, 0x04, 0xd3, 0x1d, 0xf4, 0x30, 0xc6, 0x3f, 0x1e, - 0xf4, 0x02, 0x62, 0xe5, 0x13, 0x10, 0x9b, 0x3a, 0x09, 0xb1, 0xca, 0x21, 0xc4, 0x6e, 0x41, 0x33, - 0xa2, 0x8e, 0x6f, 0xd1, 0xb1, 0xe9, 0xe2, 0x98, 0xb5, 0xab, 0x02, 0xb7, 0x46, 0xa2, 0x7b, 0x84, - 0x63, 0xa6, 0xbf, 0x52, 0x60, 0xba, 0x87, 0x16, 0xed, 0x0f, 0xcf, 0x0c, 0x86, 0x09, 0xff, 0xd5, - 0xa2, 0xff, 0x85, 0xfd, 0x57, 0x3e, 0xb8, 0xff, 0xfe, 0x0a, 0x1a, 0x45, 0x36, 0xf2, 0x62, 0x33, - 0x07, 0x47, 0x02, 0x30, 0x23, 0xf5, 0x6b, 0x19, 0x44, 0x2b, 0x30, 0xb5, 0x33, 0x42, 0x3a, 0x16, - 0xe9, 0x76, 0x22, 0xff, 0x72, 0x9e, 0xfe, 0x9d, 0x02, 0x5a, 0x6f, 0xcc, 0xd6, 0xc2, 0x60, 0xdb, - 0xb1, 0xcf, 0x5d, 0xe4, 0x04, 0xca, 0x82, 0xaf, 0xa9, 0x45, 0x75, 0xa9, 0x6e, 0x88, 0x31, 0xe7, - 0xd2, 0xc5, 0xb1, 0x19, 0x51, 0xdc, 0x76, 0x9e, 0xa2, 0x64, 0xbb, 0x6e, 0x34, 0x5c, 0x1c, 0x6f, - 0x24, 0x2a, 0xfd, 0x79, 0x09, 0x9a, 0x29, 0x97, 0x1c, 0x9f, 0x37, 0x09, 0x28, 0xbf, 0x6f, 0x96, - 0x5e, 0xff, 0xbe, 0x99, 0xa1, 0xa0, 0x1e, 0x83, 0xc2, 0x81, 0x3a, 0x7a, 0x0b, 0x9a, 0x82, 0x0e, - 0x33, 0x08, 0x07, 0x98, 0xb1, 0xdb, 0x10, 0xba, 0xc7, 0x42, 0x55, 0x04, 0xaa, 0xf2, 0x3a, 0x29, - 0x52, 0x3d, 0x3a, 0x45, 0x08, 0x94, 0x87, 0x4e, 0x2c, 0xeb, 0x4a, 0xd3, 0x10, 0x63, 0xfd, 0x7d, - 0x68, 0x6c, 0x3a, 0x3e, 0x6e, 0x3a, 0x7d, 0x77, 0x9d, 0xd9, 0x6f, 0x02, 0x57, 0xfe, 0xe0, 0x51, - 0x2a, 0x3c, 0x78, 0x9c, 0x78, 0xc2, 0xe8, 0x1f, 0x2b, 0x50, 0x7d, 0x84, 0xe3, 0xd5, 0x1e, 0xda, - 0x02, 0x3b, 0x5e, 0xcf, 0xd2, 0x47, 0x08, 0x21, 0x90, 0x9b, 0xd0, 0x98, 0xd8, 0xc1, 0xc9, 0xe2, - 0x90, 0x6f, 0xe0, 0x53, 0x8e, 0xb0, 0xab, 0x50, 0x73, 0x98, 0xb9, 0x6b, 0x79, 0xce, 0x40, 0x60, - 0x5f, 0x33, 0xaa, 0x0e, 0x7b, 0x87, 0x8b, 0xbc, 0x76, 0x64, 0x25, 0x5b, 0x66, 0x9a, 0x6a, 0x4c, - 0x68, 0xf4, 0x27, 0x00, 0x89, 0x6b, 0x1c, 0x9a, 0x8c, 0x59, 0x65, 0x92, 0xd9, 0x7f, 0x41, 0xd5, - 0xc5, 0xf1, 0x2a, 0x43, 0xbb, 0x5d, 0x12, 0x85, 0xf7, 0x38, 0xbc, 0x92, 0x95, 0x8c, 0x74, 0xba, - 0xfe, 0x91, 0x7c, 0xbe, 0xe2, 0xc6, 0x78, 0x0e, 0xb1, 0xe2, 0x91, 0xa2, 0x1c, 0x3c, 0x52, 0x6e, - 0x42, 0xc3, 0x47, 0x3f, 0xa4, 0x63, 0x93, 0x39, 0xfb, 0x98, 0xc2, 0x20, 0x55, 0x3d, 0x67, 0x1f, - 0x79, 0xa0, 0xc1, 0xc8, 0x37, 0x69, 0xb8, 0xc7, 0xd2, 0xad, 0x16, 0x8c, 0x7c, 0x23, 0xdc, 0x63, - 0xe4, 0x6f, 0x30, 0x4b, 0xb1, 0x8f, 0x41, 0xec, 0x8d, 0x4d, 0x3f, 0x1c, 0x38, 0xdb, 0x0e, 0xa6, - 0x60, 0x68, 0xe9, 0x87, 0xf5, 0x44, 0xaf, 0x7f, 0xa6, 0xc0, 0xec, 0xff, 0xd3, 0xf4, 0xeb, 0xa1, - 0x2d, 0x9d, 0x3b, 0x83, 0xc4, 0xf8, 0xb7, 0x88, 0xd7, 0xe4, 0x1b, 0x87, 0x9d, 0xfe, 0x9c, 0x95, - 0xe1, 0x64, 0xd4, 0x58, 0xe2, 0xd4, 0xdd, 0x17, 0x25, 0xa8, 0x26, 0xe6, 0x48, 0x1d, 0xa6, 0xdc, - 0xc7, 0x61, 0x80, 0xda, 0x25, 0xb2, 0x00, 0xb3, 0xee, 0xc1, 0xe7, 0x14, 0x6d, 0x40, 0xe6, 0x60, - 0xc6, 0x2d, 0xbe, 0x46, 0x68, 0x48, 0x08, 0xb4, 0xdc, 0xc2, 0x85, 0x5d, 0xdb, 0x26, 0x57, 0x60, - 0xce, 0x3d, 0x7c, 0xa7, 0xd5, 0x78, 0x0a, 0x68, 0x6e, 0xf1, 0xda, 0xc7, 0xb4, 0xa1, 0x58, 0xe2, - 0xbf, 0x18, 0x67, 0xb5, 0x94, 0x69, 0x0e, 0x59, 0x00, 0xcd, 0x3d, 0x70, 0xfb, 0xd2, 0xbe, 0x54, - 0xc8, 0x1c, 0xb4, 0xdc, 0xc2, 0x25, 0x45, 0xfb, 0x4a, 0x21, 0x04, 0xa6, 0xdd, 0xc9, 0x5e, 0x5e, - 0xfb, 0x5a, 0x21, 0x57, 0x80, 0xb8, 0x87, 0x5a, 0x5e, 0xed, 0x1b, 0x85, 0xcc, 0xc3, 0x8c, 0x5b, - 0xe8, 0x0c, 0x99, 0xf6, 0xad, 0x42, 0x9a, 0x50, 0x75, 0x65, 0xfb, 0xa4, 0x7d, 0xa0, 0x0a, 0x49, - 0x9e, 0xeb, 0xda, 0x87, 0x52, 0x92, 0x25, 0x51, 0x7b, 0xa5, 0x0a, 0x63, 0x93, 0x05, 0x52, 0xfb, - 0x55, 0x25, 0x2d, 0xa8, 0xbb, 0x69, 0x11, 0xd0, 0x3e, 0xaf, 0x0b, 0xe3, 0x87, 0x32, 0x40, 0xfb, - 0xa2, 0x7e, 0xf7, 0x3e, 0xd4, 0xd2, 0x87, 0x49, 0x02, 0x50, 0x59, 0xb7, 0x58, 0x8c, 0x54, 0xbb, - 0xc4, 0xc7, 0x06, 0x5a, 0x03, 0xa4, 0x9a, 0xc2, 0xc7, 0xef, 0x52, 0x87, 0xeb, 0x4b, 0x9c, 0x93, - 0x0d, 0x5e, 0xe7, 0x34, 0xf5, 0x41, 0xe7, 0xbd, 0x07, 0xb6, 0x13, 0x0f, 0x47, 0x5b, 0xbc, 0x6e, - 0xae, 0xec, 0x3b, 0x9e, 0xe7, 0xec, 0xc7, 0xd8, 0x1f, 0xae, 0x48, 0xc2, 0xff, 0x3e, 0x70, 0x58, - 0x4c, 0x9d, 0xad, 0x51, 0x8c, 0x83, 0x95, 0x94, 0xf6, 0x15, 0x91, 0x05, 0x99, 0x18, 0x6d, 0x6d, - 0x55, 0x84, 0xe6, 0xde, 0x6f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x9e, 0x9f, 0x3d, 0x09, 0x18, - 0x00, 0x00, + // 1416 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x58, 0x4b, 0x6f, 0x1c, 0xc5, + 0x13, 0x4f, 0xef, 0xac, 0xf7, 0x51, 0x6b, 0xaf, 0xc7, 0x6d, 0x3b, 0xd9, 0x24, 0x7f, 0x25, 0xce, + 0xe4, 0x2f, 0x62, 0x82, 0xb0, 0x85, 0xc3, 0x81, 0xdc, 0x20, 0xde, 0x43, 0x96, 0xc8, 0x51, 0x18, + 0x5b, 0x20, 0xa1, 0x48, 0xa3, 0xf1, 0x6e, 0x65, 0x77, 0x34, 0x4f, 0x77, 0xcf, 0xda, 0x59, 0x1f, + 0x38, 0xe5, 0x03, 0xc0, 0x81, 0x03, 0x07, 0x24, 0x8e, 0x9c, 0x22, 0xf8, 0x16, 0xbc, 0xae, 0x1c, + 0xf8, 0x0a, 0x20, 0x88, 0x04, 0xe1, 0x8e, 0xba, 0x7b, 0x1e, 0x3b, 0x7e, 0x46, 0x4a, 0x0c, 0x96, + 0x7c, 0x9b, 0xaa, 0xe9, 0xe9, 0xaa, 0xfa, 0xfd, 0xaa, 0x6b, 0xaa, 0x1a, 0xa8, 0x13, 0xc4, 0xc8, + 0x02, 0xdb, 0xb3, 0x7c, 0xde, 0x5f, 0x8a, 0x58, 0x18, 0x87, 0x74, 0xde, 0x77, 0xbc, 0xed, 0x21, + 0x57, 0xd2, 0x52, 0xba, 0xe0, 0xd2, 0x64, 0x37, 0xf4, 0xfd, 0x30, 0x50, 0xea, 0x4b, 0x33, 0x1c, + 0xd9, 0xb6, 0xd3, 0xc5, 0xfc, 0x3b, 0x23, 0x80, 0x7a, 0xa7, 0x6d, 0xe2, 0xd6, 0x10, 0x79, 0x4c, + 0xcf, 0x43, 0x25, 0x42, 0x64, 0x9d, 0x76, 0x8b, 0x2c, 0x90, 0x45, 0xcd, 0x4c, 0x24, 0x7a, 0x0b, + 0xca, 0x2c, 0xf4, 0xb0, 0x55, 0x5a, 0x20, 0x8b, 0xcd, 0x95, 0xab, 0x4b, 0x07, 0xda, 0x5a, 0x7a, + 0x80, 0xc8, 0xcc, 0xd0, 0x43, 0x53, 0x2e, 0xa6, 0x73, 0x30, 0xd1, 0x0d, 0x87, 0x41, 0xdc, 0xd2, + 0x16, 0xc8, 0xe2, 0x94, 0xa9, 0x04, 0xa3, 0x0f, 0x20, 0xec, 0xf1, 0x28, 0x0c, 0x38, 0xd2, 0x5b, + 0x50, 0xe1, 0xb1, 0x1d, 0x0f, 0xb9, 0x34, 0xd8, 0x58, 0xb9, 0x5c, 0xdc, 0x3a, 0x71, 0x7e, 0x5d, + 0x2e, 0x31, 0x93, 0xa5, 0xb4, 0x09, 0xa5, 0x4e, 0x5b, 0xfa, 0xa2, 0x99, 0xa5, 0x4e, 0xfb, 0x10, + 0x43, 0x21, 0xc0, 0x06, 0x0f, 0xff, 0xc5, 0xc8, 0xb6, 0xa1, 0x21, 0x0d, 0xbe, 0x4c, 0x68, 0xff, + 0x83, 0x7a, 0xec, 0xf8, 0xc8, 0x63, 0xdb, 0x8f, 0xa4, 0x4f, 0x65, 0x33, 0x57, 0x1c, 0x62, 0xf7, + 0x09, 0x81, 0xc9, 0x75, 0xec, 0xe7, 0x2c, 0x66, 0xcb, 0xc8, 0xd8, 0x32, 0xb1, 0x75, 0x77, 0x60, + 0x07, 0x01, 0x7a, 0x09, 0x78, 0x13, 0x66, 0xae, 0xa0, 0x97, 0xa1, 0xde, 0x0d, 0x3d, 0xcf, 0x0a, + 0x6c, 0x1f, 0xe5, 0xf6, 0x75, 0xb3, 0x26, 0x14, 0xf7, 0x6d, 0x1f, 0xe9, 0x75, 0x98, 0x8a, 0x6c, + 0x16, 0x3b, 0xb1, 0x13, 0x06, 0x56, 0x6c, 0xf7, 0x5b, 0x65, 0xb9, 0x60, 0x32, 0x53, 0x6e, 0xd8, + 0x7d, 0xe3, 0x29, 0x01, 0xfa, 0x1e, 0xe7, 0x4e, 0x3f, 0x28, 0x38, 0xf3, 0x4a, 0x81, 0xbf, 0x07, + 0xd3, 0x11, 0x32, 0x2b, 0x71, 0xdb, 0x62, 0xb8, 0xd5, 0xd2, 0x16, 0xb4, 0xc5, 0xc6, 0xca, 0xf5, + 0x43, 0xbe, 0x1f, 0x77, 0xc5, 0x9c, 0x8a, 0x90, 0xad, 0xaa, 0x4f, 0x4d, 0xdc, 0x32, 0xbe, 0x24, + 0x30, 0x2d, 0xdf, 0x2b, 0xaf, 0x7d, 0x0c, 0x24, 0x74, 0x5c, 0xa8, 0x12, 0x67, 0x95, 0x70, 0x0c, + 0x74, 0x07, 0xb2, 0x52, 0x04, 0xb4, 0x7c, 0x1c, 0xa0, 0x13, 0x07, 0x00, 0xfa, 0x8c, 0xc0, 0x6c, + 0x01, 0xd0, 0x93, 0x4b, 0xac, 0x1b, 0x30, 0x8d, 0x8f, 0x23, 0x87, 0xa1, 0xd5, 0x1b, 0x32, 0x5b, + 0x38, 0x20, 0x83, 0x29, 0x9b, 0x4d, 0xa5, 0x6e, 0x27, 0x5a, 0xfa, 0x10, 0xce, 0x8f, 0x13, 0x60, + 0x67, 0xc8, 0xb5, 0xca, 0x92, 0x87, 0xd7, 0x8e, 0xe2, 0x21, 0xc7, 0xd9, 0x9c, 0xcb, 0xa9, 0xc8, + 0xb5, 0xc6, 0xcf, 0x04, 0x2e, 0xac, 0x32, 0xb4, 0x63, 0x5c, 0x0d, 0x3d, 0x0f, 0xbb, 0xc2, 0x64, + 0x9a, 0x47, 0xb7, 0xa1, 0xe6, 0xf3, 0xbe, 0x15, 0x8f, 0x22, 0x94, 0x71, 0x37, 0x57, 0xae, 0x1c, + 0x62, 0x6b, 0x8d, 0xf7, 0x37, 0x46, 0x11, 0x9a, 0x55, 0x5f, 0x3d, 0x08, 0x82, 0x18, 0x6e, 0x65, + 0x25, 0x43, 0x09, 0x45, 0x44, 0xb4, 0xbd, 0x88, 0xb4, 0xa0, 0x1a, 0xb1, 0xf0, 0xf1, 0xa8, 0xd3, + 0x96, 0xe4, 0x69, 0x66, 0x2a, 0xd2, 0xb7, 0xa0, 0xc2, 0xbb, 0x03, 0xf4, 0x6d, 0x49, 0x5a, 0x63, + 0xe5, 0xe2, 0x81, 0xf0, 0xdf, 0xf1, 0xc2, 0x4d, 0x33, 0x59, 0x28, 0x98, 0x9c, 0x6f, 0xb3, 0x30, + 0x3a, 0xc5, 0x51, 0xad, 0xc1, 0x74, 0x37, 0xf3, 0x4e, 0x25, 0xad, 0x0a, 0xef, 0xff, 0x45, 0x7f, + 0x92, 0x1f, 0xc8, 0x52, 0x1e, 0x8a, 0x48, 0x68, 0xb3, 0xd9, 0x2d, 0xc8, 0xc6, 0x1f, 0x04, 0xe6, + 0xee, 0xda, 0xfc, 0xec, 0x04, 0xfc, 0x17, 0x81, 0x8b, 0x6d, 0xe4, 0x5d, 0xe6, 0x6c, 0xe2, 0xd9, + 0x89, 0xfa, 0x2b, 0x02, 0xf3, 0xeb, 0x83, 0x70, 0xe7, 0xf4, 0x46, 0x6c, 0xfc, 0x4e, 0xe0, 0xbc, + 0xaa, 0x29, 0x0f, 0xd2, 0xe2, 0x7a, 0xea, 0x58, 0x79, 0x1f, 0x9a, 0xf9, 0xef, 0x60, 0x8c, 0x94, + 0xeb, 0x07, 0x93, 0x92, 0x05, 0x22, 0x39, 0xc9, 0xff, 0x24, 0x92, 0x92, 0xdf, 0x08, 0xcc, 0x89, + 0x5a, 0x73, 0x36, 0xa2, 0xfd, 0x95, 0xc0, 0xec, 0x5d, 0x9b, 0x9f, 0x8d, 0x60, 0x9f, 0x11, 0x68, + 0xa5, 0x35, 0xe6, 0x6c, 0x44, 0x2c, 0x7e, 0x23, 0xa2, 0xbe, 0x9c, 0xde, 0x68, 0x5f, 0x71, 0x41, + 0xfd, 0xb3, 0x04, 0x53, 0x9d, 0x80, 0x23, 0x8b, 0x4f, 0x2c, 0xd2, 0x1b, 0xfb, 0x3d, 0x56, 0xfd, + 0xfe, 0x1e, 0x5f, 0x5e, 0xa8, 0xeb, 0x17, 0xb8, 0x71, 0xec, 0x8b, 0xee, 0xad, 0xd3, 0x96, 0x91, + 0x6b, 0x66, 0xae, 0x28, 0x36, 0xce, 0x15, 0xf5, 0x36, 0x6f, 0x9c, 0xc7, 0x50, 0xad, 0x16, 0x51, + 0xbd, 0x02, 0x90, 0x81, 0xcf, 0x5b, 0xb5, 0x05, 0x6d, 0xb1, 0x6c, 0x8e, 0x69, 0xc4, 0x50, 0xc1, + 0xc2, 0x9d, 0x4e, 0x9b, 0xb7, 0xea, 0x0b, 0x9a, 0x18, 0x2a, 0x94, 0x44, 0xdf, 0x86, 0x1a, 0x0b, + 0x77, 0xac, 0x9e, 0x1d, 0xdb, 0x2d, 0x90, 0x0d, 0xe9, 0x11, 0xdd, 0x59, 0x95, 0x85, 0x3b, 0x6d, + 0x3b, 0xb6, 0x8d, 0x27, 0x25, 0x98, 0x6a, 0xa3, 0x87, 0x31, 0xfe, 0xf7, 0xa0, 0x17, 0x10, 0x2b, + 0x1f, 0x81, 0xd8, 0xc4, 0x51, 0x88, 0x55, 0xf6, 0x21, 0x76, 0x0d, 0x26, 0x23, 0xe6, 0xf8, 0x36, + 0x1b, 0x59, 0x2e, 0x8e, 0x78, 0xab, 0x2a, 0x71, 0x6b, 0x24, 0xba, 0x7b, 0x38, 0xe2, 0xc6, 0x73, + 0x02, 0x53, 0xeb, 0x68, 0xb3, 0xee, 0xe0, 0xc4, 0x60, 0x18, 0xf3, 0x5f, 0x2b, 0xfa, 0x5f, 0x38, + 0x7f, 0xe5, 0xbd, 0xe7, 0xef, 0x75, 0xd0, 0x19, 0xf2, 0xa1, 0x17, 0x5b, 0x39, 0x38, 0x0a, 0x80, + 0x69, 0xa5, 0x5f, 0xcd, 0x20, 0x5a, 0x86, 0x89, 0xad, 0x21, 0xb2, 0x91, 0x4c, 0xb7, 0x23, 0xf9, + 0x57, 0xeb, 0x8c, 0xa7, 0x25, 0x31, 0x3e, 0xab, 0xb0, 0xc5, 0x56, 0x2f, 0x13, 0x75, 0x3e, 0x9a, + 0x95, 0x5e, 0x7c, 0x34, 0xcb, 0xa0, 0xd2, 0x0e, 0x81, 0x6a, 0x4f, 0xc9, 0xb9, 0x06, 0x93, 0xd2, + 0x73, 0x2b, 0x08, 0x7b, 0x98, 0x01, 0xd1, 0x90, 0xba, 0xfb, 0x52, 0x55, 0x44, 0xb3, 0xf2, 0x22, + 0x68, 0x56, 0x0f, 0x46, 0x93, 0x42, 0x79, 0xe0, 0xc4, 0xea, 0x08, 0x4e, 0x9a, 0xf2, 0xd9, 0xf8, + 0x04, 0x1a, 0x1b, 0x8e, 0x8f, 0x1b, 0x4e, 0xd7, 0x5d, 0xe3, 0xfd, 0x97, 0x81, 0x2b, 0xbf, 0x1b, + 0x28, 0x15, 0xee, 0x06, 0x8e, 0x2c, 0xc6, 0xc6, 0x17, 0x04, 0xaa, 0xf7, 0x70, 0xb4, 0xb2, 0x8e, + 0x7d, 0x89, 0x9d, 0x38, 0xfa, 0xe9, 0xbc, 0x2e, 0x05, 0x7a, 0x15, 0x1a, 0x63, 0xc9, 0x9e, 0x6c, + 0x0e, 0x79, 0xae, 0x1f, 0x53, 0xed, 0x2f, 0x42, 0xcd, 0xe1, 0xd6, 0xb6, 0xed, 0x39, 0x3d, 0x89, + 0x7d, 0xcd, 0xac, 0x3a, 0xfc, 0x43, 0x21, 0x8a, 0x63, 0x96, 0x55, 0x37, 0xde, 0x9a, 0x90, 0x87, + 0x68, 0x4c, 0x63, 0x3c, 0x04, 0x48, 0x5c, 0x13, 0xd0, 0x64, 0xcc, 0x92, 0x71, 0x66, 0xdf, 0x81, + 0xaa, 0x8b, 0xa3, 0x15, 0x8e, 0xfd, 0x56, 0x49, 0xd6, 0xa8, 0xc3, 0xf0, 0x4a, 0x76, 0x32, 0xd3, + 0xe5, 0xc6, 0xe7, 0xea, 0xa6, 0x47, 0x18, 0x13, 0x39, 0xc4, 0x8b, 0xd5, 0x97, 0xec, 0xad, 0xbe, + 0x57, 0xa1, 0xe1, 0xa3, 0x1f, 0xb2, 0x91, 0xc5, 0x9d, 0x5d, 0x4c, 0x61, 0x50, 0xaa, 0x75, 0x67, + 0x17, 0x45, 0xa0, 0xc1, 0xd0, 0xb7, 0x58, 0xb8, 0xc3, 0xd3, 0xf3, 0x18, 0x0c, 0x7d, 0x33, 0xdc, + 0xe1, 0xf4, 0x0d, 0x98, 0x61, 0xd8, 0xc5, 0x20, 0xf6, 0x46, 0x96, 0x1f, 0xf6, 0x9c, 0x47, 0x0e, + 0xa6, 0x60, 0xe8, 0xe9, 0x8b, 0xb5, 0x44, 0x6f, 0x7c, 0x4d, 0x60, 0xe6, 0x83, 0x34, 0xfd, 0xd6, + 0xb1, 0xaf, 0x9c, 0x3b, 0x81, 0xc4, 0x78, 0x57, 0xc6, 0x6b, 0x89, 0x83, 0xc3, 0x8f, 0xbf, 0xf9, + 0xc9, 0x70, 0x32, 0x6b, 0x3c, 0x71, 0xea, 0xe6, 0x2f, 0x25, 0xa8, 0x26, 0xe6, 0x68, 0x1d, 0x26, + 0xdc, 0xfb, 0x61, 0x80, 0xfa, 0x39, 0x3a, 0x0f, 0x33, 0xee, 0xde, 0x9b, 0x07, 0xbd, 0x47, 0x67, + 0x61, 0xda, 0x2d, 0x0e, 0xee, 0x3a, 0x52, 0x0a, 0x4d, 0xb7, 0x30, 0xdb, 0xea, 0x8f, 0xe8, 0x05, + 0x98, 0x75, 0xf7, 0x8f, 0x7f, 0xba, 0x48, 0x01, 0xdd, 0x2d, 0x4e, 0x48, 0x5c, 0x1f, 0xd0, 0x79, + 0xd0, 0xdd, 0x3d, 0x43, 0x89, 0xfe, 0x1d, 0xa1, 0xb3, 0xd0, 0x74, 0x0b, 0xbd, 0xbb, 0xfe, 0x3d, + 0xa1, 0x14, 0xa6, 0xdc, 0xf1, 0x16, 0x57, 0xff, 0x81, 0xd0, 0x0b, 0x40, 0xdd, 0x7d, 0x9d, 0xa0, + 0xfe, 0x23, 0xa1, 0x73, 0x30, 0xed, 0x16, 0x1a, 0x26, 0xae, 0xff, 0x44, 0xe8, 0x24, 0x54, 0x5d, + 0xd5, 0x55, 0xe8, 0x9f, 0x6a, 0x52, 0x52, 0xbf, 0x3b, 0xfd, 0x33, 0x25, 0xa9, 0xf2, 0xa7, 0x3f, + 0xd7, 0xa4, 0xb1, 0xf1, 0x62, 0xa8, 0xff, 0xad, 0xd1, 0x26, 0xd4, 0xdd, 0xf4, 0xc0, 0xeb, 0xdf, + 0xd4, 0xa5, 0xf1, 0x7d, 0x6c, 0xeb, 0xdf, 0xd6, 0x6f, 0xde, 0x86, 0x5a, 0x7a, 0x5f, 0x47, 0x01, + 0x2a, 0x6b, 0x36, 0x8f, 0x91, 0xe9, 0xe7, 0xc4, 0xb3, 0x89, 0x76, 0x0f, 0x99, 0x4e, 0xc4, 0xf3, + 0x47, 0xcc, 0x11, 0xfa, 0x92, 0xc0, 0xff, 0x81, 0xa8, 0x69, 0xba, 0x76, 0xa7, 0xfd, 0xf1, 0x9d, + 0xbe, 0x13, 0x0f, 0x86, 0x9b, 0xa2, 0x46, 0x2e, 0xef, 0x3a, 0x9e, 0xe7, 0xec, 0xc6, 0xd8, 0x1d, + 0x2c, 0x2b, 0x72, 0xdf, 0xec, 0x39, 0x3c, 0x66, 0xce, 0xe6, 0x30, 0xc6, 0xde, 0x72, 0x4a, 0xf1, + 0xb2, 0x64, 0x3c, 0x13, 0xa3, 0xcd, 0xcd, 0x8a, 0xd4, 0xdc, 0xfa, 0x27, 0x00, 0x00, 0xff, 0xff, + 0x56, 0x57, 0x32, 0x28, 0x20, 0x17, 0x00, 0x00, } diff --git a/internal/proto/master.proto b/internal/proto/master.proto index 453a0ae28b..8a4d301aad 100644 --- a/internal/proto/master.proto +++ b/internal/proto/master.proto @@ -89,15 +89,6 @@ service Master { rpc ShowPartitions(internal.ShowPartitionRequest) returns (service.StringListResponse) {} - /** - * @brief This method is used to get system configs - * - * @param SysConfigRequest, keys or key_prefixes of the configs. - * - * @return SysConfigResponse - */ - rpc GetSysConfigs(internal.SysConfigRequest) returns (service.SysConfigResponse) {} - rpc AllocTimestamp(internal.TsoRequest) returns (internal.TsoResponse) {} rpc AllocID(internal.IDRequest) returns (internal.IDResponse) {} diff --git a/internal/proto/masterpb/master.pb.go b/internal/proto/masterpb/master.pb.go index 48d4b6b455..b4a08b948d 100644 --- a/internal/proto/masterpb/master.pb.go +++ b/internal/proto/masterpb/master.pb.go @@ -30,38 +30,36 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("master.proto", fileDescriptor_f9c348dec43a6705) } var fileDescriptor_f9c348dec43a6705 = []byte{ - // 484 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x95, 0xdf, 0x6e, 0xd3, 0x30, - 0x14, 0x87, 0x7b, 0x35, 0x24, 0xd3, 0x3f, 0xcc, 0xdc, 0x95, 0x1b, 0xd6, 0x9b, 0x41, 0xcb, 0x12, - 0x04, 0x2f, 0xc0, 0xda, 0x48, 0xac, 0x12, 0x48, 0xd3, 0xb2, 0x1b, 0x40, 0x68, 0x24, 0xd9, 0x21, - 0x35, 0x24, 0x76, 0xf0, 0x39, 0x19, 0xa2, 0x2f, 0xc1, 0x2b, 0xa3, 0x26, 0x75, 0x52, 0xd3, 0xba, - 0x94, 0xdd, 0xd5, 0xf6, 0xe7, 0xdf, 0x57, 0x9f, 0x73, 0xa4, 0xb0, 0x6e, 0x1e, 0x21, 0x81, 0xf6, - 0x0a, 0xad, 0x48, 0xf1, 0xc7, 0xb9, 0xc8, 0xee, 0x4a, 0xac, 0x57, 0x5e, 0x7d, 0x34, 0xec, 0x26, - 0x2a, 0xcf, 0x95, 0xac, 0x37, 0x87, 0x5c, 0x48, 0x02, 0x2d, 0xa3, 0xec, 0x26, 0xc7, 0x74, 0xbd, - 0x77, 0x8c, 0xa0, 0xef, 0x44, 0x02, 0xed, 0xd6, 0xab, 0xdf, 0x0f, 0xd9, 0xd1, 0xfb, 0xea, 0x3e, - 0x8f, 0xd8, 0xa3, 0x99, 0x86, 0x88, 0x60, 0xa6, 0xb2, 0x0c, 0x12, 0x12, 0x4a, 0x72, 0xcf, 0xb3, - 0x4c, 0x26, 0xd3, 0xfb, 0x1b, 0xbc, 0x82, 0x1f, 0x25, 0x20, 0x0d, 0x9f, 0xd8, 0xfc, 0xfa, 0x1f, - 0x85, 0x14, 0x51, 0x89, 0xa3, 0x0e, 0xff, 0xcc, 0xfa, 0x81, 0x56, 0xc5, 0x86, 0xe0, 0x85, 0x43, - 0x60, 0x63, 0x07, 0xc6, 0xc7, 0xac, 0x77, 0x11, 0xe1, 0x46, 0xfa, 0xc4, 0x91, 0x6e, 0x51, 0x26, - 0x7c, 0x64, 0xc3, 0xeb, 0x5a, 0x79, 0x53, 0xa5, 0xb2, 0x2b, 0xc0, 0x42, 0x49, 0x84, 0x51, 0x87, - 0x97, 0x8c, 0x07, 0x80, 0x89, 0x16, 0xf1, 0x66, 0x9d, 0x5e, 0xba, 0x9e, 0xb1, 0x85, 0x1a, 0xdb, - 0x64, 0xb7, 0xad, 0x05, 0xeb, 0xab, 0xc5, 0xea, 0xe7, 0xa8, 0xc3, 0xbf, 0xb3, 0x41, 0xb8, 0x50, - 0x3f, 0xdb, 0x63, 0x74, 0x96, 0xce, 0xe6, 0x8c, 0xef, 0xd9, 0x6e, 0x5f, 0x48, 0x5a, 0xc8, 0xf4, - 0x9d, 0x40, 0xda, 0x78, 0xe3, 0x0d, 0x1b, 0xd4, 0x0d, 0xbe, 0x8c, 0x34, 0x89, 0xea, 0x81, 0x67, - 0x7b, 0x07, 0xa1, 0xe1, 0x0e, 0x6c, 0xd4, 0x27, 0xd6, 0x5b, 0x35, 0xb8, 0x8d, 0x9f, 0xec, 0x19, - 0x83, 0xff, 0x0d, 0xff, 0xc2, 0xba, 0x17, 0x11, 0xb6, 0xd9, 0x63, 0xf7, 0x10, 0x6c, 0x45, 0x1f, - 0x36, 0x03, 0x9a, 0x1d, 0x9b, 0xc6, 0xb6, 0x1a, 0xff, 0x1f, 0x23, 0xb0, 0xe5, 0x1a, 0xef, 0x76, - 0x35, 0x9c, 0x3d, 0x00, 0x82, 0xf5, 0x57, 0x8d, 0x6d, 0x4e, 0xd1, 0x59, 0x33, 0x0b, 0xbb, 0x4f, - 0xfb, 0x13, 0xd6, 0x7b, 0x0b, 0x14, 0xfe, 0xc2, 0x99, 0x92, 0x5f, 0x45, 0x8a, 0xfc, 0xd4, 0x65, - 0x32, 0x88, 0xb1, 0x9c, 0x3a, 0x2c, 0x2d, 0xd7, 0x48, 0x3e, 0xb0, 0xfe, 0x79, 0x96, 0xa9, 0xe4, - 0x5a, 0xe4, 0x80, 0x14, 0xe5, 0x05, 0x3f, 0x71, 0x58, 0xae, 0x51, 0x39, 0xda, 0x63, 0x23, 0x4d, - 0xf4, 0x25, 0x7b, 0x50, 0x45, 0xcf, 0x03, 0xfe, 0xd4, 0x71, 0x61, 0x1e, 0x98, 0xc8, 0x93, 0x3d, - 0x44, 0x93, 0xf8, 0x8d, 0x0d, 0xce, 0x11, 0x45, 0x2a, 0x43, 0x48, 0x73, 0x90, 0x34, 0x0f, 0xf8, - 0x73, 0xc7, 0xbd, 0x86, 0x6b, 0x15, 0xe3, 0x43, 0x50, 0xe3, 0x9a, 0x4e, 0x3f, 0xbe, 0x49, 0x05, - 0x2d, 0xca, 0x78, 0x35, 0xd8, 0xfe, 0x52, 0x64, 0x99, 0x58, 0x12, 0x24, 0x0b, 0xbf, 0x0e, 0x39, - 0xbb, 0x15, 0x48, 0x5a, 0xc4, 0x25, 0xc1, 0xad, 0x6f, 0xa2, 0xfc, 0x2a, 0xd9, 0xaf, 0x3f, 0x04, - 0x45, 0x1c, 0x1f, 0x55, 0xeb, 0xd7, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x8d, 0x75, 0x7d, 0xec, - 0x36, 0x06, 0x00, 0x00, + // 458 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x94, 0x41, 0x6f, 0xd3, 0x30, + 0x14, 0xc7, 0x7b, 0x1a, 0x92, 0xd5, 0xb5, 0xcc, 0xdc, 0xca, 0x85, 0xf5, 0x04, 0x2d, 0x4b, 0x10, + 0x7c, 0x01, 0xd6, 0xe5, 0xb0, 0x4a, 0x20, 0x4d, 0xeb, 0x2e, 0x80, 0xd0, 0x70, 0xb2, 0xa7, 0xf4, + 0x81, 0x13, 0x07, 0xbf, 0x97, 0x21, 0xed, 0x23, 0xf1, 0x29, 0x51, 0x93, 0x26, 0xa9, 0x69, 0x5d, + 0xca, 0x6e, 0xb5, 0xfd, 0xf3, 0xef, 0x5f, 0xbf, 0xf7, 0x14, 0xd1, 0xcf, 0x14, 0x31, 0xd8, 0xa0, + 0xb0, 0x86, 0x8d, 0x7c, 0x96, 0xa1, 0xbe, 0x2f, 0xa9, 0x5e, 0x05, 0xf5, 0xd1, 0xa8, 0x9f, 0x98, + 0x2c, 0x33, 0x79, 0xbd, 0x39, 0x92, 0x98, 0x33, 0xd8, 0x5c, 0xe9, 0xdb, 0x8c, 0xd2, 0xf5, 0xde, + 0x09, 0x81, 0xbd, 0xc7, 0x04, 0xba, 0xad, 0xb7, 0xbf, 0x85, 0x38, 0xfa, 0x58, 0xdd, 0x97, 0x4a, + 0x3c, 0xbd, 0xb0, 0xa0, 0x18, 0x2e, 0x8c, 0xd6, 0x90, 0x30, 0x9a, 0x5c, 0x06, 0x81, 0x93, 0xd4, + 0x38, 0x83, 0xbf, 0xc1, 0x6b, 0xf8, 0x59, 0x02, 0xf1, 0xe8, 0xb9, 0xcb, 0xaf, 0xff, 0xd1, 0x82, + 0x15, 0x97, 0x34, 0xee, 0xc9, 0xaf, 0x62, 0x10, 0x59, 0x53, 0x6c, 0x04, 0xbc, 0xf6, 0x04, 0xb8, + 0xd8, 0x81, 0xfa, 0x58, 0x1c, 0x5f, 0x2a, 0xda, 0xb0, 0x4f, 0x3d, 0x76, 0x87, 0x6a, 0xe4, 0x63, + 0x17, 0x5e, 0xd7, 0x2a, 0x98, 0x19, 0xa3, 0xaf, 0x81, 0x0a, 0x93, 0x13, 0x8c, 0x7b, 0xb2, 0x14, + 0x32, 0x02, 0x4a, 0x2c, 0xc6, 0x9b, 0x75, 0x7a, 0xe3, 0x7b, 0xc6, 0x16, 0xda, 0xa4, 0x4d, 0x77, + 0xa7, 0x75, 0x60, 0x7d, 0xb5, 0x58, 0xfd, 0x1c, 0xf7, 0xe4, 0x0f, 0x31, 0x5c, 0x2c, 0xcd, 0xaf, + 0xee, 0x98, 0xbc, 0xa5, 0x73, 0xb9, 0x26, 0xef, 0xe5, 0xee, 0xbc, 0x05, 0x5b, 0xcc, 0xd3, 0x0f, + 0x48, 0xbc, 0xf1, 0xc6, 0x5b, 0x31, 0xac, 0x1b, 0x7c, 0xa5, 0x2c, 0x63, 0xf5, 0xc0, 0xb3, 0xbd, + 0x83, 0xd0, 0x72, 0x07, 0x36, 0xea, 0x8b, 0x38, 0x5e, 0x35, 0xb8, 0xd3, 0x4f, 0xf7, 0x8c, 0xc1, + 0xff, 0xca, 0xbf, 0x89, 0xfe, 0xa5, 0xa2, 0xce, 0x3d, 0xf1, 0x0f, 0xc1, 0x96, 0xfa, 0xb0, 0x19, + 0xb0, 0xe2, 0xa4, 0x69, 0x6c, 0x17, 0x13, 0xfe, 0x63, 0x04, 0xb6, 0xb2, 0x26, 0xbb, 0xb3, 0x5a, + 0xce, 0x1d, 0x00, 0x14, 0x83, 0x55, 0x63, 0xdb, 0x53, 0xf2, 0xd6, 0xcc, 0xc1, 0x1e, 0xd3, 0xfe, + 0x4f, 0x62, 0x70, 0xae, 0xb5, 0x49, 0x6e, 0x30, 0x03, 0x62, 0x95, 0x15, 0xf2, 0xd4, 0x13, 0x75, + 0x43, 0xc6, 0x53, 0x39, 0x17, 0x69, 0xd5, 0x57, 0xe2, 0x49, 0xa5, 0x9e, 0x47, 0xf2, 0x85, 0xe7, + 0xc2, 0x3c, 0x6a, 0x94, 0xa7, 0x7b, 0x88, 0xd6, 0xf8, 0x5d, 0x0c, 0xcf, 0x89, 0x30, 0xcd, 0x17, + 0x90, 0x66, 0x90, 0xf3, 0x3c, 0x92, 0xaf, 0x3c, 0xf7, 0x5a, 0xae, 0x8b, 0x98, 0x1c, 0x82, 0x36, + 0x59, 0xb3, 0xd9, 0xe7, 0xf7, 0x29, 0xf2, 0xb2, 0x8c, 0x57, 0x33, 0x17, 0x3e, 0xa0, 0xd6, 0xf8, + 0xc0, 0x90, 0x2c, 0xc3, 0x5a, 0x72, 0x76, 0x87, 0xc4, 0x16, 0xe3, 0x92, 0xe1, 0x2e, 0x6c, 0x54, + 0x61, 0x65, 0x0e, 0xeb, 0x6f, 0x74, 0x11, 0xc7, 0x47, 0xd5, 0xfa, 0xdd, 0x9f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xa0, 0xb5, 0xeb, 0xf6, 0xd1, 0x05, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -136,13 +134,6 @@ type MasterClient interface { // // @return StringListResponse ShowPartitions(ctx context.Context, in *internalpb.ShowPartitionRequest, opts ...grpc.CallOption) (*servicepb.StringListResponse, error) - //* - // @brief This method is used to get system configs - // - // @param SysConfigRequest, keys or key_prefixes of the configs. - // - // @return SysConfigResponse - GetSysConfigs(ctx context.Context, in *internalpb.SysConfigRequest, opts ...grpc.CallOption) (*servicepb.SysConfigResponse, error) AllocTimestamp(ctx context.Context, in *internalpb.TsoRequest, opts ...grpc.CallOption) (*internalpb.TsoResponse, error) AllocID(ctx context.Context, in *internalpb.IDRequest, opts ...grpc.CallOption) (*internalpb.IDResponse, error) AssignSegmentID(ctx context.Context, in *internalpb.AssignSegIDRequest, opts ...grpc.CallOption) (*internalpb.AssignSegIDResponse, error) @@ -246,15 +237,6 @@ func (c *masterClient) ShowPartitions(ctx context.Context, in *internalpb.ShowPa return out, nil } -func (c *masterClient) GetSysConfigs(ctx context.Context, in *internalpb.SysConfigRequest, opts ...grpc.CallOption) (*servicepb.SysConfigResponse, error) { - out := new(servicepb.SysConfigResponse) - err := c.cc.Invoke(ctx, "/milvus.proto.master.Master/GetSysConfigs", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *masterClient) AllocTimestamp(ctx context.Context, in *internalpb.TsoRequest, opts ...grpc.CallOption) (*internalpb.TsoResponse, error) { out := new(internalpb.TsoResponse) err := c.cc.Invoke(ctx, "/milvus.proto.master.Master/AllocTimestamp", in, out, opts...) @@ -344,13 +326,6 @@ type MasterServer interface { // // @return StringListResponse ShowPartitions(context.Context, *internalpb.ShowPartitionRequest) (*servicepb.StringListResponse, error) - //* - // @brief This method is used to get system configs - // - // @param SysConfigRequest, keys or key_prefixes of the configs. - // - // @return SysConfigResponse - GetSysConfigs(context.Context, *internalpb.SysConfigRequest) (*servicepb.SysConfigResponse, error) AllocTimestamp(context.Context, *internalpb.TsoRequest) (*internalpb.TsoResponse, error) AllocID(context.Context, *internalpb.IDRequest) (*internalpb.IDResponse, error) AssignSegmentID(context.Context, *internalpb.AssignSegIDRequest) (*internalpb.AssignSegIDResponse, error) @@ -390,9 +365,6 @@ func (*UnimplementedMasterServer) DescribePartition(ctx context.Context, req *in func (*UnimplementedMasterServer) ShowPartitions(ctx context.Context, req *internalpb.ShowPartitionRequest) (*servicepb.StringListResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ShowPartitions not implemented") } -func (*UnimplementedMasterServer) GetSysConfigs(ctx context.Context, req *internalpb.SysConfigRequest) (*servicepb.SysConfigResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetSysConfigs not implemented") -} func (*UnimplementedMasterServer) AllocTimestamp(ctx context.Context, req *internalpb.TsoRequest) (*internalpb.TsoResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AllocTimestamp not implemented") } @@ -587,24 +559,6 @@ func _Master_ShowPartitions_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } -func _Master_GetSysConfigs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(internalpb.SysConfigRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MasterServer).GetSysConfigs(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/milvus.proto.master.Master/GetSysConfigs", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MasterServer).GetSysConfigs(ctx, req.(*internalpb.SysConfigRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Master_AllocTimestamp_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(internalpb.TsoRequest) if err := dec(in); err != nil { @@ -703,10 +657,6 @@ var _Master_serviceDesc = grpc.ServiceDesc{ MethodName: "ShowPartitions", Handler: _Master_ShowPartitions_Handler, }, - { - MethodName: "GetSysConfigs", - Handler: _Master_GetSysConfigs_Handler, - }, { MethodName: "AllocTimestamp", Handler: _Master_AllocTimestamp_Handler, diff --git a/internal/proto/schema.proto b/internal/proto/schema.proto index 07b19a6fa9..4ef9997fe0 100644 --- a/internal/proto/schema.proto +++ b/internal/proto/schema.proto @@ -30,10 +30,11 @@ enum DataType { */ message FieldSchema { string name = 1; - string description = 2; - DataType data_type = 3; - repeated common.KeyValuePair type_params = 4; - repeated common.KeyValuePair index_params = 5; + bool is_primary_key = 2; + string description = 3; + DataType data_type = 4; + repeated common.KeyValuePair type_params = 5; + repeated common.KeyValuePair index_params = 6; } /** diff --git a/internal/proto/schemapb/schema.pb.go b/internal/proto/schemapb/schema.pb.go index 4305c3a745..7550e9314e 100644 --- a/internal/proto/schemapb/schema.pb.go +++ b/internal/proto/schemapb/schema.pb.go @@ -79,10 +79,11 @@ func (DataType) EnumDescriptor() ([]byte, []int) { // @brief Field schema type FieldSchema struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` - DataType DataType `protobuf:"varint,3,opt,name=data_type,json=dataType,proto3,enum=milvus.proto.schema.DataType" json:"data_type,omitempty"` - TypeParams []*commonpb.KeyValuePair `protobuf:"bytes,4,rep,name=type_params,json=typeParams,proto3" json:"type_params,omitempty"` - IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,5,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"` + IsPrimaryKey bool `protobuf:"varint,2,opt,name=is_primary_key,json=isPrimaryKey,proto3" json:"is_primary_key,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + DataType DataType `protobuf:"varint,4,opt,name=data_type,json=dataType,proto3,enum=milvus.proto.schema.DataType" json:"data_type,omitempty"` + TypeParams []*commonpb.KeyValuePair `protobuf:"bytes,5,rep,name=type_params,json=typeParams,proto3" json:"type_params,omitempty"` + IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,6,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -120,6 +121,13 @@ func (m *FieldSchema) GetName() string { return "" } +func (m *FieldSchema) GetIsPrimaryKey() bool { + if m != nil { + return m.IsPrimaryKey + } + return false +} + func (m *FieldSchema) GetDescription() string { if m != nil { return m.Description @@ -222,31 +230,34 @@ func init() { func init() { proto.RegisterFile("schema.proto", fileDescriptor_1c5fb4d8cc22d66a) } var fileDescriptor_1c5fb4d8cc22d66a = []byte{ - // 416 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x91, 0x5f, 0x8b, 0xd4, 0x30, - 0x14, 0xc5, 0xed, 0xfc, 0x63, 0xe6, 0x76, 0x94, 0x18, 0x45, 0x06, 0x41, 0xa8, 0xfb, 0x34, 0x08, - 0xb6, 0x38, 0x2b, 0xcb, 0xe2, 0x93, 0xdb, 0xed, 0xac, 0x14, 0x87, 0x76, 0xc9, 0xd6, 0x05, 0x7d, - 0x19, 0xd2, 0x26, 0x3a, 0x81, 0xf4, 0x0f, 0x6d, 0x2a, 0xce, 0x7e, 0x03, 0x1f, 0x7d, 0xf5, 0xd3, - 0x4a, 0xd2, 0x0e, 0xac, 0x30, 0x0f, 0x82, 0x6f, 0xbf, 0x7b, 0x6f, 0xce, 0x4d, 0xce, 0x09, 0xcc, - 0x9b, 0x6c, 0xc7, 0x73, 0xea, 0x56, 0x75, 0xa9, 0x4a, 0xfc, 0x24, 0x17, 0xf2, 0x7b, 0xdb, 0x74, - 0x95, 0xdb, 0x8d, 0x9e, 0xcf, 0xb3, 0x32, 0xcf, 0xcb, 0xa2, 0x6b, 0x9e, 0xfc, 0x1c, 0x80, 0x7d, - 0x25, 0xb8, 0x64, 0x37, 0x66, 0x8a, 0x31, 0x8c, 0x0a, 0x9a, 0xf3, 0x85, 0xe5, 0x58, 0xcb, 0x19, - 0x31, 0x8c, 0x1d, 0xb0, 0x19, 0x6f, 0xb2, 0x5a, 0x54, 0x4a, 0x94, 0xc5, 0x62, 0x60, 0x46, 0xf7, - 0x5b, 0xf8, 0x1d, 0xcc, 0x18, 0x55, 0x74, 0xab, 0xf6, 0x15, 0x5f, 0x0c, 0x1d, 0x6b, 0xf9, 0x68, - 0xf5, 0xc2, 0x3d, 0x72, 0xb9, 0x1b, 0x50, 0x45, 0x93, 0x7d, 0xc5, 0xc9, 0x94, 0xf5, 0x84, 0x7d, - 0xb0, 0xb5, 0x6c, 0x5b, 0xd1, 0x9a, 0xe6, 0xcd, 0x62, 0xe4, 0x0c, 0x97, 0xf6, 0xea, 0xe5, 0xdf, - 0xea, 0xfe, 0xc9, 0x1f, 0xf9, 0xfe, 0x96, 0xca, 0x96, 0x5f, 0x53, 0x51, 0x13, 0xd0, 0xaa, 0x6b, - 0x23, 0xc2, 0x01, 0xcc, 0x45, 0xc1, 0xf8, 0x8f, 0xc3, 0x92, 0xf1, 0xbf, 0x2e, 0xb1, 0x8d, 0xac, - 0xdb, 0x72, 0xf2, 0xdb, 0x02, 0x74, 0x59, 0x4a, 0xc9, 0x33, 0x6d, 0xea, 0xbf, 0x02, 0x79, 0x06, - 0x13, 0xda, 0xaa, 0x32, 0x0c, 0x4c, 0x1a, 0x53, 0xd2, 0x57, 0xf8, 0x1c, 0x26, 0x5f, 0x75, 0xda, - 0x07, 0x9f, 0xce, 0xd1, 0x94, 0xee, 0x7d, 0x08, 0xe9, 0xcf, 0xbf, 0xfa, 0x65, 0xc1, 0xf4, 0x90, - 0x1e, 0x9e, 0xc2, 0x28, 0x8a, 0xa3, 0x35, 0x7a, 0xa0, 0xc9, 0x8f, 0xe3, 0x0d, 0xb2, 0x34, 0x85, - 0x51, 0x72, 0x8e, 0x06, 0x78, 0x06, 0xe3, 0x30, 0x4a, 0xde, 0x9c, 0xa1, 0x61, 0x8f, 0xa7, 0x2b, - 0x34, 0xea, 0xf1, 0xec, 0x2d, 0x1a, 0x6b, 0xbc, 0xda, 0xc4, 0x17, 0x09, 0x02, 0x0c, 0x30, 0x09, - 0xe2, 0x4f, 0xfe, 0x66, 0x8d, 0x6c, 0xcd, 0x37, 0x09, 0x09, 0xa3, 0x0f, 0xe8, 0x29, 0x7e, 0x0c, - 0x0f, 0x6f, 0xd7, 0x97, 0x49, 0x4c, 0xb6, 0x7e, 0x18, 0x5d, 0x90, 0xcf, 0x88, 0x61, 0x04, 0xf3, - 0xbe, 0xd5, 0x89, 0xb9, 0xef, 0x7f, 0x79, 0xff, 0x4d, 0xa8, 0x5d, 0x9b, 0xea, 0x6c, 0xbd, 0x3b, - 0x21, 0xa5, 0xb8, 0x53, 0x3c, 0xdb, 0x79, 0x9d, 0xa9, 0xd7, 0x4c, 0x34, 0xaa, 0x16, 0x69, 0xab, - 0x38, 0xf3, 0x44, 0xa1, 0x78, 0x5d, 0x50, 0xe9, 0x19, 0xa7, 0x5e, 0xe7, 0xb4, 0x4a, 0xd3, 0x89, - 0xa9, 0x4f, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x74, 0xb3, 0xe8, 0x78, 0xba, 0x02, 0x00, 0x00, + // 451 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x91, 0xdf, 0x6a, 0xd4, 0x40, + 0x18, 0xc5, 0xcd, 0xfe, 0x09, 0xd9, 0x2f, 0xb1, 0x8c, 0xa3, 0x48, 0x10, 0x84, 0x58, 0xbc, 0x58, + 0x04, 0x37, 0xb8, 0x95, 0x52, 0xbc, 0xb2, 0xe9, 0x6e, 0x25, 0x74, 0x49, 0x96, 0x69, 0x2c, 0xe8, + 0x4d, 0x98, 0x4d, 0x46, 0x77, 0x30, 0xff, 0x48, 0x26, 0x62, 0xfa, 0x16, 0xde, 0xfa, 0x12, 0xbe, + 0xa2, 0xcc, 0x24, 0x85, 0xaa, 0x45, 0xbc, 0xfb, 0x7d, 0xdf, 0xcc, 0x39, 0xcc, 0x39, 0x03, 0x56, + 0x93, 0xec, 0x59, 0x4e, 0x17, 0x55, 0x5d, 0x8a, 0x12, 0x3f, 0xcc, 0x79, 0xf6, 0xb5, 0x6d, 0xfa, + 0x69, 0xd1, 0x1f, 0x3d, 0xb1, 0x92, 0x32, 0xcf, 0xcb, 0xa2, 0x5f, 0x1e, 0xfe, 0x1c, 0x81, 0x79, + 0xce, 0x59, 0x96, 0x5e, 0xaa, 0x53, 0x8c, 0x61, 0x52, 0xd0, 0x9c, 0xd9, 0x9a, 0xa3, 0xcd, 0x67, + 0x44, 0x31, 0x7e, 0x0e, 0x07, 0xbc, 0x89, 0xab, 0x9a, 0xe7, 0xb4, 0xee, 0xe2, 0x2f, 0xac, 0xb3, + 0x47, 0x8e, 0x36, 0x37, 0x88, 0xc5, 0x9b, 0x6d, 0xbf, 0xbc, 0x60, 0x1d, 0x76, 0xc0, 0x4c, 0x59, + 0x93, 0xd4, 0xbc, 0x12, 0xbc, 0x2c, 0xec, 0xb1, 0x32, 0xb8, 0xbd, 0xc2, 0x6f, 0x60, 0x96, 0x52, + 0x41, 0x63, 0xd1, 0x55, 0xcc, 0x9e, 0x38, 0xda, 0xfc, 0x60, 0xf9, 0x74, 0x71, 0xc7, 0x13, 0x17, + 0x2b, 0x2a, 0x68, 0xd4, 0x55, 0x8c, 0x18, 0xe9, 0x40, 0xd8, 0x03, 0x53, 0xca, 0xe2, 0x8a, 0xd6, + 0x34, 0x6f, 0xec, 0xa9, 0x33, 0x9e, 0x9b, 0xcb, 0x67, 0xbf, 0xab, 0x87, 0x60, 0x17, 0xac, 0xbb, + 0xa2, 0x59, 0xcb, 0xb6, 0x94, 0xd7, 0x04, 0xa4, 0x6a, 0xab, 0x44, 0x78, 0x05, 0x16, 0x2f, 0x52, + 0xf6, 0xed, 0xc6, 0x44, 0xff, 0x5f, 0x13, 0x53, 0xc9, 0x7a, 0x97, 0xc3, 0x1f, 0x1a, 0xa0, 0xb3, + 0x32, 0xcb, 0x58, 0x22, 0x43, 0xfd, 0xa3, 0xb6, 0x3f, 0x0a, 0x19, 0xfd, 0x5d, 0xc8, 0x63, 0xd0, + 0x69, 0x2b, 0x4a, 0x7f, 0xa5, 0xda, 0x32, 0xc8, 0x30, 0xe1, 0x13, 0xd0, 0x3f, 0xc9, 0x3f, 0x69, + 0xec, 0x89, 0x7a, 0xa2, 0x73, 0x67, 0x4b, 0xb7, 0xbe, 0x8d, 0x0c, 0xf7, 0x5f, 0x7c, 0xd7, 0xc0, + 0xb8, 0x69, 0x0f, 0x1b, 0x30, 0x09, 0xc2, 0x60, 0x8d, 0xee, 0x49, 0xf2, 0xc2, 0x70, 0x83, 0x34, + 0x49, 0x7e, 0x10, 0x9d, 0xa0, 0x11, 0x9e, 0xc1, 0xd4, 0x0f, 0xa2, 0x57, 0xc7, 0x68, 0x3c, 0xe0, + 0xd1, 0x12, 0x4d, 0x06, 0x3c, 0x7e, 0x8d, 0xa6, 0x12, 0xcf, 0x37, 0xe1, 0x69, 0x84, 0x00, 0x03, + 0xe8, 0xab, 0xf0, 0xbd, 0xb7, 0x59, 0x23, 0x53, 0xf2, 0x65, 0x44, 0xfc, 0xe0, 0x1d, 0x7a, 0x84, + 0x1f, 0xc0, 0xfd, 0xab, 0xf5, 0x59, 0x14, 0x92, 0xd8, 0xf3, 0x83, 0x53, 0xf2, 0x01, 0xa5, 0x18, + 0x81, 0x35, 0xac, 0x7a, 0x31, 0xf3, 0xbc, 0x8f, 0x6f, 0x3f, 0x73, 0xb1, 0x6f, 0x77, 0xb2, 0x5b, + 0xf7, 0x9a, 0x67, 0x19, 0xbf, 0x16, 0x2c, 0xd9, 0xbb, 0x7d, 0xa8, 0x97, 0x29, 0x6f, 0x44, 0xcd, + 0x77, 0xad, 0x60, 0xa9, 0xcb, 0x0b, 0xc1, 0xea, 0x82, 0x66, 0xae, 0x4a, 0xea, 0xf6, 0x49, 0xab, + 0xdd, 0x4e, 0x57, 0xf3, 0xd1, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x14, 0xa6, 0xc9, 0x55, 0xe0, + 0x02, 0x00, 0x00, } diff --git a/internal/proto/service_msg.proto b/internal/proto/service_msg.proto index a2779a01a5..b480b7c13b 100644 --- a/internal/proto/service_msg.proto +++ b/internal/proto/service_msg.proto @@ -135,14 +135,6 @@ message PartitionDescription { repeated common.KeyValuePair statistics = 3; } -/** - * @brief Response of GetSysConfig - */ -message SysConfigResponse { - common.Status status = 1; - repeated string keys = 2; - repeated string values = 3; -} /** * @brief Entities hit by query diff --git a/internal/proto/servicepb/service_msg.pb.go b/internal/proto/servicepb/service_msg.pb.go index 14abfa45bb..48b44e7817 100644 --- a/internal/proto/servicepb/service_msg.pb.go +++ b/internal/proto/servicepb/service_msg.pb.go @@ -737,63 +737,6 @@ func (m *PartitionDescription) GetStatistics() []*commonpb.KeyValuePair { return nil } -//* -// @brief Response of GetSysConfig -type SysConfigResponse struct { - Status *commonpb.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` - Keys []string `protobuf:"bytes,2,rep,name=keys,proto3" json:"keys,omitempty"` - Values []string `protobuf:"bytes,3,rep,name=values,proto3" json:"values,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SysConfigResponse) Reset() { *m = SysConfigResponse{} } -func (m *SysConfigResponse) String() string { return proto.CompactTextString(m) } -func (*SysConfigResponse) ProtoMessage() {} -func (*SysConfigResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_b4b40b84dd2f74cb, []int{13} -} - -func (m *SysConfigResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SysConfigResponse.Unmarshal(m, b) -} -func (m *SysConfigResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SysConfigResponse.Marshal(b, m, deterministic) -} -func (m *SysConfigResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SysConfigResponse.Merge(m, src) -} -func (m *SysConfigResponse) XXX_Size() int { - return xxx_messageInfo_SysConfigResponse.Size(m) -} -func (m *SysConfigResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SysConfigResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SysConfigResponse proto.InternalMessageInfo - -func (m *SysConfigResponse) GetStatus() *commonpb.Status { - if m != nil { - return m.Status - } - return nil -} - -func (m *SysConfigResponse) GetKeys() []string { - if m != nil { - return m.Keys - } - return nil -} - -func (m *SysConfigResponse) GetValues() []string { - if m != nil { - return m.Values - } - return nil -} - //* // @brief Entities hit by query type Hits struct { @@ -809,7 +752,7 @@ func (m *Hits) Reset() { *m = Hits{} } func (m *Hits) String() string { return proto.CompactTextString(m) } func (*Hits) ProtoMessage() {} func (*Hits) Descriptor() ([]byte, []int) { - return fileDescriptor_b4b40b84dd2f74cb, []int{14} + return fileDescriptor_b4b40b84dd2f74cb, []int{13} } func (m *Hits) XXX_Unmarshal(b []byte) error { @@ -865,7 +808,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_b4b40b84dd2f74cb, []int{15} + return fileDescriptor_b4b40b84dd2f74cb, []int{14} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { @@ -915,7 +858,6 @@ func init() { proto.RegisterType((*IntegerRangeResponse)(nil), "milvus.proto.service.IntegerRangeResponse") proto.RegisterType((*CollectionDescription)(nil), "milvus.proto.service.CollectionDescription") proto.RegisterType((*PartitionDescription)(nil), "milvus.proto.service.PartitionDescription") - proto.RegisterType((*SysConfigResponse)(nil), "milvus.proto.service.SysConfigResponse") proto.RegisterType((*Hits)(nil), "milvus.proto.service.Hits") proto.RegisterType((*QueryResult)(nil), "milvus.proto.service.QueryResult") } @@ -923,53 +865,52 @@ func init() { func init() { proto.RegisterFile("service_msg.proto", fileDescriptor_b4b40b84dd2f74cb) } var fileDescriptor_b4b40b84dd2f74cb = []byte{ - // 762 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xdd, 0x4e, 0xe3, 0x46, - 0x14, 0xae, 0xe3, 0x90, 0x86, 0x13, 0x27, 0x24, 0xd3, 0x14, 0x19, 0xb8, 0x49, 0x8d, 0x68, 0xa3, - 0x56, 0x4d, 0x24, 0xa8, 0x54, 0x71, 0x51, 0xa9, 0x49, 0xa0, 0x2d, 0x3f, 0x0a, 0x74, 0x12, 0x21, - 0xd1, 0x4a, 0x8d, 0x26, 0xf6, 0xd4, 0x1e, 0xd5, 0xf1, 0x58, 0x9e, 0x31, 0x51, 0x78, 0x90, 0xbe, - 0xc4, 0x3e, 0xc8, 0xde, 0xed, 0x33, 0xad, 0x3c, 0x36, 0xf9, 0x61, 0x59, 0x2d, 0x4b, 0xb8, 0x3b, - 0xe7, 0xcc, 0x9c, 0xf3, 0x9d, 0xdf, 0x0f, 0x6a, 0x82, 0x46, 0x77, 0xcc, 0xa6, 0xa3, 0x89, 0x70, - 0x5b, 0x61, 0xc4, 0x25, 0x47, 0xf5, 0x09, 0xf3, 0xef, 0x62, 0x91, 0x6a, 0xad, 0xec, 0x7d, 0xd7, - 0xb0, 0xf9, 0x64, 0xc2, 0x83, 0xd4, 0xba, 0x6b, 0x08, 0xdb, 0xa3, 0x13, 0x92, 0x6a, 0xd6, 0x31, - 0x54, 0x7a, 0xdc, 0xf7, 0xa9, 0x2d, 0x19, 0x0f, 0xfa, 0x64, 0x42, 0xd1, 0x77, 0xb0, 0x65, 0xcf, - 0x2d, 0xa3, 0x80, 0x4c, 0xa8, 0xa9, 0x35, 0xb4, 0xe6, 0x26, 0xae, 0xd8, 0x2b, 0x1f, 0xad, 0x73, - 0x28, 0x5f, 0x93, 0x48, 0xb2, 0xcf, 0xf6, 0x44, 0x55, 0xd0, 0x25, 0x71, 0xcd, 0x9c, 0x7a, 0x4c, - 0x44, 0xeb, 0x8d, 0x06, 0x45, 0xcc, 0xa7, 0x5d, 0x22, 0x6d, 0xef, 0xf9, 0x71, 0xf6, 0xa1, 0x1c, - 0x3e, 0x64, 0x30, 0x5a, 0x44, 0x34, 0xe6, 0xc6, 0x21, 0x71, 0xd1, 0x4f, 0x50, 0x8c, 0xf8, 0x74, - 0xe4, 0x10, 0x49, 0x4c, 0xbd, 0xa1, 0x37, 0x4b, 0x87, 0x3b, 0xad, 0x95, 0x36, 0x65, 0xdd, 0xe9, - 0xfa, 0x7c, 0x8c, 0xbf, 0x8c, 0xf8, 0xf4, 0x84, 0x48, 0x82, 0xf6, 0x60, 0xd3, 0x23, 0xc2, 0x1b, - 0xfd, 0x47, 0x67, 0xc2, 0xcc, 0x37, 0xf4, 0xe6, 0x06, 0x2e, 0x26, 0x86, 0x0b, 0x3a, 0x13, 0xd6, - 0x14, 0xaa, 0xd7, 0x3e, 0xb1, 0xa9, 0xc7, 0x7d, 0x87, 0x46, 0x37, 0xc4, 0x8f, 0xe7, 0x35, 0x69, - 0xf3, 0x9a, 0xd0, 0x31, 0xe4, 0xe5, 0x2c, 0xa4, 0x2a, 0xa9, 0xca, 0xe1, 0x41, 0xeb, 0xa9, 0xd9, - 0xb4, 0x96, 0xe2, 0x0c, 0x67, 0x21, 0xc5, 0xca, 0x05, 0x6d, 0x43, 0xe1, 0x2e, 0x89, 0x2a, 0x54, - 0xc6, 0x06, 0xce, 0x34, 0xeb, 0x9f, 0x15, 0xe0, 0xdf, 0x23, 0x1e, 0x87, 0xe8, 0x1c, 0x8c, 0x70, - 0x61, 0x13, 0xa6, 0xa6, 0x6a, 0xfc, 0xf6, 0x93, 0x70, 0x2a, 0x6d, 0xbc, 0xe2, 0x6b, 0xfd, 0xaf, - 0xc1, 0xc6, 0x9f, 0x31, 0x8d, 0x66, 0xcf, 0x9f, 0xc1, 0x01, 0x54, 0x56, 0x66, 0x20, 0xcc, 0x5c, - 0x43, 0x6f, 0x6e, 0xe2, 0xf2, 0xf2, 0x10, 0x44, 0xd2, 0x1e, 0x47, 0xf8, 0xa6, 0x9e, 0xb6, 0xc7, - 0x11, 0x3e, 0xfa, 0x01, 0x6a, 0x4b, 0xd8, 0x23, 0x37, 0x29, 0xc6, 0xcc, 0x37, 0xb4, 0xa6, 0x81, - 0xab, 0xe1, 0xa3, 0x22, 0xad, 0xbf, 0xa1, 0x32, 0x90, 0x11, 0x0b, 0x5c, 0x4c, 0x45, 0xc8, 0x03, - 0x41, 0xd1, 0x11, 0x14, 0x84, 0x24, 0x32, 0x16, 0x2a, 0xaf, 0xd2, 0xe1, 0xde, 0x93, 0x43, 0x1d, - 0xa8, 0x2f, 0x38, 0xfb, 0x8a, 0xea, 0xb0, 0xa1, 0x3a, 0x99, 0x2d, 0x4a, 0xaa, 0x58, 0xb7, 0x60, - 0x74, 0x39, 0xf7, 0x5f, 0x31, 0x74, 0xf1, 0x21, 0x34, 0x01, 0x94, 0xe6, 0x7d, 0xc9, 0x84, 0x5c, - 0x0f, 0x60, 0xb1, 0x13, 0x69, 0x83, 0x1f, 0x76, 0x62, 0x0c, 0x5f, 0x9d, 0x05, 0x92, 0xba, 0x34, - 0x7a, 0x6d, 0x0c, 0x7d, 0x8e, 0x21, 0xa0, 0x9e, 0x61, 0x60, 0x12, 0xb8, 0x74, 0xed, 0x4e, 0x8d, - 0xa9, 0xcb, 0x02, 0xd5, 0x29, 0x1d, 0xa7, 0x4a, 0xb2, 0x20, 0x34, 0x70, 0xd4, 0x82, 0xe8, 0x38, - 0x11, 0xad, 0x77, 0x1a, 0x7c, 0xbd, 0xe0, 0xa6, 0x13, 0x2a, 0xec, 0x88, 0x85, 0x89, 0xf8, 0x32, - 0xd8, 0x5f, 0xa0, 0x90, 0x32, 0x9f, 0xc2, 0x2d, 0x7d, 0x70, 0x90, 0x29, 0x2b, 0x2e, 0x00, 0x07, - 0xca, 0x80, 0x33, 0x27, 0xd4, 0x01, 0x48, 0x02, 0x31, 0x21, 0x99, 0x2d, 0x32, 0x22, 0xf9, 0xe6, - 0x49, 0xdc, 0x0b, 0x3a, 0x53, 0xb7, 0x75, 0x4d, 0x58, 0x84, 0x97, 0x9c, 0xac, 0xb7, 0x1a, 0xd4, - 0xe7, 0x8c, 0xb9, 0x76, 0x3d, 0x3f, 0x43, 0x5e, 0x9d, 0x65, 0x5a, 0xcd, 0xfe, 0x47, 0xee, 0x7d, - 0x99, 0xa0, 0xb1, 0x72, 0x78, 0x8d, 0x4a, 0x24, 0xd4, 0x06, 0x33, 0xd1, 0xe3, 0xc1, 0xbf, 0x6c, - 0xcd, 0x8b, 0x44, 0x90, 0x57, 0x14, 0x9b, 0xee, 0xb4, 0x92, 0x1f, 0xb1, 0xdf, 0x62, 0xd3, 0x2f, - 0x20, 0xff, 0x07, 0x93, 0x8a, 0x4b, 0xce, 0x4e, 0x52, 0xa2, 0xd3, 0x71, 0x22, 0xa2, 0x9d, 0x25, - 0x8e, 0xcf, 0x29, 0xc6, 0x9c, 0x13, 0xf9, 0x76, 0x32, 0x76, 0x1e, 0x65, 0xc1, 0x72, 0x38, 0xd3, - 0xac, 0x1b, 0x28, 0x29, 0xa6, 0xc3, 0x54, 0xc4, 0xbe, 0x7c, 0x71, 0xf2, 0x1e, 0x93, 0x22, 0x83, - 0x54, 0xf2, 0xf7, 0xbf, 0xc2, 0xd6, 0x23, 0x4e, 0x47, 0x45, 0xc8, 0xf7, 0xaf, 0xfa, 0xa7, 0xd5, - 0x2f, 0x50, 0x0d, 0xca, 0x37, 0xa7, 0xbd, 0xe1, 0x15, 0x1e, 0x75, 0xcf, 0xfa, 0x1d, 0x7c, 0x5b, - 0x75, 0x50, 0x15, 0x8c, 0xcc, 0xf4, 0xdb, 0xe5, 0x55, 0x67, 0x58, 0xa5, 0xdd, 0xde, 0x5f, 0x1d, - 0x97, 0x49, 0x2f, 0x1e, 0x27, 0xa8, 0xed, 0x7b, 0xe6, 0xfb, 0xec, 0x5e, 0x52, 0xdb, 0x6b, 0xa7, - 0x19, 0xfd, 0xe8, 0x30, 0x21, 0x23, 0x36, 0x8e, 0x25, 0x75, 0xda, 0x2c, 0x90, 0x34, 0x0a, 0x88, - 0xdf, 0x56, 0x69, 0xb6, 0xb3, 0xb1, 0x87, 0xe3, 0x71, 0x41, 0x19, 0x8e, 0xde, 0x07, 0x00, 0x00, - 0xff, 0xff, 0x05, 0xd2, 0x35, 0xf0, 0x25, 0x08, 0x00, 0x00, + // 739 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xdd, 0x4e, 0xdb, 0x4a, + 0x10, 0x3e, 0x8e, 0x43, 0x4e, 0x98, 0x38, 0x21, 0xec, 0xc9, 0x41, 0x06, 0x6e, 0x72, 0x8c, 0x38, + 0x8d, 0x5a, 0x35, 0x91, 0xa0, 0x52, 0xc5, 0x45, 0xa5, 0x26, 0x40, 0x5b, 0x7e, 0x14, 0xe8, 0x12, + 0x21, 0xd1, 0x4a, 0x8d, 0x36, 0xf6, 0xca, 0x5e, 0xd5, 0xf1, 0x5a, 0xde, 0x35, 0x51, 0x78, 0x90, + 0xbe, 0x44, 0x1f, 0xa4, 0x77, 0x7d, 0xa6, 0xca, 0x6b, 0x93, 0x1f, 0x4a, 0x55, 0x0a, 0xdc, 0xcd, + 0xcc, 0xee, 0xcc, 0x37, 0xbf, 0x1f, 0x2c, 0x0b, 0x1a, 0x5d, 0x32, 0x9b, 0xf6, 0x87, 0xc2, 0x6d, + 0x86, 0x11, 0x97, 0x1c, 0xd5, 0x86, 0xcc, 0xbf, 0x8c, 0x45, 0xaa, 0x35, 0xb3, 0xf7, 0x35, 0xc3, + 0xe6, 0xc3, 0x21, 0x0f, 0x52, 0xeb, 0x9a, 0x21, 0x6c, 0x8f, 0x0e, 0x49, 0xaa, 0x59, 0x3b, 0x50, + 0xd9, 0xe5, 0xbe, 0x4f, 0x6d, 0xc9, 0x78, 0xd0, 0x25, 0x43, 0x8a, 0x9e, 0xc0, 0x92, 0x3d, 0xb1, + 0xf4, 0x03, 0x32, 0xa4, 0xa6, 0x56, 0xd7, 0x1a, 0x8b, 0xb8, 0x62, 0xcf, 0x7d, 0xb4, 0x0e, 0xa1, + 0x7c, 0x4a, 0x22, 0xc9, 0xfe, 0xd8, 0x13, 0x55, 0x41, 0x97, 0xc4, 0x35, 0x73, 0xea, 0x31, 0x11, + 0xad, 0xaf, 0x1a, 0x14, 0x31, 0x1f, 0x75, 0x88, 0xb4, 0xbd, 0xbb, 0xc7, 0xd9, 0x80, 0x72, 0x78, + 0x9d, 0x41, 0x7f, 0x1a, 0xd1, 0x98, 0x18, 0x7b, 0xc4, 0x45, 0x2f, 0xa0, 0x18, 0xf1, 0x51, 0xdf, + 0x21, 0x92, 0x98, 0x7a, 0x5d, 0x6f, 0x94, 0xb6, 0x56, 0x9b, 0x73, 0x6d, 0xca, 0xba, 0xd3, 0xf1, + 0xf9, 0x00, 0xff, 0x1d, 0xf1, 0xd1, 0x1e, 0x91, 0x04, 0xad, 0xc3, 0xa2, 0x47, 0x84, 0xd7, 0xff, + 0x4c, 0xc7, 0xc2, 0xcc, 0xd7, 0xf5, 0xc6, 0x02, 0x2e, 0x26, 0x86, 0x23, 0x3a, 0x16, 0xd6, 0x08, + 0xaa, 0xa7, 0x3e, 0xb1, 0xa9, 0xc7, 0x7d, 0x87, 0x46, 0xe7, 0xc4, 0x8f, 0x27, 0x35, 0x69, 0x93, + 0x9a, 0xd0, 0x0e, 0xe4, 0xe5, 0x38, 0xa4, 0x2a, 0xa9, 0xca, 0xd6, 0x66, 0xf3, 0xb6, 0xd9, 0x34, + 0x67, 0xe2, 0xf4, 0xc6, 0x21, 0xc5, 0xca, 0x05, 0xad, 0x40, 0xe1, 0x32, 0x89, 0x2a, 0x54, 0xc6, + 0x06, 0xce, 0x34, 0xeb, 0xd3, 0x1c, 0xf0, 0xdb, 0x88, 0xc7, 0x21, 0x3a, 0x04, 0x23, 0x9c, 0xda, + 0x84, 0xa9, 0xa9, 0x1a, 0xff, 0xff, 0x2d, 0x9c, 0x4a, 0x1b, 0xcf, 0xf9, 0x5a, 0x5f, 0x34, 0x58, + 0x78, 0x1f, 0xd3, 0x68, 0x7c, 0xf7, 0x19, 0x6c, 0x42, 0x65, 0x6e, 0x06, 0xc2, 0xcc, 0xd5, 0xf5, + 0xc6, 0x22, 0x2e, 0xcf, 0x0e, 0x41, 0x24, 0xed, 0x71, 0x84, 0x6f, 0xea, 0x69, 0x7b, 0x1c, 0xe1, + 0xa3, 0x67, 0xb0, 0x3c, 0x83, 0xdd, 0x77, 0x93, 0x62, 0xcc, 0x7c, 0x5d, 0x6b, 0x18, 0xb8, 0x1a, + 0xde, 0x28, 0xd2, 0xfa, 0x08, 0x95, 0x33, 0x19, 0xb1, 0xc0, 0xc5, 0x54, 0x84, 0x3c, 0x10, 0x14, + 0x6d, 0x43, 0x41, 0x48, 0x22, 0x63, 0xa1, 0xf2, 0x2a, 0x6d, 0xad, 0xdf, 0x3a, 0xd4, 0x33, 0xf5, + 0x05, 0x67, 0x5f, 0x51, 0x0d, 0x16, 0x54, 0x27, 0xb3, 0x45, 0x49, 0x15, 0xeb, 0x02, 0x8c, 0x0e, + 0xe7, 0xfe, 0x23, 0x86, 0x2e, 0x5e, 0x87, 0x26, 0x80, 0xd2, 0xbc, 0x8f, 0x99, 0x90, 0x0f, 0x03, + 0x98, 0xee, 0x44, 0xda, 0xe0, 0xeb, 0x9d, 0x18, 0xc0, 0x3f, 0x07, 0x81, 0xa4, 0x2e, 0x8d, 0x1e, + 0x1b, 0x43, 0x9f, 0x60, 0x08, 0xa8, 0x65, 0x18, 0x98, 0x04, 0x2e, 0x7d, 0x70, 0xa7, 0x06, 0xd4, + 0x65, 0x81, 0xea, 0x94, 0x8e, 0x53, 0x25, 0x59, 0x10, 0x1a, 0x38, 0x6a, 0x41, 0x74, 0x9c, 0x88, + 0xd6, 0x77, 0x0d, 0xfe, 0x9d, 0x72, 0xd3, 0x1e, 0x15, 0x76, 0xc4, 0xc2, 0x44, 0xbc, 0x1f, 0xec, + 0x2b, 0x28, 0xa4, 0xcc, 0xa7, 0x70, 0x4b, 0x3f, 0x1d, 0x64, 0xca, 0x8a, 0x53, 0xc0, 0x33, 0x65, + 0xc0, 0x99, 0x13, 0x6a, 0x03, 0x24, 0x81, 0x98, 0x90, 0xcc, 0x16, 0x19, 0x91, 0xfc, 0x77, 0x2b, + 0xee, 0x11, 0x1d, 0xab, 0xdb, 0x3a, 0x25, 0x2c, 0xc2, 0x33, 0x4e, 0xd6, 0x37, 0x0d, 0x6a, 0x13, + 0xc6, 0x7c, 0x70, 0x3d, 0x2f, 0x21, 0xaf, 0xce, 0x32, 0xad, 0x66, 0xe3, 0x17, 0xf7, 0x3e, 0x4b, + 0xd0, 0x58, 0x39, 0x3c, 0x46, 0x25, 0x47, 0x90, 0x7f, 0xc7, 0xa4, 0xba, 0xea, 0x83, 0xbd, 0x94, + 0x72, 0x74, 0x9c, 0x88, 0x68, 0x75, 0x86, 0x6d, 0x73, 0x8a, 0xbb, 0x26, 0x94, 0xba, 0x92, 0x0c, + 0x80, 0x47, 0x19, 0xa9, 0xe5, 0x70, 0xa6, 0x59, 0xe7, 0x50, 0x52, 0x9c, 0x83, 0xa9, 0x88, 0x7d, + 0x79, 0xbf, 0x66, 0x20, 0xc8, 0x7b, 0x4c, 0x8a, 0x0c, 0x52, 0xc9, 0x4f, 0x5f, 0xc3, 0xd2, 0x0d, + 0x76, 0x45, 0x45, 0xc8, 0x77, 0x4f, 0xba, 0xfb, 0xd5, 0xbf, 0xd0, 0x32, 0x94, 0xcf, 0xf7, 0x77, + 0x7b, 0x27, 0xb8, 0xdf, 0x39, 0xe8, 0xb6, 0xf1, 0x45, 0xd5, 0x41, 0x55, 0x30, 0x32, 0xd3, 0x9b, + 0xe3, 0x93, 0x76, 0xaf, 0x4a, 0x3b, 0xbb, 0x1f, 0xda, 0x2e, 0x93, 0x5e, 0x3c, 0x48, 0x50, 0x5b, + 0x57, 0xcc, 0xf7, 0xd9, 0x95, 0xa4, 0xb6, 0xd7, 0x4a, 0x33, 0x7a, 0xee, 0x30, 0x21, 0x23, 0x36, + 0x88, 0x25, 0x75, 0x5a, 0x2c, 0x90, 0x34, 0x0a, 0x88, 0xdf, 0x52, 0x69, 0xb6, 0xb2, 0x01, 0x84, + 0x83, 0x41, 0x41, 0x19, 0xb6, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0x33, 0xc8, 0x08, 0xe2, 0xaf, + 0x07, 0x00, 0x00, } diff --git a/internal/proxy/grpc_service.go b/internal/proxy/grpc_service.go index 2a0597e7f0..fdf138b8a8 100644 --- a/internal/proxy/grpc_service.go +++ b/internal/proxy/grpc_service.go @@ -4,10 +4,8 @@ import ( "context" "errors" "log" - "strconv" "time" - "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" @@ -122,8 +120,8 @@ func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.Qu qt := &QueryTask{ Condition: NewTaskCondition(ctx), SearchRequest: internalpb.SearchRequest{ - MsgType: internalpb.MsgType_kSearch, - Query: &commonpb.Blob{}, + ProxyID: Params.ProxyID(), + ResultChannelID: Params.ProxyID(), }, queryMsgStream: p.queryMsgStream, resultBuf: make(chan []*internalpb.SearchResult), @@ -131,11 +129,6 @@ func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.Qu } var cancel func() qt.ctx, cancel = context.WithTimeout(ctx, reqTimeoutInterval) - // Hack with test, shit here but no other ways - reqID, _ := strconv.Atoi(req.CollectionName[len(req.CollectionName)-1:]) - qt.ReqID = int64(reqID) - queryBytes, _ := proto.Marshal(req) - qt.SearchRequest.Query.Value = queryBytes log.Printf("grpc address of query task: %p", qt) defer cancel() diff --git a/internal/proxy/paramtable.go b/internal/proxy/paramtable.go index ce36f7c628..0328493730 100644 --- a/internal/proxy/paramtable.go +++ b/internal/proxy/paramtable.go @@ -38,7 +38,12 @@ func (pt *ParamTable) Init() { proxyIDStr := os.Getenv("PROXY_ID") if proxyIDStr == "" { - proxyIDStr = "2" + proxyIDList := pt.ProxyIDList() + if len(proxyIDList) <= 0 { + proxyIDStr = "0" + } else { + proxyIDStr = strconv.Itoa(int(proxyIDList[0])) + } } pt.Save("_proxyID", proxyIDStr) } diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 4ffcaf18e0..9553cdff48 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -37,13 +37,13 @@ var testNum = 10 func startMaster(ctx context.Context) { master.Init() etcdAddr := master.Params.EtcdAddress - rootPath := master.Params.EtcdRootPath + metaRootPath := master.Params.MetaRootPath etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}}) if err != nil { panic(err) } - _, err = etcdCli.Delete(context.TODO(), rootPath, clientv3.WithPrefix()) + _, err = etcdCli.Delete(context.TODO(), metaRootPath, clientv3.WithPrefix()) if err != nil { panic(err) } diff --git a/internal/proxy/task.go b/internal/proxy/task.go index b523813d15..250c2894fe 100644 --- a/internal/proxy/task.go +++ b/internal/proxy/task.go @@ -334,6 +334,17 @@ func (qt *QueryTask) PreExecute() error { return err } } + qt.MsgType = internalpb.MsgType_kSearch + if qt.query.PartitionTags == nil || len(qt.query.PartitionTags) <= 0 { + qt.query.PartitionTags = []string{Params.defaultPartitionTag()} + } + queryBytes, err := proto.Marshal(qt.query) + if err != nil { + return err + } + qt.Query = &commonpb.Blob{ + Value: queryBytes, + } return nil } @@ -341,6 +352,7 @@ func (qt *QueryTask) Execute() error { var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{ SearchRequest: qt.SearchRequest, BaseMsg: msgstream.BaseMsg{ + HashValues: []int32{int32(Params.ProxyID())}, BeginTimestamp: qt.Timestamp, EndTimestamp: qt.Timestamp, }, @@ -351,8 +363,12 @@ func (qt *QueryTask) Execute() error { Msgs: make([]msgstream.TsMsg, 1), } msgPack.Msgs[0] = tsMsg - qt.queryMsgStream.Produce(msgPack) - return nil + err := qt.queryMsgStream.Produce(msgPack) + log.Printf("[Proxy] length of searchMsg: %v", len(msgPack.Msgs)) + if err != nil { + log.Printf("[Proxy] send search request failed: %v", err) + } + return err } func (qt *QueryTask) PostExecute() error { @@ -367,35 +383,41 @@ func (qt *QueryTask) PostExecute() error { qt.result = &servicepb.QueryResult{} return nil } + n := len(searchResults[0].Hits) // n if n <= 0 { qt.result = &servicepb.QueryResult{} return nil } - var hits [][]*servicepb.Hits = make([][]*servicepb.Hits, rlen) - for i, sr := range searchResults { + + hits := make([][]*servicepb.Hits, rlen) + for i, searchResult := range searchResults { hits[i] = make([]*servicepb.Hits, n) - for j, bs := range sr.Hits { + for j, bs := range searchResult.Hits { + hits[i][j] = &servicepb.Hits{} err := proto.Unmarshal(bs, hits[i][j]) if err != nil { return err } } } + k := len(hits[0][0].IDs) - queryResult := &servicepb.QueryResult{ + qt.result = &servicepb.QueryResult{ Status: &commonpb.Status{ ErrorCode: 0, }, + Hits: make([][]byte, 0), } - // reduce by score, TODO: use better algorithm - // use merge-sort here, the number of ways to merge is `rlen` - // in this process, we must make sure: - // len(queryResult.Hits) == n - // len(queryResult.Hits[i].Ids) == k for i in range(n) - for i := 0; i < n; n++ { // n + + for i := 0; i < n; i++ { // n locs := make([]int, rlen) - reducedHits := &servicepb.Hits{} + reducedHits := &servicepb.Hits{ + IDs: make([]int64, 0), + RowData: make([][]byte, 0), + Scores: make([]float32, 0), + } + for j := 0; j < k; j++ { // k choice, minDistance := 0, float32(math.MaxFloat32) for q, loc := range locs { // query num, the number of ways to merge @@ -407,7 +429,9 @@ func (qt *QueryTask) PostExecute() error { } choiceOffset := locs[choice] reducedHits.IDs = append(reducedHits.IDs, hits[choice][i].IDs[choiceOffset]) - reducedHits.RowData = append(reducedHits.RowData, hits[choice][i].RowData[choiceOffset]) + if hits[choice][i].RowData != nil && len(hits[choice][i].RowData) > 0 { + reducedHits.RowData = append(reducedHits.RowData, hits[choice][i].RowData[choiceOffset]) + } reducedHits.Scores = append(reducedHits.Scores, hits[choice][i].Scores[choiceOffset]) locs[choice]++ } @@ -415,12 +439,11 @@ func (qt *QueryTask) PostExecute() error { if err != nil { return err } - queryResult.Hits = append(queryResult.Hits, reducedHitsBs) + qt.result.Hits = append(qt.result.Hits, reducedHitsBs) } - qt.result = queryResult + return nil } } - //return nil } type HasCollectionTask struct { diff --git a/internal/proxy/task_scheduler.go b/internal/proxy/task_scheduler.go index f08d4672ff..b3a23272a8 100644 --- a/internal/proxy/task_scheduler.go +++ b/internal/proxy/task_scheduler.go @@ -293,7 +293,7 @@ func (sched *TaskScheduler) processTask(t task, q TaskQueue) { } q.AddActiveTask(t) - log.Printf("query task add to active list ...") + log.Printf("task add to active list ...") defer func() { q.PopActiveTask(t.EndTs()) log.Printf("pop from active list ...") @@ -304,8 +304,9 @@ func (sched *TaskScheduler) processTask(t task, q TaskQueue) { log.Printf("execute definition task failed, error = %v", err) return } - log.Printf("scheduler task done ...") + log.Printf("task execution done ...") err = t.PostExecute() + log.Printf("post execute task done ...") } func (sched *TaskScheduler) definitionLoop() { @@ -437,6 +438,6 @@ func (sched *TaskScheduler) Close() { func (sched *TaskScheduler) TaskDoneTest(ts Timestamp) bool { ddTaskDone := sched.DdQueue.TaskDoneTest(ts) dmTaskDone := sched.DmQueue.TaskDoneTest(ts) - dqTaskDone := sched.DqQueue.TaskDoneTest(ts) - return ddTaskDone && dmTaskDone && dqTaskDone + //dqTaskDone := sched.DqQueue.TaskDoneTest(ts) + return ddTaskDone && dmTaskDone && true } diff --git a/internal/proxy/timetick.go b/internal/proxy/timetick.go index 721c3a276c..6a4436dd1e 100644 --- a/internal/proxy/timetick.go +++ b/internal/proxy/timetick.go @@ -43,7 +43,7 @@ func newTimeTick(ctx context.Context, cancel: cancel, tsoAllocator: tsoAllocator, interval: interval, - peerID: 1, + peerID: Params.ProxyID(), checkFunc: checkFunc, } @@ -56,7 +56,6 @@ func newTimeTick(ctx context.Context, } func (tt *timeTick) tick() error { - if tt.lastTick == tt.currentTick { ts, err := tt.tsoAllocator.AllocOne() if err != nil { @@ -84,7 +83,7 @@ func (tt *timeTick) tick() error { if err != nil { log.Printf("proxy send time tick error: %v", err) } else { - log.Printf("proxy send time tick message") + //log.Printf("proxy send time tick message") } tt.lastTick = tt.currentTick return nil diff --git a/internal/querynode/collection_replica_test.go b/internal/querynode/collection_replica_test.go index b133aa642a..31d26238cb 100644 --- a/internal/querynode/collection_replica_test.go +++ b/internal/querynode/collection_replica_test.go @@ -19,8 +19,9 @@ func TestCollectionReplica_getCollectionNum(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -30,8 +31,9 @@ func TestCollectionReplica_getCollectionNum(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -41,7 +43,8 @@ func TestCollectionReplica_getCollectionNum(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -72,8 +75,9 @@ func TestCollectionReplica_addCollection(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -83,8 +87,9 @@ func TestCollectionReplica_addCollection(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -94,7 +99,8 @@ func TestCollectionReplica_addCollection(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -130,8 +136,9 @@ func TestCollectionReplica_removeCollection(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -141,8 +148,9 @@ func TestCollectionReplica_removeCollection(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -152,7 +160,8 @@ func TestCollectionReplica_removeCollection(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -192,8 +201,9 @@ func TestCollectionReplica_getCollectionByID(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -203,8 +213,9 @@ func TestCollectionReplica_getCollectionByID(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -214,7 +225,8 @@ func TestCollectionReplica_getCollectionByID(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -256,8 +268,9 @@ func TestCollectionReplica_getCollectionByName(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -267,8 +280,9 @@ func TestCollectionReplica_getCollectionByName(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -278,7 +292,8 @@ func TestCollectionReplica_getCollectionByName(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -320,8 +335,9 @@ func TestCollectionReplica_hasCollection(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -331,8 +347,9 @@ func TestCollectionReplica_hasCollection(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -342,7 +359,8 @@ func TestCollectionReplica_hasCollection(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -378,8 +396,9 @@ func TestCollectionReplica_getPartitionNum(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -389,8 +408,9 @@ func TestCollectionReplica_getPartitionNum(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -400,7 +420,8 @@ func TestCollectionReplica_getPartitionNum(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -449,8 +470,9 @@ func TestCollectionReplica_addPartition(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -460,8 +482,9 @@ func TestCollectionReplica_addPartition(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -471,7 +494,8 @@ func TestCollectionReplica_addPartition(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -517,8 +541,9 @@ func TestCollectionReplica_removePartition(t *testing.T) { collectionID := UniqueID(0) partitionTag := "default" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -528,8 +553,9 @@ func TestCollectionReplica_removePartition(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -539,7 +565,8 @@ func TestCollectionReplica_removePartition(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -586,8 +613,9 @@ func TestCollectionReplica_addPartitionsByCollectionMeta(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -597,8 +625,9 @@ func TestCollectionReplica_addPartitionsByCollectionMeta(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -608,7 +637,8 @@ func TestCollectionReplica_addPartitionsByCollectionMeta(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -659,8 +689,9 @@ func TestCollectionReplica_removePartitionsByCollectionMeta(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -670,8 +701,9 @@ func TestCollectionReplica_removePartitionsByCollectionMeta(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -681,7 +713,8 @@ func TestCollectionReplica_removePartitionsByCollectionMeta(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -732,8 +765,9 @@ func TestCollectionReplica_getPartitionByTag(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -743,8 +777,9 @@ func TestCollectionReplica_getPartitionByTag(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -754,7 +789,8 @@ func TestCollectionReplica_getPartitionByTag(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -800,8 +836,9 @@ func TestCollectionReplica_hasPartition(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -811,8 +848,9 @@ func TestCollectionReplica_hasPartition(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -822,7 +860,8 @@ func TestCollectionReplica_hasPartition(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -867,8 +906,9 @@ func TestCollectionReplica_addSegment(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -878,8 +918,9 @@ func TestCollectionReplica_addSegment(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -889,7 +930,8 @@ func TestCollectionReplica_addSegment(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -938,8 +980,9 @@ func TestCollectionReplica_removeSegment(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -949,8 +992,9 @@ func TestCollectionReplica_removeSegment(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -960,7 +1004,8 @@ func TestCollectionReplica_removeSegment(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -1011,8 +1056,9 @@ func TestCollectionReplica_getSegmentByID(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -1022,8 +1068,9 @@ func TestCollectionReplica_getSegmentByID(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -1033,7 +1080,8 @@ func TestCollectionReplica_getSegmentByID(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -1082,8 +1130,9 @@ func TestCollectionReplica_hasSegment(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -1093,8 +1142,9 @@ func TestCollectionReplica_hasSegment(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -1104,7 +1154,8 @@ func TestCollectionReplica_hasSegment(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -1157,8 +1208,9 @@ func TestCollectionReplica_freeAll(t *testing.T) { collectionName := "collection0" collectionID := UniqueID(0) fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -1168,8 +1220,9 @@ func TestCollectionReplica_freeAll(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -1179,7 +1232,8 @@ func TestCollectionReplica_freeAll(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/collection_test.go b/internal/querynode/collection_test.go index 5fa8cee1fa..fdce3e7565 100644 --- a/internal/querynode/collection_test.go +++ b/internal/querynode/collection_test.go @@ -17,8 +17,9 @@ func TestCollection_Partitions(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -28,8 +29,9 @@ func TestCollection_Partitions(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -39,7 +41,8 @@ func TestCollection_Partitions(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -77,8 +80,9 @@ func TestCollection_Partitions(t *testing.T) { func TestCollection_newCollection(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -88,8 +92,9 @@ func TestCollection_newCollection(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -99,7 +104,8 @@ func TestCollection_newCollection(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -123,8 +129,9 @@ func TestCollection_newCollection(t *testing.T) { func TestCollection_deleteCollection(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -134,8 +141,9 @@ func TestCollection_deleteCollection(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -145,7 +153,8 @@ func TestCollection_deleteCollection(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/data_sync_service_test.go b/internal/querynode/data_sync_service_test.go index 6661a1d1ac..e833c5e08a 100644 --- a/internal/querynode/data_sync_service_test.go +++ b/internal/querynode/data_sync_service_test.go @@ -38,8 +38,9 @@ func TestDataSyncService_Start(t *testing.T) { // init meta collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -49,8 +50,9 @@ func TestDataSyncService_Start(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -60,7 +62,8 @@ func TestDataSyncService_Start(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/flow_graph_service_time_node.go b/internal/querynode/flow_graph_service_time_node.go index fe0d26b151..e7dfe1a89c 100644 --- a/internal/querynode/flow_graph_service_time_node.go +++ b/internal/querynode/flow_graph_service_time_node.go @@ -14,7 +14,7 @@ func (stNode *serviceTimeNode) Name() string { } func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg { - // fmt.Println("Do serviceTimeNode operation") + //fmt.Println("Do serviceTimeNode operation") if len(in) != 1 { log.Println("Invalid operate message input in serviceTimeNode, input length = ", len(in)) @@ -29,6 +29,7 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg { // update service time (*(*stNode.replica).getTSafe()).set(serviceTimeMsg.timeRange.timestampMax) + //fmt.Println("update tSafe to:", getPhysicalTime(serviceTimeMsg.timeRange.timestampMax)) return nil } diff --git a/internal/querynode/meta_service.go b/internal/querynode/meta_service.go index f95ec6776d..1a51a31891 100644 --- a/internal/querynode/meta_service.go +++ b/internal/querynode/meta_service.go @@ -31,7 +31,7 @@ type metaService struct { func newMetaService(ctx context.Context, replica *collectionReplica) *metaService { ETCDAddr := Params.etcdAddress() - ETCDRootPath := Params.etcdRootPath() + MetaRootPath := Params.metaRootPath() cli, _ := clientv3.New(clientv3.Config{ Endpoints: []string{ETCDAddr}, @@ -40,7 +40,7 @@ func newMetaService(ctx context.Context, replica *collectionReplica) *metaServic return &metaService{ ctx: ctx, - kvBase: kv.NewEtcdKV(cli, ETCDRootPath), + kvBase: kv.NewEtcdKV(cli, MetaRootPath), replica: replica, } } @@ -71,21 +71,21 @@ func (mService *metaService) start() { } func GetCollectionObjID(key string) string { - ETCDRootPath := Params.etcdRootPath() + ETCDRootPath := Params.metaRootPath() prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/" return strings.TrimPrefix(key, prefix) } func GetSegmentObjID(key string) string { - ETCDRootPath := Params.etcdRootPath() + ETCDRootPath := Params.metaRootPath() prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/" return strings.TrimPrefix(key, prefix) } func isCollectionObj(key string) bool { - ETCDRootPath := Params.etcdRootPath() + ETCDRootPath := Params.metaRootPath() prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/" prefix = strings.TrimSpace(prefix) @@ -95,7 +95,7 @@ func isCollectionObj(key string) bool { } func isSegmentObj(key string) bool { - ETCDRootPath := Params.etcdRootPath() + ETCDRootPath := Params.metaRootPath() prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/" prefix = strings.TrimSpace(prefix) diff --git a/internal/querynode/meta_service_test.go b/internal/querynode/meta_service_test.go index 66c0c8e535..bdc2f8835c 100644 --- a/internal/querynode/meta_service_test.go +++ b/internal/querynode/meta_service_test.go @@ -64,24 +64,24 @@ func TestMetaService_getSegmentObjId(t *testing.T) { } func TestMetaService_isCollectionObj(t *testing.T) { - var key = "by-dev/collection/collection0" + var key = "by-dev/meta/collection/collection0" var b1 = isCollectionObj(key) assert.Equal(t, b1, true) - key = "by-dev/segment/segment0" + key = "by-dev/meta/segment/segment0" var b2 = isCollectionObj(key) assert.Equal(t, b2, false) } func TestMetaService_isSegmentObj(t *testing.T) { - var key = "by-dev/segment/segment0" + var key = "by-dev/meta/segment/segment0" var b1 = isSegmentObj(key) assert.Equal(t, b1, true) - key = "by-dev/collection/collection0" + key = "by-dev/meta/collection/collection0" var b2 = isSegmentObj(key) assert.Equal(t, b2, false) @@ -120,8 +120,9 @@ func TestMetaService_isSegmentChannelRangeInQueryNodeChannelRange(t *testing.T) func TestMetaService_printCollectionStruct(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -131,8 +132,9 @@ func TestMetaService_printCollectionStruct(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -142,7 +144,8 @@ func TestMetaService_printCollectionStruct(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -228,8 +231,9 @@ func TestMetaService_processSegmentCreate(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -239,8 +243,9 @@ func TestMetaService_processSegmentCreate(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -250,7 +255,8 @@ func TestMetaService_processSegmentCreate(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -295,7 +301,7 @@ func TestMetaService_processCreate(t *testing.T) { node := NewQueryNode(ctx, 0) node.metaService = newMetaService(ctx, node.replica) - key1 := "by-dev/collection/0" + key1 := "by-dev/meta/collection/0" msg1 := `schema: < name: "test" fields: < @@ -327,7 +333,7 @@ func TestMetaService_processCreate(t *testing.T) { assert.NoError(t, err) assert.Equal(t, collection.ID(), UniqueID(0)) - key2 := "by-dev/segment/0" + key2 := "by-dev/meta/segment/0" msg2 := `partition_tag: "default" channel_start: 0 channel_end: 1 @@ -351,8 +357,9 @@ func TestMetaService_processSegmentModify(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -362,8 +369,9 @@ func TestMetaService_processSegmentModify(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -373,7 +381,8 @@ func TestMetaService_processSegmentModify(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -529,7 +538,7 @@ func TestMetaService_processModify(t *testing.T) { node := NewQueryNode(ctx, 0) node.metaService = newMetaService(ctx, node.replica) - key1 := "by-dev/collection/0" + key1 := "by-dev/meta/collection/0" msg1 := `schema: < name: "test" fields: < @@ -576,7 +585,7 @@ func TestMetaService_processModify(t *testing.T) { hasPartition = (*node.replica).hasPartition(UniqueID(0), "p3") assert.Equal(t, hasPartition, false) - key2 := "by-dev/segment/0" + key2 := "by-dev/meta/segment/0" msg2 := `partition_tag: "p1" channel_start: 0 channel_end: 1 @@ -656,8 +665,9 @@ func TestMetaService_processSegmentDelete(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -667,8 +677,9 @@ func TestMetaService_processSegmentDelete(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -678,7 +689,8 @@ func TestMetaService_processSegmentDelete(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -772,7 +784,7 @@ func TestMetaService_processDelete(t *testing.T) { node := NewQueryNode(ctx, 0) node.metaService = newMetaService(ctx, node.replica) - key1 := "by-dev/collection/0" + key1 := "by-dev/meta/collection/0" msg1 := `schema: < name: "test" fields: < @@ -804,7 +816,7 @@ func TestMetaService_processDelete(t *testing.T) { assert.NoError(t, err) assert.Equal(t, collection.ID(), UniqueID(0)) - key2 := "by-dev/segment/0" + key2 := "by-dev/meta/segment/0" msg2 := `partition_tag: "default" channel_start: 0 channel_end: 1 diff --git a/internal/querynode/param_table.go b/internal/querynode/param_table.go index be8bef8b65..026d116149 100644 --- a/internal/querynode/param_table.go +++ b/internal/querynode/param_table.go @@ -199,12 +199,16 @@ func (p *ParamTable) etcdAddress() string { return etcdAddress } -func (p *ParamTable) etcdRootPath() string { - etcdRootPath, err := p.Load("etcd.rootpath") +func (p *ParamTable) metaRootPath() string { + rootPath, err := p.Load("etcd.rootPath") if err != nil { panic(err) } - return etcdRootPath + subPath, err := p.Load("etcd.metaSubPath") + if err != nil { + panic(err) + } + return rootPath + "/" + subPath } func (p *ParamTable) gracefulTime() int64 { diff --git a/internal/querynode/param_table_test.go b/internal/querynode/param_table_test.go index e9405f8a84..544659134d 100644 --- a/internal/querynode/param_table_test.go +++ b/internal/querynode/param_table_test.go @@ -120,3 +120,9 @@ func TestParamTable_statsChannelName(t *testing.T) { name := Params.statsChannelName() assert.Equal(t, name, "query-node-stats") } + +func TestParamTable_metaRootPath(t *testing.T) { + Params.Init() + path := Params.metaRootPath() + assert.Equal(t, path, "by-dev/meta") +} diff --git a/internal/querynode/partition_test.go b/internal/querynode/partition_test.go index fe7484a8f8..f9aa7d324c 100644 --- a/internal/querynode/partition_test.go +++ b/internal/querynode/partition_test.go @@ -17,8 +17,9 @@ func TestPartition_Segments(t *testing.T) { collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -28,8 +29,9 @@ func TestPartition_Segments(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -39,7 +41,8 @@ func TestPartition_Segments(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/plan_test.go b/internal/querynode/plan_test.go index a01ce7558c..714f7edbe2 100644 --- a/internal/querynode/plan_test.go +++ b/internal/querynode/plan_test.go @@ -16,8 +16,9 @@ import ( func TestPlan_Plan(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -27,8 +28,9 @@ func TestPlan_Plan(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -38,7 +40,8 @@ func TestPlan_Plan(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -71,8 +74,9 @@ func TestPlan_Plan(t *testing.T) { func TestPlan_PlaceholderGroup(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -82,8 +86,9 @@ func TestPlan_PlaceholderGroup(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -93,7 +98,8 @@ func TestPlan_PlaceholderGroup(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/reduce_test.go b/internal/querynode/reduce_test.go index b66a11a551..241a3001bb 100644 --- a/internal/querynode/reduce_test.go +++ b/internal/querynode/reduce_test.go @@ -17,8 +17,9 @@ import ( func TestReduce_AllFunc(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -28,8 +29,9 @@ func TestReduce_AllFunc(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -39,7 +41,8 @@ func TestReduce_AllFunc(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/search_service.go b/internal/querynode/search_service.go index 589e3567d6..9ecf3d78c2 100644 --- a/internal/querynode/search_service.go +++ b/internal/querynode/search_service.go @@ -4,11 +4,10 @@ import "C" import ( "context" "errors" + "github.com/golang/protobuf/proto" "log" "sync" - "github.com/golang/protobuf/proto" - "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" @@ -146,7 +145,7 @@ func (ss *searchService) receiveSearchMsg() { log.Println("publish FailedSearchResult failed, error message: ", err) } } - log.Println("Do search done, num of searchMsg = ", len(searchMsg)) + log.Println("ReceiveSearchMsg, do search done, num of searchMsg = ", len(searchMsg)) } } } @@ -173,17 +172,16 @@ func (ss *searchService) doUnsolvedMsgSearch() { } for { + msgBufferLength := len(ss.msgBuffer) + if msgBufferLength <= 0 { + break + } msg := <-ss.msgBuffer if msg.EndTs() <= serviceTime { searchMsg = append(searchMsg, msg) continue } ss.unsolvedMsg = append(ss.unsolvedMsg, msg) - - msgBufferLength := len(ss.msgBuffer) - if msgBufferLength <= 0 { - break - } } if len(searchMsg) <= 0 { @@ -199,7 +197,7 @@ func (ss *searchService) doUnsolvedMsgSearch() { log.Println("publish FailedSearchResult failed, error message: ", err) } } - log.Println("Do search done, num of searchMsg = ", len(searchMsg)) + log.Println("doUnsolvedMsgSearch, do search done, num of searchMsg = ", len(searchMsg)) } } } @@ -241,7 +239,10 @@ func (ss *searchService) search(msg msgstream.TsMsg) error { return err } for _, segment := range partition.segments { + //fmt.Println("dsl = ", dsl) + searchResult, err := segment.segmentSearch(plan, placeholderGroups, []Timestamp{searchTimestamp}) + if err != nil { return err } @@ -249,6 +250,15 @@ func (ss *searchService) search(msg msgstream.TsMsg) error { } } + if len(searchResults) <= 0 { + log.Println("search Failed, invalid partitionTag") + err = ss.publishFailedSearchResult(msg) + if err != nil { + log.Println("publish FailedSearchResult failed, error message: ", err) + } + return err + } + reducedSearchResult := reduceSearchResults(searchResults, int64(len(searchResults))) marshaledHits := reducedSearchResult.reorganizeQueryResults(plan, placeholderGroups) hitsBlob, err := marshaledHits.getHitsBlob() diff --git a/internal/querynode/search_service_test.go b/internal/querynode/search_service_test.go index 86f02fdc4c..422cef945e 100644 --- a/internal/querynode/search_service_test.go +++ b/internal/querynode/search_service_test.go @@ -30,8 +30,9 @@ func TestSearch_Search(t *testing.T) { // init meta collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -41,8 +42,9 @@ func TestSearch_Search(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -52,7 +54,8 @@ func TestSearch_Search(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/segment_test.go b/internal/querynode/segment_test.go index ef7f22f23e..a9100ca21b 100644 --- a/internal/querynode/segment_test.go +++ b/internal/querynode/segment_test.go @@ -20,8 +20,9 @@ import ( //-------------------------------------------------------------------------------------- constructor and destructor func TestSegment_newSegment(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -31,8 +32,9 @@ func TestSegment_newSegment(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -42,7 +44,8 @@ func TestSegment_newSegment(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -72,8 +75,9 @@ func TestSegment_newSegment(t *testing.T) { func TestSegment_deleteSegment(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -83,8 +87,9 @@ func TestSegment_deleteSegment(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -94,7 +99,8 @@ func TestSegment_deleteSegment(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -126,8 +132,9 @@ func TestSegment_deleteSegment(t *testing.T) { //-------------------------------------------------------------------------------------- stats functions func TestSegment_getRowCount(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -137,8 +144,9 @@ func TestSegment_getRowCount(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -148,7 +156,8 @@ func TestSegment_getRowCount(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -211,8 +220,9 @@ func TestSegment_getRowCount(t *testing.T) { func TestSegment_getDeletedCount(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -222,8 +232,9 @@ func TestSegment_getDeletedCount(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -233,7 +244,8 @@ func TestSegment_getDeletedCount(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -302,8 +314,9 @@ func TestSegment_getDeletedCount(t *testing.T) { func TestSegment_getMemSize(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -313,8 +326,9 @@ func TestSegment_getMemSize(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -324,7 +338,8 @@ func TestSegment_getMemSize(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -388,8 +403,9 @@ func TestSegment_getMemSize(t *testing.T) { //-------------------------------------------------------------------------------------- dm & search functions func TestSegment_segmentInsert(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -399,8 +415,9 @@ func TestSegment_segmentInsert(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -410,7 +427,8 @@ func TestSegment_segmentInsert(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -469,8 +487,9 @@ func TestSegment_segmentInsert(t *testing.T) { func TestSegment_segmentDelete(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -480,8 +499,9 @@ func TestSegment_segmentDelete(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -491,7 +511,8 @@ func TestSegment_segmentDelete(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -559,8 +580,9 @@ func TestSegment_segmentSearch(t *testing.T) { defer cancel() fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -570,8 +592,9 @@ func TestSegment_segmentSearch(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -581,7 +604,8 @@ func TestSegment_segmentSearch(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -683,8 +707,9 @@ func TestSegment_segmentSearch(t *testing.T) { //-------------------------------------------------------------------------------------- preDm functions func TestSegment_segmentPreInsert(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -694,8 +719,9 @@ func TestSegment_segmentPreInsert(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -705,7 +731,8 @@ func TestSegment_segmentPreInsert(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -759,8 +786,9 @@ func TestSegment_segmentPreInsert(t *testing.T) { func TestSegment_segmentPreDelete(t *testing.T) { fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -770,8 +798,9 @@ func TestSegment_segmentPreDelete(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -781,7 +810,8 @@ func TestSegment_segmentPreDelete(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: "collection0", + Name: "collection0", + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/stats_service_test.go b/internal/querynode/stats_service_test.go index 523e589148..68347928a6 100644 --- a/internal/querynode/stats_service_test.go +++ b/internal/querynode/stats_service_test.go @@ -33,8 +33,8 @@ func TestStatsService_start(t *testing.T) { // init meta collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -44,8 +44,9 @@ func TestStatsService_start(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -55,7 +56,8 @@ func TestStatsService_start(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, @@ -114,8 +116,9 @@ func TestSegmentManagement_SegmentStatisticService(t *testing.T) { // init meta collectionName := "collection0" fieldVec := schemapb.FieldSchema{ - Name: "vec", - DataType: schemapb.DataType_VECTOR_FLOAT, + Name: "vec", + IsPrimaryKey: false, + DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -125,8 +128,9 @@ func TestSegmentManagement_SegmentStatisticService(t *testing.T) { } fieldInt := schemapb.FieldSchema{ - Name: "age", - DataType: schemapb.DataType_INT32, + Name: "age", + IsPrimaryKey: false, + DataType: schemapb.DataType_INT32, TypeParams: []*commonpb.KeyValuePair{ { Key: "dim", @@ -136,7 +140,8 @@ func TestSegmentManagement_SegmentStatisticService(t *testing.T) { } schema := schemapb.CollectionSchema{ - Name: collectionName, + Name: collectionName, + AutoID: true, Fields: []*schemapb.FieldSchema{ &fieldVec, &fieldInt, }, diff --git a/internal/querynode/tsafe.go b/internal/querynode/tsafe.go index 056c47b811..bbafc10e46 100644 --- a/internal/querynode/tsafe.go +++ b/internal/querynode/tsafe.go @@ -10,7 +10,7 @@ type tSafeWatcher struct { func newTSafeWatcher() *tSafeWatcher { return &tSafeWatcher{ - notifyChan: make(chan bool), + notifyChan: make(chan bool, 1), } } diff --git a/internal/util/paramtable/paramtable.go b/internal/util/paramtable/paramtable.go index b8334c27f3..04b24bbf84 100644 --- a/internal/util/paramtable/paramtable.go +++ b/internal/util/paramtable/paramtable.go @@ -82,17 +82,8 @@ func (gp *BaseTable) LoadRange(key, endKey string, limit int) ([]string, []strin func (gp *BaseTable) LoadYaml(fileName string) error { config := viper.New() _, fpath, _, _ := runtime.Caller(0) - configFile := path.Dir(fpath) + "/../../../configs/" + fileName - _, err := os.Stat(configFile) - if os.IsNotExist(err) { - runPath, err := os.Getwd() - if err != nil { - panic(err) - } - configFile = runPath + "/configs/" + fileName - } - - config.SetConfigFile(configFile) + configPath := path.Dir(fpath) + "/../../../configs/" + config.SetConfigFile(configPath + fileName) if err := config.ReadInConfig(); err != nil { panic(err) }