milvus/internal/proto/query_coord.proto
yihao.dai e49df11942
Fix QueryNode is not able to recovery (#24335)
Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
2023-05-24 14:29:29 +08:00

616 lines
17 KiB
Protocol Buffer

syntax = "proto3";
package milvus.proto.query;
option go_package = "github.com/milvus-io/milvus/internal/proto/querypb";
import "common.proto";
import "milvus.proto";
import "internal.proto";
import "schema.proto";
import "msg.proto";
import "data_coord.proto";
import "index_coord.proto";
service QueryCoord {
rpc GetComponentStates(milvus.GetComponentStatesRequest) returns (milvus.ComponentStates) {}
rpc GetTimeTickChannel(internal.GetTimeTickChannelRequest) returns(milvus.StringResponse) {}
rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}
rpc ShowCollections(ShowCollectionsRequest) returns (ShowCollectionsResponse) {}
rpc ShowPartitions(ShowPartitionsRequest) returns (ShowPartitionsResponse) {}
rpc LoadPartitions(LoadPartitionsRequest) returns (common.Status) {}
rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {}
rpc LoadCollection(LoadCollectionRequest) returns (common.Status) {}
rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {}
rpc SyncNewCreatedPartition(SyncNewCreatedPartitionRequest) returns (common.Status) {}
rpc GetPartitionStates(GetPartitionStatesRequest) returns (GetPartitionStatesResponse) {}
rpc GetSegmentInfo(GetSegmentInfoRequest) returns (GetSegmentInfoResponse) {}
rpc LoadBalance(LoadBalanceRequest) returns (common.Status) {}
rpc ShowConfigurations(internal.ShowConfigurationsRequest) returns (internal.ShowConfigurationsResponse){}
// https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}
// https://wiki.lfaidata.foundation/display/MIL/MEP+23+--+Multiple+memory+replication+design
rpc GetReplicas(milvus.GetReplicasRequest) returns (milvus.GetReplicasResponse) {}
rpc GetShardLeaders(GetShardLeadersRequest) returns (GetShardLeadersResponse) {}
rpc CheckHealth(milvus.CheckHealthRequest) returns (milvus.CheckHealthResponse) {}
rpc CreateResourceGroup(milvus.CreateResourceGroupRequest) returns (common.Status) {}
rpc DropResourceGroup(milvus.DropResourceGroupRequest) returns (common.Status) {}
rpc TransferNode(milvus.TransferNodeRequest) returns (common.Status) {}
rpc TransferReplica(TransferReplicaRequest) returns (common.Status) {}
rpc ListResourceGroups(milvus.ListResourceGroupsRequest) returns (milvus.ListResourceGroupsResponse) {}
rpc DescribeResourceGroup(DescribeResourceGroupRequest) returns (DescribeResourceGroupResponse) {}
}
service QueryNode {
rpc GetComponentStates(milvus.GetComponentStatesRequest) returns (milvus.ComponentStates) {}
rpc GetTimeTickChannel(internal.GetTimeTickChannelRequest) returns(milvus.StringResponse) {}
rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}
rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {}
rpc UnsubDmChannel(UnsubDmChannelRequest) returns (common.Status) {}
rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {}
rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {}
rpc LoadPartitions(LoadPartitionsRequest) returns (common.Status) {}
rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {}
rpc ReleaseSegments(ReleaseSegmentsRequest) returns (common.Status) {}
rpc GetSegmentInfo(GetSegmentInfoRequest) returns (GetSegmentInfoResponse) {}
rpc SyncReplicaSegments(SyncReplicaSegmentsRequest) returns (common.Status) {}
rpc GetStatistics(GetStatisticsRequest) returns (internal.GetStatisticsResponse) {}
rpc Search(SearchRequest) returns (internal.SearchResults) {}
rpc SearchSegments(SearchRequest) returns (internal.SearchResults) {}
rpc Query(QueryRequest) returns (internal.RetrieveResults) {}
rpc QuerySegments(QueryRequest) returns (internal.RetrieveResults) {}
rpc ShowConfigurations(internal.ShowConfigurationsRequest) returns (internal.ShowConfigurationsResponse){}
// https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}
rpc GetDataDistribution(GetDataDistributionRequest) returns (GetDataDistributionResponse) {}
rpc SyncDistribution(SyncDistributionRequest) returns (common.Status) {}
rpc Delete(DeleteRequest) returns (common.Status) {}
}
//--------------------QueryCoord grpc request and response proto------------------
message ShowCollectionsRequest {
common.MsgBase base = 1;
// Not useful for now
int64 dbID = 2;
repeated int64 collectionIDs = 3;
}
message ShowCollectionsResponse {
common.Status status = 1;
repeated int64 collectionIDs = 2;
repeated int64 inMemory_percentages = 3;
repeated bool query_service_available = 4;
repeated int64 refresh_progress = 5;
}
message ShowPartitionsRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
repeated int64 partitionIDs = 4;
}
message ShowPartitionsResponse {
common.Status status = 1;
repeated int64 partitionIDs = 2;
repeated int64 inMemory_percentages = 3;
repeated int64 refresh_progress = 4;
}
message LoadCollectionRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
schema.CollectionSchema schema = 4;
int32 replica_number = 5;
// fieldID -> indexID
map<int64, int64> field_indexID = 6;
bool refresh = 7;
// resource group names
repeated string resource_groups = 8;
}
message ReleaseCollectionRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
int64 nodeID = 4;
}
message GetStatisticsRequest {
internal.GetStatisticsRequest req = 1;
repeated string dml_channels = 2;
repeated int64 segmentIDs = 3;
bool from_shard_leader = 4;
DataScope scope = 5; // All, Streaming, Historical
}
message LoadPartitionsRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
repeated int64 partitionIDs = 4;
schema.CollectionSchema schema = 5;
int32 replica_number = 6;
// fieldID -> indexID
map<int64, int64> field_indexID = 7;
bool refresh = 8;
// resource group names
repeated string resource_groups = 9;
repeated index.IndexInfo index_info_list = 10;
}
message ReleasePartitionsRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
repeated int64 partitionIDs = 4;
int64 nodeID = 5;
}
message GetPartitionStatesRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
repeated int64 partitionIDs = 4;
}
message GetPartitionStatesResponse {
common.Status status = 1;
repeated PartitionStates partition_descriptions = 2;
}
message GetSegmentInfoRequest {
common.MsgBase base = 1;
repeated int64 segmentIDs = 2; // deprecated
int64 collectionID = 3;
}
message GetSegmentInfoResponse {
common.Status status = 1;
repeated SegmentInfo infos = 2;
}
message GetShardLeadersRequest {
common.MsgBase base = 1;
int64 collectionID = 2;
}
message GetShardLeadersResponse {
common.Status status = 1;
repeated ShardLeadersList shards = 2;
}
message ShardLeadersList { // All leaders of all replicas of one shard
string channel_name = 1;
repeated int64 node_ids = 2;
repeated string node_addrs = 3;
}
message SyncNewCreatedPartitionRequest {
common.MsgBase base = 1;
int64 collectionID = 2;
int64 partitionID = 3;
}
//-----------------query node grpc request and response proto----------------
message LoadMetaInfo {
LoadType load_type = 1;
int64 collectionID = 2;
repeated int64 partitionIDs = 3;
}
message WatchDmChannelsRequest {
common.MsgBase base = 1;
int64 nodeID = 2;
int64 collectionID = 3;
repeated int64 partitionIDs = 4;
repeated data.VchannelInfo infos = 5;
schema.CollectionSchema schema = 6;
repeated data.SegmentInfo exclude_infos = 7;
LoadMetaInfo load_meta = 8;
int64 replicaID = 9;
map<int64, data.SegmentInfo> segment_infos = 10;
// Deprecated
// for node down load balance, need to remove offline node in time after every watchDmChannel finish.
int64 offlineNodeID = 11;
int64 version = 12;
repeated index.IndexInfo index_info_list = 13;
}
message UnsubDmChannelRequest {
common.MsgBase base = 1;
int64 nodeID = 2;
int64 collectionID = 3;
string channel_name = 4;
}
message SegmentLoadInfo {
int64 segmentID = 1;
int64 partitionID = 2;
int64 collectionID = 3;
int64 dbID = 4;
int64 flush_time = 5;
repeated data.FieldBinlog binlog_paths = 6;
int64 num_of_rows = 7;
repeated data.FieldBinlog statslogs = 8;
repeated data.FieldBinlog deltalogs = 9;
repeated int64 compactionFrom = 10; // segmentIDs compacted from
repeated FieldIndexInfo index_infos = 11;
int64 segment_size = 12;
string insert_channel = 13;
msg.MsgPosition start_position = 14;
msg.MsgPosition delta_position = 15;
}
message FieldIndexInfo {
int64 fieldID =1;
// deprecated
bool enable_index = 2;
string index_name = 3;
int64 indexID = 4;
int64 buildID = 5;
repeated common.KeyValuePair index_params = 6;
repeated string index_file_paths = 7;
int64 index_size = 8;
int64 index_version = 9;
int64 num_rows = 10;
}
enum LoadScope {
Full = 0;
Delta = 1;
}
message LoadSegmentsRequest {
common.MsgBase base = 1;
int64 dst_nodeID = 2;
repeated SegmentLoadInfo infos = 3;
schema.CollectionSchema schema = 4;
int64 source_nodeID = 5;
int64 collectionID = 6;
LoadMetaInfo load_meta = 7;
int64 replicaID = 8;
repeated msg.MsgPosition delta_positions = 9; // keep it for compatibility of rolling upgrade from 2.2.x to 2.3
int64 version = 10;
bool need_transfer = 11;
LoadScope load_scope = 12;
}
message ReleaseSegmentsRequest {
common.MsgBase base = 1;
int64 nodeID = 2;
// Not useful for now
int64 dbID = 3;
int64 collectionID = 4;
repeated int64 partitionIDs = 5;
repeated int64 segmentIDs = 6;
DataScope scope = 7; // All, Streaming, Historical
string shard = 8;
bool need_transfer = 11;
}
message SearchRequest {
internal.SearchRequest req = 1;
repeated string dml_channels = 2;
repeated int64 segmentIDs = 3;
bool from_shard_leader = 4;
DataScope scope = 5; // All, Streaming, Historical
int32 total_channel_num = 6;
}
message QueryRequest {
internal.RetrieveRequest req = 1;
repeated string dml_channels = 2;
repeated int64 segmentIDs = 3;
bool from_shard_leader = 4;
DataScope scope = 5; // All, Streaming, Historical
}
message SyncReplicaSegmentsRequest {
common.MsgBase base = 1;
string vchannel_name = 2;
repeated ReplicaSegmentsInfo replica_segments = 3;
}
message ReplicaSegmentsInfo {
int64 node_id = 1;
int64 partition_id = 2;
repeated int64 segment_ids = 3;
repeated int64 versions = 4;
}
message GetLoadInfoRequest {
common.MsgBase base = 1;
int64 collection_id = 2;
}
message GetLoadInfoResponse {
common.Status status = 1;
schema.CollectionSchema schema = 2;
LoadType load_type = 3;
repeated int64 partitions = 4;
}
//----------------request auto triggered by QueryCoord-----------------
message HandoffSegmentsRequest {
common.MsgBase base = 1;
repeated SegmentInfo segmentInfos = 2;
repeated int64 released_segments = 3;
}
message LoadBalanceRequest {
common.MsgBase base = 1;
repeated int64 source_nodeIDs = 2;
TriggerCondition balance_reason = 3;
repeated int64 dst_nodeIDs = 4;
repeated int64 sealed_segmentIDs = 5;
int64 collectionID = 6;
}
//-------------------- internal meta proto------------------
enum DataScope {
UnKnown = 0;
All = 1;
Streaming = 2;
Historical = 3;
}
enum PartitionState {
NotExist = 0;
NotPresent = 1;
OnDisk = 2;
PartialInMemory = 3;
InMemory = 4;
PartialInGPU = 5;
InGPU = 6;
}
enum TriggerCondition {
UnKnowCondition = 0;
Handoff = 1;
LoadBalance = 2;
GrpcRequest = 3;
NodeDown = 4;
}
enum LoadType {
UnKnownType = 0;
LoadPartition = 1;
LoadCollection = 2;
}
message DmChannelWatchInfo {
int64 collectionID = 1;
string dmChannel = 2;
int64 nodeID_loaded = 3;
int64 replicaID = 4;
repeated int64 node_ids = 5;
}
message QueryChannelInfo {
int64 collectionID = 1;
string query_channel = 2;
string query_result_channel = 3;
repeated SegmentInfo global_sealed_segments = 4;
msg.MsgPosition seek_position = 5;
}
message PartitionStates {
int64 partitionID = 1;
PartitionState state = 2;
int64 inMemory_percentage = 3;
}
message SegmentInfo {
int64 segmentID = 1;
int64 collectionID = 2;
int64 partitionID = 3;
// deprecated, check node_ids(NodeIds) field
int64 nodeID = 4;
int64 mem_size = 5;
int64 num_rows = 6;
string index_name = 7;
int64 indexID = 8;
string dmChannel = 9;
repeated int64 compactionFrom = 10;
bool createdByCompaction = 11;
common.SegmentState segment_state = 12;
repeated FieldIndexInfo index_infos = 13;
repeated int64 replica_ids = 14;
repeated int64 node_ids = 15;
bool enable_index = 16;
bool is_fake = 17;
}
message CollectionInfo {
int64 collectionID = 1;
repeated int64 partitionIDs = 2;
repeated PartitionStates partition_states = 3;
LoadType load_type = 4;
schema.CollectionSchema schema = 5;
repeated int64 released_partitionIDs = 6;
int64 inMemory_percentage = 7;
repeated int64 replica_ids = 8;
int32 replica_number = 9;
}
message UnsubscribeChannels {
int64 collectionID = 1;
repeated string channels = 2;
}
message UnsubscribeChannelInfo {
int64 nodeID = 1;
repeated UnsubscribeChannels collection_channels = 2;
}
//---- synchronize messages proto between QueryCoord and QueryNode -----
message SegmentChangeInfo {
int64 online_nodeID = 1;
repeated SegmentInfo online_segments = 2;
int64 offline_nodeID = 3;
repeated SegmentInfo offline_segments = 4;
}
message SealedSegmentsChangeInfo {
common.MsgBase base = 1;
repeated SegmentChangeInfo infos = 2;
}
message GetDataDistributionRequest {
common.MsgBase base = 1;
map<string, msg.MsgPosition> checkpoints = 2;
}
message GetDataDistributionResponse {
common.Status status = 1;
int64 nodeID = 2;
repeated SegmentVersionInfo segments = 3;
repeated ChannelVersionInfo channels = 4;
repeated LeaderView leader_views = 5;
}
message LeaderView {
int64 collection = 1;
string channel = 2;
map<int64, SegmentDist> segment_dist = 3;
repeated int64 growing_segmentIDs = 4;
map<int64, msg.MsgPosition> growing_segments = 5;
}
message SegmentDist {
int64 nodeID = 1;
int64 version = 2;
}
message SegmentVersionInfo {
int64 ID = 1;
int64 collection = 2;
int64 partition = 3;
string channel = 4;
int64 version = 5;
uint64 last_delta_timestamp = 6;
}
message ChannelVersionInfo {
string channel = 1;
int64 collection = 2;
int64 version = 3;
}
enum LoadStatus {
Invalid = 0;
Loading = 1;
Loaded = 2;
}
message CollectionLoadInfo {
int64 collectionID = 1;
repeated int64 released_partitions = 2; // Deprecated: No longer used; kept for compatibility.
int32 replica_number = 3;
LoadStatus status = 4;
map<int64, int64> field_indexID = 5;
LoadType load_type = 6;
}
message PartitionLoadInfo {
int64 collectionID = 1;
int64 partitionID = 2;
int32 replica_number = 3; // Deprecated: No longer used; kept for compatibility.
LoadStatus status = 4;
map<int64, int64> field_indexID = 5; // Deprecated: No longer used; kept for compatibility.
}
message Replica {
int64 ID = 1;
int64 collectionID = 2;
repeated int64 nodes = 3;
string resource_group = 4;
}
enum SyncType {
Remove = 0;
Set = 1;
Amend = 2;
}
message SyncAction {
SyncType type = 1;
int64 partitionID = 2;
int64 segmentID = 3;
int64 nodeID = 4;
int64 version = 5;
SegmentLoadInfo info = 6;
}
message SyncDistributionRequest {
common.MsgBase base = 1;
int64 collectionID = 2;
string channel = 3;
repeated SyncAction actions = 4;
schema.CollectionSchema schema = 5;
LoadMetaInfo load_meta = 6;
int64 replicaID = 7;
int64 version = 8;
}
message ResourceGroup {
string name = 1;
int32 capacity = 2;
repeated int64 nodes = 3;
}
// transfer `replicaNum` replicas in `collectionID` from `source_resource_group` to `target_resource_groups`
message TransferReplicaRequest {
common.MsgBase base = 1;
string source_resource_group = 2;
string target_resource_group = 3;
int64 collectionID = 4;
int64 num_replica = 5;
}
message DescribeResourceGroupRequest {
common.MsgBase base = 1;
string resource_group = 2;
}
message DescribeResourceGroupResponse {
common.Status status = 1;
ResourceGroupInfo resource_group = 2;
}
message ResourceGroupInfo {
string name = 1;
int32 capacity = 2;
int32 num_available_node = 3;
// collection id -> loaded replica num
map<int64, int32> num_loaded_replica = 4;
// collection id -> accessed other rg's node num
map<int64, int32> num_outgoing_node = 5;
// collection id -> be accessed node num by other rg
map<int64, int32> num_incoming_node = 6;
}
message DeleteRequest {
common.MsgBase base = 1;
int64 collection_id = 2;
int64 partition_id = 3;
string vchannel_name = 4;
int64 segment_id = 5;
schema.IDs primary_keys = 6;
repeated uint64 timestamps = 7;
}