milvus/internal/proto/query_coord.proto

syntax = "proto3";

package milvus.proto.query;

option go_package = "github.com/milvus-io/milvus/internal/proto/querypb";

import "common.proto";
import "milvus.proto";
import "internal.proto";
import "schema.proto";
import "msg.proto";
import "data_coord.proto";
import "index_coord.proto";

service QueryCoord {
  rpc GetComponentStates(milvus.GetComponentStatesRequest) returns (milvus.ComponentStates) {}
  rpc GetTimeTickChannel(internal.GetTimeTickChannelRequest) returns(milvus.StringResponse) {}
  rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}

  rpc ShowCollections(ShowCollectionsRequest) returns (ShowCollectionsResponse) {}
  rpc ShowPartitions(ShowPartitionsRequest) returns (ShowPartitionsResponse) {}

  rpc LoadPartitions(LoadPartitionsRequest) returns (common.Status) {}
  rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {}
  rpc LoadCollection(LoadCollectionRequest) returns (common.Status) {}
  rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {}
  rpc SyncNewCreatedPartition(SyncNewCreatedPartitionRequest) returns (common.Status) {}

  rpc GetPartitionStates(GetPartitionStatesRequest) returns (GetPartitionStatesResponse) {}
  rpc GetSegmentInfo(GetSegmentInfoRequest) returns (GetSegmentInfoResponse) {}
  rpc LoadBalance(LoadBalanceRequest) returns (common.Status) {}

  rpc ShowConfigurations(internal.ShowConfigurationsRequest) returns (internal.ShowConfigurationsResponse){}
  // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
  rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}

  // https://wiki.lfaidata.foundation/display/MIL/MEP+23+--+Multiple+memory+replication+design
  rpc GetReplicas(milvus.GetReplicasRequest) returns (milvus.GetReplicasResponse) {}
  rpc GetShardLeaders(GetShardLeadersRequest) returns (GetShardLeadersResponse) {}

  rpc CheckHealth(milvus.CheckHealthRequest) returns (milvus.CheckHealthResponse) {}

  rpc CreateResourceGroup(milvus.CreateResourceGroupRequest) returns (common.Status) {}
  rpc DropResourceGroup(milvus.DropResourceGroupRequest) returns (common.Status) {}
  rpc TransferNode(milvus.TransferNodeRequest) returns (common.Status) {}
  rpc TransferReplica(TransferReplicaRequest) returns (common.Status) {}
  rpc ListResourceGroups(milvus.ListResourceGroupsRequest) returns (milvus.ListResourceGroupsResponse) {}
  rpc DescribeResourceGroup(DescribeResourceGroupRequest) returns (DescribeResourceGroupResponse) {}
}

service QueryNode {
  rpc GetComponentStates(milvus.GetComponentStatesRequest) returns (milvus.ComponentStates) {}
  rpc GetTimeTickChannel(internal.GetTimeTickChannelRequest) returns(milvus.StringResponse) {}
  rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}

  rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {}
  rpc UnsubDmChannel(UnsubDmChannelRequest) returns (common.Status) {}
  rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {}
  rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {}
  rpc LoadPartitions(LoadPartitionsRequest) returns (common.Status) {}
  rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {}
  rpc ReleaseSegments(ReleaseSegmentsRequest) returns (common.Status) {}
  rpc GetSegmentInfo(GetSegmentInfoRequest) returns (GetSegmentInfoResponse) {}
  rpc SyncReplicaSegments(SyncReplicaSegmentsRequest) returns (common.Status) {}

  rpc GetStatistics(GetStatisticsRequest) returns (internal.GetStatisticsResponse) {}
  rpc Search(SearchRequest) returns (internal.SearchResults) {}
  rpc Query(QueryRequest) returns (internal.RetrieveResults) {}

  rpc ShowConfigurations(internal.ShowConfigurationsRequest) returns (internal.ShowConfigurationsResponse){}
  // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
  rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}

  rpc GetDataDistribution(GetDataDistributionRequest) returns (GetDataDistributionResponse) {}
  rpc SyncDistribution(SyncDistributionRequest) returns (common.Status) {}
  rpc Delete(DeleteRequest) returns (common.Status) {}
}

//--------------------QueryCoord grpc request and response proto------------------
message ShowCollectionsRequest {
  common.MsgBase base = 1;
  // Not useful for now
  int64 dbID = 2;
  repeated int64 collectionIDs = 3;
}

message ShowCollectionsResponse {
  common.Status status = 1;
  repeated int64 collectionIDs = 2;
  repeated int64 inMemory_percentages = 3;
  repeated bool query_service_available = 4;
  repeated int64 refresh_progress = 5;
}

message ShowPartitionsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 partitionIDs = 4;
}

message ShowPartitionsResponse {
  common.Status status = 1;
  repeated int64 partitionIDs = 2;
  repeated int64 inMemory_percentages = 3;
  repeated int64 refresh_progress = 4;
}

message LoadCollectionRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  schema.CollectionSchema schema = 4;
  int32 replica_number = 5;
  // fieldID -> indexID
  map<int64, int64> field_indexID = 6;
  bool refresh = 7;
  // resource group names
  repeated string resource_groups = 8;
}

message ReleaseCollectionRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  int64 nodeID = 4;
}

message GetStatisticsRequest {
  internal.GetStatisticsRequest req = 1;
  repeated string dml_channels = 2;
  repeated int64 segmentIDs = 3;
  bool from_shard_leader = 4;
  DataScope scope = 5; // All, Streaming, Historical
}

message LoadPartitionsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 partitionIDs = 4;
  schema.CollectionSchema schema = 5;
  int32 replica_number = 6;
  // fieldID -> indexID
  map<int64, int64> field_indexID = 7;
  bool refresh = 8;
  // resource group names
  repeated string resource_groups = 9;
}

message ReleasePartitionsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 partitionIDs = 4;
  int64 nodeID = 5;
}

message GetPartitionStatesRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 partitionIDs = 4;
}

message GetPartitionStatesResponse {
  common.Status status = 1;
  repeated PartitionStates partition_descriptions = 2;
}

message GetSegmentInfoRequest {
  common.MsgBase base = 1;
  repeated int64 segmentIDs = 2; // deprecated
  int64 collectionID = 3;
}

message GetSegmentInfoResponse {
  common.Status status = 1;
  repeated SegmentInfo infos = 2;
}

message GetShardLeadersRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
}

message GetShardLeadersResponse {
  common.Status status = 1;
  repeated ShardLeadersList shards = 2;
}

message ShardLeadersList {  // All leaders of all replicas of one shard
  string channel_name = 1;
  repeated int64 node_ids = 2;
  repeated string node_addrs = 3;
}

message SyncNewCreatedPartitionRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
  schema.CollectionSchema schema = 4;
}

//-----------------query node grpc request and response proto----------------
message LoadMetaInfo {
  LoadType load_type = 1;
  int64 collectionID = 2;
  repeated int64 partitionIDs = 3;
}

message WatchDmChannelsRequest {
  common.MsgBase base = 1;
  int64 nodeID = 2;
  int64 collectionID = 3;
  repeated int64 partitionIDs = 4;
  repeated data.VchannelInfo infos = 5;
  schema.CollectionSchema schema = 6;
  repeated data.SegmentInfo exclude_infos = 7;
  LoadMetaInfo load_meta = 8;
  int64 replicaID = 9;
  map<int64, data.SegmentInfo> segment_infos = 10;
  repeated index.IndexInfo index_info_list = 11;
  // Deprecated
  // for node down load balance, need to remove offline node in time after every watchDmChannel finish.
  int64 offlineNodeID = 12;
  int64 version = 13;
}

message UnsubDmChannelRequest {
    common.MsgBase base = 1;
    int64 nodeID = 2;
    int64 collectionID = 3;
    string channel_name = 4;
}

message SegmentLoadInfo {
  int64 segmentID = 1;
  int64 partitionID = 2;
  int64 collectionID = 3;
  int64 dbID = 4;
  int64 flush_time = 5;
  repeated data.FieldBinlog binlog_paths = 6;
  int64 num_of_rows = 7;
  repeated data.FieldBinlog statslogs = 8;
  repeated data.FieldBinlog deltalogs = 9;
  repeated int64 compactionFrom = 10; // segmentIDs compacted from
  repeated FieldIndexInfo index_infos = 11;
  int64 segment_size = 12;
  string insert_channel = 13;
  msg.MsgPosition start_position = 14;
  msg.MsgPosition end_position = 15;
}

message FieldIndexInfo {
  int64 fieldID =1;
  // deprecated
  bool enable_index = 2;
  string index_name = 3;
  int64 indexID = 4;
  int64 buildID = 5;
  repeated common.KeyValuePair index_params = 6;
  repeated string index_file_paths = 7;
  int64 index_size = 8;
  int64 index_version = 9;
  int64 num_rows = 10;
}

enum LoadScope {
  Full = 0;
  Delta = 1;
}

message LoadSegmentsRequest {
  common.MsgBase base = 1;
  int64 dst_nodeID = 2;
  repeated SegmentLoadInfo infos = 3;
  schema.CollectionSchema schema = 4;
  int64 source_nodeID = 5;
  int64 collectionID = 6;
  LoadMetaInfo load_meta = 7;
  int64 replicaID = 8;
  repeated msg.MsgPosition delta_positions = 9;
  int64 version = 10;
  bool need_transfer = 11;
  LoadScope load_scope = 12;
}

message ReleaseSegmentsRequest {
  common.MsgBase base = 1;
  int64 nodeID = 2;
  // Not useful for now
  int64 dbID = 3;
  int64 collectionID = 4;
  repeated int64 partitionIDs = 5;
  repeated int64 segmentIDs = 6;
  DataScope scope = 7; // All, Streaming, Historical
  string shard = 8;
  bool need_transfer = 11;
}

message SearchRequest {
  internal.SearchRequest req = 1;
  repeated string dml_channels = 2;
  repeated int64 segmentIDs = 3;
  bool from_shard_leader = 4;
  DataScope scope = 5; // All, Streaming, Historical
  int32 total_channel_num = 6;
}

message QueryRequest {
  internal.RetrieveRequest req = 1;
  repeated string dml_channels = 2;
  repeated int64 segmentIDs = 3;
  bool from_shard_leader = 4;
  DataScope scope = 5; // All, Streaming, Historical
}

message SyncReplicaSegmentsRequest {
  common.MsgBase base = 1;
  string vchannel_name = 2;
  repeated ReplicaSegmentsInfo replica_segments = 3;
}

message ReplicaSegmentsInfo {
  int64 node_id = 1;
  int64 partition_id = 2;
  repeated int64 segment_ids = 3;
  repeated int64 versions = 4;
}

message GetLoadInfoRequest {
  common.MsgBase base = 1;
  int64 collection_id = 2;
}

message GetLoadInfoResponse {
  common.Status status = 1;
  schema.CollectionSchema schema = 2;
  LoadType load_type = 3;
  repeated int64 partitions = 4;
}

//----------------request auto triggered by QueryCoord-----------------
message HandoffSegmentsRequest {
  common.MsgBase base = 1;
  repeated SegmentInfo segmentInfos = 2;
  repeated int64 released_segments = 3;
}

message LoadBalanceRequest {
  common.MsgBase base = 1;
  repeated int64 source_nodeIDs = 2;
  TriggerCondition balance_reason = 3;
  repeated int64 dst_nodeIDs = 4;
  repeated int64 sealed_segmentIDs = 5;
  int64 collectionID = 6;
}

//-------------------- internal meta proto------------------

enum DataScope {
  UnKnown = 0;
  All = 1;
  Streaming = 2;
  Historical = 3;
}

enum PartitionState {
  NotExist = 0;
  NotPresent = 1;
  OnDisk = 2;
  PartialInMemory = 3;
  InMemory = 4;
  PartialInGPU = 5;
  InGPU = 6;
}

enum TriggerCondition {
  UnKnowCondition = 0;
  Handoff = 1;
  LoadBalance = 2;
  GrpcRequest = 3;
  NodeDown = 4;
}

enum LoadType {
  UnKnownType = 0;
  LoadPartition = 1;
  LoadCollection = 2;
}

message DmChannelWatchInfo {
  int64 collectionID = 1;
  string dmChannel = 2;
  int64 nodeID_loaded = 3;
  int64 replicaID = 4;
  repeated int64 node_ids = 5;
}

message QueryChannelInfo {
  int64 collectionID = 1;
  string query_channel = 2;
  string query_result_channel = 3;
  repeated SegmentInfo global_sealed_segments = 4;
  msg.MsgPosition seek_position = 5;
}

message PartitionStates {
  int64 partitionID = 1;
  PartitionState state = 2;
  int64 inMemory_percentage = 3;
}

message SegmentInfo {
  int64 segmentID = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
  // deprecated, check node_ids(NodeIds) field
  int64 nodeID = 4;
  int64 mem_size = 5;
  int64 num_rows = 6;
  string index_name = 7;
  int64 indexID = 8;
  string dmChannel = 9;
  repeated int64 compactionFrom = 10;
  bool createdByCompaction = 11;
  common.SegmentState segment_state = 12;
  repeated FieldIndexInfo index_infos = 13;
  repeated int64 replica_ids = 14;
  repeated int64 node_ids = 15;
  bool enable_index = 16;
  bool is_fake = 17;
}

message CollectionInfo {
  int64 collectionID = 1;
  repeated int64 partitionIDs = 2;
  repeated PartitionStates partition_states = 3;
  LoadType load_type = 4;
  schema.CollectionSchema schema = 5;
  repeated int64 released_partitionIDs = 6;
  int64 inMemory_percentage = 7;
  repeated int64 replica_ids = 8;
  int32 replica_number = 9;
}

message UnsubscribeChannels {
  int64 collectionID = 1;
  repeated string channels = 2;
}

message UnsubscribeChannelInfo {
  int64 nodeID = 1;
  repeated UnsubscribeChannels collection_channels = 2;
}

//---- synchronize messages proto between QueryCoord and QueryNode -----
message SegmentChangeInfo {
  int64 online_nodeID = 1;
  repeated SegmentInfo online_segments = 2;
  int64 offline_nodeID = 3;
  repeated SegmentInfo offline_segments = 4;
}

message SealedSegmentsChangeInfo {
  common.MsgBase base = 1;
  repeated SegmentChangeInfo infos = 2;
}

message GetDataDistributionRequest {
  common.MsgBase base = 1;
  map<string, msg.MsgPosition> checkpoints = 2;
}

message GetDataDistributionResponse {
  common.Status status = 1;
  int64 nodeID = 2;
  repeated SegmentVersionInfo segments = 3;
  repeated ChannelVersionInfo channels = 4;
  repeated LeaderView leader_views = 5;
}

message LeaderView {
  int64 collection = 1;
  string channel = 2;
  map<int64, SegmentDist> segment_dist = 3;
  repeated int64 growing_segmentIDs = 4;
  map<int64, msg.MsgPosition> growing_segments = 5;
}

message SegmentDist {
  int64 nodeID = 1;
  int64 version = 2;
}


message SegmentVersionInfo {
  int64 ID = 1;
  int64 collection = 2;
  int64 partition = 3;
  string channel = 4;
  int64 version = 5;
  uint64 last_delta_timestamp = 6;
}

message ChannelVersionInfo {
  string channel = 1;
  int64 collection = 2;
  int64 version = 3;
}

enum LoadStatus {
  Invalid = 0;
  Loading = 1;
  Loaded = 2;
}

message CollectionLoadInfo {
  int64 collectionID = 1;
  repeated int64 released_partitions = 2; // Deprecated: No longer used; kept for compatibility.
  int32 replica_number = 3;
  LoadStatus status = 4;
  map<int64, int64> field_indexID = 5;
  LoadType load_type = 6;
}

message PartitionLoadInfo {
  int64 collectionID = 1;
  int64 partitionID = 2;
  int32 replica_number = 3; // Deprecated: No longer used; kept for compatibility.
  LoadStatus status = 4;
  map<int64, int64> field_indexID = 5; // Deprecated: No longer used; kept for compatibility.
}

message Replica {
  int64 ID = 1;
  int64 collectionID = 2;
  repeated int64 nodes = 3;
  string resource_group = 4;
}

enum SyncType {
  Remove = 0;
  Set = 1;
  Amend = 2;
}

message SyncAction {
  SyncType type = 1;
  int64 partitionID = 2;
  int64 segmentID = 3;
  int64 nodeID = 4;
  int64 version = 5;
  SegmentLoadInfo info = 6;
}

message SyncDistributionRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
  string channel = 3;
  repeated SyncAction actions = 4;
  schema.CollectionSchema schema = 5;
  LoadMetaInfo load_meta = 6;
  int64 replicaID = 7;
  int64 version = 8;
}

message ResourceGroup {
  string name = 1;
  int32 capacity = 2;
  repeated int64 nodes = 3;
}

// transfer `replicaNum` replicas in `collectionID` from `source_resource_group` to `target_resource_groups`
message TransferReplicaRequest {
  common.MsgBase base = 1;
  string source_resource_group = 2;
  string target_resource_group = 3;
  int64  collectionID = 4;
  int64  num_replica = 5;
}

message DescribeResourceGroupRequest {
  common.MsgBase base = 1;
  string resource_group = 2;
}

message DescribeResourceGroupResponse {
  common.Status status = 1;
  ResourceGroupInfo resource_group = 2;
}

message ResourceGroupInfo {
  string name = 1;
  int32 capacity = 2;
  int32 num_available_node = 3;
  // collection id -> loaded replica num
  map<int64, int32> num_loaded_replica = 4;
  // collection id -> accessed other rg's node num 
  map<int64, int32> num_outgoing_node = 5;
   // collection id -> be accessed node num by other rg
  map<int64, int32> num_incoming_node = 6;
}
message DeleteRequest {
  common.MsgBase base = 1;
  int64 collection_id = 2;
  int64 partition_id = 3;
  string vchannel_name = 4;
  int64 segment_id = 5;
  schema.IDs primary_keys = 6;
  repeated uint64 timestamps = 7; 
}