milvus/internal/proto/data_service.proto

syntax = "proto3";

package milvus.proto.data;

option go_package = "github.com/milvus-io/milvus/internal/proto/datapb";

import "common.proto";
import "internal.proto";
import "milvus.proto";
import "schema.proto";

service DataService {
  rpc GetComponentStates(internal.GetComponentStatesRequest) returns (internal.ComponentStates) {}
  rpc GetTimeTickChannel(internal.GetTimeTickChannelRequest) returns(milvus.StringResponse) {}
  rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}

  rpc RegisterNode(RegisterNodeRequest) returns (RegisterNodeResponse) {}
  rpc Flush(FlushRequest) returns (common.Status) {}

  rpc AssignSegmentID(AssignSegmentIDRequest) returns (AssignSegmentIDResponse) {}
  rpc ShowSegments(ShowSegmentsRequest) returns (ShowSegmentsResponse) {}

  rpc GetSegmentInfo(GetSegmentInfoRequest) returns (GetSegmentInfoResponse) {}
  rpc GetSegmentStates(GetSegmentStatesRequest) returns (GetSegmentStatesResponse) {}
  rpc GetInsertBinlogPaths(GetInsertBinlogPathsRequest) returns (GetInsertBinlogPathsResponse) {}

  rpc GetInsertChannels(GetInsertChannelsRequest) returns (internal.StringList) {}
  rpc GetCollectionStatistics(GetCollectionStatisticsRequest) returns (GetCollectionStatisticsResponse) {}
  rpc GetPartitionStatistics(GetPartitionStatisticsRequest) returns (GetPartitionStatisticsResponse) {}

  rpc GetSegmentInfoChannel(GetSegmentInfoChannelRequest) returns (milvus.StringResponse){}

  rpc SaveBinlogPaths(SaveBinlogPathsRequest) returns (common.Status){}
  rpc GetRecoveryInfo(GetRecoveryInfoRequest) returns (GetRecoveryInfoResponse){}
}

service DataNode {
  rpc GetComponentStates(internal.GetComponentStatesRequest) returns (internal.ComponentStates) {}
  rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}

  rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {}
  rpc FlushSegments(FlushSegmentsRequest) returns(common.Status) {}
}

message RegisterNodeRequest {
  common.MsgBase base = 1;
  common.Address address = 2;
}

message RegisterNodeResponse {
  internal.InitParams init_params = 1;
  common.Status status = 2;
}

message FlushRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 4;
}

message SegmentIDRequest {
  uint32 count = 1;
  string channel_name = 2;
  int64 collectionID = 3;
  int64 partitionID = 4;
}

message AssignSegmentIDRequest {
  int64 nodeID = 1;
  string peer_role = 2;
  repeated SegmentIDRequest segmentIDRequests = 3;
}

message SegmentIDAssignment {
  int64 segID = 1;
  string channel_name = 2;
  uint32 count = 3;
  int64 collectionID = 4;
  int64 partitionID = 5;
  uint64 expire_time = 6;
  common.Status status = 7;
}

message AssignSegmentIDResponse {
  repeated SegmentIDAssignment segIDAssignments = 1;
  common.Status status = 2;
}

message ShowSegmentsRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
  int64 dbID = 4;
}

message ShowSegmentsResponse {
  repeated int64 segmentIDs = 1;
  common.Status status = 2;
}

message GetSegmentStatesRequest {
  common.MsgBase base = 1;
  repeated int64 segmentIDs = 2;
}

message SegmentStateInfo {
  int64 segmentID = 1;
  common.SegmentState state = 2;
  internal.MsgPosition start_position = 3;
  internal.MsgPosition end_position = 4;
  common.Status status = 5;
}

message GetSegmentStatesResponse {
  common.Status status = 1;
  repeated SegmentStateInfo states = 2;
}

message GetSegmentInfoRequest {
  common.MsgBase base = 1;
  repeated int64 segmentIDs = 2;
}

message GetSegmentInfoResponse {
  common.Status status = 1;
  repeated SegmentInfo infos = 2;
}

message GetInsertBinlogPathsRequest {
  common.MsgBase base = 1;
  int64 segmentID = 2;
}

message GetInsertBinlogPathsResponse {
  repeated int64 fieldIDs = 1;
  repeated internal.StringList paths = 2;
  common.Status status = 3;
}

message GetInsertChannelsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
}

message GetCollectionStatisticsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
}

message GetCollectionStatisticsResponse {
  repeated common.KeyValuePair stats = 1;
  common.Status status = 2;
}

message GetPartitionStatisticsRequest{
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  int64 partitionID = 4;
}

message GetPartitionStatisticsResponse {
  repeated common.KeyValuePair stats = 1;
  common.Status status = 2;
}

message GetSegmentInfoChannelRequest {
}

message VchannelPair {
  int64 collectionID = 1;
  string dml_vchannel_name = 2;
  string ddl_vchannel_name = 3;
  PositionPair ddl_position = 4;
  PositionPair dml_position = 5;
}

message WatchDmChannelsRequest {
  common.MsgBase base = 1;
  repeated VchannelPair vchannels = 2;
}

message FlushSegmentsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 segmentIDs = 4;
}

message SegmentMsg{
  common.MsgBase base = 1;
  SegmentInfo segment = 2;
}

// key: ${prefix}/${segmentID}/${fieldID}/${idx}
message SegmentFieldBinlogMeta {
    int64  fieldID = 1;
    string binlog_path = 2;
}

// key: ${prefix}/${collectionID}/${idx}
message DDLBinlogMeta {
    string ddl_binlog_path = 1;
    string ts_binlog_path = 2;
}

message FieldFlushMeta {
    int64 fieldID = 1;
    repeated string binlog_paths = 2;
}

message SegmentFlushMeta{
    int64 segmentID = 1;
    bool is_flushed = 2;
    repeated FieldFlushMeta fields = 5;
}

message DDLFlushMeta {
    int64 collectionID = 1;
    repeated string binlog_paths = 2;
}

message CollectionInfo {
  int64 ID = 1;
  schema.CollectionSchema schema = 2;
  repeated int64 partitions = 3;
}
message SegmentInfo {
  int64 ID = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
  string insert_channel = 4;
  int64 num_rows = 5;
  common.SegmentState state = 6;
  internal.MsgPosition start_position = 7;
  internal.MsgPosition end_position = 8;
  int64 max_row_num = 9;
  uint64 last_expire_time = 10;
}

message ID2PathList {
  int64 ID = 1;
  repeated string Paths = 2;
}

message PositionPair {
  internal.MsgPosition start_position = 1;
  internal.MsgPosition end_position = 2;
}

message SaveBinlogPathsRequest {
  common.MsgBase base = 1;
  int64 segmentID = 2;
  int64 collectionID = 3;
  ID2PathList field2BinlogPaths = 4;
  repeated DDLBinlogMeta ddlBinlogPaths = 5;
  PositionPair dml_position = 6;
  PositionPair ddl_position =7;
}

message DataNodeTtMsg {
    common.MsgBase base =1;
    string channel_name = 2;
    uint64 timestamp = 3;
}

message CheckPoint {
  int64 segmentID = 1;
  internal.MsgPosition position = 2;
  int64 num_of_rows = 3;
}

message VchannelInfo {
  int64 collectionID = 1;
  string channelName = 2;
  internal.MsgPosition seek_position = 3;
  repeated CheckPoint checkPoints = 4;
  repeated int64 flushedSegments = 5;
}

message SegmentBinlogs {
  int64 segmentID = 1;
  repeated FieldBinlog fieldBinlogs = 2;
}

message FieldBinlog{
  int64 fieldID = 1;
  repeated string binlogs = 2;
}

enum ChannelWatchState {
  Uncomplete = 0;
  Complete = 1;
}

message ChannelStatus {
  string name = 1;
  ChannelWatchState state=2;
}

message DataNodeInfo {
  string address = 1;
  int64 version  = 2;
  repeated ChannelStatus channels = 3;
}

message GetRecoveryInfoResponse {
  common.MsgBase base = 1;
  repeated VchannelInfo channels = 2;
  repeated SegmentBinlogs binlogs = 3;
}

message GetRecoveryInfoRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
}