milvus/internal/proto/data_coord.proto

syntax = "proto3";

package milvus.proto.data;

option go_package = "github.com/milvus-io/milvus/internal/proto/datapb";

import "common.proto";
import "internal.proto";
import "milvus.proto";
import "schema.proto";

service DataCoord {
  rpc GetComponentStates(internal.GetComponentStatesRequest) returns (internal.ComponentStates) {}
  rpc GetTimeTickChannel(internal.GetTimeTickChannelRequest) returns(milvus.StringResponse) {}
  rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}

  rpc Flush(FlushRequest) returns (FlushResponse) {}

  rpc AssignSegmentID(AssignSegmentIDRequest) returns (AssignSegmentIDResponse) {}

  rpc GetSegmentInfo(GetSegmentInfoRequest) returns (GetSegmentInfoResponse) {}
  rpc GetSegmentStates(GetSegmentStatesRequest) returns (GetSegmentStatesResponse) {}
  rpc GetInsertBinlogPaths(GetInsertBinlogPathsRequest) returns (GetInsertBinlogPathsResponse) {}

  rpc GetCollectionStatistics(GetCollectionStatisticsRequest) returns (GetCollectionStatisticsResponse) {}
  rpc GetPartitionStatistics(GetPartitionStatisticsRequest) returns (GetPartitionStatisticsResponse) {}

  rpc GetSegmentInfoChannel(GetSegmentInfoChannelRequest) returns (milvus.StringResponse){}

  rpc SaveBinlogPaths(SaveBinlogPathsRequest) returns (common.Status){}
  rpc GetRecoveryInfo(GetRecoveryInfoRequest) returns (GetRecoveryInfoResponse){}
  rpc GetFlushedSegments(GetFlushedSegmentsRequest) returns(GetFlushedSegmentsResponse){}

  // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
  rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}
  rpc CompleteCompaction(CompactionResult) returns (common.Status) {}
  rpc ManualCompaction(ManualCompactionRequest) returns (ManualCompactionResponse) {}
  rpc GetCompactionState(GetCompactionStateRequest) returns (GetCompactionStateResponse) {}
}

service DataNode {
  rpc GetComponentStates(internal.GetComponentStatesRequest) returns (internal.ComponentStates) {}
  rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}

  rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {}
  rpc FlushSegments(FlushSegmentsRequest) returns(common.Status) {}

  // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
  rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}
  rpc Compaction(CompactionPlan) returns (common.Status) {}
}

message FlushRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 4;
}

message FlushResponse {
  common.Status status = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 segmentIDs = 4;
}

message SegmentIDRequest {
  uint32 count = 1;
  string channel_name = 2;
  int64 collectionID = 3;
  int64 partitionID = 4;
}

message AssignSegmentIDRequest {
  int64 nodeID = 1;
  string peer_role = 2;
  repeated SegmentIDRequest segmentIDRequests = 3;
}

message SegmentIDAssignment {
  int64 segID = 1;
  string channel_name = 2;
  uint32 count = 3;
  int64 collectionID = 4;
  int64 partitionID = 5;
  uint64 expire_time = 6;
  common.Status status = 7;
}

message AssignSegmentIDResponse {
  repeated SegmentIDAssignment segIDAssignments = 1;
  common.Status status = 2;
}

message GetSegmentStatesRequest {
  common.MsgBase base = 1;
  repeated int64 segmentIDs = 2;
}

message SegmentStateInfo {
  int64 segmentID = 1;
  common.SegmentState state = 2;
  internal.MsgPosition start_position = 3;
  internal.MsgPosition end_position = 4;
  common.Status status = 5;
}

message GetSegmentStatesResponse {
  common.Status status = 1;
  repeated SegmentStateInfo states = 2;
}

message GetSegmentInfoRequest {
  common.MsgBase base = 1;
  repeated int64 segmentIDs = 2;
}

message GetSegmentInfoResponse {
  common.Status status = 1;
  repeated SegmentInfo infos = 2;
}

message GetInsertBinlogPathsRequest {
  common.MsgBase base = 1;
  int64 segmentID = 2;
}

message GetInsertBinlogPathsResponse {
  repeated int64 fieldIDs = 1;
  repeated internal.StringList paths = 2;
  common.Status status = 3;
}

message GetCollectionStatisticsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
}

message GetCollectionStatisticsResponse {
  repeated common.KeyValuePair stats = 1;
  common.Status status = 2;
}

message GetPartitionStatisticsRequest{
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  int64 partitionID = 4;
}

message GetPartitionStatisticsResponse {
  repeated common.KeyValuePair stats = 1;
  common.Status status = 2;
}

message GetSegmentInfoChannelRequest {
}


message VchannelInfo {
  int64 collectionID = 1;
  string channelName = 2;
  internal.MsgPosition seek_position = 3;
  repeated SegmentInfo unflushedSegments = 4;
  repeated SegmentInfo flushedSegments = 5;
}

message WatchDmChannelsRequest {
  common.MsgBase base = 1;
  repeated VchannelInfo vchannels = 2;
}

message FlushSegmentsRequest {
  common.MsgBase base = 1;
  int64 dbID = 2;
  int64 collectionID = 3;
  repeated int64 segmentIDs = 4; // segments to flush
  repeated int64 markSegmentIDs = 5; // segments to clean buffer and mark segment position, but NOT flushed
}

message SegmentMsg{
  common.MsgBase base = 1;
  SegmentInfo segment = 2;
}

message CollectionInfo {
  int64 ID = 1;
  schema.CollectionSchema schema = 2;
  repeated int64 partitions = 3;
  repeated common.KeyDataPair start_positions = 4;
}

message SegmentInfo {
  int64 ID = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
  string insert_channel = 4;
  int64 num_of_rows = 5;
  common.SegmentState state = 6;
  int64 max_row_num = 7;
  uint64 last_expire_time = 8;
  internal.MsgPosition start_position = 9;
  internal.MsgPosition dml_position = 10;
  // binlogs consist of insert binlogs
  repeated FieldBinlog binlogs = 11;
  repeated FieldBinlog statslogs = 12;
  repeated DeltaLogInfo deltalogs = 13;
  
  bool createdByCompaction = 14;
  repeated int64 compactionFrom = 15;
}

message SegmentStartPosition {
  internal.MsgPosition start_position = 1;
  int64 segmentID = 2;
}

message SaveBinlogPathsRequest {
  common.MsgBase base = 1;
  int64 segmentID = 2;
  int64 collectionID = 3;
  repeated FieldBinlog field2BinlogPaths = 4;
  repeated CheckPoint checkPoints = 5;
  repeated SegmentStartPosition start_positions = 6;
  bool flushed = 7;
  repeated FieldBinlog field2StatslogPaths = 8;
  repeated DeltaLogInfo deltalogs = 9;
}

message CheckPoint {
  int64 segmentID = 1;
  internal.MsgPosition position = 2;
  int64 num_of_rows = 3;
}

message DeltaLogInfo {
  uint64 record_entries = 1;
  uint64 timestamp_from = 2;
  uint64 timestamp_to = 3;
  string delta_log_path = 4;
  int64 delta_log_size = 5; 
}

message DataNodeTtMsg {
    common.MsgBase base =1;
    string channel_name = 2;
    uint64 timestamp = 3;
}

enum ChannelWatchState {
  Uncomplete = 0;
  Complete = 1;
}

message ChannelStatus {
  string name = 1;
  ChannelWatchState state=2;
  int64 collectionID = 3;
}

message DataNodeInfo {
  string address = 1;
  int64 version  = 2;
  repeated ChannelStatus channels = 3;
}

message SegmentBinlogs {
  int64 segmentID = 1;
  repeated FieldBinlog fieldBinlogs = 2;
  int64 num_of_rows = 3;
  repeated FieldBinlog statslogs = 4;
  repeated DeltaLogInfo deltalogs = 5;
}

message FieldBinlog{
  int64 fieldID = 1;
  repeated string binlogs = 2;
}

message GetRecoveryInfoResponse {
  common.Status status = 1;
  repeated VchannelInfo channels = 2;
  repeated SegmentBinlogs binlogs = 3;
}

message GetRecoveryInfoRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
}

message GetFlushedSegmentsRequest {
  common.MsgBase base = 1;
  int64 collectionID = 2;
  int64 partitionID = 3;
}

message GetFlushedSegmentsResponse {
  common.Status status = 1;
  repeated int64 segments = 2;
}

message SegmentFlushCompletedMsg {
  common.MsgBase base = 1;
  SegmentInfo segment = 2;
}

message ChannelWatchInfo {
    VchannelInfo vchan= 1;
    int64 startTs = 2;
    ChannelWatchState state = 3;
}

enum CompactionType {
  UndefinedCompaction = 0;
  InnerCompaction = 1;
  MergeCompaction = 2;
}

message CompactionSegmentBinlogs {
  int64 segmentID = 1;
  repeated FieldBinlog fieldBinlogs = 2;
  repeated FieldBinlog field2StatslogPaths = 3;
  repeated DeltaLogInfo deltalogs = 4;
}

message CompactionPlan {
  int64 planID = 1;
  repeated CompactionSegmentBinlogs segmentBinlogs = 2;
  uint64 start_time = 3;
  int32 timeout_in_seconds = 4;
  CompactionType type = 5;
  uint64 timetravel = 6;
  string channel = 7;
}

message CompactionResult {
  int64 planID = 1;
  int64 segmentID = 2;
  int64 num_of_rows = 3;
  repeated FieldBinlog insert_logs = 4;
  repeated FieldBinlog field2StatslogPaths = 5;
  repeated DeltaLogInfo deltalogs = 6;
}

message ManualCompactionRequest {
  int64 collectionID = 1;
  uint64 timetravel = 2;
}

message ManualCompactionResponse {
  common.Status status = 1;
  int64 compactionID = 2;
}

message GetCompactionStateRequest {
  int64 compactionID = 1;
}

enum CompactionState {
  UndefiedState = 0;
  Executing = 1;
  Completed = 2;
}

message GetCompactionStateResponse {
  common.Status status = 1;
  CompactionState state = 2;
  int64 executingPlanNo = 3;
  int64 timeoutPlanNo = 4;
  int64 completedPlanNo = 5;
}

// Deprecated
message SegmentFieldBinlogMeta {
  int64  fieldID = 1;
  string binlog_path = 2;
}