milvus/internal/proto/index_coord.proto

syntax = "proto3";

package milvus.proto.index;

option go_package = "github.com/milvus-io/milvus/internal/proto/indexpb";

import "common.proto";
import "internal.proto";
import "milvus.proto";
import "schema.proto";

service IndexCoord {
    rpc GetComponentStates(milvus.GetComponentStatesRequest) returns (milvus.ComponentStates) {}
    rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}
    rpc CreateIndex(CreateIndexRequest) returns (common.Status){}
    rpc AlterIndex(AlterIndexRequest) returns (common.Status){}
    // Deprecated: use DescribeIndex instead
    rpc GetIndexState(GetIndexStateRequest) returns (GetIndexStateResponse) {}
    rpc GetSegmentIndexState(GetSegmentIndexStateRequest) returns (GetSegmentIndexStateResponse) {}
    rpc GetIndexInfos(GetIndexInfoRequest) returns (GetIndexInfoResponse){}
    rpc DropIndex(DropIndexRequest) returns (common.Status) {}
    rpc DescribeIndex(DescribeIndexRequest) returns (DescribeIndexResponse) {}
    rpc GetIndexStatistics(GetIndexStatisticsRequest) returns (GetIndexStatisticsResponse) {}
    // Deprecated: use DescribeIndex instead
    rpc GetIndexBuildProgress(GetIndexBuildProgressRequest) returns (GetIndexBuildProgressResponse) {}

    rpc ShowConfigurations(internal.ShowConfigurationsRequest)
        returns (internal.ShowConfigurationsResponse) {
    }
    // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
    rpc GetMetrics(milvus.GetMetricsRequest)
        returns (milvus.GetMetricsResponse) {
    }

    rpc CheckHealth(milvus.CheckHealthRequest)
        returns (milvus.CheckHealthResponse) {
    }
}

service IndexNode {
    rpc GetComponentStates(milvus.GetComponentStatesRequest)
        returns (milvus.ComponentStates) {
    }
    rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest)
        returns (milvus.StringResponse) {
    }
    rpc CreateJob(CreateJobRequest) returns (common.Status) {
    }
    rpc QueryJobs(QueryJobsRequest) returns (QueryJobsResponse) {
    }
    rpc DropJobs(DropJobsRequest) returns (common.Status) {
    }
    rpc GetJobStats(GetJobStatsRequest) returns (GetJobStatsResponse) {
    }

    rpc ShowConfigurations(internal.ShowConfigurationsRequest)
        returns (internal.ShowConfigurationsResponse) {
    }
    // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
    rpc GetMetrics(milvus.GetMetricsRequest)
        returns (milvus.GetMetricsResponse) {
    }

    rpc CreateJobV2(CreateJobV2Request) returns (common.Status) {
    }
    rpc QueryJobsV2(QueryJobsV2Request) returns (QueryJobsV2Response) {
    }
    rpc DropJobsV2(DropJobsV2Request) returns (common.Status) {
    }
}

message IndexInfo {
    int64 collectionID = 1;
    int64 fieldID = 2;
    string index_name = 3;
    int64 indexID = 4;
    repeated common.KeyValuePair type_params = 5;
    repeated common.KeyValuePair index_params = 6;
    // index build progress
    // The real-time statistics may not be expected due to the existence of the compaction mechanism.
    int64 indexed_rows = 7;
    int64 total_rows = 8;
    // index state
    common.IndexState state = 9;
    string index_state_fail_reason = 10;
    bool is_auto_index = 11;
    repeated common.KeyValuePair user_index_params = 12;
    int64 pending_index_rows = 13;
}

message FieldIndex {
    IndexInfo index_info = 1;
    bool deleted = 2;
    uint64 create_time = 3;
}

message SegmentIndex {
    int64 collectionID = 1;
    int64 partitionID = 2;
    int64 segmentID = 3;
    int64 num_rows = 4;
    int64 indexID = 5;
    int64 buildID = 6;
    int64 nodeID = 7;
    int64 index_version = 8;
    common.IndexState state = 9;
    string fail_reason = 10;
    repeated string index_file_keys = 11;
    bool deleted = 12;
    uint64 create_time = 13;
    uint64 serialize_size = 14;
    bool write_handoff = 15;
    int32 current_index_version = 16;
    int64 index_store_version = 17;
}

message RegisterNodeRequest {
    common.MsgBase base = 1;
    common.Address address = 2;
    int64 nodeID = 3;
}

message RegisterNodeResponse {
    common.Status status = 1;
    internal.InitParams init_params = 2;
}

message GetIndexStateRequest {
    int64 collectionID = 1;
    string index_name = 2;
}

message GetIndexStateResponse {
    common.Status status = 1;
    common.IndexState state = 2;
    string fail_reason = 3;
}

message GetSegmentIndexStateRequest {
    int64 collectionID = 1;
    string index_name = 2;
    repeated int64 segmentIDs = 3;
}

message SegmentIndexState {
    int64 segmentID = 1;
    common.IndexState state = 2;
    string fail_reason = 3;
    string index_name = 4;
}

message GetSegmentIndexStateResponse {
    common.Status status = 1;
    repeated SegmentIndexState states = 2;
}

message CreateIndexRequest {
    int64 collectionID = 1;
    int64 fieldID = 2;
    string index_name = 3;
    repeated common.KeyValuePair type_params = 4;
    repeated common.KeyValuePair index_params = 5;
    uint64 timestamp = 6;
    bool is_auto_index = 7;
    repeated common.KeyValuePair user_index_params = 8;
    bool user_autoindex_metric_type_specified = 9;
}

message AlterIndexRequest {
    int64 collectionID = 1;
    string index_name = 2;
    repeated common.KeyValuePair params = 3;
}

message GetIndexInfoRequest {
    int64 collectionID = 1;
    repeated int64 segmentIDs = 2;
    string index_name = 3;
}

message IndexFilePathInfo {
    int64 segmentID = 1;
    int64 fieldID = 2;
    int64 indexID = 3;
    int64 buildID = 4;
    string index_name = 5;
    repeated common.KeyValuePair index_params = 6;
    repeated string index_file_paths = 7;
    uint64 serialized_size = 8;
    int64 index_version = 9;
    int64 num_rows = 10;
    int32 current_index_version = 11;
}

message SegmentInfo {
    int64 collectionID = 1;
    int64 segmentID = 2;
    bool enable_index = 3;
    repeated IndexFilePathInfo index_infos = 4;
}

message GetIndexInfoResponse {
    common.Status status = 1;
    map<int64, SegmentInfo> segment_info = 2;
}

message DropIndexRequest {
    int64 collectionID = 1;
    repeated int64 partitionIDs = 2;
    string index_name = 3;
    bool drop_all = 4;
}

message DescribeIndexRequest {
    int64 collectionID = 1;
    string index_name = 2;
    uint64 timestamp = 3;
}

message DescribeIndexResponse {
    common.Status status = 1;
    repeated IndexInfo index_infos = 2;
}

message GetIndexBuildProgressRequest {
    int64 collectionID = 1;
    string index_name = 2;
}

message GetIndexBuildProgressResponse {
    common.Status status = 1;
    int64 indexed_rows = 2;
    int64 total_rows = 3;
    int64 pending_index_rows = 4;
}

// Synchronously modify StorageConfig in index_cgo_msg.proto/clustering.proto file
message StorageConfig {
    string address = 1;
    string access_keyID = 2;
    string secret_access_key = 3;
    bool useSSL = 4;
    string bucket_name = 5;
    string root_path = 6;
    bool useIAM = 7;
    string IAMEndpoint = 8;
    string storage_type = 9;
    bool use_virtual_host = 10;
    string region = 11;
    string cloud_provider = 12;
    int64 request_timeout_ms = 13;
    string sslCACert = 14;
}

// Synchronously modify OptionalFieldInfo in index_cgo_msg.proto file
message OptionalFieldInfo {
    int64 fieldID = 1;
    string field_name = 2;
    int32 field_type = 3;
    repeated string data_paths = 4;
    repeated int64 data_ids = 5;
}

message CreateJobRequest {
    string clusterID = 1;
    string index_file_prefix = 2;
    int64 buildID = 3;
    repeated string data_paths = 4;
    int64 index_version = 5;
    int64 indexID = 6;
    string index_name = 7;
    StorageConfig storage_config = 8;
    repeated common.KeyValuePair index_params = 9;
    repeated common.KeyValuePair type_params = 10;
    int64 num_rows = 11;
    int32 current_index_version = 12;
    int64 collectionID = 13;
    int64 partitionID = 14;
    int64 segmentID = 15;
    int64 fieldID = 16;
    string field_name = 17;
    schema.DataType field_type = 18;
    string store_path = 19;
    int64 store_version = 20;
    string index_store_path = 21;
    int64 dim = 22;
    repeated int64 data_ids = 23;
    repeated OptionalFieldInfo optional_scalar_fields = 24;
    schema.FieldSchema field = 25;
    bool partition_key_isolation = 26;
}

message QueryJobsRequest {
    string clusterID = 1;
    repeated int64 buildIDs = 2;
}

message IndexTaskInfo {
    int64 buildID = 1;
    common.IndexState state = 2;
    repeated string index_file_keys = 3;
    uint64 serialized_size = 4;
    string fail_reason = 5;
    int32 current_index_version = 6;
    int64 index_store_version = 7;
}

message QueryJobsResponse {
    common.Status status = 1;
    string clusterID = 2;
    repeated IndexTaskInfo index_infos = 3;
}

message DropJobsRequest {
    string clusterID = 1;
    repeated int64 buildIDs = 2;
}

message JobInfo {
    int64 num_rows = 1;
    int64 dim = 2;
    int64 start_time = 3;
    int64 end_time = 4;
    repeated common.KeyValuePair index_params = 5;
    int64 podID = 6;
}

message GetJobStatsRequest {
}

message GetJobStatsResponse {
    common.Status status = 1;
    int64 total_job_num = 2;
    int64 in_progress_job_num = 3;
    int64 enqueue_job_num = 4;
    int64 task_slots = 5;
    repeated JobInfo job_infos = 6;
    bool enable_disk = 7;
}

message GetIndexStatisticsRequest {
    int64 collectionID = 1;
    string index_name = 2;
}

message GetIndexStatisticsResponse {
    common.Status status = 1;
    repeated IndexInfo index_infos = 2;
}

message ListIndexesRequest {
    int64 collectionID = 1;
}

message ListIndexesResponse {
    common.Status status = 1;
    repeated IndexInfo index_infos = 2;
}

message AnalyzeTask {
    int64 collectionID = 1;
    int64 partitionID = 2;
    int64 fieldID = 3;
    string field_name = 4;
    schema.DataType field_type = 5;
    int64 taskID = 6;
    int64 version = 7;
    repeated int64 segmentIDs = 8;
    int64 nodeID = 9;
    JobState state = 10;
    string fail_reason = 11;
    int64 dim = 12;
    string centroids_file = 13;
}

message SegmentStats {
    int64 ID = 1;
    int64 num_rows = 2;
    repeated int64 logIDs = 3;
}

message AnalyzeRequest {
    string clusterID = 1;
    int64 taskID = 2;
    int64 collectionID = 3;
    int64 partitionID = 4;
    int64 fieldID = 5;
    string fieldName = 6;
    schema.DataType field_type = 7;
    map<int64, SegmentStats> segment_stats = 8;
    int64 version = 9;
    StorageConfig storage_config = 10;
    int64 dim = 11;
    double max_train_size_ratio = 12;
    int64 num_clusters = 13;
    schema.FieldSchema field = 14;
    double min_cluster_size_ratio = 15;
    double max_cluster_size_ratio = 16;
    int64 max_cluster_size = 17;
}

message AnalyzeResult {
    int64 taskID = 1;
    JobState state = 2;
    string fail_reason = 3;
    string centroids_file = 4;
}

enum JobType {
    JobTypeNone = 0;
    JobTypeIndexJob = 1;
    JobTypeAnalyzeJob = 2;
}

message CreateJobV2Request {
    string clusterID = 1;
    int64 taskID = 2;
    JobType job_type = 3;
    oneof request {
        AnalyzeRequest analyze_request = 4;
        CreateJobRequest index_request = 5;
    }
    //    JobDescriptor job = 3;
}

message QueryJobsV2Request {
    string clusterID = 1;
    repeated int64 taskIDs = 2;
    JobType job_type = 3;
}

message IndexJobResults {
    repeated IndexTaskInfo results = 1;
}

message AnalyzeResults {
    repeated AnalyzeResult results = 1;
}

message QueryJobsV2Response {
    common.Status status = 1;
    string clusterID = 2;
    oneof result {
        IndexJobResults index_job_results = 3;
        AnalyzeResults analyze_job_results = 4;
    }
}

message DropJobsV2Request {
    string clusterID = 1;
    repeated int64 taskIDs = 2;
    JobType job_type = 3;
}


enum JobState {
    JobStateNone = 0;
    JobStateInit = 1;
    JobStateInProgress = 2;
    JobStateFinished = 3;
    JobStateFailed = 4;
    JobStateRetry = 5;
}