From 141a3bbe004159983eeb411a7b2627dbe2eb4f3d Mon Sep 17 00:00:00 2001 From: "cai.zhang" Date: Fri, 17 Jun 2022 18:08:12 +0800 Subject: [PATCH] Mark deleted before drop index (#17490) Signed-off-by: Cai.Zhang --- .../distributed/indexcoord/client/client.go | 13 + .../indexcoord/client/client_test.go | 9 + internal/distributed/indexcoord/service.go | 4 + .../distributed/indexcoord/service_test.go | 9 + internal/distributed/proxy/service_test.go | 4 + .../distributed/rootcoord/service_test.go | 19 +- internal/indexcoord/index_coord.go | 30 ++ internal/indexcoord/index_coord_mock.go | 11 + internal/indexcoord/index_coord_mock_test.go | 18 ++ internal/indexcoord/index_coord_test.go | 15 + internal/indexcoord/meta_table.go | 36 +++ internal/indexcoord/meta_table_test.go | 12 + internal/proto/etcd_meta.proto | 1 + internal/proto/etcdpb/etcd_meta.pb.go | 104 ++++--- internal/proto/index_coord.proto | 5 + internal/proto/indexpb/index_coord.pb.go | 216 ++++++++----- internal/proxy/index_coord_mock_test.go | 7 + internal/rootcoord/meta_table.go | 276 +++++++++++++++-- internal/rootcoord/meta_table_test.go | 220 ++++++++++++- internal/rootcoord/root_coord.go | 291 ++++++++++-------- internal/rootcoord/root_coord_test.go | 20 +- internal/rootcoord/task.go | 9 +- internal/rootcoord/task_test.go | 4 +- internal/types/types.go | 3 + 24 files changed, 1053 insertions(+), 283 deletions(-) diff --git a/internal/distributed/indexcoord/client/client.go b/internal/distributed/indexcoord/client/client.go index 13ed815da8..0b821c883b 100644 --- a/internal/distributed/indexcoord/client/client.go +++ b/internal/distributed/indexcoord/client/client.go @@ -181,6 +181,19 @@ func (c *Client) DropIndex(ctx context.Context, req *indexpb.DropIndexRequest) ( return ret.(*commonpb.Status), err } +func (c *Client) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) { + ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { + if !funcutil.CheckCtxValid(ctx) { + return nil, ctx.Err() + } + return client.(indexpb.IndexCoordClient).RemoveIndex(ctx, req) + }) + if err != nil || ret == nil { + return nil, err + } + return ret.(*commonpb.Status), err +} + // GetIndexStates gets the index states from IndexCoord. func (c *Client) GetIndexStates(ctx context.Context, req *indexpb.GetIndexStatesRequest) (*indexpb.GetIndexStatesResponse, error) { ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { diff --git a/internal/distributed/indexcoord/client/client_test.go b/internal/distributed/indexcoord/client/client_test.go index b859dab439..45352dc16e 100644 --- a/internal/distributed/indexcoord/client/client_test.go +++ b/internal/distributed/indexcoord/client/client_test.go @@ -102,6 +102,15 @@ func TestIndexCoordClient(t *testing.T) { assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode) }) + t.Run("RemoveIndex", func(t *testing.T) { + req := &indexpb.RemoveIndexRequest{ + BuildIDs: []int64{0}, + } + resp, err := icc.RemoveIndex(ctx, req) + assert.Nil(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode) + }) + t.Run("GetIndexStates", func(t *testing.T) { req := &indexpb.GetIndexStatesRequest{ IndexBuildIDs: []int64{0}, diff --git a/internal/distributed/indexcoord/service.go b/internal/distributed/indexcoord/service.go index 3269a51f2a..f8e9227431 100644 --- a/internal/distributed/indexcoord/service.go +++ b/internal/distributed/indexcoord/service.go @@ -227,6 +227,10 @@ func (s *Server) DropIndex(ctx context.Context, request *indexpb.DropIndexReques return s.indexcoord.DropIndex(ctx, request) } +func (s *Server) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) { + return s.indexcoord.RemoveIndex(ctx, req) +} + // GetIndexFilePaths gets the index file paths from IndexCoord. func (s *Server) GetIndexFilePaths(ctx context.Context, req *indexpb.GetIndexFilePathsRequest) (*indexpb.GetIndexFilePathsResponse, error) { return s.indexcoord.GetIndexFilePaths(ctx, req) diff --git a/internal/distributed/indexcoord/service_test.go b/internal/distributed/indexcoord/service_test.go index 3282ec777f..50bea362c5 100644 --- a/internal/distributed/indexcoord/service_test.go +++ b/internal/distributed/indexcoord/service_test.go @@ -102,6 +102,15 @@ func TestIndexCoordinateServer(t *testing.T) { assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode) }) + t.Run("RemoveIndex", func(t *testing.T) { + req := &indexpb.RemoveIndexRequest{ + BuildIDs: []UniqueID{0}, + } + resp, err := server.RemoveIndex(ctx, req) + assert.Nil(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode) + }) + t.Run("GetIndexFilePaths", func(t *testing.T) { req := &indexpb.GetIndexFilePathsRequest{ IndexBuildIDs: []UniqueID{0, 1}, diff --git a/internal/distributed/proxy/service_test.go b/internal/distributed/proxy/service_test.go index aaf25a4343..f5120ab595 100644 --- a/internal/distributed/proxy/service_test.go +++ b/internal/distributed/proxy/service_test.go @@ -291,6 +291,10 @@ func (m *MockIndexCoord) DropIndex(ctx context.Context, req *indexpb.DropIndexRe return nil, nil } +func (m *MockIndexCoord) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) { + return nil, nil +} + func (m *MockIndexCoord) GetIndexStates(ctx context.Context, req *indexpb.GetIndexStatesRequest) (*indexpb.GetIndexStatesResponse, error) { return nil, nil } diff --git a/internal/distributed/rootcoord/service_test.go b/internal/distributed/rootcoord/service_test.go index 7c73d4c802..eb8a0cd220 100644 --- a/internal/distributed/rootcoord/service_test.go +++ b/internal/distributed/rootcoord/service_test.go @@ -208,6 +208,10 @@ func TestGrpcService(t *testing.T) { return nil } + core.CallRemoveIndexService = func(ctx context.Context, buildIDs []rootcoord.UniqueID) error { + return nil + } + collectionMetaCache := make([]string, 0, 16) pnm := proxyMock{} core.NewProxyClient = func(*sessionutil.Session) (types.Proxy, error) { @@ -738,10 +742,17 @@ func TestGrpcService(t *testing.T) { assert.Nil(t, err) assert.Equal(t, commonpb.ErrorCode_Success, rsp.ErrorCode) - dropIDLock.Lock() - assert.Equal(t, 1, len(dropID)) - assert.Equal(t, idx[0].IndexID, dropID[0]) - dropIDLock.Unlock() + for { + dropIDLock.Lock() + if len(dropID) == 1 { + assert.Equal(t, idx[0].IndexID, dropID[0]) + dropIDLock.Unlock() + break + } + dropIDLock.Unlock() + time.Sleep(time.Second) + } + }) t.Run("drop partition", func(t *testing.T) { diff --git a/internal/indexcoord/index_coord.go b/internal/indexcoord/index_coord.go index ab282e7dc0..8a97139e33 100644 --- a/internal/indexcoord/index_coord.go +++ b/internal/indexcoord/index_coord.go @@ -567,6 +567,36 @@ func (i *IndexCoord) DropIndex(ctx context.Context, req *indexpb.DropIndexReques return ret, nil } +func (i *IndexCoord) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) { + log.Info("IndexCoord receive RemoveIndex", zap.Int64s("buildIDs", req.BuildIDs)) + + if !i.isHealthy() { + errMsg := "IndexCoord is not healthy" + log.Warn(errMsg) + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: errMsg, + }, nil + } + + sp, _ := trace.StartSpanFromContextWithOperationName(ctx, "IndexCoord-RemoveIndex") + defer sp.Finish() + + ret := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + } + err := i.metaTable.MarkIndexAsDeletedByBuildIDs(req.GetBuildIDs()) + if err != nil { + log.Error("IndexCoord MarkIndexAsDeletedByBuildIDs failed", zap.Int64s("buildIDs", req.GetBuildIDs()), + zap.Error(err)) + ret.ErrorCode = commonpb.ErrorCode_UnexpectedError + ret.Reason = err.Error() + return ret, nil + } + + return ret, nil +} + // GetIndexFilePaths gets the index file paths from IndexCoord. func (i *IndexCoord) GetIndexFilePaths(ctx context.Context, req *indexpb.GetIndexFilePathsRequest) (*indexpb.GetIndexFilePathsResponse, error) { log.Debug("IndexCoord GetIndexFilePaths", zap.Int("number of IndexBuildIds", len(req.IndexBuildIDs))) diff --git a/internal/indexcoord/index_coord_mock.go b/internal/indexcoord/index_coord_mock.go index 7cd1911968..c72f1e18ef 100644 --- a/internal/indexcoord/index_coord_mock.go +++ b/internal/indexcoord/index_coord_mock.go @@ -252,6 +252,17 @@ func (icm *Mock) GetMetrics(ctx context.Context, request *milvuspb.GetMetricsReq }, nil } +func (icm *Mock) RemoveIndex(ctx context.Context, request *indexpb.RemoveIndexRequest) (*commonpb.Status, error) { + if icm.Failure { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + }, errors.New("IndexCoordinator RemoveIndex failed") + } + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, nil +} + type DataCoordMock struct { types.DataCoord diff --git a/internal/indexcoord/index_coord_mock_test.go b/internal/indexcoord/index_coord_mock_test.go index f20917ede1..d58c913dbd 100644 --- a/internal/indexcoord/index_coord_mock_test.go +++ b/internal/indexcoord/index_coord_mock_test.go @@ -117,6 +117,15 @@ func TestIndexCoordMock(t *testing.T) { assert.Equal(t, "IndexCoord", resp.ComponentName) }) + t.Run("RemoveIndex", func(t *testing.T) { + req := &indexpb.RemoveIndexRequest{ + BuildIDs: []UniqueID{}, + } + status, err := icm.RemoveIndex(ctx, req) + assert.Nil(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode()) + }) + err = icm.Stop() assert.Nil(t, err) } @@ -200,6 +209,15 @@ func TestIndexCoordMockError(t *testing.T) { assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode) }) + t.Run("RemoveIndex", func(t *testing.T) { + req := &indexpb.RemoveIndexRequest{ + BuildIDs: []UniqueID{}, + } + status, err := icm.RemoveIndex(ctx, req) + assert.Error(t, err) + assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.GetErrorCode()) + }) + err = icm.Stop() assert.NotNil(t, err) } diff --git a/internal/indexcoord/index_coord_test.go b/internal/indexcoord/index_coord_test.go index 0dfd5ddb41..2b3da6d26f 100644 --- a/internal/indexcoord/index_coord_test.go +++ b/internal/indexcoord/index_coord_test.go @@ -362,6 +362,11 @@ func TestIndexCoord_NotHealthy(t *testing.T) { resp4, err := ic.GetIndexFilePaths(context.Background(), req4) assert.Nil(t, err) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp4.Status.ErrorCode) + + req5 := &indexpb.RemoveIndexRequest{} + resp5, err := ic.RemoveIndex(context.Background(), req5) + assert.Nil(t, err) + assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp5.GetErrorCode()) } func TestIndexCoord_GetIndexFilePaths(t *testing.T) { @@ -470,3 +475,13 @@ func Test_tryReleaseSegmentReferLock(t *testing.T) { assert.NoError(t, err) }) } + +func TestIndexCoord_RemoveIndex(t *testing.T) { + ic := &IndexCoord{ + metaTable: &metaTable{}, + } + ic.stateCode.Store(internalpb.StateCode_Healthy) + status, err := ic.RemoveIndex(context.Background(), &indexpb.RemoveIndexRequest{BuildIDs: []UniqueID{0}}) + assert.Nil(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode()) +} diff --git a/internal/indexcoord/meta_table.go b/internal/indexcoord/meta_table.go index 97114367aa..1bfa6e1a76 100644 --- a/internal/indexcoord/meta_table.go +++ b/internal/indexcoord/meta_table.go @@ -281,6 +281,42 @@ func (mt *metaTable) MarkIndexAsDeleted(indexID UniqueID) error { return nil } +func (mt *metaTable) MarkIndexAsDeletedByBuildIDs(buildIDs []UniqueID) error { + mt.lock.Lock() + defer mt.lock.Unlock() + + log.Debug("IndexCoord metaTable MarkIndexAsDeletedByBuildIDs ", zap.Int64s("buildIDs", buildIDs)) + + for _, buildID := range buildIDs { + if meta, ok := mt.indexBuildID2Meta[buildID]; ok { + clonedMeta := &Meta{ + indexMeta: proto.Clone(meta.indexMeta).(*indexpb.IndexMeta), + revision: meta.revision, + } + clonedMeta.indexMeta.MarkDeleted = true + // marshal inside + /* #nosec G601 */ + if err := mt.saveIndexMeta(clonedMeta); err != nil { + log.Error("IndexCoord metaTable MarkIndexAsDeleted saveIndexMeta failed", zap.Error(err)) + fn := func() error { + m, err := mt.reloadMeta(meta.indexMeta.IndexBuildID) + if m == nil { + return err + } + + m.indexMeta.MarkDeleted = true + return mt.saveIndexMeta(m) + } + err2 := retry.Do(context.TODO(), fn, retry.Attempts(5)) + if err2 != nil { + return err2 + } + } + } + } + return nil +} + // GetIndexStates gets the index states from meta table. func (mt *metaTable) GetIndexStates(indexBuildIDs []UniqueID) []*indexpb.IndexInfo { mt.lock.Lock() diff --git a/internal/indexcoord/meta_table_test.go b/internal/indexcoord/meta_table_test.go index eaf26c2612..8cee5a7528 100644 --- a/internal/indexcoord/meta_table_test.go +++ b/internal/indexcoord/meta_table_test.go @@ -17,6 +17,7 @@ package indexcoord import ( + "path" "strconv" "testing" @@ -145,6 +146,17 @@ func TestMetaTable(t *testing.T) { assert.Equal(t, "index 1 has been deleted", indexInfos[1].Reason) }) + t.Run("MarkSegmentIndexAsDeleted", func(t *testing.T) { + indexMeta1.Version = indexMeta1.Version + 1 + value, err = proto.Marshal(indexMeta1) + assert.Nil(t, err) + key = path.Join(indexFilePrefix, strconv.FormatInt(indexMeta1.IndexBuildID, 10)) + err = etcdKV.Save(key, string(value)) + assert.Nil(t, err) + err = metaTable.MarkIndexAsDeletedByBuildIDs([]UniqueID{indexMeta1.IndexBuildID}) + assert.Nil(t, err) + }) + t.Run("GetIndexFilePathInfo", func(t *testing.T) { indexFilePathInfo, err := metaTable.GetIndexFilePathInfo(0) assert.Nil(t, indexFilePathInfo) diff --git a/internal/proto/etcd_meta.proto b/internal/proto/etcd_meta.proto index 2fb4608d10..b653322be9 100644 --- a/internal/proto/etcd_meta.proto +++ b/internal/proto/etcd_meta.proto @@ -15,6 +15,7 @@ message IndexInfo { string index_name = 1; int64 indexID = 2; repeated common.KeyValuePair index_params = 3; + bool deleted = 4; } message FieldIndexInfo{ diff --git a/internal/proto/etcdpb/etcd_meta.pb.go b/internal/proto/etcdpb/etcd_meta.pb.go index 50f332db1c..0b4e56987b 100644 --- a/internal/proto/etcdpb/etcd_meta.pb.go +++ b/internal/proto/etcdpb/etcd_meta.pb.go @@ -81,6 +81,7 @@ type IndexInfo struct { IndexName string `protobuf:"bytes,1,opt,name=index_name,json=indexName,proto3" json:"index_name,omitempty"` IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"` IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,3,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"` + Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -132,6 +133,13 @@ func (m *IndexInfo) GetIndexParams() []*commonpb.KeyValuePair { return nil } +func (m *IndexInfo) GetDeleted() bool { + if m != nil { + return m.Deleted + } + return false +} + type FieldIndexInfo struct { FiledID int64 `protobuf:"varint,1,opt,name=filedID,proto3" json:"filedID,omitempty"` IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"` @@ -492,52 +500,52 @@ func init() { func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) } var fileDescriptor_975d306d62b73e88 = []byte{ - // 737 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4d, 0x6b, 0xdb, 0x4a, - 0x14, 0x45, 0x96, 0x63, 0x47, 0xd7, 0x8a, 0x93, 0xcc, 0xfb, 0x60, 0x08, 0x79, 0xef, 0x29, 0x82, - 0x3c, 0x04, 0xa5, 0x36, 0x4d, 0x4a, 0x77, 0x85, 0xa6, 0x16, 0x01, 0xd3, 0x36, 0x98, 0x49, 0xe8, - 0xa2, 0x1b, 0x31, 0x96, 0xc6, 0xf6, 0x80, 0x3e, 0x8c, 0x66, 0x14, 0xe2, 0x5d, 0xd7, 0xed, 0x4f, - 0xed, 0x7f, 0x28, 0x45, 0x33, 0x92, 0xfc, 0x91, 0x64, 0xd9, 0x9d, 0xef, 0x99, 0x7b, 0xae, 0xee, - 0x9c, 0x39, 0xc7, 0x70, 0xc8, 0x64, 0x18, 0x05, 0x09, 0x93, 0x74, 0xb0, 0xcc, 0x33, 0x99, 0xa1, - 0xe3, 0x84, 0xc7, 0xf7, 0x85, 0xd0, 0xd5, 0xa0, 0x3c, 0x3d, 0xb1, 0xc3, 0x2c, 0x49, 0xb2, 0x54, - 0x43, 0x27, 0xb6, 0x08, 0x17, 0x2c, 0xa9, 0xda, 0xdd, 0xaf, 0x06, 0x58, 0x93, 0x3c, 0x7b, 0x58, - 0x7d, 0x62, 0x92, 0xa2, 0x3e, 0xb4, 0xc6, 0x3e, 0x36, 0x1c, 0xc3, 0x33, 0x49, 0x6b, 0xec, 0xa3, - 0x37, 0xd0, 0xa5, 0x51, 0x94, 0x33, 0x21, 0x70, 0xcb, 0x31, 0xbc, 0xde, 0xc5, 0xe9, 0x60, 0x6b, - 0x7c, 0x35, 0xf8, 0x4a, 0xf7, 0x90, 0xba, 0x19, 0xbd, 0x80, 0xe3, 0x9c, 0x89, 0x22, 0x96, 0x41, - 0xb8, 0xa0, 0x69, 0xca, 0xe2, 0xb1, 0x2f, 0xb0, 0xe9, 0x98, 0x9e, 0x45, 0x8e, 0xf4, 0xc1, 0xa8, - 0xc1, 0xdd, 0x6f, 0x06, 0x58, 0xe3, 0x34, 0x62, 0x0f, 0xe3, 0x74, 0x96, 0xa1, 0x7f, 0x00, 0x78, - 0x59, 0x04, 0x29, 0x4d, 0x98, 0x5a, 0xc5, 0x22, 0x96, 0x42, 0x6e, 0x68, 0xc2, 0x10, 0x86, 0xae, - 0x2a, 0xc6, 0xbe, 0xda, 0xc8, 0x24, 0x75, 0x89, 0x7c, 0xb0, 0x35, 0x71, 0x49, 0x73, 0x9a, 0xe8, - 0xcf, 0xf5, 0x2e, 0xce, 0x9e, 0x5c, 0xf8, 0x03, 0x5b, 0x7d, 0xa6, 0x71, 0xc1, 0x26, 0x94, 0xe7, - 0xa4, 0xa7, 0x68, 0x13, 0xc5, 0x72, 0x7d, 0xe8, 0x5f, 0x73, 0x16, 0x47, 0xeb, 0x85, 0x30, 0x74, - 0x67, 0x3c, 0x66, 0x51, 0x23, 0x4c, 0x5d, 0x3e, 0xbf, 0x8b, 0xfb, 0xb3, 0x0d, 0xfd, 0x51, 0x16, - 0xc7, 0x2c, 0x94, 0x3c, 0x4b, 0xd5, 0x98, 0x5d, 0x69, 0xdf, 0x42, 0x47, 0x3f, 0x44, 0xa5, 0xec, - 0xf9, 0xf6, 0xa2, 0xd5, 0x23, 0xad, 0x87, 0xdc, 0x2a, 0x80, 0x54, 0x24, 0xf4, 0x1f, 0xf4, 0xc2, - 0x9c, 0x51, 0xc9, 0x02, 0xc9, 0x13, 0x86, 0x4d, 0xc7, 0xf0, 0xda, 0x04, 0x34, 0x74, 0xc7, 0x13, - 0x86, 0x5c, 0xb0, 0x97, 0x34, 0x97, 0x5c, 0x2d, 0xe0, 0x0b, 0xdc, 0x76, 0x4c, 0xcf, 0x24, 0x5b, - 0x18, 0xfa, 0x1f, 0xfa, 0x4d, 0x5d, 0xaa, 0x2b, 0xf0, 0x9e, 0x7a, 0xa3, 0x1d, 0x14, 0x5d, 0xc3, - 0xc1, 0xac, 0x14, 0x25, 0x50, 0xf7, 0x63, 0x02, 0x77, 0x9e, 0xd2, 0xb6, 0xf4, 0xda, 0x60, 0x5b, - 0x3c, 0x62, 0xcf, 0x9a, 0x9a, 0x09, 0x74, 0x01, 0x7f, 0xdd, 0xf3, 0x5c, 0x16, 0x34, 0xae, 0x7d, - 0xa1, 0x5e, 0x59, 0xe0, 0xae, 0xfa, 0xec, 0x1f, 0xd5, 0x61, 0xe5, 0x0d, 0xfd, 0xed, 0xd7, 0xf0, - 0xf7, 0x72, 0xb1, 0x12, 0x3c, 0x7c, 0x44, 0xda, 0x57, 0xa4, 0x3f, 0xeb, 0xd3, 0x2d, 0xd6, 0x3b, - 0x38, 0x6d, 0xee, 0x10, 0x68, 0x55, 0x22, 0xa5, 0x94, 0x90, 0x34, 0x59, 0x0a, 0x6c, 0x39, 0xa6, - 0xd7, 0x26, 0x27, 0x4d, 0xcf, 0x48, 0xb7, 0xdc, 0x35, 0x1d, 0xa5, 0x0f, 0xc5, 0x82, 0xe6, 0x91, - 0x08, 0xd2, 0x22, 0xc1, 0xe0, 0x18, 0xde, 0x1e, 0xb1, 0x34, 0x72, 0x53, 0x24, 0x68, 0x0c, 0x87, - 0x42, 0xd2, 0x5c, 0x06, 0xcb, 0x4c, 0xa8, 0x09, 0x02, 0xf7, 0x94, 0x28, 0xce, 0x73, 0x86, 0xf3, - 0xa9, 0xa4, 0xca, 0x6f, 0x7d, 0x45, 0x9c, 0xd4, 0x3c, 0x44, 0xe0, 0x38, 0xcc, 0x52, 0xc1, 0x85, - 0x64, 0x69, 0xb8, 0x0a, 0x62, 0x76, 0xcf, 0x62, 0x6c, 0x3b, 0x86, 0xd7, 0xdf, 0x35, 0x45, 0x35, - 0x6c, 0xb4, 0xee, 0xfe, 0x58, 0x36, 0x93, 0xa3, 0x70, 0x07, 0x71, 0xbf, 0xb7, 0xe0, 0xe8, 0x96, - 0xcd, 0x13, 0x96, 0xca, 0xb5, 0x93, 0x5d, 0xb0, 0xc3, 0xb5, 0x29, 0x6b, 0x33, 0x6e, 0x61, 0xc8, - 0x81, 0xde, 0x86, 0x45, 0x2a, 0x5f, 0x6f, 0x42, 0xe8, 0x14, 0x2c, 0x51, 0x4d, 0xf6, 0x95, 0xef, - 0x4c, 0xb2, 0x06, 0x74, 0x5a, 0xca, 0x27, 0xf7, 0x71, 0xbb, 0x4e, 0x8b, 0x2a, 0x37, 0xd3, 0xb2, - 0xb7, 0x9d, 0x5c, 0x0c, 0xdd, 0x69, 0xc1, 0x15, 0xa7, 0xa3, 0x4f, 0xaa, 0x12, 0x9d, 0x81, 0xcd, - 0x52, 0x3a, 0x8d, 0x99, 0x76, 0x1e, 0xee, 0x3a, 0x86, 0xb7, 0x4f, 0x7a, 0x1a, 0x53, 0x17, 0x43, - 0x2e, 0x1c, 0x4c, 0x57, 0x01, 0x2d, 0x64, 0x16, 0xcc, 0xe2, 0x42, 0x2c, 0xf0, 0xbe, 0xee, 0x99, - 0xae, 0xae, 0x0a, 0x99, 0x5d, 0x97, 0x90, 0xfb, 0xc3, 0xd8, 0x8c, 0xe3, 0x93, 0xff, 0x74, 0xbf, - 0x3b, 0x8e, 0xff, 0x02, 0x34, 0x22, 0xd5, 0x61, 0xdc, 0x40, 0xd0, 0xf9, 0x46, 0x14, 0x03, 0x49, - 0xe7, 0x75, 0x14, 0x0f, 0x1a, 0xf4, 0x8e, 0xce, 0xc5, 0xa3, 0x54, 0x77, 0x1e, 0xa7, 0xfa, 0xfd, - 0xe5, 0x97, 0x57, 0x73, 0x2e, 0x17, 0xc5, 0xb4, 0xf4, 0xcb, 0x50, 0x5f, 0xe3, 0x25, 0xcf, 0xaa, - 0x5f, 0x43, 0x9e, 0x4a, 0x96, 0xa7, 0x34, 0x1e, 0xaa, 0x9b, 0x0d, 0xcb, 0xd4, 0x2e, 0xa7, 0xd3, - 0x8e, 0xaa, 0x2e, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0x24, 0xa2, 0x1e, 0x16, 0x50, 0x06, 0x00, - 0x00, + // 751 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4d, 0x6b, 0xeb, 0x46, + 0x14, 0x45, 0x96, 0x63, 0x47, 0xd7, 0x8a, 0x5f, 0x32, 0xfd, 0x60, 0x08, 0x69, 0xab, 0x27, 0x78, + 0x45, 0x50, 0x6a, 0xd3, 0xbc, 0xd2, 0x5d, 0xa1, 0xaf, 0x16, 0x01, 0xd3, 0xf6, 0x61, 0x26, 0xa1, + 0x8b, 0x6e, 0xc4, 0x58, 0xba, 0xb6, 0x07, 0xf4, 0x61, 0x34, 0xa3, 0x10, 0xef, 0xba, 0xef, 0xdf, + 0xe8, 0xbf, 0xeb, 0x7f, 0x28, 0x45, 0x33, 0x92, 0xfc, 0x91, 0x64, 0xd9, 0x9d, 0xcf, 0x99, 0x7b, + 0xaf, 0xe6, 0x9e, 0x39, 0xc7, 0xf0, 0x06, 0x55, 0x9c, 0x44, 0x19, 0x2a, 0x3e, 0xd9, 0x96, 0x85, + 0x2a, 0xc8, 0x55, 0x26, 0xd2, 0xc7, 0x4a, 0x1a, 0x34, 0xa9, 0x4f, 0xaf, 0xdd, 0xb8, 0xc8, 0xb2, + 0x22, 0x37, 0xd4, 0xb5, 0x2b, 0xe3, 0x0d, 0x66, 0x4d, 0xb9, 0xff, 0xa7, 0x05, 0xce, 0xa2, 0x2c, + 0x9e, 0x76, 0xbf, 0xa1, 0xe2, 0x64, 0x0c, 0xbd, 0x79, 0x48, 0x2d, 0xcf, 0x0a, 0x6c, 0xd6, 0x9b, + 0x87, 0xe4, 0x07, 0x18, 0xf2, 0x24, 0x29, 0x51, 0x4a, 0xda, 0xf3, 0xac, 0x60, 0x74, 0x7b, 0x33, + 0x39, 0x1a, 0xdf, 0x0c, 0xfe, 0x60, 0x6a, 0x58, 0x5b, 0x4c, 0xbe, 0x81, 0xab, 0x12, 0x65, 0x95, + 0xaa, 0x28, 0xde, 0xf0, 0x3c, 0xc7, 0x74, 0x1e, 0x4a, 0x6a, 0x7b, 0x76, 0xe0, 0xb0, 0x4b, 0x73, + 0x30, 0xeb, 0x78, 0xff, 0x6f, 0x0b, 0x9c, 0x79, 0x9e, 0xe0, 0xd3, 0x3c, 0x5f, 0x15, 0xe4, 0x0b, + 0x00, 0x51, 0x83, 0x28, 0xe7, 0x19, 0xea, 0xab, 0x38, 0xcc, 0xd1, 0xcc, 0x47, 0x9e, 0x21, 0xa1, + 0x30, 0xd4, 0x60, 0x1e, 0xea, 0x1b, 0xd9, 0xac, 0x85, 0x24, 0x04, 0xd7, 0x34, 0x6e, 0x79, 0xc9, + 0x33, 0xf3, 0xb9, 0xd1, 0xed, 0xdb, 0x17, 0x2f, 0xfc, 0x0b, 0xee, 0x7e, 0xe7, 0x69, 0x85, 0x0b, + 0x2e, 0x4a, 0x36, 0xd2, 0x6d, 0x0b, 0xdd, 0x55, 0xcf, 0x4f, 0x30, 0x45, 0x85, 0x09, 0xed, 0x7b, + 0x56, 0x70, 0xce, 0x5a, 0xe8, 0x87, 0x30, 0xbe, 0x13, 0x98, 0x26, 0xfb, 0xab, 0x52, 0x18, 0xae, + 0x44, 0x8a, 0x49, 0x27, 0x59, 0x0b, 0x5f, 0xbf, 0xa5, 0xff, 0x6f, 0x1f, 0xc6, 0xb3, 0x22, 0x4d, + 0x31, 0x56, 0xa2, 0xc8, 0xf5, 0x98, 0x53, 0xd1, 0x7f, 0x84, 0x81, 0x79, 0xa2, 0x46, 0xf3, 0x77, + 0xc7, 0x2b, 0x34, 0xcf, 0xb7, 0x1f, 0x72, 0xaf, 0x09, 0xd6, 0x34, 0x91, 0xaf, 0x60, 0x14, 0x97, + 0xc8, 0x15, 0x46, 0x4a, 0x64, 0x48, 0x6d, 0xcf, 0x0a, 0xfa, 0x0c, 0x0c, 0xf5, 0x20, 0x32, 0x24, + 0x3e, 0xb8, 0x5b, 0x5e, 0x2a, 0xa1, 0x2f, 0x10, 0x4a, 0xda, 0xf7, 0xec, 0xc0, 0x66, 0x47, 0x1c, + 0xf9, 0x1a, 0xc6, 0x1d, 0xae, 0x75, 0x97, 0xf4, 0x4c, 0xbf, 0xde, 0x09, 0x4b, 0xee, 0xe0, 0x62, + 0x55, 0x8b, 0x12, 0xe9, 0xfd, 0x50, 0xd2, 0xc1, 0x4b, 0xaa, 0xd7, 0x2e, 0x9c, 0x1c, 0x8b, 0xc7, + 0xdc, 0x55, 0x87, 0x51, 0x92, 0x5b, 0xf8, 0xec, 0x51, 0x94, 0xaa, 0xe2, 0x69, 0xeb, 0x18, 0xfd, + 0xfe, 0x92, 0x0e, 0xf5, 0x67, 0x3f, 0x69, 0x0e, 0x1b, 0xd7, 0x98, 0x6f, 0x7f, 0x0f, 0x9f, 0x6f, + 0x37, 0x3b, 0x29, 0xe2, 0x67, 0x4d, 0xe7, 0xba, 0xe9, 0xd3, 0xf6, 0xf4, 0xa8, 0xeb, 0x27, 0xb8, + 0xe9, 0x76, 0x88, 0x8c, 0x2a, 0x89, 0x56, 0x4a, 0x2a, 0x9e, 0x6d, 0x25, 0x75, 0x3c, 0x3b, 0xe8, + 0xb3, 0xeb, 0xae, 0x66, 0x66, 0x4a, 0x1e, 0xba, 0x8a, 0xda, 0xa1, 0x72, 0xc3, 0xcb, 0x44, 0x46, + 0x79, 0x95, 0x51, 0xf0, 0xac, 0xe0, 0x8c, 0x39, 0x86, 0xf9, 0x58, 0x65, 0x64, 0x0e, 0x6f, 0xa4, + 0xe2, 0xa5, 0x8a, 0xb6, 0x85, 0xd4, 0x13, 0x24, 0x1d, 0x69, 0x51, 0xbc, 0xd7, 0xac, 0x18, 0x72, + 0xc5, 0xb5, 0x13, 0xc7, 0xba, 0x71, 0xd1, 0xf6, 0x11, 0x06, 0x57, 0x71, 0x91, 0x4b, 0x21, 0x15, + 0xe6, 0xf1, 0x2e, 0x4a, 0xf1, 0x11, 0x53, 0xea, 0x7a, 0x56, 0x30, 0x3e, 0x35, 0x45, 0x33, 0x6c, + 0xb6, 0xaf, 0xfe, 0xb5, 0x2e, 0x66, 0x97, 0xf1, 0x09, 0xe3, 0xff, 0xd5, 0x83, 0xcb, 0x7b, 0x5c, + 0x67, 0x98, 0xab, 0xbd, 0x93, 0x7d, 0x70, 0xe3, 0xbd, 0x29, 0x5b, 0x33, 0x1e, 0x71, 0xc4, 0x83, + 0xd1, 0x81, 0x45, 0x1a, 0x5f, 0x1f, 0x52, 0xe4, 0x06, 0x1c, 0xd9, 0x4c, 0x0e, 0xb5, 0xef, 0x6c, + 0xb6, 0x27, 0x4c, 0x5a, 0xea, 0x27, 0x0f, 0x75, 0xb2, 0x74, 0x5a, 0x34, 0x3c, 0x4c, 0xcb, 0xd9, + 0x71, 0xa6, 0x29, 0x0c, 0x97, 0x95, 0xd0, 0x3d, 0x03, 0x73, 0xd2, 0x40, 0xf2, 0x16, 0x5c, 0xcc, + 0xf9, 0x32, 0x45, 0xe3, 0x3c, 0x3a, 0xd4, 0x61, 0x1d, 0x19, 0x4e, 0x2f, 0x46, 0x7c, 0xb8, 0x58, + 0xee, 0x22, 0x5e, 0xa9, 0x22, 0x5a, 0xa5, 0x95, 0xdc, 0xd0, 0x73, 0x53, 0xb3, 0xdc, 0x7d, 0xa8, + 0x54, 0x71, 0x57, 0x53, 0xfe, 0x3f, 0xd6, 0x61, 0x1c, 0x5f, 0xfc, 0x0f, 0xfc, 0xbf, 0xe3, 0xf8, + 0x25, 0x40, 0x27, 0x52, 0x1b, 0xc6, 0x03, 0x86, 0xbc, 0x3b, 0x88, 0x62, 0xa4, 0xf8, 0xba, 0x8d, + 0xe2, 0x45, 0xc7, 0x3e, 0xf0, 0xb5, 0x7c, 0x96, 0xea, 0xc1, 0xf3, 0x54, 0xff, 0xfc, 0xfe, 0x8f, + 0xef, 0xd6, 0x42, 0x6d, 0xaa, 0x65, 0xed, 0x97, 0xa9, 0x59, 0xe3, 0x5b, 0x51, 0x34, 0xbf, 0xa6, + 0x22, 0x57, 0x58, 0xe6, 0x3c, 0x9d, 0xea, 0xcd, 0xa6, 0x75, 0x6a, 0xb7, 0xcb, 0xe5, 0x40, 0xa3, + 0xf7, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x19, 0xa3, 0x49, 0xe3, 0x6a, 0x06, 0x00, 0x00, } diff --git a/internal/proto/index_coord.proto b/internal/proto/index_coord.proto index d17512670e..3f6a1d657b 100644 --- a/internal/proto/index_coord.proto +++ b/internal/proto/index_coord.proto @@ -17,6 +17,7 @@ service IndexCoord { rpc GetIndexStates(GetIndexStatesRequest) returns (GetIndexStatesResponse) {} rpc GetIndexFilePaths(GetIndexFilePathsRequest) returns (GetIndexFilePathsResponse){} rpc DropIndex(DropIndexRequest) returns (common.Status) {} + rpc RemoveIndex(RemoveIndexRequest) returns (common.Status) {} // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {} @@ -120,3 +121,7 @@ message IndexMeta { message DropIndexRequest { int64 indexID = 1; } + +message RemoveIndexRequest { + repeated int64 buildIDs = 1; +} diff --git a/internal/proto/indexpb/index_coord.pb.go b/internal/proto/indexpb/index_coord.pb.go index 1672a1069f..fbca77f857 100644 --- a/internal/proto/indexpb/index_coord.pb.go +++ b/internal/proto/indexpb/index_coord.pb.go @@ -831,6 +831,45 @@ func (m *DropIndexRequest) GetIndexID() int64 { return 0 } +type RemoveIndexRequest struct { + BuildIDs []int64 `protobuf:"varint,1,rep,packed,name=buildIDs,proto3" json:"buildIDs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RemoveIndexRequest) Reset() { *m = RemoveIndexRequest{} } +func (m *RemoveIndexRequest) String() string { return proto.CompactTextString(m) } +func (*RemoveIndexRequest) ProtoMessage() {} +func (*RemoveIndexRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_f9e019eb3fda53c2, []int{13} +} + +func (m *RemoveIndexRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_RemoveIndexRequest.Unmarshal(m, b) +} +func (m *RemoveIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_RemoveIndexRequest.Marshal(b, m, deterministic) +} +func (m *RemoveIndexRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_RemoveIndexRequest.Merge(m, src) +} +func (m *RemoveIndexRequest) XXX_Size() int { + return xxx_messageInfo_RemoveIndexRequest.Size(m) +} +func (m *RemoveIndexRequest) XXX_DiscardUnknown() { + xxx_messageInfo_RemoveIndexRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_RemoveIndexRequest proto.InternalMessageInfo + +func (m *RemoveIndexRequest) GetBuildIDs() []int64 { + if m != nil { + return m.BuildIDs + } + return nil +} + func init() { proto.RegisterType((*RegisterNodeRequest)(nil), "milvus.proto.index.RegisterNodeRequest") proto.RegisterType((*RegisterNodeResponse)(nil), "milvus.proto.index.RegisterNodeResponse") @@ -845,80 +884,83 @@ func init() { proto.RegisterType((*GetIndexFilePathsResponse)(nil), "milvus.proto.index.GetIndexFilePathsResponse") proto.RegisterType((*IndexMeta)(nil), "milvus.proto.index.IndexMeta") proto.RegisterType((*DropIndexRequest)(nil), "milvus.proto.index.DropIndexRequest") + proto.RegisterType((*RemoveIndexRequest)(nil), "milvus.proto.index.RemoveIndexRequest") } func init() { proto.RegisterFile("index_coord.proto", fileDescriptor_f9e019eb3fda53c2) } var fileDescriptor_f9e019eb3fda53c2 = []byte{ - // 1087 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x57, 0xcd, 0x6e, 0x1b, 0x37, - 0x10, 0xf6, 0x7a, 0x6d, 0xfd, 0x8c, 0x14, 0x37, 0x66, 0xd3, 0x60, 0xa3, 0x24, 0x88, 0xbc, 0xcd, - 0x8f, 0x5a, 0x24, 0x72, 0xa0, 0x34, 0xed, 0xa9, 0x40, 0x6b, 0x0b, 0x31, 0x84, 0xc2, 0x81, 0x41, - 0x1b, 0x3d, 0x14, 0x28, 0x04, 0x5a, 0x3b, 0xb2, 0x88, 0xec, 0x8f, 0xbc, 0xa4, 0x92, 0xda, 0xc7, - 0xa2, 0xf7, 0xde, 0x92, 0x47, 0xe9, 0xb1, 0xcf, 0x90, 0x37, 0x2a, 0xc8, 0xe5, 0x4a, 0x5a, 0x69, - 0xe5, 0xc8, 0x75, 0xd3, 0x5e, 0x7a, 0xd3, 0xcc, 0x7e, 0xc3, 0x8f, 0xf3, 0xcd, 0x70, 0x48, 0xc1, - 0x26, 0x0f, 0x3d, 0xfc, 0xa5, 0xdb, 0x8b, 0xa2, 0xd8, 0x6b, 0x0e, 0xe3, 0x48, 0x46, 0x84, 0x04, - 0xdc, 0x7f, 0x3d, 0x12, 0x89, 0xd5, 0xd4, 0xdf, 0x6b, 0xd5, 0x5e, 0x14, 0x04, 0x51, 0x98, 0xf8, - 0x6a, 0x1b, 0x3c, 0x94, 0x18, 0x87, 0xcc, 0x37, 0x76, 0x75, 0x3a, 0xa2, 0x56, 0x15, 0xbd, 0x01, - 0x06, 0x2c, 0xb1, 0xdc, 0x77, 0x16, 0x7c, 0x4a, 0xf1, 0x84, 0x0b, 0x89, 0xf1, 0xcb, 0xc8, 0x43, - 0x8a, 0xa7, 0x23, 0x14, 0x92, 0x3c, 0x85, 0xb5, 0x63, 0x26, 0xd0, 0xb1, 0xea, 0x56, 0xa3, 0xd2, - 0xba, 0xd3, 0xcc, 0x90, 0x1a, 0xb6, 0x7d, 0x71, 0xb2, 0xc3, 0x04, 0x52, 0x8d, 0x24, 0x5f, 0x43, - 0x91, 0x79, 0x5e, 0x8c, 0x42, 0x38, 0xab, 0x17, 0x04, 0x7d, 0x9f, 0x60, 0x68, 0x0a, 0x26, 0x37, - 0xa1, 0x10, 0x46, 0x1e, 0x76, 0xda, 0x8e, 0x5d, 0xb7, 0x1a, 0x36, 0x35, 0x96, 0xfb, 0xbb, 0x05, - 0x37, 0xb2, 0x3b, 0x13, 0xc3, 0x28, 0x14, 0x48, 0x9e, 0x41, 0x41, 0x48, 0x26, 0x47, 0xc2, 0x6c, - 0xee, 0x76, 0x2e, 0xcf, 0xa1, 0x86, 0x50, 0x03, 0x25, 0x3b, 0x50, 0xe1, 0x21, 0x97, 0xdd, 0x21, - 0x8b, 0x59, 0x90, 0xee, 0x70, 0xab, 0x39, 0xa3, 0xa5, 0x91, 0xad, 0x13, 0x72, 0x79, 0xa0, 0x81, - 0x14, 0xf8, 0xf8, 0xb7, 0xfb, 0x2d, 0x7c, 0xb6, 0x87, 0xb2, 0xa3, 0x14, 0x57, 0xab, 0xa3, 0x48, - 0xc5, 0xba, 0x0f, 0xd7, 0x74, 0x1d, 0x76, 0x46, 0xdc, 0xf7, 0x3a, 0x6d, 0xb5, 0x31, 0xbb, 0x61, - 0xd3, 0xac, 0xd3, 0xfd, 0xc3, 0x82, 0xb2, 0x0e, 0xee, 0x84, 0xfd, 0x88, 0x3c, 0x87, 0x75, 0xb5, - 0xb5, 0x44, 0xe1, 0x8d, 0xd6, 0xbd, 0xdc, 0x24, 0x26, 0x5c, 0x34, 0x41, 0x13, 0x17, 0xaa, 0xd3, - 0xab, 0xea, 0x44, 0x6c, 0x9a, 0xf1, 0x11, 0x07, 0x8a, 0xda, 0x1e, 0x4b, 0x9a, 0x9a, 0xe4, 0x2e, - 0x40, 0xd2, 0x50, 0x21, 0x0b, 0xd0, 0x59, 0xab, 0x5b, 0x8d, 0x32, 0x2d, 0x6b, 0xcf, 0x4b, 0x16, - 0xa0, 0x2a, 0x45, 0x8c, 0x4c, 0x44, 0xa1, 0xb3, 0xae, 0x3f, 0x19, 0xcb, 0xfd, 0xcd, 0x82, 0x9b, - 0xb3, 0x99, 0x5f, 0xa5, 0x18, 0xcf, 0x93, 0x20, 0x54, 0x75, 0xb0, 0x1b, 0x95, 0xd6, 0xdd, 0xe6, - 0x7c, 0x4f, 0x37, 0xc7, 0x52, 0x51, 0x03, 0x76, 0xdf, 0xaf, 0x02, 0xd9, 0x8d, 0x91, 0x49, 0xd4, - 0xdf, 0x52, 0xf5, 0x67, 0x25, 0xb1, 0x72, 0x24, 0xc9, 0x26, 0xbe, 0x3a, 0x9b, 0xf8, 0x62, 0xc5, - 0x1c, 0x28, 0xbe, 0xc6, 0x58, 0xf0, 0x28, 0xd4, 0x72, 0xd9, 0x34, 0x35, 0xc9, 0x6d, 0x28, 0x07, - 0x28, 0x59, 0x77, 0xc8, 0xe4, 0xc0, 0xe8, 0x55, 0x52, 0x8e, 0x03, 0x26, 0x07, 0x8a, 0xcf, 0x63, - 0xe6, 0xa3, 0x70, 0x0a, 0x75, 0x5b, 0xf1, 0x29, 0x8f, 0xfa, 0xaa, 0xbb, 0x51, 0x9e, 0x0d, 0x31, - 0xed, 0xc6, 0xa2, 0x56, 0x61, 0x2b, 0x57, 0xba, 0x1f, 0xf0, 0xec, 0x47, 0xe6, 0x8f, 0xf0, 0x80, - 0xf1, 0x98, 0x82, 0x8a, 0x4a, 0xba, 0x91, 0xb4, 0x4d, 0xda, 0xe9, 0x22, 0xa5, 0x65, 0x17, 0xa9, - 0xe8, 0x30, 0xd3, 0xd3, 0xef, 0x6c, 0xd8, 0x4c, 0x44, 0xfa, 0xd7, 0x24, 0xcd, 0x6a, 0xb3, 0xfe, - 0x01, 0x6d, 0x0a, 0xff, 0x84, 0x36, 0xc5, 0xbf, 0xa3, 0x0d, 0xb9, 0x05, 0xa5, 0x70, 0x14, 0x74, - 0xe3, 0xe8, 0x8d, 0x52, 0x57, 0xe7, 0x10, 0x8e, 0x02, 0x1a, 0xbd, 0x11, 0x64, 0x17, 0xaa, 0x7d, - 0x8e, 0xbe, 0xd7, 0x4d, 0x86, 0xa9, 0x53, 0xd6, 0xcd, 0x5f, 0xcf, 0x12, 0x98, 0x41, 0xfb, 0x42, - 0x01, 0x0f, 0xf5, 0x6f, 0x5a, 0xe9, 0x4f, 0x0c, 0x72, 0x07, 0xca, 0x02, 0x4f, 0x02, 0x0c, 0x65, - 0xa7, 0xed, 0x80, 0x26, 0x98, 0x38, 0xdc, 0x00, 0xc8, 0x74, 0x61, 0xae, 0x72, 0xde, 0x96, 0x18, - 0x1a, 0xee, 0x77, 0xe0, 0xa4, 0x47, 0xfc, 0x05, 0xf7, 0x51, 0xd7, 0xe2, 0x72, 0xf3, 0xed, 0x4f, - 0x0b, 0x36, 0x33, 0xf1, 0x7a, 0xce, 0x7d, 0xac, 0x0d, 0x93, 0x06, 0x5c, 0x4f, 0x6a, 0xdc, 0xe7, - 0x3e, 0x9a, 0x66, 0xb2, 0x75, 0x33, 0x6d, 0xf0, 0x4c, 0x16, 0xe4, 0x11, 0x7c, 0x22, 0x30, 0xe6, - 0xcc, 0xe7, 0xe7, 0xe8, 0x75, 0x05, 0x3f, 0x4f, 0x46, 0xdf, 0x1a, 0xdd, 0x98, 0xb8, 0x0f, 0xf9, - 0x39, 0xba, 0x6f, 0x2d, 0xb8, 0x95, 0x23, 0xc2, 0x55, 0xa4, 0x6f, 0x03, 0x4c, 0xed, 0x2f, 0x19, - 0x77, 0x0f, 0x16, 0x8e, 0xbb, 0x69, 0xe5, 0x68, 0xb9, 0x9f, 0x6e, 0xc1, 0xfd, 0xd5, 0x36, 0x57, - 0xc7, 0x3e, 0x4a, 0xb6, 0xd4, 0xe9, 0x1c, 0x5f, 0x2f, 0xab, 0x97, 0xba, 0x5e, 0xee, 0x41, 0xa5, - 0xcf, 0xb8, 0xdf, 0x35, 0xd7, 0x80, 0xad, 0x4f, 0x35, 0x28, 0x17, 0xd5, 0x1e, 0xf2, 0x0d, 0xd8, - 0x31, 0x9e, 0x6a, 0xfd, 0x16, 0x24, 0x32, 0x37, 0x4d, 0xa8, 0x8a, 0xc8, 0x2d, 0xd7, 0x7a, 0x6e, - 0xb9, 0xb6, 0xa0, 0x1a, 0xb0, 0xf8, 0x55, 0xd7, 0x43, 0x1f, 0x25, 0x7a, 0x4e, 0xa1, 0x6e, 0x35, - 0x4a, 0xb4, 0xa2, 0x7c, 0xed, 0xc4, 0x35, 0xf5, 0x66, 0x28, 0x4e, 0xbf, 0x19, 0xa6, 0xa7, 0x75, - 0x29, 0x3b, 0xad, 0x6b, 0x50, 0x8a, 0xb1, 0x77, 0xd6, 0xf3, 0xd1, 0xd3, 0x87, 0xb5, 0x44, 0xc7, - 0x36, 0x79, 0x00, 0x93, 0x46, 0x48, 0xda, 0x03, 0x74, 0x7b, 0x5c, 0x1b, 0x7b, 0x75, 0x77, 0x3c, - 0x86, 0xeb, 0xed, 0x38, 0x1a, 0x66, 0x06, 0xe5, 0xd4, 0x94, 0xb3, 0x32, 0x53, 0xae, 0xf5, 0xbe, - 0x00, 0xa0, 0xa1, 0xbb, 0xea, 0xed, 0x46, 0x86, 0x40, 0xf6, 0x50, 0xee, 0x46, 0xc1, 0x30, 0x0a, - 0x31, 0x94, 0xc9, 0x2d, 0x4a, 0x9e, 0x2e, 0x78, 0x80, 0xcc, 0x43, 0x0d, 0x61, 0xed, 0xe1, 0x82, - 0x88, 0x19, 0xb8, 0xbb, 0x42, 0x02, 0xcd, 0x78, 0xc4, 0x03, 0x3c, 0xe2, 0xbd, 0x57, 0xbb, 0x03, - 0x16, 0x86, 0xe8, 0x5f, 0xc4, 0x38, 0x03, 0x4d, 0x19, 0x3f, 0xcf, 0x46, 0x18, 0xe3, 0x50, 0xc6, - 0x3c, 0x3c, 0x49, 0xcf, 0x86, 0xbb, 0x42, 0x4e, 0xe1, 0xc6, 0x1e, 0x6a, 0x76, 0x2e, 0x24, 0xef, - 0x89, 0x94, 0xb0, 0xb5, 0x98, 0x70, 0x0e, 0x7c, 0x49, 0xca, 0x9f, 0x01, 0x26, 0xcd, 0x46, 0x96, - 0x6b, 0xc6, 0x79, 0x01, 0x67, 0x61, 0xe3, 0xe5, 0x39, 0x6c, 0x64, 0x1f, 0x3d, 0xe4, 0x8b, 0xbc, - 0xd8, 0xdc, 0x27, 0x61, 0xed, 0xcb, 0x65, 0xa0, 0x63, 0xaa, 0x18, 0x36, 0xe7, 0xe6, 0x0e, 0x79, - 0x7c, 0xd1, 0x12, 0xb3, 0x33, 0xba, 0xf6, 0x64, 0x49, 0xf4, 0x98, 0xf3, 0x00, 0xca, 0xe3, 0x76, - 0x26, 0xf7, 0xf3, 0xa2, 0x67, 0xbb, 0xbd, 0x76, 0xd1, 0xc4, 0x73, 0x57, 0x48, 0x17, 0x60, 0x0f, - 0xe5, 0x3e, 0xca, 0x98, 0xf7, 0x04, 0x79, 0x98, 0x5b, 0xc4, 0x09, 0x20, 0x5d, 0xf4, 0xd1, 0x07, - 0x71, 0xe9, 0x96, 0x5b, 0x6f, 0xd7, 0xcc, 0x18, 0x54, 0xff, 0x07, 0xfe, 0x3f, 0x52, 0x1f, 0xe1, - 0x48, 0x1d, 0x41, 0x65, 0xea, 0x85, 0x4d, 0x72, 0x0f, 0xcb, 0xfc, 0x13, 0xfc, 0xbf, 0x6e, 0x8c, - 0x9d, 0xaf, 0x7e, 0x6a, 0x9d, 0x70, 0x39, 0x18, 0x1d, 0x2b, 0xea, 0xed, 0x04, 0xf9, 0x84, 0x47, - 0xe6, 0xd7, 0x76, 0xaa, 0xd0, 0xb6, 0x5e, 0x69, 0x5b, 0xa7, 0x31, 0x3c, 0x3e, 0x2e, 0x68, 0xf3, - 0xd9, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x71, 0x6a, 0x92, 0xdd, 0x65, 0x0f, 0x00, 0x00, + // 1118 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x57, 0xcd, 0x6e, 0xdb, 0xc6, + 0x13, 0x37, 0x4d, 0x5b, 0x1f, 0x23, 0xc5, 0xff, 0x78, 0xff, 0x69, 0xc0, 0x28, 0x09, 0x22, 0xb3, + 0x49, 0xa3, 0x16, 0x89, 0x6c, 0x28, 0x4d, 0x7b, 0x2a, 0xd0, 0xda, 0x42, 0x0c, 0xa1, 0x70, 0x60, + 0xac, 0x8d, 0x1e, 0x0a, 0x14, 0xc2, 0x5a, 0x1c, 0xd9, 0x8b, 0xf0, 0x43, 0xe6, 0xae, 0x9c, 0xda, + 0xc7, 0xa2, 0xf7, 0xde, 0x92, 0x47, 0xe8, 0x23, 0xf4, 0xd8, 0x67, 0xe8, 0x1b, 0x15, 0x5c, 0x2e, + 0x29, 0x52, 0xa2, 0x6c, 0xb9, 0x6e, 0xda, 0x4b, 0x6f, 0x9c, 0xd9, 0xdf, 0xcc, 0xec, 0xfc, 0x76, + 0x66, 0x76, 0x09, 0xeb, 0xdc, 0x77, 0xf0, 0xc7, 0xfe, 0x20, 0x08, 0x42, 0xa7, 0x3d, 0x0a, 0x03, + 0x19, 0x10, 0xe2, 0x71, 0xf7, 0x6c, 0x2c, 0x62, 0xa9, 0xad, 0xd6, 0x1b, 0xf5, 0x41, 0xe0, 0x79, + 0x81, 0x1f, 0xeb, 0x1a, 0x6b, 0xdc, 0x97, 0x18, 0xfa, 0xcc, 0xd5, 0x72, 0x3d, 0x6b, 0xd1, 0xa8, + 0x8b, 0xc1, 0x09, 0x7a, 0x2c, 0x96, 0xec, 0xf7, 0x06, 0xfc, 0x9f, 0xe2, 0x31, 0x17, 0x12, 0xc3, + 0xd7, 0x81, 0x83, 0x14, 0x4f, 0xc7, 0x28, 0x24, 0xd9, 0x82, 0x95, 0x23, 0x26, 0xd0, 0x32, 0x9a, + 0x46, 0xab, 0xd6, 0x79, 0xd0, 0xce, 0x05, 0xd5, 0xd1, 0xf6, 0xc4, 0xf1, 0x36, 0x13, 0x48, 0x15, + 0x92, 0x7c, 0x01, 0x65, 0xe6, 0x38, 0x21, 0x0a, 0x61, 0x2d, 0x5f, 0x62, 0xf4, 0x4d, 0x8c, 0xa1, + 0x09, 0x98, 0xdc, 0x85, 0x92, 0x1f, 0x38, 0xd8, 0xeb, 0x5a, 0x66, 0xd3, 0x68, 0x99, 0x54, 0x4b, + 0xf6, 0x2f, 0x06, 0xdc, 0xc9, 0xef, 0x4c, 0x8c, 0x02, 0x5f, 0x20, 0x79, 0x01, 0x25, 0x21, 0x99, + 0x1c, 0x0b, 0xbd, 0xb9, 0xfb, 0x85, 0x71, 0x0e, 0x14, 0x84, 0x6a, 0x28, 0xd9, 0x86, 0x1a, 0xf7, + 0xb9, 0xec, 0x8f, 0x58, 0xc8, 0xbc, 0x64, 0x87, 0x1b, 0xed, 0x29, 0x2e, 0x35, 0x6d, 0x3d, 0x9f, + 0xcb, 0x7d, 0x05, 0xa4, 0xc0, 0xd3, 0x6f, 0xfb, 0x2b, 0xf8, 0x68, 0x17, 0x65, 0x2f, 0x62, 0x3c, + 0xf2, 0x8e, 0x22, 0x21, 0xeb, 0x31, 0xdc, 0x52, 0xe7, 0xb0, 0x3d, 0xe6, 0xae, 0xd3, 0xeb, 0x46, + 0x1b, 0x33, 0x5b, 0x26, 0xcd, 0x2b, 0xed, 0xdf, 0x0c, 0xa8, 0x2a, 0xe3, 0x9e, 0x3f, 0x0c, 0xc8, + 0x4b, 0x58, 0x8d, 0xb6, 0x16, 0x33, 0xbc, 0xd6, 0x79, 0x54, 0x98, 0xc4, 0x24, 0x16, 0x8d, 0xd1, + 0xc4, 0x86, 0x7a, 0xd6, 0xab, 0x4a, 0xc4, 0xa4, 0x39, 0x1d, 0xb1, 0xa0, 0xac, 0xe4, 0x94, 0xd2, + 0x44, 0x24, 0x0f, 0x01, 0xe2, 0x82, 0xf2, 0x99, 0x87, 0xd6, 0x4a, 0xd3, 0x68, 0x55, 0x69, 0x55, + 0x69, 0x5e, 0x33, 0x0f, 0xa3, 0xa3, 0x08, 0x91, 0x89, 0xc0, 0xb7, 0x56, 0xd5, 0x92, 0x96, 0xec, + 0x9f, 0x0d, 0xb8, 0x3b, 0x9d, 0xf9, 0x4d, 0x0e, 0xe3, 0x65, 0x6c, 0x84, 0xd1, 0x39, 0x98, 0xad, + 0x5a, 0xe7, 0x61, 0x7b, 0xb6, 0xa6, 0xdb, 0x29, 0x55, 0x54, 0x83, 0xed, 0x3f, 0x96, 0x81, 0xec, + 0x84, 0xc8, 0x24, 0xaa, 0xb5, 0x84, 0xfd, 0x69, 0x4a, 0x8c, 0x02, 0x4a, 0xf2, 0x89, 0x2f, 0x4f, + 0x27, 0x3e, 0x9f, 0x31, 0x0b, 0xca, 0x67, 0x18, 0x0a, 0x1e, 0xf8, 0x8a, 0x2e, 0x93, 0x26, 0x22, + 0xb9, 0x0f, 0x55, 0x0f, 0x25, 0xeb, 0x8f, 0x98, 0x3c, 0xd1, 0x7c, 0x55, 0x22, 0xc5, 0x3e, 0x93, + 0x27, 0x51, 0x3c, 0x87, 0xe9, 0x45, 0x61, 0x95, 0x9a, 0x66, 0x14, 0x2f, 0xd2, 0x44, 0xab, 0xaa, + 0x1a, 0xe5, 0xf9, 0x08, 0x93, 0x6a, 0x2c, 0x2b, 0x16, 0x36, 0x0a, 0xa9, 0xfb, 0x16, 0xcf, 0xbf, + 0x63, 0xee, 0x18, 0xf7, 0x19, 0x0f, 0x29, 0x44, 0x56, 0x71, 0x35, 0x92, 0xae, 0x4e, 0x3b, 0x71, + 0x52, 0x59, 0xd4, 0x49, 0x4d, 0x99, 0xe9, 0x9a, 0x7e, 0x6f, 0xc2, 0x7a, 0x4c, 0xd2, 0x3f, 0x46, + 0x69, 0x9e, 0x9b, 0xd5, 0x2b, 0xb8, 0x29, 0xfd, 0x1d, 0xdc, 0x94, 0xff, 0x0a, 0x37, 0xe4, 0x1e, + 0x54, 0xfc, 0xb1, 0xd7, 0x0f, 0x83, 0xb7, 0x11, 0xbb, 0x2a, 0x07, 0x7f, 0xec, 0xd1, 0xe0, 0xad, + 0x20, 0x3b, 0x50, 0x1f, 0x72, 0x74, 0x9d, 0x7e, 0x3c, 0x4c, 0xad, 0xaa, 0x2a, 0xfe, 0x66, 0x3e, + 0x80, 0x1e, 0xb4, 0xaf, 0x22, 0xe0, 0x81, 0xfa, 0xa6, 0xb5, 0xe1, 0x44, 0x20, 0x0f, 0xa0, 0x2a, + 0xf0, 0xd8, 0x43, 0x5f, 0xf6, 0xba, 0x16, 0xa8, 0x00, 0x13, 0x85, 0xed, 0x01, 0xc9, 0x1e, 0xcc, + 0x4d, 0xfa, 0x6d, 0x81, 0xa1, 0x61, 0x7f, 0x0d, 0x56, 0xd2, 0xe2, 0xaf, 0xb8, 0x8b, 0xea, 0x2c, + 0xae, 0x37, 0xdf, 0x7e, 0x37, 0x60, 0x3d, 0x67, 0xaf, 0xe6, 0xdc, 0x87, 0xda, 0x30, 0x69, 0xc1, + 0xed, 0xf8, 0x8c, 0x87, 0xdc, 0x45, 0x5d, 0x4c, 0xa6, 0x2a, 0xa6, 0x35, 0x9e, 0xcb, 0x82, 0x3c, + 0x85, 0xff, 0x09, 0x0c, 0x39, 0x73, 0xf9, 0x05, 0x3a, 0x7d, 0xc1, 0x2f, 0xe2, 0xd1, 0xb7, 0x42, + 0xd7, 0x26, 0xea, 0x03, 0x7e, 0x81, 0xf6, 0x3b, 0x03, 0xee, 0x15, 0x90, 0x70, 0x13, 0xea, 0xbb, + 0x00, 0x99, 0xfd, 0xc5, 0xe3, 0xee, 0xc9, 0xdc, 0x71, 0x97, 0x65, 0x8e, 0x56, 0x87, 0xc9, 0x16, + 0xec, 0x9f, 0x4c, 0x7d, 0x75, 0xec, 0xa1, 0x64, 0x0b, 0x75, 0x67, 0x7a, 0xbd, 0x2c, 0x5f, 0xeb, + 0x7a, 0x79, 0x04, 0xb5, 0x21, 0xe3, 0x6e, 0x5f, 0x5f, 0x03, 0xa6, 0xea, 0x6a, 0x88, 0x54, 0x54, + 0x69, 0xc8, 0x97, 0x60, 0x86, 0x78, 0xaa, 0xf8, 0x9b, 0x93, 0xc8, 0xcc, 0x34, 0xa1, 0x91, 0x45, + 0xe1, 0x71, 0xad, 0x16, 0x1e, 0xd7, 0x06, 0xd4, 0x3d, 0x16, 0xbe, 0xe9, 0x3b, 0xe8, 0xa2, 0x44, + 0xc7, 0x2a, 0x35, 0x8d, 0x56, 0x85, 0xd6, 0x22, 0x5d, 0x37, 0x56, 0x65, 0xde, 0x0c, 0xe5, 0xec, + 0x9b, 0x21, 0x3b, 0xad, 0x2b, 0xf9, 0x69, 0xdd, 0x80, 0x4a, 0x88, 0x83, 0xf3, 0x81, 0x8b, 0x8e, + 0x6a, 0xd6, 0x0a, 0x4d, 0x65, 0xf2, 0x04, 0x26, 0x85, 0x10, 0x97, 0x07, 0xa8, 0xf2, 0xb8, 0x95, + 0x6a, 0x55, 0x75, 0x3c, 0x83, 0xdb, 0xdd, 0x30, 0x18, 0xe5, 0x06, 0x65, 0x66, 0xca, 0x19, 0xb9, + 0x29, 0x67, 0x6f, 0x01, 0xa1, 0xe8, 0x05, 0x67, 0xf9, 0xbb, 0xaa, 0x01, 0x95, 0xa3, 0x7c, 0x13, + 0xa5, 0x72, 0xe7, 0xd7, 0x32, 0x80, 0x02, 0xef, 0x44, 0xaf, 0x3d, 0x32, 0x02, 0xb2, 0x8b, 0x72, + 0x27, 0xf0, 0x46, 0x81, 0x8f, 0xbe, 0x8c, 0xef, 0x5d, 0xb2, 0x35, 0xe7, 0xc9, 0x32, 0x0b, 0xd5, + 0x21, 0x1b, 0x9f, 0xcc, 0xb1, 0x98, 0x82, 0xdb, 0x4b, 0xc4, 0x53, 0x11, 0x0f, 0xb9, 0x87, 0x87, + 0x7c, 0xf0, 0x66, 0xe7, 0x84, 0xf9, 0x3e, 0xba, 0x97, 0x45, 0x9c, 0x82, 0x26, 0x11, 0x3f, 0xce, + 0x5b, 0x68, 0xe1, 0x40, 0x86, 0xdc, 0x3f, 0x4e, 0xba, 0xc9, 0x5e, 0x22, 0xa7, 0x70, 0x67, 0x17, + 0x55, 0x74, 0x2e, 0x24, 0x1f, 0x88, 0x24, 0x60, 0x67, 0x7e, 0xc0, 0x19, 0xf0, 0x35, 0x43, 0xfe, + 0x00, 0x30, 0x29, 0x4f, 0xb2, 0x58, 0xf9, 0xce, 0x12, 0x38, 0x0d, 0x4b, 0xdd, 0x73, 0x58, 0xcb, + 0x3f, 0x93, 0xc8, 0xa7, 0x45, 0xb6, 0x85, 0x8f, 0xc8, 0xc6, 0x67, 0x8b, 0x40, 0xd3, 0x50, 0x21, + 0xac, 0xcf, 0x4c, 0x2a, 0xf2, 0xec, 0x32, 0x17, 0xd3, 0x53, 0xbd, 0xf1, 0x7c, 0x41, 0x74, 0x1a, + 0x73, 0x1f, 0xaa, 0x69, 0x03, 0x90, 0xc7, 0x45, 0xd6, 0xd3, 0xfd, 0xd1, 0xb8, 0x6c, 0x46, 0xda, + 0x4b, 0xe4, 0x10, 0x6a, 0x99, 0x26, 0x21, 0x85, 0x4c, 0xcf, 0x76, 0xd1, 0x55, 0x5e, 0xfb, 0x00, + 0xbb, 0x28, 0xf7, 0x50, 0x86, 0x7c, 0x20, 0xa6, 0x9d, 0x6a, 0x61, 0x02, 0x48, 0x9c, 0x3e, 0xbd, + 0x12, 0x97, 0x10, 0xd1, 0x79, 0xb7, 0xa2, 0xc7, 0x71, 0xf4, 0x5f, 0xf2, 0x5f, 0xa3, 0x7e, 0x80, + 0x46, 0x3d, 0x84, 0x5a, 0xe6, 0xa5, 0x5f, 0x5c, 0x18, 0xb3, 0xbf, 0x02, 0xff, 0x76, 0x61, 0x6c, + 0x7f, 0xfe, 0x7d, 0xe7, 0x98, 0xcb, 0x93, 0xf1, 0x51, 0x14, 0x7a, 0x33, 0x46, 0x3e, 0xe7, 0x81, + 0xfe, 0xda, 0x4c, 0x18, 0xda, 0x54, 0x9e, 0x36, 0x55, 0x1a, 0xa3, 0xa3, 0xa3, 0x92, 0x12, 0x5f, + 0xfc, 0x19, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x8e, 0x84, 0xfb, 0xed, 0x0f, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -940,6 +982,7 @@ type IndexCoordClient interface { GetIndexStates(ctx context.Context, in *GetIndexStatesRequest, opts ...grpc.CallOption) (*GetIndexStatesResponse, error) GetIndexFilePaths(ctx context.Context, in *GetIndexFilePathsRequest, opts ...grpc.CallOption) (*GetIndexFilePathsResponse, error) DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + RemoveIndex(ctx context.Context, in *RemoveIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error) // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) } @@ -1015,6 +1058,15 @@ func (c *indexCoordClient) DropIndex(ctx context.Context, in *DropIndexRequest, return out, nil } +func (c *indexCoordClient) RemoveIndex(ctx context.Context, in *RemoveIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { + out := new(commonpb.Status) + err := c.cc.Invoke(ctx, "/milvus.proto.index.IndexCoord/RemoveIndex", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *indexCoordClient) GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) { out := new(milvuspb.GetMetricsResponse) err := c.cc.Invoke(ctx, "/milvus.proto.index.IndexCoord/GetMetrics", in, out, opts...) @@ -1033,6 +1085,7 @@ type IndexCoordServer interface { GetIndexStates(context.Context, *GetIndexStatesRequest) (*GetIndexStatesResponse, error) GetIndexFilePaths(context.Context, *GetIndexFilePathsRequest) (*GetIndexFilePathsResponse, error) DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error) + RemoveIndex(context.Context, *RemoveIndexRequest) (*commonpb.Status, error) // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy GetMetrics(context.Context, *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) } @@ -1062,6 +1115,9 @@ func (*UnimplementedIndexCoordServer) GetIndexFilePaths(ctx context.Context, req func (*UnimplementedIndexCoordServer) DropIndex(ctx context.Context, req *DropIndexRequest) (*commonpb.Status, error) { return nil, status.Errorf(codes.Unimplemented, "method DropIndex not implemented") } +func (*UnimplementedIndexCoordServer) RemoveIndex(ctx context.Context, req *RemoveIndexRequest) (*commonpb.Status, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoveIndex not implemented") +} func (*UnimplementedIndexCoordServer) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetMetrics not implemented") } @@ -1196,6 +1252,24 @@ func _IndexCoord_DropIndex_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } +func _IndexCoord_RemoveIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RemoveIndexRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IndexCoordServer).RemoveIndex(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/milvus.proto.index.IndexCoord/RemoveIndex", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IndexCoordServer).RemoveIndex(ctx, req.(*RemoveIndexRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _IndexCoord_GetMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(milvuspb.GetMetricsRequest) if err := dec(in); err != nil { @@ -1246,6 +1320,10 @@ var _IndexCoord_serviceDesc = grpc.ServiceDesc{ MethodName: "DropIndex", Handler: _IndexCoord_DropIndex_Handler, }, + { + MethodName: "RemoveIndex", + Handler: _IndexCoord_RemoveIndex_Handler, + }, { MethodName: "GetMetrics", Handler: _IndexCoord_GetMetrics_Handler, diff --git a/internal/proxy/index_coord_mock_test.go b/internal/proxy/index_coord_mock_test.go index a0fb77ba20..4df3155059 100644 --- a/internal/proxy/index_coord_mock_test.go +++ b/internal/proxy/index_coord_mock_test.go @@ -133,6 +133,13 @@ func (coord *IndexCoordMock) DropIndex(ctx context.Context, req *indexpb.DropInd }, nil } +func (coord *IndexCoordMock) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, nil +} + func (coord *IndexCoordMock) GetIndexStates(ctx context.Context, req *indexpb.GetIndexStatesRequest) (*indexpb.GetIndexStatesResponse, error) { return &indexpb.GetIndexStatesResponse{ Status: &commonpb.Status{ diff --git a/internal/rootcoord/meta_table.go b/internal/rootcoord/meta_table.go index 0dc7931f02..1aca5ef297 100644 --- a/internal/rootcoord/meta_table.go +++ b/internal/rootcoord/meta_table.go @@ -429,7 +429,6 @@ func (mt *MetaTable) GetCollectionIDByName(cName string) (typeutil.UniqueID, err func (mt *MetaTable) GetCollectionNameByID(collectionID typeutil.UniqueID) (string, error) { mt.ddLock.RLock() defer mt.ddLock.RUnlock() - log.Info("xxxxxxxxx", zap.Any("collID2Meta", mt.collID2Meta)) col, ok := mt.collID2Meta[collectionID] if !ok { return "", fmt.Errorf("can't find collection id : %d", collectionID) @@ -794,23 +793,13 @@ func (mt *MetaTable) DeletePartition(collID typeutil.UniqueID, partitionName str return partID, nil } -// AddIndex add index +// AddIndex add index; before calling this function, you must check whether indexing is required func (mt *MetaTable) AddIndex(segIdxInfo *pb.SegmentIndexInfo) error { mt.ddLock.Lock() defer mt.ddLock.Unlock() - collMeta, ok := mt.collID2Meta[segIdxInfo.CollectionID] - if !ok { - return fmt.Errorf("collection id = %d not found", segIdxInfo.CollectionID) - } - exist := false - for _, fidx := range collMeta.FieldIndexes { - if fidx.IndexID == segIdxInfo.IndexID { - exist = true - break - } - } - if !exist { + if idxInfo, ok := mt.indexID2Meta[segIdxInfo.IndexID]; !ok || idxInfo.GetDeleted() { + log.Error("index id not found or has been deleted", zap.Int64("indexID", segIdxInfo.IndexID)) return fmt.Errorf("index id = %d not found", segIdxInfo.IndexID) } @@ -856,7 +845,70 @@ func (mt *MetaTable) AddIndex(segIdxInfo *pb.SegmentIndexInfo) error { return nil } +func (mt *MetaTable) MarkIndexDeleted(collName, fieldName, indexName string) error { + mt.ddLock.Lock() + defer mt.ddLock.Unlock() + + collMeta, err := mt.unlockGetCollectionInfo(collName) + if err != nil { + log.Error("get collection meta failed", zap.String("collName", collName), zap.Error(err)) + return fmt.Errorf("collection name = %s not has meta", collName) + } + fieldSch, err := mt.unlockGetFieldSchema(collName, fieldName) + if err != nil { + return err + } + + var clonedIndex *pb.IndexInfo + var dropIdxID typeutil.UniqueID + for _, info := range collMeta.FieldIndexes { + if info.FiledID != fieldSch.FieldID { + continue + } + idxMeta, ok := mt.indexID2Meta[info.IndexID] + if !ok { + errMsg := fmt.Errorf("index not has meta with ID = %d", info.IndexID) + log.Error("index id not has meta", zap.Int64("index id", info.IndexID)) + return errMsg + } + if idxMeta.IndexName != indexName { + continue + } + dropIdxID = info.IndexID + clonedIndex = proto.Clone(&idxMeta).(*pb.IndexInfo) + clonedIndex.Deleted = true + break + } + + if dropIdxID == 0 { + log.Warn("index not found", zap.String("collName", collName), zap.String("fieldName", fieldName), + zap.String("indexName", indexName)) + return nil + } + log.Info("MarkIndexDeleted", zap.String("collName", collName), zap.String("fieldName", fieldName), + zap.String("indexName", indexName), zap.Int64("dropIndexID", dropIdxID)) + + k := fmt.Sprintf("%s/%d/%d", IndexMetaPrefix, collMeta.ID, dropIdxID) + v, err := proto.Marshal(clonedIndex) + if err != nil { + log.Error("MetaTable MarkIndexDeleted Marshal idxInfo fail", + zap.String("key", k), zap.Error(err)) + return err + } + + err = mt.txn.Save(k, string(v)) + if err != nil { + log.Error("MetaTable MarkIndexDeleted txn MultiSave failed", zap.Error(err)) + return err + } + + // update meta cache + mt.indexID2Meta[dropIdxID] = *clonedIndex + return nil +} + // DropIndex drop index +// Deprecated, only ut are used. func (mt *MetaTable) DropIndex(collName, fieldName, indexName string) (typeutil.UniqueID, bool, error) { mt.ddLock.Lock() defer mt.ddLock.Unlock() @@ -884,15 +936,11 @@ func (mt *MetaTable) DropIndex(collName, fieldName, indexName string) (typeutil. continue } idxMeta, ok := mt.indexID2Meta[info.IndexID] - if !ok { + if !ok || idxMeta.GetDeleted() || idxMeta.IndexName != indexName { fieldIdxInfo = append(fieldIdxInfo, info) log.Warn("index id not has meta", zap.Int64("index id", info.IndexID)) continue } - if idxMeta.IndexName != indexName { - fieldIdxInfo = append(fieldIdxInfo, info) - continue - } dropIdxID = info.IndexID fieldIdxInfo = append(fieldIdxInfo, collMeta.FieldIndexes[i+1:]...) break @@ -973,6 +1021,161 @@ func (mt *MetaTable) GetInitBuildIDs(collName, indexName string) ([]UniqueID, er return initBuildIDs, nil } +func (mt *MetaTable) GetBuildIDsBySegIDs(segIDs []UniqueID) []UniqueID { + mt.ddLock.RLock() + defer mt.ddLock.RUnlock() + + buildIDs := make([]UniqueID, 0) + for _, segID := range segIDs { + if idxID2segIdx, ok := mt.segID2IndexMeta[segID]; ok { + for _, segIndex := range idxID2segIdx { + buildIDs = append(buildIDs, segIndex.BuildID) + } + } + } + return buildIDs +} + +func (mt *MetaTable) AlignSegmentsMeta(collID, partID UniqueID, segIDs map[UniqueID]struct{}) ([]UniqueID, []UniqueID) { + mt.ddLock.Lock() + defer mt.ddLock.Unlock() + + allIndexID := make([]UniqueID, 0) + if collMeta, ok := mt.collID2Meta[collID]; ok { + for _, fieldIndex := range collMeta.FieldIndexes { + allIndexID = append(allIndexID, fieldIndex.IndexID) + } + } + + recycledSegIDs := make([]UniqueID, 0) + recycledBuildIDs := make([]UniqueID, 0) + if segMap, ok := mt.partID2SegID[partID]; ok { + for segID := range segMap { + if _, ok := segIDs[segID]; !ok { + recycledSegIDs = append(recycledSegIDs, segID) + } + if idxID2segIndex, ok := mt.segID2IndexMeta[segID]; ok { + for _, segIndex := range idxID2segIndex { + recycledBuildIDs = append(recycledBuildIDs, segIndex.BuildID) + } + } + } + } + + return recycledSegIDs, recycledBuildIDs +} + +func (mt *MetaTable) RemoveSegments(collID, partID UniqueID, segIDs []UniqueID) error { + mt.ddLock.Lock() + defer mt.ddLock.Unlock() + + log.Info("RootCoord MetaTable remove segments", zap.Int64("collID", collID), zap.Int64("partID", partID), + zap.Int64s("segIDs", segIDs)) + allIndexID := make([]UniqueID, 0) + if collMeta, ok := mt.collID2Meta[collID]; ok { + for _, fieldIndex := range collMeta.FieldIndexes { + allIndexID = append(allIndexID, fieldIndex.IndexID) + } + } + + for _, segID := range segIDs { + delMeta := make([]string, 0) + for _, indexID := range allIndexID { + delMeta = append(delMeta, fmt.Sprintf("%s/%d/%d/%d/%d", SegmentIndexMetaPrefix, collID, indexID, partID, segID)) + } + if err := mt.txn.MultiRemove(delMeta); err != nil { + log.Error("remove redundant segment failed, wait to retry", zap.Int64("collID", collID), zap.Int64("part", partID), + zap.Int64("segID", segID), zap.Error(err)) + return err + } + delete(mt.partID2SegID[partID], segID) + delete(mt.segID2IndexMeta, segID) + } + + return nil +} + +func (mt *MetaTable) GetDroppedIndex() map[UniqueID][]*pb.FieldIndexInfo { + mt.ddLock.Lock() + defer mt.ddLock.Unlock() + + droppedIndex := make(map[UniqueID][]*pb.FieldIndexInfo) + for collID, meta := range mt.collID2Meta { + for _, fieldIndex := range meta.FieldIndexes { + if indexMeta, ok := mt.indexID2Meta[fieldIndex.IndexID]; ok && indexMeta.Deleted { + droppedIndex[collID] = append(droppedIndex[collID], proto.Clone(fieldIndex).(*pb.FieldIndexInfo)) + } + } + } + return droppedIndex +} + +// RecycleDroppedIndex remove the meta about index which is deleted. +func (mt *MetaTable) RecycleDroppedIndex() error { + mt.ddLock.Lock() + defer mt.ddLock.Unlock() + + for collID, collMeta := range mt.collID2Meta { + meta := collMeta + fieldIndexes := make([]*pb.FieldIndexInfo, 0) + delMeta := make([]string, 0) + deletedIndexIDs := make(map[UniqueID]struct{}) + for _, fieldIndex := range meta.FieldIndexes { + // Prevent the number of transaction operations from exceeding the etcd limit (128), where a maximum of 100 are processed each time + if len(delMeta) >= 100 { + break + } + if idxInfo, ok := mt.indexID2Meta[fieldIndex.IndexID]; !ok || idxInfo.GetDeleted() { + deletedIndexIDs[fieldIndex.IndexID] = struct{}{} + delMeta = append(delMeta, fmt.Sprintf("%s/%d/%d", IndexMetaPrefix, collID, fieldIndex.IndexID)) + delMeta = append(delMeta, fmt.Sprintf("%s/%d/%d", SegmentIndexMetaPrefix, collID, fieldIndex.IndexID)) + continue + } + fieldIndexes = append(fieldIndexes, fieldIndex) + } + // node index is deleted + if len(fieldIndexes) == len(meta.FieldIndexes) { + continue + } + clonedCollMeta := proto.Clone(&meta).(*pb.CollectionInfo) + clonedCollMeta.FieldIndexes = fieldIndexes + + saveMeta := make(map[string]string) + k := path.Join(CollectionMetaPrefix, strconv.FormatInt(collID, 10)) + v, err := proto.Marshal(clonedCollMeta) + if err != nil { + log.Error("MetaTable RecycleDroppedIndex Marshal collMeta failed", + zap.String("key", k), zap.Error(err)) + return err + } + saveMeta[k] = string(v) + + if err = mt.txn.MultiSaveAndRemoveWithPrefix(saveMeta, delMeta); err != nil { + log.Error("MetaTable RecycleDroppedIndex MultiSaveAndRemoveWithPrefix failed", zap.Error(err)) + return err + } + mt.collID2Meta[collID] = *clonedCollMeta + for indexID := range deletedIndexIDs { + delete(mt.indexID2Meta, indexID) + } + // update segID2IndexMeta + for _, partID := range meta.PartitionIDs { + if segIDMap, ok := mt.partID2SegID[partID]; ok { + for segID := range segIDMap { + if segIndexInfos, ok := mt.segID2IndexMeta[segID]; ok { + for indexID := range segIndexInfos { + if _, ok := deletedIndexIDs[indexID]; ok { + delete(mt.segID2IndexMeta[segID], indexID) + } + } + } + } + } + } + } + return nil +} + // GetSegmentIndexInfoByID return segment index info by segment id func (mt *MetaTable) GetSegmentIndexInfoByID(segID typeutil.UniqueID, fieldID int64, idxName string) (pb.SegmentIndexInfo, error) { mt.ddLock.RLock() @@ -995,20 +1198,14 @@ func (mt *MetaTable) GetSegmentIndexInfoByID(segID typeutil.UniqueID, fieldID in if fieldID == -1 && idxName == "" { // return default index for _, seg := range segIdxMap { info, ok := mt.indexID2Meta[seg.IndexID] - if ok && info.IndexName == Params.CommonCfg.DefaultIndexName { + if ok && !info.GetDeleted() && info.IndexName == Params.CommonCfg.DefaultIndexName { return seg, nil } } } else { for idxID, seg := range segIdxMap { idxMeta, ok := mt.indexID2Meta[idxID] - if ok { - if idxMeta.IndexName != idxName { - continue - } - if seg.FieldID != fieldID { - continue - } + if ok && !idxMeta.GetDeleted() && idxMeta.IndexName == idxName && seg.FieldID == fieldID { return seg, nil } } @@ -1075,7 +1272,7 @@ func (mt *MetaTable) unlockIsSegmentIndexed(segID typeutil.UniqueID, fieldSchema continue } idxMeta, ok := mt.indexID2Meta[idxID] - if !ok { + if !ok || idxMeta.GetDeleted() { continue } if EqualKeyPairArray(indexParams, idxMeta.IndexParams) { @@ -1105,6 +1302,9 @@ func (mt *MetaTable) checkFieldCanBeIndexed(collMeta pb.CollectionInfo, fieldSch for _, f := range collMeta.FieldIndexes { if f.GetFiledID() == fieldSchema.GetFieldID() { if info, ok := mt.indexID2Meta[f.GetIndexID()]; ok { + if info.GetDeleted() { + continue + } if idxInfo.GetIndexName() != info.GetIndexName() { return fmt.Errorf( "creating multiple indexes on same field is not supported, "+ @@ -1128,7 +1328,7 @@ func (mt *MetaTable) checkFieldCanBeIndexed(collMeta pb.CollectionInfo, fieldSch func (mt *MetaTable) checkFieldIndexDuplicate(collMeta pb.CollectionInfo, fieldSchema schemapb.FieldSchema, idxInfo *pb.IndexInfo) (duplicate bool, err error) { for _, f := range collMeta.FieldIndexes { - if info, ok := mt.indexID2Meta[f.IndexID]; ok { + if info, ok := mt.indexID2Meta[f.IndexID]; ok && !info.GetDeleted() { if info.IndexName == idxInfo.IndexName { // the index name must be different for different indexes if f.FiledID != fieldSchema.FieldID || !EqualKeyPairArray(info.IndexParams, idxInfo.IndexParams) { @@ -1229,6 +1429,7 @@ func (mt *MetaTable) GetNotIndexedSegments(collName string, fieldName string, id } // GetIndexByName return index info by index name +// TODO: IndexName is unique on collection, should return (pb.CollectionInfo, pb.IndexInfo, error) func (mt *MetaTable) GetIndexByName(collName, indexName string) (pb.CollectionInfo, []pb.IndexInfo, error) { mt.ddLock.RLock() defer mt.ddLock.RUnlock() @@ -1251,6 +1452,9 @@ func (mt *MetaTable) GetIndexByName(collName, indexName string) (pb.CollectionIn if !ok { return pb.CollectionInfo{}, nil, fmt.Errorf("index id = %d not found", idx.IndexID) } + if idxInfo.GetDeleted() { + continue + } if indexName == "" || idxInfo.IndexName == indexName { rstIndex = append(rstIndex, idxInfo) } @@ -1264,12 +1468,24 @@ func (mt *MetaTable) GetIndexByID(indexID typeutil.UniqueID) (*pb.IndexInfo, err defer mt.ddLock.RUnlock() indexInfo, ok := mt.indexID2Meta[indexID] - if !ok { + if !ok || indexInfo.GetDeleted() { return nil, fmt.Errorf("cannot find index, id = %d", indexID) } return &indexInfo, nil } +func (mt *MetaTable) dupCollectionMeta() map[typeutil.UniqueID]pb.CollectionInfo { + mt.ddLock.RLock() + defer mt.ddLock.RUnlock() + + collID2Meta := map[typeutil.UniqueID]pb.CollectionInfo{} + for k, v := range mt.collID2Meta { + v := v + collID2Meta[k] = *proto.Clone(&v).(*pb.CollectionInfo) + } + return collID2Meta +} + func (mt *MetaTable) dupMeta() ( map[typeutil.UniqueID]pb.CollectionInfo, map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo, diff --git a/internal/rootcoord/meta_table_test.go b/internal/rootcoord/meta_table_test.go index 61ed5635ec..5f80e3048d 100644 --- a/internal/rootcoord/meta_table_test.go +++ b/internal/rootcoord/meta_table_test.go @@ -75,6 +75,7 @@ type mockTestTxnKV struct { multiSave func(kvs map[string]string) error multiSaveAndRemoveWithPrefix func(saves map[string]string, removals []string) error remove func(key string) error + multiRemove func(keys []string) error } func (m *mockTestTxnKV) LoadWithPrefix(key string) ([]string, []string, error) { @@ -97,6 +98,10 @@ func (m *mockTestTxnKV) Remove(key string) error { return m.remove(key) } +func (m *mockTestTxnKV) MultiRemove(keys []string) error { + return m.multiRemove(keys) +} + func Test_MockKV(t *testing.T) { k1 := &mockTestKV{} kt := &mockTestTxnKV{} @@ -799,7 +804,7 @@ func TestMetaTable(t *testing.T) { mt.collID2Meta = make(map[int64]pb.CollectionInfo) err = mt.AddIndex(&segIdxInfo) assert.NotNil(t, err) - assert.EqualError(t, err, fmt.Sprintf("collection id = %d not found", collInfo.ID)) + assert.EqualError(t, err, fmt.Sprintf("index id = %d not found", segIdxInfo.IndexID)) err = mt.reloadFromKV() assert.Nil(t, err) @@ -1588,3 +1593,216 @@ func TestMetaTable_GetInitBuildIDs(t *testing.T) { assert.Equal(t, 1, len(buildIDs)) }) } + +func TestMetaTable_GetDroppedIndex(t *testing.T) { + mt := &MetaTable{ + collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{ + 1: { + FieldIndexes: []*pb.FieldIndexInfo{ + { + FiledID: 1, + IndexID: 1, + }, + { + FiledID: 2, + IndexID: 2, + }, + { + FiledID: 3, + IndexID: 3, + }, + }, + }, + }, + indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{ + 1: { + IndexName: "GetInitBuildID-Index-1", + IndexID: 1, + Deleted: true, + }, + 2: { + IndexName: "GetInitBuildID-Index-2", + IndexID: 2, + Deleted: false, + }, + }, + } + + droppedIndexID := mt.GetDroppedIndex() + indexInfo, ok := droppedIndexID[1] + assert.True(t, ok) + assert.Equal(t, 1, len(indexInfo)) +} + +func TestMetaTable_AlignSegmentsMeta(t *testing.T) { + var ( + indexName = "GetDroppedIndex-Index" + collID = UniqueID(1) + partID = UniqueID(10) + indexID = UniqueID(1000) + ) + mt := &MetaTable{ + txn: &mockTestTxnKV{ + multiRemove: func(keys []string) error { + return nil + }, + }, + collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{ + collID: { + FieldIndexes: []*pb.FieldIndexInfo{ + { + FiledID: 1, + IndexID: 1, + }, + }, + }, + }, + partID2SegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{ + partID: { + 100: true, + 101: true, + 102: true, + }, + }, + indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{ + indexID: { + IndexName: indexName, + IndexID: 1, + Deleted: true, + }, + }, + } + t.Run("success", func(t *testing.T) { + mt.AlignSegmentsMeta(collID, partID, map[UniqueID]struct{}{101: {}, 102: {}, 103: {}}) + }) + + t.Run("txn error", func(t *testing.T) { + txn := &mockTestTxnKV{ + multiRemove: func(keys []string) error { + return fmt.Errorf("error occurred") + }, + } + mt.txn = txn + mt.AlignSegmentsMeta(collID, partID, map[UniqueID]struct{}{103: {}, 104: {}, 105: {}}) + }) +} + +func TestMetaTable_MarkIndexDeleted(t *testing.T) { + var ( + collName = "MarkIndexDeleted-Coll" + fieldName = "MarkIndexDeleted-Field" + indexName = "MarkIndexDeleted-Index" + collID = UniqueID(1) + partID = UniqueID(10) + fieldID = UniqueID(100) + indexID = UniqueID(1000) + ) + mt := &MetaTable{ + txn: &mockTestTxnKV{ + multiRemove: func(keys []string) error { + return nil + }, + save: func(key, value string) error { + return nil + }, + }, + collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{ + collID: { + FieldIndexes: []*pb.FieldIndexInfo{ + { + FiledID: 101, + IndexID: 1001, + }, + }, + Schema: &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + FieldID: fieldID, + Name: fieldName, + }, + }, + }, + }, + }, + collName2ID: map[string]typeutil.UniqueID{ + collName: collID, + }, + partID2SegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{ + partID: { + 100: true, + 101: true, + 102: true, + }, + }, + indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{ + 1001: { + IndexName: indexName, + IndexID: indexID, + Deleted: true, + }, + }, + } + + t.Run("get collection meta failed", func(t *testing.T) { + err := mt.MarkIndexDeleted("collName", fieldName, indexName) + assert.Error(t, err) + }) + + t.Run("get field meta failed", func(t *testing.T) { + err := mt.MarkIndexDeleted(collName, "fieldName", indexName) + assert.Error(t, err) + + err = mt.MarkIndexDeleted(collName, fieldName, indexName) + assert.NoError(t, err) + }) + + t.Run("indexMeta error", func(t *testing.T) { + collMeta := pb.CollectionInfo{ + FieldIndexes: []*pb.FieldIndexInfo{ + { + FiledID: fieldID, + IndexID: indexID, + }, + }, + Schema: &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + FieldID: fieldID, + Name: fieldName, + }, + }, + }, + } + mt.collID2Meta[collID] = collMeta + + err := mt.MarkIndexDeleted(collName, fieldName, indexName) + assert.Error(t, err) + + mt.indexID2Meta[indexID] = pb.IndexInfo{ + IndexName: "indexName", + IndexID: indexID, + } + + err = mt.MarkIndexDeleted(collName, fieldName, indexName) + assert.NoError(t, err) + + mt.indexID2Meta[indexID] = pb.IndexInfo{ + IndexName: indexName, + IndexID: indexID, + } + + err = mt.MarkIndexDeleted(collName, fieldName, indexName) + assert.NoError(t, err) + }) + + t.Run("txn save failed", func(t *testing.T) { + txn := &mockTestTxnKV{ + save: func(key, value string) error { + return fmt.Errorf("error occurred") + }, + } + mt.txn = txn + err := mt.MarkIndexDeleted(collName, fieldName, indexName) + assert.Error(t, err) + }) +} diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go index 8e988dcb42..bd87393b13 100644 --- a/internal/rootcoord/root_coord.go +++ b/internal/rootcoord/root_coord.go @@ -135,6 +135,7 @@ type Core struct { //call index builder's client to build index, return build id or get index state. CallBuildIndexService func(ctx context.Context, segID UniqueID, binlog []string, field *schemapb.FieldSchema, idxInfo *etcdpb.IndexInfo, numRows int64) (typeutil.UniqueID, error) CallDropIndexService func(ctx context.Context, indexID typeutil.UniqueID) error + CallRemoveIndexService func(ctx context.Context, buildIDs []UniqueID) error CallGetIndexStatesService func(ctx context.Context, IndexBuildIDs []int64) ([]*indexpb.IndexInfo, error) NewProxyClient func(sess *sessionutil.Session) (types.Proxy, error) @@ -281,6 +282,9 @@ func (c *Core) checkInit() error { if c.CallDropIndexService == nil { return fmt.Errorf("callDropIndexService is nil") } + if c.CallRemoveIndexService == nil { + return fmt.Errorf("callDropIndexService is nil") + } if c.CallWatchChannels == nil { return fmt.Errorf("callWatchChannels is nil") } @@ -369,84 +373,145 @@ func (c *Core) checkFlushedSegmentsLoop() { } } +func (c *Core) recycleDroppedIndex() { + defer c.wg.Done() + ticker := time.NewTicker(3 * time.Second) + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + droppedIndex := c.MetaTable.GetDroppedIndex() + for collID, fieldIndexes := range droppedIndex { + for _, fieldIndex := range fieldIndexes { + indexID := fieldIndex.GetIndexID() + fieldID := fieldIndex.GetFiledID() + if err := c.CallDropIndexService(c.ctx, indexID); err != nil { + log.Warn("Notify IndexCoord to drop index failed, wait to retry", zap.Int64("collID", collID), + zap.Int64("fieldID", fieldID), zap.Int64("indexID", indexID)) + } + } + } + err := c.MetaTable.RecycleDroppedIndex() + if err != nil { + log.Warn("Remove index meta failed, wait to retry", zap.Error(err)) + } + } + } +} + +func (c *Core) createIndexForSegment(ctx context.Context, collID, partID, segID UniqueID, numRows int64, binlogs []*datapb.FieldBinlog) error { + collID2Meta, _, indexID2Meta := c.MetaTable.dupMeta() + collMeta, ok := collID2Meta[collID] + if !ok { + log.Error("collection meta is not exist", zap.Int64("collID", collID)) + return fmt.Errorf("collection meta is not exist with ID = %d", collID) + } + if len(collMeta.FieldIndexes) == 0 { + log.Info("collection has no index, no need to build index on segment", zap.Int64("collID", collID), + zap.Int64("segID", segID)) + return nil + } + for _, fieldIndex := range collMeta.FieldIndexes { + indexMeta, ok := indexID2Meta[fieldIndex.IndexID] + if !ok { + log.Warn("index has no meta", zap.Int64("collID", collID), zap.Int64("indexID", fieldIndex.IndexID)) + return fmt.Errorf("index has no meta with ID = %d in collection %d", fieldIndex.IndexID, collID) + } + if indexMeta.Deleted { + log.Info("index has been deleted, no need to build index on segment") + continue + } + + field, err := GetFieldSchemaByID(&collMeta, fieldIndex.FiledID) + if err != nil { + log.Debug("GetFieldSchemaByID failed", + zap.Int64("collectionID", collID), + zap.Int64("fieldID", fieldIndex.FiledID)) + return err + } + if c.MetaTable.IsSegmentIndexed(segID, field, indexMeta.IndexParams) { + continue + } + + segIndexInfo := etcdpb.SegmentIndexInfo{ + CollectionID: collMeta.ID, + PartitionID: partID, + SegmentID: segID, + FieldID: fieldIndex.FiledID, + IndexID: fieldIndex.IndexID, + EnableIndex: false, + ByAutoFlush: true, + } + buildID, err := c.BuildIndex(ctx, segID, numRows, binlogs, field, &indexMeta, false) + if err != nil { + log.Debug("build index failed", + zap.Int64("segmentID", segID), + zap.Int64("fieldID", field.FieldID), + zap.Int64("indexID", indexMeta.IndexID)) + return err + } + // if buildID == 0, means it's no need to build index. + if buildID != 0 { + segIndexInfo.BuildID = buildID + segIndexInfo.EnableIndex = true + } + + if err := c.MetaTable.AddIndex(&segIndexInfo); err != nil { + log.Error("Add index into meta table failed, need remove index with buildID", + zap.Int64("collectionID", collID), zap.Int64("indexID", fieldIndex.IndexID), + zap.Int64("buildID", buildID), zap.Error(err)) + if err = retry.Do(ctx, func() error { + return c.CallRemoveIndexService(ctx, []UniqueID{buildID}) + }); err != nil { + log.Error("remove index failed, need to be resolved manually", zap.Int64("collectionID", collID), + zap.Int64("indexID", fieldIndex.IndexID), zap.Int64("buildID", buildID), zap.Error(err)) + return err + } + return err + } + } + return nil +} + func (c *Core) checkFlushedSegments(ctx context.Context) { - collID2Meta, segID2IndexMeta, indexID2Meta := c.MetaTable.dupMeta() - for _, collMeta := range collID2Meta { + collID2Meta := c.MetaTable.dupCollectionMeta() + for collID, collMeta := range collID2Meta { if len(collMeta.FieldIndexes) == 0 { continue } for _, partID := range collMeta.PartitionIDs { - ctx2, cancel2 := context.WithTimeout(ctx, 3*time.Minute) - segBinlogs, err := c.CallGetRecoveryInfoService(ctx2, collMeta.ID, partID) + segBinlogs, err := c.CallGetRecoveryInfoService(ctx, collMeta.ID, partID) if err != nil { log.Debug("failed to get flushed segments from dataCoord", zap.Int64("collection ID", collMeta.GetID()), zap.Int64("partition ID", partID), zap.Error(err)) - cancel2() continue } + segIDs := make(map[UniqueID]struct{}) for _, segBinlog := range segBinlogs { - segID := segBinlog.SegmentID - var indexInfos []*etcdpb.FieldIndexInfo - indexMeta, ok := segID2IndexMeta[segID] - if !ok { - indexInfos = append(indexInfos, collMeta.FieldIndexes...) - } else { - for _, idx := range collMeta.FieldIndexes { - if _, ok := indexMeta[idx.IndexID]; !ok { - indexInfos = append(indexInfos, idx) - } - } - } - for _, idxInfo := range indexInfos { - /* #nosec G601 */ - field, err := GetFieldSchemaByID(&collMeta, idxInfo.FiledID) - if err != nil { - log.Debug("GetFieldSchemaByID", - zap.Any("collection_meta", collMeta), - zap.Int64("field id", idxInfo.FiledID)) - continue - } - indexMeta, ok := indexID2Meta[idxInfo.IndexID] - if !ok { - log.Debug("index meta does not exist", zap.Int64("index_id", idxInfo.IndexID)) - continue - } - info := etcdpb.SegmentIndexInfo{ - CollectionID: collMeta.ID, - PartitionID: partID, - SegmentID: segID, - FieldID: idxInfo.FiledID, - IndexID: idxInfo.IndexID, - EnableIndex: false, - ByAutoFlush: true, - } - log.Debug("building index by background checker", - zap.Int64("segment_id", segID), - zap.Int64("index_id", indexMeta.IndexID), - zap.Int64("collection_id", collMeta.ID)) - info.BuildID, err = c.BuildIndex(ctx2, segID, segBinlog.GetNumOfRows(), segBinlog.GetFieldBinlogs(), field, &indexMeta, false) - if err != nil { - log.Debug("build index failed", - zap.Int64("segment_id", segID), - zap.Int64("field_id", field.FieldID), - zap.Int64("index_id", indexMeta.IndexID)) - continue - } - if info.BuildID != 0 { - info.EnableIndex = true - } - if err := c.MetaTable.AddIndex(&info); err != nil { - log.Debug("Add index into meta table failed", - zap.Int64("collection_id", collMeta.ID), - zap.Int64("index_id", info.IndexID), - zap.Int64("build_id", info.BuildID), - zap.Error(err)) - } + segIDs[segBinlog.GetSegmentID()] = struct{}{} + err = c.createIndexForSegment(ctx, collID, partID, segBinlog.GetSegmentID(), segBinlog.GetNumOfRows(), segBinlog.GetFieldBinlogs()) + if err != nil { + log.Error("createIndexForSegment failed, wait to retry", zap.Int64("collID", collID), + zap.Int64("partID", partID), zap.Int64("segID", segBinlog.GetSegmentID()), zap.Error(err)) + continue } } - cancel2() + recycledSegIDs, recycledBuildIDs := c.MetaTable.AlignSegmentsMeta(collID, partID, segIDs) + log.Info("there buildIDs should be remove index", zap.Int64s("buildIDs", recycledBuildIDs)) + if err := c.CallRemoveIndexService(ctx, recycledBuildIDs); err != nil { + log.Error("CallRemoveIndexService remove indexes on segments failed", + zap.Int64s("need dropped buildIDs", recycledBuildIDs), zap.Error(err)) + continue + } + if err := c.MetaTable.RemoveSegments(collID, partID, recycledSegIDs); err != nil { + log.Warn("remove segments failed, wait to retry", zap.Int64("collID", collID), zap.Int64("partID", partID), + zap.Int64s("segIDs", recycledSegIDs), zap.Error(err)) + continue + } } } } @@ -833,6 +898,27 @@ func (c *Core) SetIndexCoord(s types.IndexCoord) error { } return res.GetStates(), nil } + + c.CallRemoveIndexService = func(ctx context.Context, buildIDs []UniqueID) (retErr error) { + defer func() { + if err := recover(); err != nil { + retErr = fmt.Errorf("get index state from index service panic, msg = %v", err) + } + }() + <-initCh + status, err := s.RemoveIndex(ctx, &indexpb.RemoveIndexRequest{ + BuildIDs: buildIDs, + }) + if err != nil { + return err + } + + if status.GetErrorCode() != commonpb.ErrorCode_Success { + return fmt.Errorf(status.Reason) + } + + return nil + } return nil } @@ -1341,13 +1427,14 @@ func (c *Core) Start() error { log.Fatal("RootCoord Start reSendDdMsg failed", zap.Error(err)) panic(err) } - c.wg.Add(6) + c.wg.Add(7) go c.startTimeTickLoop() go c.tsLoop() go c.chanTimeTick.startWatch(&c.wg) go c.checkFlushedSegmentsLoop() go c.importManager.expireOldTasksLoop(&c.wg, c.CallReleaseSegRefLock) go c.importManager.sendOutTasksLoop(&c.wg) + go c.recycleDroppedIndex() Params.RootCoordCfg.CreatedTime = time.Now() Params.RootCoordCfg.UpdatedTime = time.Now() }) @@ -2162,71 +2249,35 @@ func (c *Core) SegmentFlushCompleted(ctx context.Context, in *datapb.SegmentFlus if in.Base.MsgType != commonpb.MsgType_SegmentFlushDone { return failStatus(commonpb.ErrorCode_UnexpectedError, "invalid msg type "+commonpb.MsgType_name[int32(in.Base.MsgType)]), nil } - segID := in.Segment.GetID() - log.Debug("SegmentFlushCompleted", zap.String("role", typeutil.RootCoordRole), - zap.Int64("collection id", in.Segment.CollectionID), zap.Int64("partition id", in.Segment.PartitionID), - zap.Int64("segment id", segID), zap.Int64("msgID", in.Base.MsgID)) - coll, err := c.MetaTable.GetCollectionByID(in.Segment.CollectionID, 0) + log.Info("SegmentFlushCompleted received", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID), + zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID), zap.Int64s("compactFrom", in.Segment.CompactionFrom)) + err := c.createIndexForSegment(ctx, in.Segment.CollectionID, in.Segment.PartitionID, in.Segment.ID, in.Segment.NumOfRows, in.Segment.Binlogs) if err != nil { - log.Error("GetCollectionByID failed", zap.String("role", typeutil.RootCoordRole), - zap.Int64("msgID", in.Base.MsgID), zap.Error(err)) - return failStatus(commonpb.ErrorCode_UnexpectedError, "GetCollectionByID failed: "+err.Error()), nil + log.Error("createIndexForSegment", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID), + zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID), zap.Error(err)) + return failStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), nil } - if len(coll.FieldIndexes) == 0 { - log.Debug("no index params on collection", zap.String("role", typeutil.RootCoordRole), - zap.String("collection_name", coll.Schema.Name), zap.Int64("msgID", in.Base.MsgID)) - } - - for _, f := range coll.FieldIndexes { - fieldSch, err := GetFieldSchemaByID(coll, f.FiledID) - if err != nil { - log.Warn("field schema not found", zap.String("role", typeutil.RootCoordRole), - zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID), - zap.Int64("msgID", in.Base.MsgID), zap.Error(err)) - continue - } - - idxInfo, err := c.MetaTable.GetIndexByID(f.IndexID) - if err != nil { - log.Warn("index not found", zap.String("role", typeutil.RootCoordRole), - zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID), - zap.Int64("index id", f.IndexID), zap.Int64("msgID", in.Base.MsgID), zap.Error(err)) - continue - } - - info := etcdpb.SegmentIndexInfo{ - CollectionID: in.Segment.CollectionID, - PartitionID: in.Segment.PartitionID, - SegmentID: segID, - FieldID: fieldSch.FieldID, - IndexID: idxInfo.IndexID, - EnableIndex: false, - ByAutoFlush: true, - } - info.BuildID, err = c.BuildIndex(ctx, segID, in.Segment.GetNumOfRows(), in.Segment.GetBinlogs(), fieldSch, idxInfo, true) - if err == nil && info.BuildID != 0 { - info.EnableIndex = true - } else { - log.Error("BuildIndex failed", zap.String("role", typeutil.RootCoordRole), - zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID), - zap.Int64("index id", f.IndexID), zap.Int64("build id", info.BuildID), - zap.Int64("msgID", in.Base.MsgID), zap.Error(err)) - continue - } - err = c.MetaTable.AddIndex(&info) - if err != nil { - log.Error("AddIndex failed", zap.String("role", typeutil.RootCoordRole), - zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID), - zap.Int64("index id", f.IndexID), zap.Int64("msgID", in.Base.MsgID), zap.Error(err)) - continue + buildIDs := c.MetaTable.GetBuildIDsBySegIDs(in.Segment.CompactionFrom) + if len(buildIDs) != 0 { + if err = c.CallRemoveIndexService(ctx, buildIDs); err != nil { + log.Error("CallRemoveIndexService failed", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID), + zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID), + zap.Int64s("compactFrom", in.Segment.CompactionFrom), zap.Int64s("buildIDs", buildIDs), zap.Error(err)) + return failStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), nil } } + if err = c.MetaTable.RemoveSegments(in.Segment.CollectionID, in.Segment.PartitionID, in.Segment.CompactionFrom); err != nil { + log.Error("RemoveSegments failed", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID), + zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID), + zap.Int64s("compactFrom", in.Segment.CompactionFrom), zap.Error(err)) + return failStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), nil + } log.Debug("SegmentFlushCompleted success", zap.String("role", typeutil.RootCoordRole), zap.Int64("collection id", in.Segment.CollectionID), zap.Int64("partition id", in.Segment.PartitionID), - zap.Int64("segment id", segID), zap.Int64("msgID", in.Base.MsgID)) + zap.Int64("segment id", in.Segment.ID), zap.Int64("msgID", in.Base.MsgID)) return succStatus(), nil } diff --git a/internal/rootcoord/root_coord_test.go b/internal/rootcoord/root_coord_test.go index 7eba86e6bc..5594aab23b 100644 --- a/internal/rootcoord/root_coord_test.go +++ b/internal/rootcoord/root_coord_test.go @@ -1722,10 +1722,16 @@ func TestRootCoord_Base(t *testing.T) { assert.NoError(t, err) assert.Equal(t, commonpb.ErrorCode_Success, rsp.ErrorCode) - im.mutex.Lock() - assert.Equal(t, 1, len(im.idxDropID)) - assert.Equal(t, idx[0].IndexID, im.idxDropID[0]) - im.mutex.Unlock() + for { + im.mutex.Lock() + if len(im.idxDropID) == 1 { + assert.Equal(t, idx[0].IndexID, im.idxDropID[0]) + im.mutex.Unlock() + break + } + im.mutex.Unlock() + time.Sleep(time.Second) + } _, idx, err = core.MetaTable.GetIndexByName(collName, Params.CommonCfg.DefaultIndexName) assert.NoError(t, err) @@ -3040,6 +3046,12 @@ func TestCheckInit(t *testing.T) { err = c.checkInit() assert.Error(t, err) + c.CallRemoveIndexService = func(ctx context.Context, buildIDs []typeutil.UniqueID) error { + return nil + } + err = c.checkInit() + assert.Error(t, err) + c.NewProxyClient = func(*sessionutil.Session) (types.Proxy, error) { return nil, nil } diff --git a/internal/rootcoord/task.go b/internal/rootcoord/task.go index 3e4b676835..820c4b679e 100644 --- a/internal/rootcoord/task.go +++ b/internal/rootcoord/task.go @@ -919,12 +919,12 @@ func (t *DescribeSegmentsReqTask) Execute(ctx context.Context) error { }) extraIndexInfo, err := t.core.MetaTable.GetIndexByID(indexID) if err != nil { - log.Error("index not found in meta table", + log.Warn("index not found in meta table, maybe index has been deleted", zap.Error(err), zap.Int64("indexID", indexID), zap.Int64("collection", collectionID), zap.Int64("segment", segID)) - return err + continue } t.Rsp.SegmentInfos[segID].ExtraIndexInfos[indexID] = extraIndexInfo } @@ -1072,11 +1072,10 @@ func (t *DropIndexReqTask) Execute(ctx context.Context) error { if t.Type() != commonpb.MsgType_DropIndex { return fmt.Errorf("drop index, msg type = %s", commonpb.MsgType_name[int32(t.Type())]) } - if err := t.core.RemoveIndex(ctx, t.Req.CollectionName, t.Req.IndexName); err != nil { + if err := t.core.MetaTable.MarkIndexDeleted(t.Req.CollectionName, t.Req.FieldName, t.Req.IndexName); err != nil { return err } - _, _, err := t.core.MetaTable.DropIndex(t.Req.CollectionName, t.Req.FieldName, t.Req.IndexName) - return err + return nil } // CreateAliasReqTask create alias request task diff --git a/internal/rootcoord/task_test.go b/internal/rootcoord/task_test.go index 39e6a633a2..7574764deb 100644 --- a/internal/rootcoord/task_test.go +++ b/internal/rootcoord/task_test.go @@ -76,7 +76,7 @@ func TestDescribeSegmentsReqTask_Execute(t *testing.T) { } assert.NoError(t, tsk.Execute(context.Background())) - // index not found in meta. + // index not found in meta. no return error c.MetaTable = &MetaTable{ segID2IndexMeta: map[typeutil.UniqueID]map[typeutil.UniqueID]etcdpb.SegmentIndexInfo{ segID: { @@ -92,7 +92,7 @@ func TestDescribeSegmentsReqTask_Execute(t *testing.T) { }, }, } - assert.Error(t, tsk.Execute(context.Background())) + assert.NoError(t, tsk.Execute(context.Background())) // success. c.MetaTable = &MetaTable{ diff --git a/internal/types/types.go b/internal/types/types.go index 9f08c6b946..181c108219 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -351,6 +351,9 @@ type IndexCoord interface { // GetMetrics gets the metrics about IndexCoord. GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) + + // RemoveIndex removes the index on specify segments. + RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) } // IndexCoordComponent is used by grpc server of IndexCoord