Mark deleted before drop index (#17490)

Signed-off-by: Cai.Zhang <cai.zhang@zilliz.com>
This commit is contained in:
cai.zhang 2022-06-17 18:08:12 +08:00 committed by GitHub
parent 92d06b2e30
commit 141a3bbe00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 1053 additions and 283 deletions

View File

@ -181,6 +181,19 @@ func (c *Client) DropIndex(ctx context.Context, req *indexpb.DropIndexRequest) (
return ret.(*commonpb.Status), err
}
func (c *Client) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) {
ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {
if !funcutil.CheckCtxValid(ctx) {
return nil, ctx.Err()
}
return client.(indexpb.IndexCoordClient).RemoveIndex(ctx, req)
})
if err != nil || ret == nil {
return nil, err
}
return ret.(*commonpb.Status), err
}
// GetIndexStates gets the index states from IndexCoord.
func (c *Client) GetIndexStates(ctx context.Context, req *indexpb.GetIndexStatesRequest) (*indexpb.GetIndexStatesResponse, error) {
ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {

View File

@ -102,6 +102,15 @@ func TestIndexCoordClient(t *testing.T) {
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("RemoveIndex", func(t *testing.T) {
req := &indexpb.RemoveIndexRequest{
BuildIDs: []int64{0},
}
resp, err := icc.RemoveIndex(ctx, req)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("GetIndexStates", func(t *testing.T) {
req := &indexpb.GetIndexStatesRequest{
IndexBuildIDs: []int64{0},

View File

@ -227,6 +227,10 @@ func (s *Server) DropIndex(ctx context.Context, request *indexpb.DropIndexReques
return s.indexcoord.DropIndex(ctx, request)
}
func (s *Server) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) {
return s.indexcoord.RemoveIndex(ctx, req)
}
// GetIndexFilePaths gets the index file paths from IndexCoord.
func (s *Server) GetIndexFilePaths(ctx context.Context, req *indexpb.GetIndexFilePathsRequest) (*indexpb.GetIndexFilePathsResponse, error) {
return s.indexcoord.GetIndexFilePaths(ctx, req)

View File

@ -102,6 +102,15 @@ func TestIndexCoordinateServer(t *testing.T) {
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("RemoveIndex", func(t *testing.T) {
req := &indexpb.RemoveIndexRequest{
BuildIDs: []UniqueID{0},
}
resp, err := server.RemoveIndex(ctx, req)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("GetIndexFilePaths", func(t *testing.T) {
req := &indexpb.GetIndexFilePathsRequest{
IndexBuildIDs: []UniqueID{0, 1},

View File

@ -291,6 +291,10 @@ func (m *MockIndexCoord) DropIndex(ctx context.Context, req *indexpb.DropIndexRe
return nil, nil
}
func (m *MockIndexCoord) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) {
return nil, nil
}
func (m *MockIndexCoord) GetIndexStates(ctx context.Context, req *indexpb.GetIndexStatesRequest) (*indexpb.GetIndexStatesResponse, error) {
return nil, nil
}

View File

@ -208,6 +208,10 @@ func TestGrpcService(t *testing.T) {
return nil
}
core.CallRemoveIndexService = func(ctx context.Context, buildIDs []rootcoord.UniqueID) error {
return nil
}
collectionMetaCache := make([]string, 0, 16)
pnm := proxyMock{}
core.NewProxyClient = func(*sessionutil.Session) (types.Proxy, error) {
@ -738,10 +742,17 @@ func TestGrpcService(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, rsp.ErrorCode)
dropIDLock.Lock()
assert.Equal(t, 1, len(dropID))
assert.Equal(t, idx[0].IndexID, dropID[0])
dropIDLock.Unlock()
for {
dropIDLock.Lock()
if len(dropID) == 1 {
assert.Equal(t, idx[0].IndexID, dropID[0])
dropIDLock.Unlock()
break
}
dropIDLock.Unlock()
time.Sleep(time.Second)
}
})
t.Run("drop partition", func(t *testing.T) {

View File

@ -567,6 +567,36 @@ func (i *IndexCoord) DropIndex(ctx context.Context, req *indexpb.DropIndexReques
return ret, nil
}
func (i *IndexCoord) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) {
log.Info("IndexCoord receive RemoveIndex", zap.Int64s("buildIDs", req.BuildIDs))
if !i.isHealthy() {
errMsg := "IndexCoord is not healthy"
log.Warn(errMsg)
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: errMsg,
}, nil
}
sp, _ := trace.StartSpanFromContextWithOperationName(ctx, "IndexCoord-RemoveIndex")
defer sp.Finish()
ret := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}
err := i.metaTable.MarkIndexAsDeletedByBuildIDs(req.GetBuildIDs())
if err != nil {
log.Error("IndexCoord MarkIndexAsDeletedByBuildIDs failed", zap.Int64s("buildIDs", req.GetBuildIDs()),
zap.Error(err))
ret.ErrorCode = commonpb.ErrorCode_UnexpectedError
ret.Reason = err.Error()
return ret, nil
}
return ret, nil
}
// GetIndexFilePaths gets the index file paths from IndexCoord.
func (i *IndexCoord) GetIndexFilePaths(ctx context.Context, req *indexpb.GetIndexFilePathsRequest) (*indexpb.GetIndexFilePathsResponse, error) {
log.Debug("IndexCoord GetIndexFilePaths", zap.Int("number of IndexBuildIds", len(req.IndexBuildIDs)))

View File

@ -252,6 +252,17 @@ func (icm *Mock) GetMetrics(ctx context.Context, request *milvuspb.GetMetricsReq
}, nil
}
func (icm *Mock) RemoveIndex(ctx context.Context, request *indexpb.RemoveIndexRequest) (*commonpb.Status, error) {
if icm.Failure {
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
}, errors.New("IndexCoordinator RemoveIndex failed")
}
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil
}
type DataCoordMock struct {
types.DataCoord

View File

@ -117,6 +117,15 @@ func TestIndexCoordMock(t *testing.T) {
assert.Equal(t, "IndexCoord", resp.ComponentName)
})
t.Run("RemoveIndex", func(t *testing.T) {
req := &indexpb.RemoveIndexRequest{
BuildIDs: []UniqueID{},
}
status, err := icm.RemoveIndex(ctx, req)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
})
err = icm.Stop()
assert.Nil(t, err)
}
@ -200,6 +209,15 @@ func TestIndexCoordMockError(t *testing.T) {
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode)
})
t.Run("RemoveIndex", func(t *testing.T) {
req := &indexpb.RemoveIndexRequest{
BuildIDs: []UniqueID{},
}
status, err := icm.RemoveIndex(ctx, req)
assert.Error(t, err)
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.GetErrorCode())
})
err = icm.Stop()
assert.NotNil(t, err)
}

View File

@ -362,6 +362,11 @@ func TestIndexCoord_NotHealthy(t *testing.T) {
resp4, err := ic.GetIndexFilePaths(context.Background(), req4)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp4.Status.ErrorCode)
req5 := &indexpb.RemoveIndexRequest{}
resp5, err := ic.RemoveIndex(context.Background(), req5)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp5.GetErrorCode())
}
func TestIndexCoord_GetIndexFilePaths(t *testing.T) {
@ -470,3 +475,13 @@ func Test_tryReleaseSegmentReferLock(t *testing.T) {
assert.NoError(t, err)
})
}
func TestIndexCoord_RemoveIndex(t *testing.T) {
ic := &IndexCoord{
metaTable: &metaTable{},
}
ic.stateCode.Store(internalpb.StateCode_Healthy)
status, err := ic.RemoveIndex(context.Background(), &indexpb.RemoveIndexRequest{BuildIDs: []UniqueID{0}})
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
}

View File

@ -281,6 +281,42 @@ func (mt *metaTable) MarkIndexAsDeleted(indexID UniqueID) error {
return nil
}
func (mt *metaTable) MarkIndexAsDeletedByBuildIDs(buildIDs []UniqueID) error {
mt.lock.Lock()
defer mt.lock.Unlock()
log.Debug("IndexCoord metaTable MarkIndexAsDeletedByBuildIDs ", zap.Int64s("buildIDs", buildIDs))
for _, buildID := range buildIDs {
if meta, ok := mt.indexBuildID2Meta[buildID]; ok {
clonedMeta := &Meta{
indexMeta: proto.Clone(meta.indexMeta).(*indexpb.IndexMeta),
revision: meta.revision,
}
clonedMeta.indexMeta.MarkDeleted = true
// marshal inside
/* #nosec G601 */
if err := mt.saveIndexMeta(clonedMeta); err != nil {
log.Error("IndexCoord metaTable MarkIndexAsDeleted saveIndexMeta failed", zap.Error(err))
fn := func() error {
m, err := mt.reloadMeta(meta.indexMeta.IndexBuildID)
if m == nil {
return err
}
m.indexMeta.MarkDeleted = true
return mt.saveIndexMeta(m)
}
err2 := retry.Do(context.TODO(), fn, retry.Attempts(5))
if err2 != nil {
return err2
}
}
}
}
return nil
}
// GetIndexStates gets the index states from meta table.
func (mt *metaTable) GetIndexStates(indexBuildIDs []UniqueID) []*indexpb.IndexInfo {
mt.lock.Lock()

View File

@ -17,6 +17,7 @@
package indexcoord
import (
"path"
"strconv"
"testing"
@ -145,6 +146,17 @@ func TestMetaTable(t *testing.T) {
assert.Equal(t, "index 1 has been deleted", indexInfos[1].Reason)
})
t.Run("MarkSegmentIndexAsDeleted", func(t *testing.T) {
indexMeta1.Version = indexMeta1.Version + 1
value, err = proto.Marshal(indexMeta1)
assert.Nil(t, err)
key = path.Join(indexFilePrefix, strconv.FormatInt(indexMeta1.IndexBuildID, 10))
err = etcdKV.Save(key, string(value))
assert.Nil(t, err)
err = metaTable.MarkIndexAsDeletedByBuildIDs([]UniqueID{indexMeta1.IndexBuildID})
assert.Nil(t, err)
})
t.Run("GetIndexFilePathInfo", func(t *testing.T) {
indexFilePathInfo, err := metaTable.GetIndexFilePathInfo(0)
assert.Nil(t, indexFilePathInfo)

View File

@ -15,6 +15,7 @@ message IndexInfo {
string index_name = 1;
int64 indexID = 2;
repeated common.KeyValuePair index_params = 3;
bool deleted = 4;
}
message FieldIndexInfo{

View File

@ -81,6 +81,7 @@ type IndexInfo struct {
IndexName string `protobuf:"bytes,1,opt,name=index_name,json=indexName,proto3" json:"index_name,omitempty"`
IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,3,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"`
Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -132,6 +133,13 @@ func (m *IndexInfo) GetIndexParams() []*commonpb.KeyValuePair {
return nil
}
func (m *IndexInfo) GetDeleted() bool {
if m != nil {
return m.Deleted
}
return false
}
type FieldIndexInfo struct {
FiledID int64 `protobuf:"varint,1,opt,name=filedID,proto3" json:"filedID,omitempty"`
IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
@ -492,52 +500,52 @@ func init() {
func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) }
var fileDescriptor_975d306d62b73e88 = []byte{
// 737 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4d, 0x6b, 0xdb, 0x4a,
0x14, 0x45, 0x96, 0x63, 0x47, 0xd7, 0x8a, 0x93, 0xcc, 0xfb, 0x60, 0x08, 0x79, 0xef, 0x29, 0x82,
0x3c, 0x04, 0xa5, 0x36, 0x4d, 0x4a, 0x77, 0x85, 0xa6, 0x16, 0x01, 0xd3, 0x36, 0x98, 0x49, 0xe8,
0xa2, 0x1b, 0x31, 0x96, 0xc6, 0xf6, 0x80, 0x3e, 0x8c, 0x66, 0x14, 0xe2, 0x5d, 0xd7, 0xed, 0x4f,
0xed, 0x7f, 0x28, 0x45, 0x33, 0x92, 0xfc, 0x91, 0x64, 0xd9, 0x9d, 0xef, 0x99, 0x7b, 0xae, 0xee,
0x9c, 0x39, 0xc7, 0x70, 0xc8, 0x64, 0x18, 0x05, 0x09, 0x93, 0x74, 0xb0, 0xcc, 0x33, 0x99, 0xa1,
0xe3, 0x84, 0xc7, 0xf7, 0x85, 0xd0, 0xd5, 0xa0, 0x3c, 0x3d, 0xb1, 0xc3, 0x2c, 0x49, 0xb2, 0x54,
0x43, 0x27, 0xb6, 0x08, 0x17, 0x2c, 0xa9, 0xda, 0xdd, 0xaf, 0x06, 0x58, 0x93, 0x3c, 0x7b, 0x58,
0x7d, 0x62, 0x92, 0xa2, 0x3e, 0xb4, 0xc6, 0x3e, 0x36, 0x1c, 0xc3, 0x33, 0x49, 0x6b, 0xec, 0xa3,
0x37, 0xd0, 0xa5, 0x51, 0x94, 0x33, 0x21, 0x70, 0xcb, 0x31, 0xbc, 0xde, 0xc5, 0xe9, 0x60, 0x6b,
0x7c, 0x35, 0xf8, 0x4a, 0xf7, 0x90, 0xba, 0x19, 0xbd, 0x80, 0xe3, 0x9c, 0x89, 0x22, 0x96, 0x41,
0xb8, 0xa0, 0x69, 0xca, 0xe2, 0xb1, 0x2f, 0xb0, 0xe9, 0x98, 0x9e, 0x45, 0x8e, 0xf4, 0xc1, 0xa8,
0xc1, 0xdd, 0x6f, 0x06, 0x58, 0xe3, 0x34, 0x62, 0x0f, 0xe3, 0x74, 0x96, 0xa1, 0x7f, 0x00, 0x78,
0x59, 0x04, 0x29, 0x4d, 0x98, 0x5a, 0xc5, 0x22, 0x96, 0x42, 0x6e, 0x68, 0xc2, 0x10, 0x86, 0xae,
0x2a, 0xc6, 0xbe, 0xda, 0xc8, 0x24, 0x75, 0x89, 0x7c, 0xb0, 0x35, 0x71, 0x49, 0x73, 0x9a, 0xe8,
0xcf, 0xf5, 0x2e, 0xce, 0x9e, 0x5c, 0xf8, 0x03, 0x5b, 0x7d, 0xa6, 0x71, 0xc1, 0x26, 0x94, 0xe7,
0xa4, 0xa7, 0x68, 0x13, 0xc5, 0x72, 0x7d, 0xe8, 0x5f, 0x73, 0x16, 0x47, 0xeb, 0x85, 0x30, 0x74,
0x67, 0x3c, 0x66, 0x51, 0x23, 0x4c, 0x5d, 0x3e, 0xbf, 0x8b, 0xfb, 0xb3, 0x0d, 0xfd, 0x51, 0x16,
0xc7, 0x2c, 0x94, 0x3c, 0x4b, 0xd5, 0x98, 0x5d, 0x69, 0xdf, 0x42, 0x47, 0x3f, 0x44, 0xa5, 0xec,
0xf9, 0xf6, 0xa2, 0xd5, 0x23, 0xad, 0x87, 0xdc, 0x2a, 0x80, 0x54, 0x24, 0xf4, 0x1f, 0xf4, 0xc2,
0x9c, 0x51, 0xc9, 0x02, 0xc9, 0x13, 0x86, 0x4d, 0xc7, 0xf0, 0xda, 0x04, 0x34, 0x74, 0xc7, 0x13,
0x86, 0x5c, 0xb0, 0x97, 0x34, 0x97, 0x5c, 0x2d, 0xe0, 0x0b, 0xdc, 0x76, 0x4c, 0xcf, 0x24, 0x5b,
0x18, 0xfa, 0x1f, 0xfa, 0x4d, 0x5d, 0xaa, 0x2b, 0xf0, 0x9e, 0x7a, 0xa3, 0x1d, 0x14, 0x5d, 0xc3,
0xc1, 0xac, 0x14, 0x25, 0x50, 0xf7, 0x63, 0x02, 0x77, 0x9e, 0xd2, 0xb6, 0xf4, 0xda, 0x60, 0x5b,
0x3c, 0x62, 0xcf, 0x9a, 0x9a, 0x09, 0x74, 0x01, 0x7f, 0xdd, 0xf3, 0x5c, 0x16, 0x34, 0xae, 0x7d,
0xa1, 0x5e, 0x59, 0xe0, 0xae, 0xfa, 0xec, 0x1f, 0xd5, 0x61, 0xe5, 0x0d, 0xfd, 0xed, 0xd7, 0xf0,
0xf7, 0x72, 0xb1, 0x12, 0x3c, 0x7c, 0x44, 0xda, 0x57, 0xa4, 0x3f, 0xeb, 0xd3, 0x2d, 0xd6, 0x3b,
0x38, 0x6d, 0xee, 0x10, 0x68, 0x55, 0x22, 0xa5, 0x94, 0x90, 0x34, 0x59, 0x0a, 0x6c, 0x39, 0xa6,
0xd7, 0x26, 0x27, 0x4d, 0xcf, 0x48, 0xb7, 0xdc, 0x35, 0x1d, 0xa5, 0x0f, 0xc5, 0x82, 0xe6, 0x91,
0x08, 0xd2, 0x22, 0xc1, 0xe0, 0x18, 0xde, 0x1e, 0xb1, 0x34, 0x72, 0x53, 0x24, 0x68, 0x0c, 0x87,
0x42, 0xd2, 0x5c, 0x06, 0xcb, 0x4c, 0xa8, 0x09, 0x02, 0xf7, 0x94, 0x28, 0xce, 0x73, 0x86, 0xf3,
0xa9, 0xa4, 0xca, 0x6f, 0x7d, 0x45, 0x9c, 0xd4, 0x3c, 0x44, 0xe0, 0x38, 0xcc, 0x52, 0xc1, 0x85,
0x64, 0x69, 0xb8, 0x0a, 0x62, 0x76, 0xcf, 0x62, 0x6c, 0x3b, 0x86, 0xd7, 0xdf, 0x35, 0x45, 0x35,
0x6c, 0xb4, 0xee, 0xfe, 0x58, 0x36, 0x93, 0xa3, 0x70, 0x07, 0x71, 0xbf, 0xb7, 0xe0, 0xe8, 0x96,
0xcd, 0x13, 0x96, 0xca, 0xb5, 0x93, 0x5d, 0xb0, 0xc3, 0xb5, 0x29, 0x6b, 0x33, 0x6e, 0x61, 0xc8,
0x81, 0xde, 0x86, 0x45, 0x2a, 0x5f, 0x6f, 0x42, 0xe8, 0x14, 0x2c, 0x51, 0x4d, 0xf6, 0x95, 0xef,
0x4c, 0xb2, 0x06, 0x74, 0x5a, 0xca, 0x27, 0xf7, 0x71, 0xbb, 0x4e, 0x8b, 0x2a, 0x37, 0xd3, 0xb2,
0xb7, 0x9d, 0x5c, 0x0c, 0xdd, 0x69, 0xc1, 0x15, 0xa7, 0xa3, 0x4f, 0xaa, 0x12, 0x9d, 0x81, 0xcd,
0x52, 0x3a, 0x8d, 0x99, 0x76, 0x1e, 0xee, 0x3a, 0x86, 0xb7, 0x4f, 0x7a, 0x1a, 0x53, 0x17, 0x43,
0x2e, 0x1c, 0x4c, 0x57, 0x01, 0x2d, 0x64, 0x16, 0xcc, 0xe2, 0x42, 0x2c, 0xf0, 0xbe, 0xee, 0x99,
0xae, 0xae, 0x0a, 0x99, 0x5d, 0x97, 0x90, 0xfb, 0xc3, 0xd8, 0x8c, 0xe3, 0x93, 0xff, 0x74, 0xbf,
0x3b, 0x8e, 0xff, 0x02, 0x34, 0x22, 0xd5, 0x61, 0xdc, 0x40, 0xd0, 0xf9, 0x46, 0x14, 0x03, 0x49,
0xe7, 0x75, 0x14, 0x0f, 0x1a, 0xf4, 0x8e, 0xce, 0xc5, 0xa3, 0x54, 0x77, 0x1e, 0xa7, 0xfa, 0xfd,
0xe5, 0x97, 0x57, 0x73, 0x2e, 0x17, 0xc5, 0xb4, 0xf4, 0xcb, 0x50, 0x5f, 0xe3, 0x25, 0xcf, 0xaa,
0x5f, 0x43, 0x9e, 0x4a, 0x96, 0xa7, 0x34, 0x1e, 0xaa, 0x9b, 0x0d, 0xcb, 0xd4, 0x2e, 0xa7, 0xd3,
0x8e, 0xaa, 0x2e, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0x24, 0xa2, 0x1e, 0x16, 0x50, 0x06, 0x00,
0x00,
// 751 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4d, 0x6b, 0xeb, 0x46,
0x14, 0x45, 0x96, 0x63, 0x47, 0xd7, 0x8a, 0x5f, 0x32, 0xfd, 0x60, 0x08, 0x69, 0xab, 0x27, 0x78,
0x45, 0x50, 0x6a, 0xd3, 0xbc, 0xd2, 0x5d, 0xa1, 0xaf, 0x16, 0x01, 0xd3, 0xf6, 0x61, 0x26, 0xa1,
0x8b, 0x6e, 0xc4, 0x58, 0xba, 0xb6, 0x07, 0xf4, 0x61, 0x34, 0xa3, 0x10, 0xef, 0xba, 0xef, 0xdf,
0xe8, 0xbf, 0xeb, 0x7f, 0x28, 0x45, 0x33, 0x92, 0xfc, 0x91, 0x64, 0xd9, 0x9d, 0xcf, 0x99, 0x7b,
0xaf, 0xe6, 0x9e, 0x39, 0xc7, 0xf0, 0x06, 0x55, 0x9c, 0x44, 0x19, 0x2a, 0x3e, 0xd9, 0x96, 0x85,
0x2a, 0xc8, 0x55, 0x26, 0xd2, 0xc7, 0x4a, 0x1a, 0x34, 0xa9, 0x4f, 0xaf, 0xdd, 0xb8, 0xc8, 0xb2,
0x22, 0x37, 0xd4, 0xb5, 0x2b, 0xe3, 0x0d, 0x66, 0x4d, 0xb9, 0xff, 0xa7, 0x05, 0xce, 0xa2, 0x2c,
0x9e, 0x76, 0xbf, 0xa1, 0xe2, 0x64, 0x0c, 0xbd, 0x79, 0x48, 0x2d, 0xcf, 0x0a, 0x6c, 0xd6, 0x9b,
0x87, 0xe4, 0x07, 0x18, 0xf2, 0x24, 0x29, 0x51, 0x4a, 0xda, 0xf3, 0xac, 0x60, 0x74, 0x7b, 0x33,
0x39, 0x1a, 0xdf, 0x0c, 0xfe, 0x60, 0x6a, 0x58, 0x5b, 0x4c, 0xbe, 0x81, 0xab, 0x12, 0x65, 0x95,
0xaa, 0x28, 0xde, 0xf0, 0x3c, 0xc7, 0x74, 0x1e, 0x4a, 0x6a, 0x7b, 0x76, 0xe0, 0xb0, 0x4b, 0x73,
0x30, 0xeb, 0x78, 0xff, 0x6f, 0x0b, 0x9c, 0x79, 0x9e, 0xe0, 0xd3, 0x3c, 0x5f, 0x15, 0xe4, 0x0b,
0x00, 0x51, 0x83, 0x28, 0xe7, 0x19, 0xea, 0xab, 0x38, 0xcc, 0xd1, 0xcc, 0x47, 0x9e, 0x21, 0xa1,
0x30, 0xd4, 0x60, 0x1e, 0xea, 0x1b, 0xd9, 0xac, 0x85, 0x24, 0x04, 0xd7, 0x34, 0x6e, 0x79, 0xc9,
0x33, 0xf3, 0xb9, 0xd1, 0xed, 0xdb, 0x17, 0x2f, 0xfc, 0x0b, 0xee, 0x7e, 0xe7, 0x69, 0x85, 0x0b,
0x2e, 0x4a, 0x36, 0xd2, 0x6d, 0x0b, 0xdd, 0x55, 0xcf, 0x4f, 0x30, 0x45, 0x85, 0x09, 0xed, 0x7b,
0x56, 0x70, 0xce, 0x5a, 0xe8, 0x87, 0x30, 0xbe, 0x13, 0x98, 0x26, 0xfb, 0xab, 0x52, 0x18, 0xae,
0x44, 0x8a, 0x49, 0x27, 0x59, 0x0b, 0x5f, 0xbf, 0xa5, 0xff, 0x6f, 0x1f, 0xc6, 0xb3, 0x22, 0x4d,
0x31, 0x56, 0xa2, 0xc8, 0xf5, 0x98, 0x53, 0xd1, 0x7f, 0x84, 0x81, 0x79, 0xa2, 0x46, 0xf3, 0x77,
0xc7, 0x2b, 0x34, 0xcf, 0xb7, 0x1f, 0x72, 0xaf, 0x09, 0xd6, 0x34, 0x91, 0xaf, 0x60, 0x14, 0x97,
0xc8, 0x15, 0x46, 0x4a, 0x64, 0x48, 0x6d, 0xcf, 0x0a, 0xfa, 0x0c, 0x0c, 0xf5, 0x20, 0x32, 0x24,
0x3e, 0xb8, 0x5b, 0x5e, 0x2a, 0xa1, 0x2f, 0x10, 0x4a, 0xda, 0xf7, 0xec, 0xc0, 0x66, 0x47, 0x1c,
0xf9, 0x1a, 0xc6, 0x1d, 0xae, 0x75, 0x97, 0xf4, 0x4c, 0xbf, 0xde, 0x09, 0x4b, 0xee, 0xe0, 0x62,
0x55, 0x8b, 0x12, 0xe9, 0xfd, 0x50, 0xd2, 0xc1, 0x4b, 0xaa, 0xd7, 0x2e, 0x9c, 0x1c, 0x8b, 0xc7,
0xdc, 0x55, 0x87, 0x51, 0x92, 0x5b, 0xf8, 0xec, 0x51, 0x94, 0xaa, 0xe2, 0x69, 0xeb, 0x18, 0xfd,
0xfe, 0x92, 0x0e, 0xf5, 0x67, 0x3f, 0x69, 0x0e, 0x1b, 0xd7, 0x98, 0x6f, 0x7f, 0x0f, 0x9f, 0x6f,
0x37, 0x3b, 0x29, 0xe2, 0x67, 0x4d, 0xe7, 0xba, 0xe9, 0xd3, 0xf6, 0xf4, 0xa8, 0xeb, 0x27, 0xb8,
0xe9, 0x76, 0x88, 0x8c, 0x2a, 0x89, 0x56, 0x4a, 0x2a, 0x9e, 0x6d, 0x25, 0x75, 0x3c, 0x3b, 0xe8,
0xb3, 0xeb, 0xae, 0x66, 0x66, 0x4a, 0x1e, 0xba, 0x8a, 0xda, 0xa1, 0x72, 0xc3, 0xcb, 0x44, 0x46,
0x79, 0x95, 0x51, 0xf0, 0xac, 0xe0, 0x8c, 0x39, 0x86, 0xf9, 0x58, 0x65, 0x64, 0x0e, 0x6f, 0xa4,
0xe2, 0xa5, 0x8a, 0xb6, 0x85, 0xd4, 0x13, 0x24, 0x1d, 0x69, 0x51, 0xbc, 0xd7, 0xac, 0x18, 0x72,
0xc5, 0xb5, 0x13, 0xc7, 0xba, 0x71, 0xd1, 0xf6, 0x11, 0x06, 0x57, 0x71, 0x91, 0x4b, 0x21, 0x15,
0xe6, 0xf1, 0x2e, 0x4a, 0xf1, 0x11, 0x53, 0xea, 0x7a, 0x56, 0x30, 0x3e, 0x35, 0x45, 0x33, 0x6c,
0xb6, 0xaf, 0xfe, 0xb5, 0x2e, 0x66, 0x97, 0xf1, 0x09, 0xe3, 0xff, 0xd5, 0x83, 0xcb, 0x7b, 0x5c,
0x67, 0x98, 0xab, 0xbd, 0x93, 0x7d, 0x70, 0xe3, 0xbd, 0x29, 0x5b, 0x33, 0x1e, 0x71, 0xc4, 0x83,
0xd1, 0x81, 0x45, 0x1a, 0x5f, 0x1f, 0x52, 0xe4, 0x06, 0x1c, 0xd9, 0x4c, 0x0e, 0xb5, 0xef, 0x6c,
0xb6, 0x27, 0x4c, 0x5a, 0xea, 0x27, 0x0f, 0x75, 0xb2, 0x74, 0x5a, 0x34, 0x3c, 0x4c, 0xcb, 0xd9,
0x71, 0xa6, 0x29, 0x0c, 0x97, 0x95, 0xd0, 0x3d, 0x03, 0x73, 0xd2, 0x40, 0xf2, 0x16, 0x5c, 0xcc,
0xf9, 0x32, 0x45, 0xe3, 0x3c, 0x3a, 0xd4, 0x61, 0x1d, 0x19, 0x4e, 0x2f, 0x46, 0x7c, 0xb8, 0x58,
0xee, 0x22, 0x5e, 0xa9, 0x22, 0x5a, 0xa5, 0x95, 0xdc, 0xd0, 0x73, 0x53, 0xb3, 0xdc, 0x7d, 0xa8,
0x54, 0x71, 0x57, 0x53, 0xfe, 0x3f, 0xd6, 0x61, 0x1c, 0x5f, 0xfc, 0x0f, 0xfc, 0xbf, 0xe3, 0xf8,
0x25, 0x40, 0x27, 0x52, 0x1b, 0xc6, 0x03, 0x86, 0xbc, 0x3b, 0x88, 0x62, 0xa4, 0xf8, 0xba, 0x8d,
0xe2, 0x45, 0xc7, 0x3e, 0xf0, 0xb5, 0x7c, 0x96, 0xea, 0xc1, 0xf3, 0x54, 0xff, 0xfc, 0xfe, 0x8f,
0xef, 0xd6, 0x42, 0x6d, 0xaa, 0x65, 0xed, 0x97, 0xa9, 0x59, 0xe3, 0x5b, 0x51, 0x34, 0xbf, 0xa6,
0x22, 0x57, 0x58, 0xe6, 0x3c, 0x9d, 0xea, 0xcd, 0xa6, 0x75, 0x6a, 0xb7, 0xcb, 0xe5, 0x40, 0xa3,
0xf7, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x19, 0xa3, 0x49, 0xe3, 0x6a, 0x06, 0x00, 0x00,
}

View File

@ -17,6 +17,7 @@ service IndexCoord {
rpc GetIndexStates(GetIndexStatesRequest) returns (GetIndexStatesResponse) {}
rpc GetIndexFilePaths(GetIndexFilePathsRequest) returns (GetIndexFilePathsResponse){}
rpc DropIndex(DropIndexRequest) returns (common.Status) {}
rpc RemoveIndex(RemoveIndexRequest) returns (common.Status) {}
// https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {}
@ -120,3 +121,7 @@ message IndexMeta {
message DropIndexRequest {
int64 indexID = 1;
}
message RemoveIndexRequest {
repeated int64 buildIDs = 1;
}

View File

@ -831,6 +831,45 @@ func (m *DropIndexRequest) GetIndexID() int64 {
return 0
}
type RemoveIndexRequest struct {
BuildIDs []int64 `protobuf:"varint,1,rep,packed,name=buildIDs,proto3" json:"buildIDs,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RemoveIndexRequest) Reset() { *m = RemoveIndexRequest{} }
func (m *RemoveIndexRequest) String() string { return proto.CompactTextString(m) }
func (*RemoveIndexRequest) ProtoMessage() {}
func (*RemoveIndexRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_f9e019eb3fda53c2, []int{13}
}
func (m *RemoveIndexRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RemoveIndexRequest.Unmarshal(m, b)
}
func (m *RemoveIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RemoveIndexRequest.Marshal(b, m, deterministic)
}
func (m *RemoveIndexRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_RemoveIndexRequest.Merge(m, src)
}
func (m *RemoveIndexRequest) XXX_Size() int {
return xxx_messageInfo_RemoveIndexRequest.Size(m)
}
func (m *RemoveIndexRequest) XXX_DiscardUnknown() {
xxx_messageInfo_RemoveIndexRequest.DiscardUnknown(m)
}
var xxx_messageInfo_RemoveIndexRequest proto.InternalMessageInfo
func (m *RemoveIndexRequest) GetBuildIDs() []int64 {
if m != nil {
return m.BuildIDs
}
return nil
}
func init() {
proto.RegisterType((*RegisterNodeRequest)(nil), "milvus.proto.index.RegisterNodeRequest")
proto.RegisterType((*RegisterNodeResponse)(nil), "milvus.proto.index.RegisterNodeResponse")
@ -845,80 +884,83 @@ func init() {
proto.RegisterType((*GetIndexFilePathsResponse)(nil), "milvus.proto.index.GetIndexFilePathsResponse")
proto.RegisterType((*IndexMeta)(nil), "milvus.proto.index.IndexMeta")
proto.RegisterType((*DropIndexRequest)(nil), "milvus.proto.index.DropIndexRequest")
proto.RegisterType((*RemoveIndexRequest)(nil), "milvus.proto.index.RemoveIndexRequest")
}
func init() { proto.RegisterFile("index_coord.proto", fileDescriptor_f9e019eb3fda53c2) }
var fileDescriptor_f9e019eb3fda53c2 = []byte{
// 1087 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x57, 0xcd, 0x6e, 0x1b, 0x37,
0x10, 0xf6, 0x7a, 0x6d, 0xfd, 0x8c, 0x14, 0x37, 0x66, 0xd3, 0x60, 0xa3, 0x24, 0x88, 0xbc, 0xcd,
0x8f, 0x5a, 0x24, 0x72, 0xa0, 0x34, 0xed, 0xa9, 0x40, 0x6b, 0x0b, 0x31, 0x84, 0xc2, 0x81, 0x41,
0x1b, 0x3d, 0x14, 0x28, 0x04, 0x5a, 0x3b, 0xb2, 0x88, 0xec, 0x8f, 0xbc, 0xa4, 0x92, 0xda, 0xc7,
0xa2, 0xf7, 0xde, 0x92, 0x47, 0xe9, 0xb1, 0xcf, 0x90, 0x37, 0x2a, 0xc8, 0xe5, 0x4a, 0x5a, 0x69,
0xe5, 0xc8, 0x75, 0xd3, 0x5e, 0x7a, 0xd3, 0xcc, 0x7e, 0xc3, 0x8f, 0xf3, 0xcd, 0x70, 0x48, 0xc1,
0x26, 0x0f, 0x3d, 0xfc, 0xa5, 0xdb, 0x8b, 0xa2, 0xd8, 0x6b, 0x0e, 0xe3, 0x48, 0x46, 0x84, 0x04,
0xdc, 0x7f, 0x3d, 0x12, 0x89, 0xd5, 0xd4, 0xdf, 0x6b, 0xd5, 0x5e, 0x14, 0x04, 0x51, 0x98, 0xf8,
0x6a, 0x1b, 0x3c, 0x94, 0x18, 0x87, 0xcc, 0x37, 0x76, 0x75, 0x3a, 0xa2, 0x56, 0x15, 0xbd, 0x01,
0x06, 0x2c, 0xb1, 0xdc, 0x77, 0x16, 0x7c, 0x4a, 0xf1, 0x84, 0x0b, 0x89, 0xf1, 0xcb, 0xc8, 0x43,
0x8a, 0xa7, 0x23, 0x14, 0x92, 0x3c, 0x85, 0xb5, 0x63, 0x26, 0xd0, 0xb1, 0xea, 0x56, 0xa3, 0xd2,
0xba, 0xd3, 0xcc, 0x90, 0x1a, 0xb6, 0x7d, 0x71, 0xb2, 0xc3, 0x04, 0x52, 0x8d, 0x24, 0x5f, 0x43,
0x91, 0x79, 0x5e, 0x8c, 0x42, 0x38, 0xab, 0x17, 0x04, 0x7d, 0x9f, 0x60, 0x68, 0x0a, 0x26, 0x37,
0xa1, 0x10, 0x46, 0x1e, 0x76, 0xda, 0x8e, 0x5d, 0xb7, 0x1a, 0x36, 0x35, 0x96, 0xfb, 0xbb, 0x05,
0x37, 0xb2, 0x3b, 0x13, 0xc3, 0x28, 0x14, 0x48, 0x9e, 0x41, 0x41, 0x48, 0x26, 0x47, 0xc2, 0x6c,
0xee, 0x76, 0x2e, 0xcf, 0xa1, 0x86, 0x50, 0x03, 0x25, 0x3b, 0x50, 0xe1, 0x21, 0x97, 0xdd, 0x21,
0x8b, 0x59, 0x90, 0xee, 0x70, 0xab, 0x39, 0xa3, 0xa5, 0x91, 0xad, 0x13, 0x72, 0x79, 0xa0, 0x81,
0x14, 0xf8, 0xf8, 0xb7, 0xfb, 0x2d, 0x7c, 0xb6, 0x87, 0xb2, 0xa3, 0x14, 0x57, 0xab, 0xa3, 0x48,
0xc5, 0xba, 0x0f, 0xd7, 0x74, 0x1d, 0x76, 0x46, 0xdc, 0xf7, 0x3a, 0x6d, 0xb5, 0x31, 0xbb, 0x61,
0xd3, 0xac, 0xd3, 0xfd, 0xc3, 0x82, 0xb2, 0x0e, 0xee, 0x84, 0xfd, 0x88, 0x3c, 0x87, 0x75, 0xb5,
0xb5, 0x44, 0xe1, 0x8d, 0xd6, 0xbd, 0xdc, 0x24, 0x26, 0x5c, 0x34, 0x41, 0x13, 0x17, 0xaa, 0xd3,
0xab, 0xea, 0x44, 0x6c, 0x9a, 0xf1, 0x11, 0x07, 0x8a, 0xda, 0x1e, 0x4b, 0x9a, 0x9a, 0xe4, 0x2e,
0x40, 0xd2, 0x50, 0x21, 0x0b, 0xd0, 0x59, 0xab, 0x5b, 0x8d, 0x32, 0x2d, 0x6b, 0xcf, 0x4b, 0x16,
0xa0, 0x2a, 0x45, 0x8c, 0x4c, 0x44, 0xa1, 0xb3, 0xae, 0x3f, 0x19, 0xcb, 0xfd, 0xcd, 0x82, 0x9b,
0xb3, 0x99, 0x5f, 0xa5, 0x18, 0xcf, 0x93, 0x20, 0x54, 0x75, 0xb0, 0x1b, 0x95, 0xd6, 0xdd, 0xe6,
0x7c, 0x4f, 0x37, 0xc7, 0x52, 0x51, 0x03, 0x76, 0xdf, 0xaf, 0x02, 0xd9, 0x8d, 0x91, 0x49, 0xd4,
0xdf, 0x52, 0xf5, 0x67, 0x25, 0xb1, 0x72, 0x24, 0xc9, 0x26, 0xbe, 0x3a, 0x9b, 0xf8, 0x62, 0xc5,
0x1c, 0x28, 0xbe, 0xc6, 0x58, 0xf0, 0x28, 0xd4, 0x72, 0xd9, 0x34, 0x35, 0xc9, 0x6d, 0x28, 0x07,
0x28, 0x59, 0x77, 0xc8, 0xe4, 0xc0, 0xe8, 0x55, 0x52, 0x8e, 0x03, 0x26, 0x07, 0x8a, 0xcf, 0x63,
0xe6, 0xa3, 0x70, 0x0a, 0x75, 0x5b, 0xf1, 0x29, 0x8f, 0xfa, 0xaa, 0xbb, 0x51, 0x9e, 0x0d, 0x31,
0xed, 0xc6, 0xa2, 0x56, 0x61, 0x2b, 0x57, 0xba, 0x1f, 0xf0, 0xec, 0x47, 0xe6, 0x8f, 0xf0, 0x80,
0xf1, 0x98, 0x82, 0x8a, 0x4a, 0xba, 0x91, 0xb4, 0x4d, 0xda, 0xe9, 0x22, 0xa5, 0x65, 0x17, 0xa9,
0xe8, 0x30, 0xd3, 0xd3, 0xef, 0x6c, 0xd8, 0x4c, 0x44, 0xfa, 0xd7, 0x24, 0xcd, 0x6a, 0xb3, 0xfe,
0x01, 0x6d, 0x0a, 0xff, 0x84, 0x36, 0xc5, 0xbf, 0xa3, 0x0d, 0xb9, 0x05, 0xa5, 0x70, 0x14, 0x74,
0xe3, 0xe8, 0x8d, 0x52, 0x57, 0xe7, 0x10, 0x8e, 0x02, 0x1a, 0xbd, 0x11, 0x64, 0x17, 0xaa, 0x7d,
0x8e, 0xbe, 0xd7, 0x4d, 0x86, 0xa9, 0x53, 0xd6, 0xcd, 0x5f, 0xcf, 0x12, 0x98, 0x41, 0xfb, 0x42,
0x01, 0x0f, 0xf5, 0x6f, 0x5a, 0xe9, 0x4f, 0x0c, 0x72, 0x07, 0xca, 0x02, 0x4f, 0x02, 0x0c, 0x65,
0xa7, 0xed, 0x80, 0x26, 0x98, 0x38, 0xdc, 0x00, 0xc8, 0x74, 0x61, 0xae, 0x72, 0xde, 0x96, 0x18,
0x1a, 0xee, 0x77, 0xe0, 0xa4, 0x47, 0xfc, 0x05, 0xf7, 0x51, 0xd7, 0xe2, 0x72, 0xf3, 0xed, 0x4f,
0x0b, 0x36, 0x33, 0xf1, 0x7a, 0xce, 0x7d, 0xac, 0x0d, 0x93, 0x06, 0x5c, 0x4f, 0x6a, 0xdc, 0xe7,
0x3e, 0x9a, 0x66, 0xb2, 0x75, 0x33, 0x6d, 0xf0, 0x4c, 0x16, 0xe4, 0x11, 0x7c, 0x22, 0x30, 0xe6,
0xcc, 0xe7, 0xe7, 0xe8, 0x75, 0x05, 0x3f, 0x4f, 0x46, 0xdf, 0x1a, 0xdd, 0x98, 0xb8, 0x0f, 0xf9,
0x39, 0xba, 0x6f, 0x2d, 0xb8, 0x95, 0x23, 0xc2, 0x55, 0xa4, 0x6f, 0x03, 0x4c, 0xed, 0x2f, 0x19,
0x77, 0x0f, 0x16, 0x8e, 0xbb, 0x69, 0xe5, 0x68, 0xb9, 0x9f, 0x6e, 0xc1, 0xfd, 0xd5, 0x36, 0x57,
0xc7, 0x3e, 0x4a, 0xb6, 0xd4, 0xe9, 0x1c, 0x5f, 0x2f, 0xab, 0x97, 0xba, 0x5e, 0xee, 0x41, 0xa5,
0xcf, 0xb8, 0xdf, 0x35, 0xd7, 0x80, 0xad, 0x4f, 0x35, 0x28, 0x17, 0xd5, 0x1e, 0xf2, 0x0d, 0xd8,
0x31, 0x9e, 0x6a, 0xfd, 0x16, 0x24, 0x32, 0x37, 0x4d, 0xa8, 0x8a, 0xc8, 0x2d, 0xd7, 0x7a, 0x6e,
0xb9, 0xb6, 0xa0, 0x1a, 0xb0, 0xf8, 0x55, 0xd7, 0x43, 0x1f, 0x25, 0x7a, 0x4e, 0xa1, 0x6e, 0x35,
0x4a, 0xb4, 0xa2, 0x7c, 0xed, 0xc4, 0x35, 0xf5, 0x66, 0x28, 0x4e, 0xbf, 0x19, 0xa6, 0xa7, 0x75,
0x29, 0x3b, 0xad, 0x6b, 0x50, 0x8a, 0xb1, 0x77, 0xd6, 0xf3, 0xd1, 0xd3, 0x87, 0xb5, 0x44, 0xc7,
0x36, 0x79, 0x00, 0x93, 0x46, 0x48, 0xda, 0x03, 0x74, 0x7b, 0x5c, 0x1b, 0x7b, 0x75, 0x77, 0x3c,
0x86, 0xeb, 0xed, 0x38, 0x1a, 0x66, 0x06, 0xe5, 0xd4, 0x94, 0xb3, 0x32, 0x53, 0xae, 0xf5, 0xbe,
0x00, 0xa0, 0xa1, 0xbb, 0xea, 0xed, 0x46, 0x86, 0x40, 0xf6, 0x50, 0xee, 0x46, 0xc1, 0x30, 0x0a,
0x31, 0x94, 0xc9, 0x2d, 0x4a, 0x9e, 0x2e, 0x78, 0x80, 0xcc, 0x43, 0x0d, 0x61, 0xed, 0xe1, 0x82,
0x88, 0x19, 0xb8, 0xbb, 0x42, 0x02, 0xcd, 0x78, 0xc4, 0x03, 0x3c, 0xe2, 0xbd, 0x57, 0xbb, 0x03,
0x16, 0x86, 0xe8, 0x5f, 0xc4, 0x38, 0x03, 0x4d, 0x19, 0x3f, 0xcf, 0x46, 0x18, 0xe3, 0x50, 0xc6,
0x3c, 0x3c, 0x49, 0xcf, 0x86, 0xbb, 0x42, 0x4e, 0xe1, 0xc6, 0x1e, 0x6a, 0x76, 0x2e, 0x24, 0xef,
0x89, 0x94, 0xb0, 0xb5, 0x98, 0x70, 0x0e, 0x7c, 0x49, 0xca, 0x9f, 0x01, 0x26, 0xcd, 0x46, 0x96,
0x6b, 0xc6, 0x79, 0x01, 0x67, 0x61, 0xe3, 0xe5, 0x39, 0x6c, 0x64, 0x1f, 0x3d, 0xe4, 0x8b, 0xbc,
0xd8, 0xdc, 0x27, 0x61, 0xed, 0xcb, 0x65, 0xa0, 0x63, 0xaa, 0x18, 0x36, 0xe7, 0xe6, 0x0e, 0x79,
0x7c, 0xd1, 0x12, 0xb3, 0x33, 0xba, 0xf6, 0x64, 0x49, 0xf4, 0x98, 0xf3, 0x00, 0xca, 0xe3, 0x76,
0x26, 0xf7, 0xf3, 0xa2, 0x67, 0xbb, 0xbd, 0x76, 0xd1, 0xc4, 0x73, 0x57, 0x48, 0x17, 0x60, 0x0f,
0xe5, 0x3e, 0xca, 0x98, 0xf7, 0x04, 0x79, 0x98, 0x5b, 0xc4, 0x09, 0x20, 0x5d, 0xf4, 0xd1, 0x07,
0x71, 0xe9, 0x96, 0x5b, 0x6f, 0xd7, 0xcc, 0x18, 0x54, 0xff, 0x07, 0xfe, 0x3f, 0x52, 0x1f, 0xe1,
0x48, 0x1d, 0x41, 0x65, 0xea, 0x85, 0x4d, 0x72, 0x0f, 0xcb, 0xfc, 0x13, 0xfc, 0xbf, 0x6e, 0x8c,
0x9d, 0xaf, 0x7e, 0x6a, 0x9d, 0x70, 0x39, 0x18, 0x1d, 0x2b, 0xea, 0xed, 0x04, 0xf9, 0x84, 0x47,
0xe6, 0xd7, 0x76, 0xaa, 0xd0, 0xb6, 0x5e, 0x69, 0x5b, 0xa7, 0x31, 0x3c, 0x3e, 0x2e, 0x68, 0xf3,
0xd9, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x71, 0x6a, 0x92, 0xdd, 0x65, 0x0f, 0x00, 0x00,
// 1118 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x57, 0xcd, 0x6e, 0xdb, 0xc6,
0x13, 0x37, 0x4d, 0x5b, 0x1f, 0x23, 0xc5, 0xff, 0x78, 0xff, 0x69, 0xc0, 0x28, 0x09, 0x22, 0xb3,
0x49, 0xa3, 0x16, 0x89, 0x6c, 0x28, 0x4d, 0x7b, 0x2a, 0xd0, 0xda, 0x42, 0x0c, 0xa1, 0x70, 0x60,
0xac, 0x8d, 0x1e, 0x0a, 0x14, 0xc2, 0x5a, 0x1c, 0xd9, 0x8b, 0xf0, 0x43, 0xe6, 0xae, 0x9c, 0xda,
0xc7, 0xa2, 0xf7, 0xde, 0x92, 0x47, 0xe8, 0x23, 0xf4, 0xd8, 0x67, 0xe8, 0x1b, 0x15, 0x5c, 0x2e,
0x29, 0x52, 0xa2, 0x6c, 0xb9, 0x6e, 0xda, 0x4b, 0x6f, 0x9c, 0xd9, 0xdf, 0xcc, 0xec, 0xfc, 0x76,
0x66, 0x76, 0x09, 0xeb, 0xdc, 0x77, 0xf0, 0xc7, 0xfe, 0x20, 0x08, 0x42, 0xa7, 0x3d, 0x0a, 0x03,
0x19, 0x10, 0xe2, 0x71, 0xf7, 0x6c, 0x2c, 0x62, 0xa9, 0xad, 0xd6, 0x1b, 0xf5, 0x41, 0xe0, 0x79,
0x81, 0x1f, 0xeb, 0x1a, 0x6b, 0xdc, 0x97, 0x18, 0xfa, 0xcc, 0xd5, 0x72, 0x3d, 0x6b, 0xd1, 0xa8,
0x8b, 0xc1, 0x09, 0x7a, 0x2c, 0x96, 0xec, 0xf7, 0x06, 0xfc, 0x9f, 0xe2, 0x31, 0x17, 0x12, 0xc3,
0xd7, 0x81, 0x83, 0x14, 0x4f, 0xc7, 0x28, 0x24, 0xd9, 0x82, 0x95, 0x23, 0x26, 0xd0, 0x32, 0x9a,
0x46, 0xab, 0xd6, 0x79, 0xd0, 0xce, 0x05, 0xd5, 0xd1, 0xf6, 0xc4, 0xf1, 0x36, 0x13, 0x48, 0x15,
0x92, 0x7c, 0x01, 0x65, 0xe6, 0x38, 0x21, 0x0a, 0x61, 0x2d, 0x5f, 0x62, 0xf4, 0x4d, 0x8c, 0xa1,
0x09, 0x98, 0xdc, 0x85, 0x92, 0x1f, 0x38, 0xd8, 0xeb, 0x5a, 0x66, 0xd3, 0x68, 0x99, 0x54, 0x4b,
0xf6, 0x2f, 0x06, 0xdc, 0xc9, 0xef, 0x4c, 0x8c, 0x02, 0x5f, 0x20, 0x79, 0x01, 0x25, 0x21, 0x99,
0x1c, 0x0b, 0xbd, 0xb9, 0xfb, 0x85, 0x71, 0x0e, 0x14, 0x84, 0x6a, 0x28, 0xd9, 0x86, 0x1a, 0xf7,
0xb9, 0xec, 0x8f, 0x58, 0xc8, 0xbc, 0x64, 0x87, 0x1b, 0xed, 0x29, 0x2e, 0x35, 0x6d, 0x3d, 0x9f,
0xcb, 0x7d, 0x05, 0xa4, 0xc0, 0xd3, 0x6f, 0xfb, 0x2b, 0xf8, 0x68, 0x17, 0x65, 0x2f, 0x62, 0x3c,
0xf2, 0x8e, 0x22, 0x21, 0xeb, 0x31, 0xdc, 0x52, 0xe7, 0xb0, 0x3d, 0xe6, 0xae, 0xd3, 0xeb, 0x46,
0x1b, 0x33, 0x5b, 0x26, 0xcd, 0x2b, 0xed, 0xdf, 0x0c, 0xa8, 0x2a, 0xe3, 0x9e, 0x3f, 0x0c, 0xc8,
0x4b, 0x58, 0x8d, 0xb6, 0x16, 0x33, 0xbc, 0xd6, 0x79, 0x54, 0x98, 0xc4, 0x24, 0x16, 0x8d, 0xd1,
0xc4, 0x86, 0x7a, 0xd6, 0xab, 0x4a, 0xc4, 0xa4, 0x39, 0x1d, 0xb1, 0xa0, 0xac, 0xe4, 0x94, 0xd2,
0x44, 0x24, 0x0f, 0x01, 0xe2, 0x82, 0xf2, 0x99, 0x87, 0xd6, 0x4a, 0xd3, 0x68, 0x55, 0x69, 0x55,
0x69, 0x5e, 0x33, 0x0f, 0xa3, 0xa3, 0x08, 0x91, 0x89, 0xc0, 0xb7, 0x56, 0xd5, 0x92, 0x96, 0xec,
0x9f, 0x0d, 0xb8, 0x3b, 0x9d, 0xf9, 0x4d, 0x0e, 0xe3, 0x65, 0x6c, 0x84, 0xd1, 0x39, 0x98, 0xad,
0x5a, 0xe7, 0x61, 0x7b, 0xb6, 0xa6, 0xdb, 0x29, 0x55, 0x54, 0x83, 0xed, 0x3f, 0x96, 0x81, 0xec,
0x84, 0xc8, 0x24, 0xaa, 0xb5, 0x84, 0xfd, 0x69, 0x4a, 0x8c, 0x02, 0x4a, 0xf2, 0x89, 0x2f, 0x4f,
0x27, 0x3e, 0x9f, 0x31, 0x0b, 0xca, 0x67, 0x18, 0x0a, 0x1e, 0xf8, 0x8a, 0x2e, 0x93, 0x26, 0x22,
0xb9, 0x0f, 0x55, 0x0f, 0x25, 0xeb, 0x8f, 0x98, 0x3c, 0xd1, 0x7c, 0x55, 0x22, 0xc5, 0x3e, 0x93,
0x27, 0x51, 0x3c, 0x87, 0xe9, 0x45, 0x61, 0x95, 0x9a, 0x66, 0x14, 0x2f, 0xd2, 0x44, 0xab, 0xaa,
0x1a, 0xe5, 0xf9, 0x08, 0x93, 0x6a, 0x2c, 0x2b, 0x16, 0x36, 0x0a, 0xa9, 0xfb, 0x16, 0xcf, 0xbf,
0x63, 0xee, 0x18, 0xf7, 0x19, 0x0f, 0x29, 0x44, 0x56, 0x71, 0x35, 0x92, 0xae, 0x4e, 0x3b, 0x71,
0x52, 0x59, 0xd4, 0x49, 0x4d, 0x99, 0xe9, 0x9a, 0x7e, 0x6f, 0xc2, 0x7a, 0x4c, 0xd2, 0x3f, 0x46,
0x69, 0x9e, 0x9b, 0xd5, 0x2b, 0xb8, 0x29, 0xfd, 0x1d, 0xdc, 0x94, 0xff, 0x0a, 0x37, 0xe4, 0x1e,
0x54, 0xfc, 0xb1, 0xd7, 0x0f, 0x83, 0xb7, 0x11, 0xbb, 0x2a, 0x07, 0x7f, 0xec, 0xd1, 0xe0, 0xad,
0x20, 0x3b, 0x50, 0x1f, 0x72, 0x74, 0x9d, 0x7e, 0x3c, 0x4c, 0xad, 0xaa, 0x2a, 0xfe, 0x66, 0x3e,
0x80, 0x1e, 0xb4, 0xaf, 0x22, 0xe0, 0x81, 0xfa, 0xa6, 0xb5, 0xe1, 0x44, 0x20, 0x0f, 0xa0, 0x2a,
0xf0, 0xd8, 0x43, 0x5f, 0xf6, 0xba, 0x16, 0xa8, 0x00, 0x13, 0x85, 0xed, 0x01, 0xc9, 0x1e, 0xcc,
0x4d, 0xfa, 0x6d, 0x81, 0xa1, 0x61, 0x7f, 0x0d, 0x56, 0xd2, 0xe2, 0xaf, 0xb8, 0x8b, 0xea, 0x2c,
0xae, 0x37, 0xdf, 0x7e, 0x37, 0x60, 0x3d, 0x67, 0xaf, 0xe6, 0xdc, 0x87, 0xda, 0x30, 0x69, 0xc1,
0xed, 0xf8, 0x8c, 0x87, 0xdc, 0x45, 0x5d, 0x4c, 0xa6, 0x2a, 0xa6, 0x35, 0x9e, 0xcb, 0x82, 0x3c,
0x85, 0xff, 0x09, 0x0c, 0x39, 0x73, 0xf9, 0x05, 0x3a, 0x7d, 0xc1, 0x2f, 0xe2, 0xd1, 0xb7, 0x42,
0xd7, 0x26, 0xea, 0x03, 0x7e, 0x81, 0xf6, 0x3b, 0x03, 0xee, 0x15, 0x90, 0x70, 0x13, 0xea, 0xbb,
0x00, 0x99, 0xfd, 0xc5, 0xe3, 0xee, 0xc9, 0xdc, 0x71, 0x97, 0x65, 0x8e, 0x56, 0x87, 0xc9, 0x16,
0xec, 0x9f, 0x4c, 0x7d, 0x75, 0xec, 0xa1, 0x64, 0x0b, 0x75, 0x67, 0x7a, 0xbd, 0x2c, 0x5f, 0xeb,
0x7a, 0x79, 0x04, 0xb5, 0x21, 0xe3, 0x6e, 0x5f, 0x5f, 0x03, 0xa6, 0xea, 0x6a, 0x88, 0x54, 0x54,
0x69, 0xc8, 0x97, 0x60, 0x86, 0x78, 0xaa, 0xf8, 0x9b, 0x93, 0xc8, 0xcc, 0x34, 0xa1, 0x91, 0x45,
0xe1, 0x71, 0xad, 0x16, 0x1e, 0xd7, 0x06, 0xd4, 0x3d, 0x16, 0xbe, 0xe9, 0x3b, 0xe8, 0xa2, 0x44,
0xc7, 0x2a, 0x35, 0x8d, 0x56, 0x85, 0xd6, 0x22, 0x5d, 0x37, 0x56, 0x65, 0xde, 0x0c, 0xe5, 0xec,
0x9b, 0x21, 0x3b, 0xad, 0x2b, 0xf9, 0x69, 0xdd, 0x80, 0x4a, 0x88, 0x83, 0xf3, 0x81, 0x8b, 0x8e,
0x6a, 0xd6, 0x0a, 0x4d, 0x65, 0xf2, 0x04, 0x26, 0x85, 0x10, 0x97, 0x07, 0xa8, 0xf2, 0xb8, 0x95,
0x6a, 0x55, 0x75, 0x3c, 0x83, 0xdb, 0xdd, 0x30, 0x18, 0xe5, 0x06, 0x65, 0x66, 0xca, 0x19, 0xb9,
0x29, 0x67, 0x6f, 0x01, 0xa1, 0xe8, 0x05, 0x67, 0xf9, 0xbb, 0xaa, 0x01, 0x95, 0xa3, 0x7c, 0x13,
0xa5, 0x72, 0xe7, 0xd7, 0x32, 0x80, 0x02, 0xef, 0x44, 0xaf, 0x3d, 0x32, 0x02, 0xb2, 0x8b, 0x72,
0x27, 0xf0, 0x46, 0x81, 0x8f, 0xbe, 0x8c, 0xef, 0x5d, 0xb2, 0x35, 0xe7, 0xc9, 0x32, 0x0b, 0xd5,
0x21, 0x1b, 0x9f, 0xcc, 0xb1, 0x98, 0x82, 0xdb, 0x4b, 0xc4, 0x53, 0x11, 0x0f, 0xb9, 0x87, 0x87,
0x7c, 0xf0, 0x66, 0xe7, 0x84, 0xf9, 0x3e, 0xba, 0x97, 0x45, 0x9c, 0x82, 0x26, 0x11, 0x3f, 0xce,
0x5b, 0x68, 0xe1, 0x40, 0x86, 0xdc, 0x3f, 0x4e, 0xba, 0xc9, 0x5e, 0x22, 0xa7, 0x70, 0x67, 0x17,
0x55, 0x74, 0x2e, 0x24, 0x1f, 0x88, 0x24, 0x60, 0x67, 0x7e, 0xc0, 0x19, 0xf0, 0x35, 0x43, 0xfe,
0x00, 0x30, 0x29, 0x4f, 0xb2, 0x58, 0xf9, 0xce, 0x12, 0x38, 0x0d, 0x4b, 0xdd, 0x73, 0x58, 0xcb,
0x3f, 0x93, 0xc8, 0xa7, 0x45, 0xb6, 0x85, 0x8f, 0xc8, 0xc6, 0x67, 0x8b, 0x40, 0xd3, 0x50, 0x21,
0xac, 0xcf, 0x4c, 0x2a, 0xf2, 0xec, 0x32, 0x17, 0xd3, 0x53, 0xbd, 0xf1, 0x7c, 0x41, 0x74, 0x1a,
0x73, 0x1f, 0xaa, 0x69, 0x03, 0x90, 0xc7, 0x45, 0xd6, 0xd3, 0xfd, 0xd1, 0xb8, 0x6c, 0x46, 0xda,
0x4b, 0xe4, 0x10, 0x6a, 0x99, 0x26, 0x21, 0x85, 0x4c, 0xcf, 0x76, 0xd1, 0x55, 0x5e, 0xfb, 0x00,
0xbb, 0x28, 0xf7, 0x50, 0x86, 0x7c, 0x20, 0xa6, 0x9d, 0x6a, 0x61, 0x02, 0x48, 0x9c, 0x3e, 0xbd,
0x12, 0x97, 0x10, 0xd1, 0x79, 0xb7, 0xa2, 0xc7, 0x71, 0xf4, 0x5f, 0xf2, 0x5f, 0xa3, 0x7e, 0x80,
0x46, 0x3d, 0x84, 0x5a, 0xe6, 0xa5, 0x5f, 0x5c, 0x18, 0xb3, 0xbf, 0x02, 0xff, 0x76, 0x61, 0x6c,
0x7f, 0xfe, 0x7d, 0xe7, 0x98, 0xcb, 0x93, 0xf1, 0x51, 0x14, 0x7a, 0x33, 0x46, 0x3e, 0xe7, 0x81,
0xfe, 0xda, 0x4c, 0x18, 0xda, 0x54, 0x9e, 0x36, 0x55, 0x1a, 0xa3, 0xa3, 0xa3, 0x92, 0x12, 0x5f,
0xfc, 0x19, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x8e, 0x84, 0xfb, 0xed, 0x0f, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.
@ -940,6 +982,7 @@ type IndexCoordClient interface {
GetIndexStates(ctx context.Context, in *GetIndexStatesRequest, opts ...grpc.CallOption) (*GetIndexStatesResponse, error)
GetIndexFilePaths(ctx context.Context, in *GetIndexFilePathsRequest, opts ...grpc.CallOption) (*GetIndexFilePathsResponse, error)
DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
RemoveIndex(ctx context.Context, in *RemoveIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error)
}
@ -1015,6 +1058,15 @@ func (c *indexCoordClient) DropIndex(ctx context.Context, in *DropIndexRequest,
return out, nil
}
func (c *indexCoordClient) RemoveIndex(ctx context.Context, in *RemoveIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.index.IndexCoord/RemoveIndex", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *indexCoordClient) GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) {
out := new(milvuspb.GetMetricsResponse)
err := c.cc.Invoke(ctx, "/milvus.proto.index.IndexCoord/GetMetrics", in, out, opts...)
@ -1033,6 +1085,7 @@ type IndexCoordServer interface {
GetIndexStates(context.Context, *GetIndexStatesRequest) (*GetIndexStatesResponse, error)
GetIndexFilePaths(context.Context, *GetIndexFilePathsRequest) (*GetIndexFilePathsResponse, error)
DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error)
RemoveIndex(context.Context, *RemoveIndexRequest) (*commonpb.Status, error)
// https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
GetMetrics(context.Context, *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error)
}
@ -1062,6 +1115,9 @@ func (*UnimplementedIndexCoordServer) GetIndexFilePaths(ctx context.Context, req
func (*UnimplementedIndexCoordServer) DropIndex(ctx context.Context, req *DropIndexRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method DropIndex not implemented")
}
func (*UnimplementedIndexCoordServer) RemoveIndex(ctx context.Context, req *RemoveIndexRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method RemoveIndex not implemented")
}
func (*UnimplementedIndexCoordServer) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetMetrics not implemented")
}
@ -1196,6 +1252,24 @@ func _IndexCoord_DropIndex_Handler(srv interface{}, ctx context.Context, dec fun
return interceptor(ctx, in, info, handler)
}
func _IndexCoord_RemoveIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RemoveIndexRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(IndexCoordServer).RemoveIndex(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/milvus.proto.index.IndexCoord/RemoveIndex",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(IndexCoordServer).RemoveIndex(ctx, req.(*RemoveIndexRequest))
}
return interceptor(ctx, in, info, handler)
}
func _IndexCoord_GetMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(milvuspb.GetMetricsRequest)
if err := dec(in); err != nil {
@ -1246,6 +1320,10 @@ var _IndexCoord_serviceDesc = grpc.ServiceDesc{
MethodName: "DropIndex",
Handler: _IndexCoord_DropIndex_Handler,
},
{
MethodName: "RemoveIndex",
Handler: _IndexCoord_RemoveIndex_Handler,
},
{
MethodName: "GetMetrics",
Handler: _IndexCoord_GetMetrics_Handler,

View File

@ -133,6 +133,13 @@ func (coord *IndexCoordMock) DropIndex(ctx context.Context, req *indexpb.DropInd
}, nil
}
func (coord *IndexCoordMock) RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error) {
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
Reason: "",
}, nil
}
func (coord *IndexCoordMock) GetIndexStates(ctx context.Context, req *indexpb.GetIndexStatesRequest) (*indexpb.GetIndexStatesResponse, error) {
return &indexpb.GetIndexStatesResponse{
Status: &commonpb.Status{

View File

@ -429,7 +429,6 @@ func (mt *MetaTable) GetCollectionIDByName(cName string) (typeutil.UniqueID, err
func (mt *MetaTable) GetCollectionNameByID(collectionID typeutil.UniqueID) (string, error) {
mt.ddLock.RLock()
defer mt.ddLock.RUnlock()
log.Info("xxxxxxxxx", zap.Any("collID2Meta", mt.collID2Meta))
col, ok := mt.collID2Meta[collectionID]
if !ok {
return "", fmt.Errorf("can't find collection id : %d", collectionID)
@ -794,23 +793,13 @@ func (mt *MetaTable) DeletePartition(collID typeutil.UniqueID, partitionName str
return partID, nil
}
// AddIndex add index
// AddIndex add index; before calling this function, you must check whether indexing is required
func (mt *MetaTable) AddIndex(segIdxInfo *pb.SegmentIndexInfo) error {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
collMeta, ok := mt.collID2Meta[segIdxInfo.CollectionID]
if !ok {
return fmt.Errorf("collection id = %d not found", segIdxInfo.CollectionID)
}
exist := false
for _, fidx := range collMeta.FieldIndexes {
if fidx.IndexID == segIdxInfo.IndexID {
exist = true
break
}
}
if !exist {
if idxInfo, ok := mt.indexID2Meta[segIdxInfo.IndexID]; !ok || idxInfo.GetDeleted() {
log.Error("index id not found or has been deleted", zap.Int64("indexID", segIdxInfo.IndexID))
return fmt.Errorf("index id = %d not found", segIdxInfo.IndexID)
}
@ -856,7 +845,70 @@ func (mt *MetaTable) AddIndex(segIdxInfo *pb.SegmentIndexInfo) error {
return nil
}
func (mt *MetaTable) MarkIndexDeleted(collName, fieldName, indexName string) error {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
collMeta, err := mt.unlockGetCollectionInfo(collName)
if err != nil {
log.Error("get collection meta failed", zap.String("collName", collName), zap.Error(err))
return fmt.Errorf("collection name = %s not has meta", collName)
}
fieldSch, err := mt.unlockGetFieldSchema(collName, fieldName)
if err != nil {
return err
}
var clonedIndex *pb.IndexInfo
var dropIdxID typeutil.UniqueID
for _, info := range collMeta.FieldIndexes {
if info.FiledID != fieldSch.FieldID {
continue
}
idxMeta, ok := mt.indexID2Meta[info.IndexID]
if !ok {
errMsg := fmt.Errorf("index not has meta with ID = %d", info.IndexID)
log.Error("index id not has meta", zap.Int64("index id", info.IndexID))
return errMsg
}
if idxMeta.IndexName != indexName {
continue
}
dropIdxID = info.IndexID
clonedIndex = proto.Clone(&idxMeta).(*pb.IndexInfo)
clonedIndex.Deleted = true
break
}
if dropIdxID == 0 {
log.Warn("index not found", zap.String("collName", collName), zap.String("fieldName", fieldName),
zap.String("indexName", indexName))
return nil
}
log.Info("MarkIndexDeleted", zap.String("collName", collName), zap.String("fieldName", fieldName),
zap.String("indexName", indexName), zap.Int64("dropIndexID", dropIdxID))
k := fmt.Sprintf("%s/%d/%d", IndexMetaPrefix, collMeta.ID, dropIdxID)
v, err := proto.Marshal(clonedIndex)
if err != nil {
log.Error("MetaTable MarkIndexDeleted Marshal idxInfo fail",
zap.String("key", k), zap.Error(err))
return err
}
err = mt.txn.Save(k, string(v))
if err != nil {
log.Error("MetaTable MarkIndexDeleted txn MultiSave failed", zap.Error(err))
return err
}
// update meta cache
mt.indexID2Meta[dropIdxID] = *clonedIndex
return nil
}
// DropIndex drop index
// Deprecated, only ut are used.
func (mt *MetaTable) DropIndex(collName, fieldName, indexName string) (typeutil.UniqueID, bool, error) {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
@ -884,15 +936,11 @@ func (mt *MetaTable) DropIndex(collName, fieldName, indexName string) (typeutil.
continue
}
idxMeta, ok := mt.indexID2Meta[info.IndexID]
if !ok {
if !ok || idxMeta.GetDeleted() || idxMeta.IndexName != indexName {
fieldIdxInfo = append(fieldIdxInfo, info)
log.Warn("index id not has meta", zap.Int64("index id", info.IndexID))
continue
}
if idxMeta.IndexName != indexName {
fieldIdxInfo = append(fieldIdxInfo, info)
continue
}
dropIdxID = info.IndexID
fieldIdxInfo = append(fieldIdxInfo, collMeta.FieldIndexes[i+1:]...)
break
@ -973,6 +1021,161 @@ func (mt *MetaTable) GetInitBuildIDs(collName, indexName string) ([]UniqueID, er
return initBuildIDs, nil
}
func (mt *MetaTable) GetBuildIDsBySegIDs(segIDs []UniqueID) []UniqueID {
mt.ddLock.RLock()
defer mt.ddLock.RUnlock()
buildIDs := make([]UniqueID, 0)
for _, segID := range segIDs {
if idxID2segIdx, ok := mt.segID2IndexMeta[segID]; ok {
for _, segIndex := range idxID2segIdx {
buildIDs = append(buildIDs, segIndex.BuildID)
}
}
}
return buildIDs
}
func (mt *MetaTable) AlignSegmentsMeta(collID, partID UniqueID, segIDs map[UniqueID]struct{}) ([]UniqueID, []UniqueID) {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
allIndexID := make([]UniqueID, 0)
if collMeta, ok := mt.collID2Meta[collID]; ok {
for _, fieldIndex := range collMeta.FieldIndexes {
allIndexID = append(allIndexID, fieldIndex.IndexID)
}
}
recycledSegIDs := make([]UniqueID, 0)
recycledBuildIDs := make([]UniqueID, 0)
if segMap, ok := mt.partID2SegID[partID]; ok {
for segID := range segMap {
if _, ok := segIDs[segID]; !ok {
recycledSegIDs = append(recycledSegIDs, segID)
}
if idxID2segIndex, ok := mt.segID2IndexMeta[segID]; ok {
for _, segIndex := range idxID2segIndex {
recycledBuildIDs = append(recycledBuildIDs, segIndex.BuildID)
}
}
}
}
return recycledSegIDs, recycledBuildIDs
}
func (mt *MetaTable) RemoveSegments(collID, partID UniqueID, segIDs []UniqueID) error {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
log.Info("RootCoord MetaTable remove segments", zap.Int64("collID", collID), zap.Int64("partID", partID),
zap.Int64s("segIDs", segIDs))
allIndexID := make([]UniqueID, 0)
if collMeta, ok := mt.collID2Meta[collID]; ok {
for _, fieldIndex := range collMeta.FieldIndexes {
allIndexID = append(allIndexID, fieldIndex.IndexID)
}
}
for _, segID := range segIDs {
delMeta := make([]string, 0)
for _, indexID := range allIndexID {
delMeta = append(delMeta, fmt.Sprintf("%s/%d/%d/%d/%d", SegmentIndexMetaPrefix, collID, indexID, partID, segID))
}
if err := mt.txn.MultiRemove(delMeta); err != nil {
log.Error("remove redundant segment failed, wait to retry", zap.Int64("collID", collID), zap.Int64("part", partID),
zap.Int64("segID", segID), zap.Error(err))
return err
}
delete(mt.partID2SegID[partID], segID)
delete(mt.segID2IndexMeta, segID)
}
return nil
}
func (mt *MetaTable) GetDroppedIndex() map[UniqueID][]*pb.FieldIndexInfo {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
droppedIndex := make(map[UniqueID][]*pb.FieldIndexInfo)
for collID, meta := range mt.collID2Meta {
for _, fieldIndex := range meta.FieldIndexes {
if indexMeta, ok := mt.indexID2Meta[fieldIndex.IndexID]; ok && indexMeta.Deleted {
droppedIndex[collID] = append(droppedIndex[collID], proto.Clone(fieldIndex).(*pb.FieldIndexInfo))
}
}
}
return droppedIndex
}
// RecycleDroppedIndex remove the meta about index which is deleted.
func (mt *MetaTable) RecycleDroppedIndex() error {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
for collID, collMeta := range mt.collID2Meta {
meta := collMeta
fieldIndexes := make([]*pb.FieldIndexInfo, 0)
delMeta := make([]string, 0)
deletedIndexIDs := make(map[UniqueID]struct{})
for _, fieldIndex := range meta.FieldIndexes {
// Prevent the number of transaction operations from exceeding the etcd limit (128), where a maximum of 100 are processed each time
if len(delMeta) >= 100 {
break
}
if idxInfo, ok := mt.indexID2Meta[fieldIndex.IndexID]; !ok || idxInfo.GetDeleted() {
deletedIndexIDs[fieldIndex.IndexID] = struct{}{}
delMeta = append(delMeta, fmt.Sprintf("%s/%d/%d", IndexMetaPrefix, collID, fieldIndex.IndexID))
delMeta = append(delMeta, fmt.Sprintf("%s/%d/%d", SegmentIndexMetaPrefix, collID, fieldIndex.IndexID))
continue
}
fieldIndexes = append(fieldIndexes, fieldIndex)
}
// node index is deleted
if len(fieldIndexes) == len(meta.FieldIndexes) {
continue
}
clonedCollMeta := proto.Clone(&meta).(*pb.CollectionInfo)
clonedCollMeta.FieldIndexes = fieldIndexes
saveMeta := make(map[string]string)
k := path.Join(CollectionMetaPrefix, strconv.FormatInt(collID, 10))
v, err := proto.Marshal(clonedCollMeta)
if err != nil {
log.Error("MetaTable RecycleDroppedIndex Marshal collMeta failed",
zap.String("key", k), zap.Error(err))
return err
}
saveMeta[k] = string(v)
if err = mt.txn.MultiSaveAndRemoveWithPrefix(saveMeta, delMeta); err != nil {
log.Error("MetaTable RecycleDroppedIndex MultiSaveAndRemoveWithPrefix failed", zap.Error(err))
return err
}
mt.collID2Meta[collID] = *clonedCollMeta
for indexID := range deletedIndexIDs {
delete(mt.indexID2Meta, indexID)
}
// update segID2IndexMeta
for _, partID := range meta.PartitionIDs {
if segIDMap, ok := mt.partID2SegID[partID]; ok {
for segID := range segIDMap {
if segIndexInfos, ok := mt.segID2IndexMeta[segID]; ok {
for indexID := range segIndexInfos {
if _, ok := deletedIndexIDs[indexID]; ok {
delete(mt.segID2IndexMeta[segID], indexID)
}
}
}
}
}
}
}
return nil
}
// GetSegmentIndexInfoByID return segment index info by segment id
func (mt *MetaTable) GetSegmentIndexInfoByID(segID typeutil.UniqueID, fieldID int64, idxName string) (pb.SegmentIndexInfo, error) {
mt.ddLock.RLock()
@ -995,20 +1198,14 @@ func (mt *MetaTable) GetSegmentIndexInfoByID(segID typeutil.UniqueID, fieldID in
if fieldID == -1 && idxName == "" { // return default index
for _, seg := range segIdxMap {
info, ok := mt.indexID2Meta[seg.IndexID]
if ok && info.IndexName == Params.CommonCfg.DefaultIndexName {
if ok && !info.GetDeleted() && info.IndexName == Params.CommonCfg.DefaultIndexName {
return seg, nil
}
}
} else {
for idxID, seg := range segIdxMap {
idxMeta, ok := mt.indexID2Meta[idxID]
if ok {
if idxMeta.IndexName != idxName {
continue
}
if seg.FieldID != fieldID {
continue
}
if ok && !idxMeta.GetDeleted() && idxMeta.IndexName == idxName && seg.FieldID == fieldID {
return seg, nil
}
}
@ -1075,7 +1272,7 @@ func (mt *MetaTable) unlockIsSegmentIndexed(segID typeutil.UniqueID, fieldSchema
continue
}
idxMeta, ok := mt.indexID2Meta[idxID]
if !ok {
if !ok || idxMeta.GetDeleted() {
continue
}
if EqualKeyPairArray(indexParams, idxMeta.IndexParams) {
@ -1105,6 +1302,9 @@ func (mt *MetaTable) checkFieldCanBeIndexed(collMeta pb.CollectionInfo, fieldSch
for _, f := range collMeta.FieldIndexes {
if f.GetFiledID() == fieldSchema.GetFieldID() {
if info, ok := mt.indexID2Meta[f.GetIndexID()]; ok {
if info.GetDeleted() {
continue
}
if idxInfo.GetIndexName() != info.GetIndexName() {
return fmt.Errorf(
"creating multiple indexes on same field is not supported, "+
@ -1128,7 +1328,7 @@ func (mt *MetaTable) checkFieldCanBeIndexed(collMeta pb.CollectionInfo, fieldSch
func (mt *MetaTable) checkFieldIndexDuplicate(collMeta pb.CollectionInfo, fieldSchema schemapb.FieldSchema, idxInfo *pb.IndexInfo) (duplicate bool, err error) {
for _, f := range collMeta.FieldIndexes {
if info, ok := mt.indexID2Meta[f.IndexID]; ok {
if info, ok := mt.indexID2Meta[f.IndexID]; ok && !info.GetDeleted() {
if info.IndexName == idxInfo.IndexName {
// the index name must be different for different indexes
if f.FiledID != fieldSchema.FieldID || !EqualKeyPairArray(info.IndexParams, idxInfo.IndexParams) {
@ -1229,6 +1429,7 @@ func (mt *MetaTable) GetNotIndexedSegments(collName string, fieldName string, id
}
// GetIndexByName return index info by index name
// TODO: IndexName is unique on collection, should return (pb.CollectionInfo, pb.IndexInfo, error)
func (mt *MetaTable) GetIndexByName(collName, indexName string) (pb.CollectionInfo, []pb.IndexInfo, error) {
mt.ddLock.RLock()
defer mt.ddLock.RUnlock()
@ -1251,6 +1452,9 @@ func (mt *MetaTable) GetIndexByName(collName, indexName string) (pb.CollectionIn
if !ok {
return pb.CollectionInfo{}, nil, fmt.Errorf("index id = %d not found", idx.IndexID)
}
if idxInfo.GetDeleted() {
continue
}
if indexName == "" || idxInfo.IndexName == indexName {
rstIndex = append(rstIndex, idxInfo)
}
@ -1264,12 +1468,24 @@ func (mt *MetaTable) GetIndexByID(indexID typeutil.UniqueID) (*pb.IndexInfo, err
defer mt.ddLock.RUnlock()
indexInfo, ok := mt.indexID2Meta[indexID]
if !ok {
if !ok || indexInfo.GetDeleted() {
return nil, fmt.Errorf("cannot find index, id = %d", indexID)
}
return &indexInfo, nil
}
func (mt *MetaTable) dupCollectionMeta() map[typeutil.UniqueID]pb.CollectionInfo {
mt.ddLock.RLock()
defer mt.ddLock.RUnlock()
collID2Meta := map[typeutil.UniqueID]pb.CollectionInfo{}
for k, v := range mt.collID2Meta {
v := v
collID2Meta[k] = *proto.Clone(&v).(*pb.CollectionInfo)
}
return collID2Meta
}
func (mt *MetaTable) dupMeta() (
map[typeutil.UniqueID]pb.CollectionInfo,
map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo,

View File

@ -75,6 +75,7 @@ type mockTestTxnKV struct {
multiSave func(kvs map[string]string) error
multiSaveAndRemoveWithPrefix func(saves map[string]string, removals []string) error
remove func(key string) error
multiRemove func(keys []string) error
}
func (m *mockTestTxnKV) LoadWithPrefix(key string) ([]string, []string, error) {
@ -97,6 +98,10 @@ func (m *mockTestTxnKV) Remove(key string) error {
return m.remove(key)
}
func (m *mockTestTxnKV) MultiRemove(keys []string) error {
return m.multiRemove(keys)
}
func Test_MockKV(t *testing.T) {
k1 := &mockTestKV{}
kt := &mockTestTxnKV{}
@ -799,7 +804,7 @@ func TestMetaTable(t *testing.T) {
mt.collID2Meta = make(map[int64]pb.CollectionInfo)
err = mt.AddIndex(&segIdxInfo)
assert.NotNil(t, err)
assert.EqualError(t, err, fmt.Sprintf("collection id = %d not found", collInfo.ID))
assert.EqualError(t, err, fmt.Sprintf("index id = %d not found", segIdxInfo.IndexID))
err = mt.reloadFromKV()
assert.Nil(t, err)
@ -1588,3 +1593,216 @@ func TestMetaTable_GetInitBuildIDs(t *testing.T) {
assert.Equal(t, 1, len(buildIDs))
})
}
func TestMetaTable_GetDroppedIndex(t *testing.T) {
mt := &MetaTable{
collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{
1: {
FieldIndexes: []*pb.FieldIndexInfo{
{
FiledID: 1,
IndexID: 1,
},
{
FiledID: 2,
IndexID: 2,
},
{
FiledID: 3,
IndexID: 3,
},
},
},
},
indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{
1: {
IndexName: "GetInitBuildID-Index-1",
IndexID: 1,
Deleted: true,
},
2: {
IndexName: "GetInitBuildID-Index-2",
IndexID: 2,
Deleted: false,
},
},
}
droppedIndexID := mt.GetDroppedIndex()
indexInfo, ok := droppedIndexID[1]
assert.True(t, ok)
assert.Equal(t, 1, len(indexInfo))
}
func TestMetaTable_AlignSegmentsMeta(t *testing.T) {
var (
indexName = "GetDroppedIndex-Index"
collID = UniqueID(1)
partID = UniqueID(10)
indexID = UniqueID(1000)
)
mt := &MetaTable{
txn: &mockTestTxnKV{
multiRemove: func(keys []string) error {
return nil
},
},
collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{
collID: {
FieldIndexes: []*pb.FieldIndexInfo{
{
FiledID: 1,
IndexID: 1,
},
},
},
},
partID2SegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{
partID: {
100: true,
101: true,
102: true,
},
},
indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{
indexID: {
IndexName: indexName,
IndexID: 1,
Deleted: true,
},
},
}
t.Run("success", func(t *testing.T) {
mt.AlignSegmentsMeta(collID, partID, map[UniqueID]struct{}{101: {}, 102: {}, 103: {}})
})
t.Run("txn error", func(t *testing.T) {
txn := &mockTestTxnKV{
multiRemove: func(keys []string) error {
return fmt.Errorf("error occurred")
},
}
mt.txn = txn
mt.AlignSegmentsMeta(collID, partID, map[UniqueID]struct{}{103: {}, 104: {}, 105: {}})
})
}
func TestMetaTable_MarkIndexDeleted(t *testing.T) {
var (
collName = "MarkIndexDeleted-Coll"
fieldName = "MarkIndexDeleted-Field"
indexName = "MarkIndexDeleted-Index"
collID = UniqueID(1)
partID = UniqueID(10)
fieldID = UniqueID(100)
indexID = UniqueID(1000)
)
mt := &MetaTable{
txn: &mockTestTxnKV{
multiRemove: func(keys []string) error {
return nil
},
save: func(key, value string) error {
return nil
},
},
collID2Meta: map[typeutil.UniqueID]pb.CollectionInfo{
collID: {
FieldIndexes: []*pb.FieldIndexInfo{
{
FiledID: 101,
IndexID: 1001,
},
},
Schema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
FieldID: fieldID,
Name: fieldName,
},
},
},
},
},
collName2ID: map[string]typeutil.UniqueID{
collName: collID,
},
partID2SegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{
partID: {
100: true,
101: true,
102: true,
},
},
indexID2Meta: map[typeutil.UniqueID]pb.IndexInfo{
1001: {
IndexName: indexName,
IndexID: indexID,
Deleted: true,
},
},
}
t.Run("get collection meta failed", func(t *testing.T) {
err := mt.MarkIndexDeleted("collName", fieldName, indexName)
assert.Error(t, err)
})
t.Run("get field meta failed", func(t *testing.T) {
err := mt.MarkIndexDeleted(collName, "fieldName", indexName)
assert.Error(t, err)
err = mt.MarkIndexDeleted(collName, fieldName, indexName)
assert.NoError(t, err)
})
t.Run("indexMeta error", func(t *testing.T) {
collMeta := pb.CollectionInfo{
FieldIndexes: []*pb.FieldIndexInfo{
{
FiledID: fieldID,
IndexID: indexID,
},
},
Schema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
FieldID: fieldID,
Name: fieldName,
},
},
},
}
mt.collID2Meta[collID] = collMeta
err := mt.MarkIndexDeleted(collName, fieldName, indexName)
assert.Error(t, err)
mt.indexID2Meta[indexID] = pb.IndexInfo{
IndexName: "indexName",
IndexID: indexID,
}
err = mt.MarkIndexDeleted(collName, fieldName, indexName)
assert.NoError(t, err)
mt.indexID2Meta[indexID] = pb.IndexInfo{
IndexName: indexName,
IndexID: indexID,
}
err = mt.MarkIndexDeleted(collName, fieldName, indexName)
assert.NoError(t, err)
})
t.Run("txn save failed", func(t *testing.T) {
txn := &mockTestTxnKV{
save: func(key, value string) error {
return fmt.Errorf("error occurred")
},
}
mt.txn = txn
err := mt.MarkIndexDeleted(collName, fieldName, indexName)
assert.Error(t, err)
})
}

View File

@ -135,6 +135,7 @@ type Core struct {
//call index builder's client to build index, return build id or get index state.
CallBuildIndexService func(ctx context.Context, segID UniqueID, binlog []string, field *schemapb.FieldSchema, idxInfo *etcdpb.IndexInfo, numRows int64) (typeutil.UniqueID, error)
CallDropIndexService func(ctx context.Context, indexID typeutil.UniqueID) error
CallRemoveIndexService func(ctx context.Context, buildIDs []UniqueID) error
CallGetIndexStatesService func(ctx context.Context, IndexBuildIDs []int64) ([]*indexpb.IndexInfo, error)
NewProxyClient func(sess *sessionutil.Session) (types.Proxy, error)
@ -281,6 +282,9 @@ func (c *Core) checkInit() error {
if c.CallDropIndexService == nil {
return fmt.Errorf("callDropIndexService is nil")
}
if c.CallRemoveIndexService == nil {
return fmt.Errorf("callDropIndexService is nil")
}
if c.CallWatchChannels == nil {
return fmt.Errorf("callWatchChannels is nil")
}
@ -369,84 +373,145 @@ func (c *Core) checkFlushedSegmentsLoop() {
}
}
func (c *Core) recycleDroppedIndex() {
defer c.wg.Done()
ticker := time.NewTicker(3 * time.Second)
for {
select {
case <-c.ctx.Done():
return
case <-ticker.C:
droppedIndex := c.MetaTable.GetDroppedIndex()
for collID, fieldIndexes := range droppedIndex {
for _, fieldIndex := range fieldIndexes {
indexID := fieldIndex.GetIndexID()
fieldID := fieldIndex.GetFiledID()
if err := c.CallDropIndexService(c.ctx, indexID); err != nil {
log.Warn("Notify IndexCoord to drop index failed, wait to retry", zap.Int64("collID", collID),
zap.Int64("fieldID", fieldID), zap.Int64("indexID", indexID))
}
}
}
err := c.MetaTable.RecycleDroppedIndex()
if err != nil {
log.Warn("Remove index meta failed, wait to retry", zap.Error(err))
}
}
}
}
func (c *Core) createIndexForSegment(ctx context.Context, collID, partID, segID UniqueID, numRows int64, binlogs []*datapb.FieldBinlog) error {
collID2Meta, _, indexID2Meta := c.MetaTable.dupMeta()
collMeta, ok := collID2Meta[collID]
if !ok {
log.Error("collection meta is not exist", zap.Int64("collID", collID))
return fmt.Errorf("collection meta is not exist with ID = %d", collID)
}
if len(collMeta.FieldIndexes) == 0 {
log.Info("collection has no index, no need to build index on segment", zap.Int64("collID", collID),
zap.Int64("segID", segID))
return nil
}
for _, fieldIndex := range collMeta.FieldIndexes {
indexMeta, ok := indexID2Meta[fieldIndex.IndexID]
if !ok {
log.Warn("index has no meta", zap.Int64("collID", collID), zap.Int64("indexID", fieldIndex.IndexID))
return fmt.Errorf("index has no meta with ID = %d in collection %d", fieldIndex.IndexID, collID)
}
if indexMeta.Deleted {
log.Info("index has been deleted, no need to build index on segment")
continue
}
field, err := GetFieldSchemaByID(&collMeta, fieldIndex.FiledID)
if err != nil {
log.Debug("GetFieldSchemaByID failed",
zap.Int64("collectionID", collID),
zap.Int64("fieldID", fieldIndex.FiledID))
return err
}
if c.MetaTable.IsSegmentIndexed(segID, field, indexMeta.IndexParams) {
continue
}
segIndexInfo := etcdpb.SegmentIndexInfo{
CollectionID: collMeta.ID,
PartitionID: partID,
SegmentID: segID,
FieldID: fieldIndex.FiledID,
IndexID: fieldIndex.IndexID,
EnableIndex: false,
ByAutoFlush: true,
}
buildID, err := c.BuildIndex(ctx, segID, numRows, binlogs, field, &indexMeta, false)
if err != nil {
log.Debug("build index failed",
zap.Int64("segmentID", segID),
zap.Int64("fieldID", field.FieldID),
zap.Int64("indexID", indexMeta.IndexID))
return err
}
// if buildID == 0, means it's no need to build index.
if buildID != 0 {
segIndexInfo.BuildID = buildID
segIndexInfo.EnableIndex = true
}
if err := c.MetaTable.AddIndex(&segIndexInfo); err != nil {
log.Error("Add index into meta table failed, need remove index with buildID",
zap.Int64("collectionID", collID), zap.Int64("indexID", fieldIndex.IndexID),
zap.Int64("buildID", buildID), zap.Error(err))
if err = retry.Do(ctx, func() error {
return c.CallRemoveIndexService(ctx, []UniqueID{buildID})
}); err != nil {
log.Error("remove index failed, need to be resolved manually", zap.Int64("collectionID", collID),
zap.Int64("indexID", fieldIndex.IndexID), zap.Int64("buildID", buildID), zap.Error(err))
return err
}
return err
}
}
return nil
}
func (c *Core) checkFlushedSegments(ctx context.Context) {
collID2Meta, segID2IndexMeta, indexID2Meta := c.MetaTable.dupMeta()
for _, collMeta := range collID2Meta {
collID2Meta := c.MetaTable.dupCollectionMeta()
for collID, collMeta := range collID2Meta {
if len(collMeta.FieldIndexes) == 0 {
continue
}
for _, partID := range collMeta.PartitionIDs {
ctx2, cancel2 := context.WithTimeout(ctx, 3*time.Minute)
segBinlogs, err := c.CallGetRecoveryInfoService(ctx2, collMeta.ID, partID)
segBinlogs, err := c.CallGetRecoveryInfoService(ctx, collMeta.ID, partID)
if err != nil {
log.Debug("failed to get flushed segments from dataCoord",
zap.Int64("collection ID", collMeta.GetID()),
zap.Int64("partition ID", partID),
zap.Error(err))
cancel2()
continue
}
segIDs := make(map[UniqueID]struct{})
for _, segBinlog := range segBinlogs {
segID := segBinlog.SegmentID
var indexInfos []*etcdpb.FieldIndexInfo
indexMeta, ok := segID2IndexMeta[segID]
if !ok {
indexInfos = append(indexInfos, collMeta.FieldIndexes...)
} else {
for _, idx := range collMeta.FieldIndexes {
if _, ok := indexMeta[idx.IndexID]; !ok {
indexInfos = append(indexInfos, idx)
}
}
}
for _, idxInfo := range indexInfos {
/* #nosec G601 */
field, err := GetFieldSchemaByID(&collMeta, idxInfo.FiledID)
if err != nil {
log.Debug("GetFieldSchemaByID",
zap.Any("collection_meta", collMeta),
zap.Int64("field id", idxInfo.FiledID))
continue
}
indexMeta, ok := indexID2Meta[idxInfo.IndexID]
if !ok {
log.Debug("index meta does not exist", zap.Int64("index_id", idxInfo.IndexID))
continue
}
info := etcdpb.SegmentIndexInfo{
CollectionID: collMeta.ID,
PartitionID: partID,
SegmentID: segID,
FieldID: idxInfo.FiledID,
IndexID: idxInfo.IndexID,
EnableIndex: false,
ByAutoFlush: true,
}
log.Debug("building index by background checker",
zap.Int64("segment_id", segID),
zap.Int64("index_id", indexMeta.IndexID),
zap.Int64("collection_id", collMeta.ID))
info.BuildID, err = c.BuildIndex(ctx2, segID, segBinlog.GetNumOfRows(), segBinlog.GetFieldBinlogs(), field, &indexMeta, false)
if err != nil {
log.Debug("build index failed",
zap.Int64("segment_id", segID),
zap.Int64("field_id", field.FieldID),
zap.Int64("index_id", indexMeta.IndexID))
continue
}
if info.BuildID != 0 {
info.EnableIndex = true
}
if err := c.MetaTable.AddIndex(&info); err != nil {
log.Debug("Add index into meta table failed",
zap.Int64("collection_id", collMeta.ID),
zap.Int64("index_id", info.IndexID),
zap.Int64("build_id", info.BuildID),
zap.Error(err))
}
segIDs[segBinlog.GetSegmentID()] = struct{}{}
err = c.createIndexForSegment(ctx, collID, partID, segBinlog.GetSegmentID(), segBinlog.GetNumOfRows(), segBinlog.GetFieldBinlogs())
if err != nil {
log.Error("createIndexForSegment failed, wait to retry", zap.Int64("collID", collID),
zap.Int64("partID", partID), zap.Int64("segID", segBinlog.GetSegmentID()), zap.Error(err))
continue
}
}
cancel2()
recycledSegIDs, recycledBuildIDs := c.MetaTable.AlignSegmentsMeta(collID, partID, segIDs)
log.Info("there buildIDs should be remove index", zap.Int64s("buildIDs", recycledBuildIDs))
if err := c.CallRemoveIndexService(ctx, recycledBuildIDs); err != nil {
log.Error("CallRemoveIndexService remove indexes on segments failed",
zap.Int64s("need dropped buildIDs", recycledBuildIDs), zap.Error(err))
continue
}
if err := c.MetaTable.RemoveSegments(collID, partID, recycledSegIDs); err != nil {
log.Warn("remove segments failed, wait to retry", zap.Int64("collID", collID), zap.Int64("partID", partID),
zap.Int64s("segIDs", recycledSegIDs), zap.Error(err))
continue
}
}
}
}
@ -833,6 +898,27 @@ func (c *Core) SetIndexCoord(s types.IndexCoord) error {
}
return res.GetStates(), nil
}
c.CallRemoveIndexService = func(ctx context.Context, buildIDs []UniqueID) (retErr error) {
defer func() {
if err := recover(); err != nil {
retErr = fmt.Errorf("get index state from index service panic, msg = %v", err)
}
}()
<-initCh
status, err := s.RemoveIndex(ctx, &indexpb.RemoveIndexRequest{
BuildIDs: buildIDs,
})
if err != nil {
return err
}
if status.GetErrorCode() != commonpb.ErrorCode_Success {
return fmt.Errorf(status.Reason)
}
return nil
}
return nil
}
@ -1341,13 +1427,14 @@ func (c *Core) Start() error {
log.Fatal("RootCoord Start reSendDdMsg failed", zap.Error(err))
panic(err)
}
c.wg.Add(6)
c.wg.Add(7)
go c.startTimeTickLoop()
go c.tsLoop()
go c.chanTimeTick.startWatch(&c.wg)
go c.checkFlushedSegmentsLoop()
go c.importManager.expireOldTasksLoop(&c.wg, c.CallReleaseSegRefLock)
go c.importManager.sendOutTasksLoop(&c.wg)
go c.recycleDroppedIndex()
Params.RootCoordCfg.CreatedTime = time.Now()
Params.RootCoordCfg.UpdatedTime = time.Now()
})
@ -2162,71 +2249,35 @@ func (c *Core) SegmentFlushCompleted(ctx context.Context, in *datapb.SegmentFlus
if in.Base.MsgType != commonpb.MsgType_SegmentFlushDone {
return failStatus(commonpb.ErrorCode_UnexpectedError, "invalid msg type "+commonpb.MsgType_name[int32(in.Base.MsgType)]), nil
}
segID := in.Segment.GetID()
log.Debug("SegmentFlushCompleted", zap.String("role", typeutil.RootCoordRole),
zap.Int64("collection id", in.Segment.CollectionID), zap.Int64("partition id", in.Segment.PartitionID),
zap.Int64("segment id", segID), zap.Int64("msgID", in.Base.MsgID))
coll, err := c.MetaTable.GetCollectionByID(in.Segment.CollectionID, 0)
log.Info("SegmentFlushCompleted received", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID),
zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID), zap.Int64s("compactFrom", in.Segment.CompactionFrom))
err := c.createIndexForSegment(ctx, in.Segment.CollectionID, in.Segment.PartitionID, in.Segment.ID, in.Segment.NumOfRows, in.Segment.Binlogs)
if err != nil {
log.Error("GetCollectionByID failed", zap.String("role", typeutil.RootCoordRole),
zap.Int64("msgID", in.Base.MsgID), zap.Error(err))
return failStatus(commonpb.ErrorCode_UnexpectedError, "GetCollectionByID failed: "+err.Error()), nil
log.Error("createIndexForSegment", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID),
zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID), zap.Error(err))
return failStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), nil
}
if len(coll.FieldIndexes) == 0 {
log.Debug("no index params on collection", zap.String("role", typeutil.RootCoordRole),
zap.String("collection_name", coll.Schema.Name), zap.Int64("msgID", in.Base.MsgID))
}
for _, f := range coll.FieldIndexes {
fieldSch, err := GetFieldSchemaByID(coll, f.FiledID)
if err != nil {
log.Warn("field schema not found", zap.String("role", typeutil.RootCoordRole),
zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID),
zap.Int64("msgID", in.Base.MsgID), zap.Error(err))
continue
}
idxInfo, err := c.MetaTable.GetIndexByID(f.IndexID)
if err != nil {
log.Warn("index not found", zap.String("role", typeutil.RootCoordRole),
zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID),
zap.Int64("index id", f.IndexID), zap.Int64("msgID", in.Base.MsgID), zap.Error(err))
continue
}
info := etcdpb.SegmentIndexInfo{
CollectionID: in.Segment.CollectionID,
PartitionID: in.Segment.PartitionID,
SegmentID: segID,
FieldID: fieldSch.FieldID,
IndexID: idxInfo.IndexID,
EnableIndex: false,
ByAutoFlush: true,
}
info.BuildID, err = c.BuildIndex(ctx, segID, in.Segment.GetNumOfRows(), in.Segment.GetBinlogs(), fieldSch, idxInfo, true)
if err == nil && info.BuildID != 0 {
info.EnableIndex = true
} else {
log.Error("BuildIndex failed", zap.String("role", typeutil.RootCoordRole),
zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID),
zap.Int64("index id", f.IndexID), zap.Int64("build id", info.BuildID),
zap.Int64("msgID", in.Base.MsgID), zap.Error(err))
continue
}
err = c.MetaTable.AddIndex(&info)
if err != nil {
log.Error("AddIndex failed", zap.String("role", typeutil.RootCoordRole),
zap.String("collection_name", coll.Schema.Name), zap.Int64("field id", f.FiledID),
zap.Int64("index id", f.IndexID), zap.Int64("msgID", in.Base.MsgID), zap.Error(err))
continue
buildIDs := c.MetaTable.GetBuildIDsBySegIDs(in.Segment.CompactionFrom)
if len(buildIDs) != 0 {
if err = c.CallRemoveIndexService(ctx, buildIDs); err != nil {
log.Error("CallRemoveIndexService failed", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID),
zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID),
zap.Int64s("compactFrom", in.Segment.CompactionFrom), zap.Int64s("buildIDs", buildIDs), zap.Error(err))
return failStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), nil
}
}
if err = c.MetaTable.RemoveSegments(in.Segment.CollectionID, in.Segment.PartitionID, in.Segment.CompactionFrom); err != nil {
log.Error("RemoveSegments failed", zap.Int64("msgID", in.Base.MsgID), zap.Int64("collID", in.Segment.CollectionID),
zap.Int64("partID", in.Segment.PartitionID), zap.Int64("segID", in.Segment.ID),
zap.Int64s("compactFrom", in.Segment.CompactionFrom), zap.Error(err))
return failStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), nil
}
log.Debug("SegmentFlushCompleted success", zap.String("role", typeutil.RootCoordRole),
zap.Int64("collection id", in.Segment.CollectionID), zap.Int64("partition id", in.Segment.PartitionID),
zap.Int64("segment id", segID), zap.Int64("msgID", in.Base.MsgID))
zap.Int64("segment id", in.Segment.ID), zap.Int64("msgID", in.Base.MsgID))
return succStatus(), nil
}

View File

@ -1722,10 +1722,16 @@ func TestRootCoord_Base(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, rsp.ErrorCode)
im.mutex.Lock()
assert.Equal(t, 1, len(im.idxDropID))
assert.Equal(t, idx[0].IndexID, im.idxDropID[0])
im.mutex.Unlock()
for {
im.mutex.Lock()
if len(im.idxDropID) == 1 {
assert.Equal(t, idx[0].IndexID, im.idxDropID[0])
im.mutex.Unlock()
break
}
im.mutex.Unlock()
time.Sleep(time.Second)
}
_, idx, err = core.MetaTable.GetIndexByName(collName, Params.CommonCfg.DefaultIndexName)
assert.NoError(t, err)
@ -3040,6 +3046,12 @@ func TestCheckInit(t *testing.T) {
err = c.checkInit()
assert.Error(t, err)
c.CallRemoveIndexService = func(ctx context.Context, buildIDs []typeutil.UniqueID) error {
return nil
}
err = c.checkInit()
assert.Error(t, err)
c.NewProxyClient = func(*sessionutil.Session) (types.Proxy, error) {
return nil, nil
}

View File

@ -919,12 +919,12 @@ func (t *DescribeSegmentsReqTask) Execute(ctx context.Context) error {
})
extraIndexInfo, err := t.core.MetaTable.GetIndexByID(indexID)
if err != nil {
log.Error("index not found in meta table",
log.Warn("index not found in meta table, maybe index has been deleted",
zap.Error(err),
zap.Int64("indexID", indexID),
zap.Int64("collection", collectionID),
zap.Int64("segment", segID))
return err
continue
}
t.Rsp.SegmentInfos[segID].ExtraIndexInfos[indexID] = extraIndexInfo
}
@ -1072,11 +1072,10 @@ func (t *DropIndexReqTask) Execute(ctx context.Context) error {
if t.Type() != commonpb.MsgType_DropIndex {
return fmt.Errorf("drop index, msg type = %s", commonpb.MsgType_name[int32(t.Type())])
}
if err := t.core.RemoveIndex(ctx, t.Req.CollectionName, t.Req.IndexName); err != nil {
if err := t.core.MetaTable.MarkIndexDeleted(t.Req.CollectionName, t.Req.FieldName, t.Req.IndexName); err != nil {
return err
}
_, _, err := t.core.MetaTable.DropIndex(t.Req.CollectionName, t.Req.FieldName, t.Req.IndexName)
return err
return nil
}
// CreateAliasReqTask create alias request task

View File

@ -76,7 +76,7 @@ func TestDescribeSegmentsReqTask_Execute(t *testing.T) {
}
assert.NoError(t, tsk.Execute(context.Background()))
// index not found in meta.
// index not found in meta. no return error
c.MetaTable = &MetaTable{
segID2IndexMeta: map[typeutil.UniqueID]map[typeutil.UniqueID]etcdpb.SegmentIndexInfo{
segID: {
@ -92,7 +92,7 @@ func TestDescribeSegmentsReqTask_Execute(t *testing.T) {
},
},
}
assert.Error(t, tsk.Execute(context.Background()))
assert.NoError(t, tsk.Execute(context.Background()))
// success.
c.MetaTable = &MetaTable{

View File

@ -351,6 +351,9 @@ type IndexCoord interface {
// GetMetrics gets the metrics about IndexCoord.
GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error)
// RemoveIndex removes the index on specify segments.
RemoveIndex(ctx context.Context, req *indexpb.RemoveIndexRequest) (*commonpb.Status, error)
}
// IndexCoordComponent is used by grpc server of IndexCoord