milvus/internal/rootcoord/meta_table_test.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rootcoord

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"path"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/milvus-io/milvus/internal/common"
	"github.com/milvus-io/milvus/internal/kv"
	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
	memkv "github.com/milvus-io/milvus/internal/kv/mem"
	"github.com/milvus-io/milvus/internal/metastore"
	"github.com/milvus-io/milvus/internal/metastore/kv/rootcoord"
	"github.com/milvus-io/milvus/internal/metastore/model"
	"github.com/milvus-io/milvus/internal/proto/commonpb"
	pb "github.com/milvus-io/milvus/internal/proto/etcdpb"
	"github.com/milvus-io/milvus/internal/proto/internalpb"
	"github.com/milvus-io/milvus/internal/proto/milvuspb"
	"github.com/milvus-io/milvus/internal/proto/schemapb"
	"github.com/milvus-io/milvus/internal/util"
	"github.com/milvus-io/milvus/internal/util/etcd"
	"github.com/milvus-io/milvus/internal/util/funcutil"
	"github.com/milvus-io/milvus/internal/util/typeutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
)

type mockTestKV struct {
	kv.SnapShotKV

	loadWithPrefix               func(key string, ts typeutil.Timestamp) ([]string, []string, error)
	save                         func(key, value string, ts typeutil.Timestamp) error
	multiSave                    func(kvs map[string]string, ts typeutil.Timestamp) error
	multiSaveAndRemoveWithPrefix func(saves map[string]string, removals []string, ts typeutil.Timestamp) error
}

func (m *mockTestKV) LoadWithPrefix(key string, ts typeutil.Timestamp) ([]string, []string, error) {
	return m.loadWithPrefix(key, ts)
}
func (m *mockTestKV) Load(key string, ts typeutil.Timestamp) (string, error) {
	return "", nil
}

func (m *mockTestKV) Save(key, value string, ts typeutil.Timestamp) error {
	return m.save(key, value, ts)
}

func (m *mockTestKV) MultiSave(kvs map[string]string, ts typeutil.Timestamp) error {
	return m.multiSave(kvs, ts)
}

func (m *mockTestKV) MultiSaveAndRemoveWithPrefix(saves map[string]string, removals []string, ts typeutil.Timestamp) error {
	return m.multiSaveAndRemoveWithPrefix(saves, removals, ts)
}

type mockTestTxnKV struct {
	kv.TxnKV
	loadWithPrefix               func(key string) ([]string, []string, error)
	save                         func(key, value string) error
	multiSave                    func(kvs map[string]string) error
	multiSaveAndRemoveWithPrefix func(saves map[string]string, removals []string) error
	remove                       func(key string) error
	multiRemove                  func(keys []string) error
	load                         func(key string) (string, error)
}

func (m *mockTestTxnKV) LoadWithPrefix(key string) ([]string, []string, error) {
	return m.loadWithPrefix(key)
}

func (m *mockTestTxnKV) Save(key, value string) error {
	return m.save(key, value)
}

func (m *mockTestTxnKV) MultiSave(kvs map[string]string) error {
	return m.multiSave(kvs)
}

func (m *mockTestTxnKV) MultiSaveAndRemoveWithPrefix(saves map[string]string, removals []string) error {
	return m.multiSaveAndRemoveWithPrefix(saves, removals)
}

func (m *mockTestTxnKV) Remove(key string) error {
	return m.remove(key)
}

func (m *mockTestTxnKV) MultiRemove(keys []string) error {
	return m.multiRemove(keys)
}

func (m *mockTestTxnKV) Load(key string) (string, error) {
	return m.load(key)
}

func generateMetaTable(t *testing.T) (*MetaTable, *mockTestKV, *mockTestTxnKV, func()) {
	rand.Seed(time.Now().UnixNano())
	randVal := rand.Int()
	Params.Init()
	rootPath := fmt.Sprintf("/test/meta/%d", randVal)
	etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
	require.Nil(t, err)

	skv, err := rootcoord.NewMetaSnapshot(etcdCli, rootPath, TimestampPrefix, 7)
	assert.Nil(t, err)
	assert.NotNil(t, skv)

	txnkv := etcdkv.NewEtcdKV(etcdCli, rootPath)
	_, err = NewMetaTable(context.TODO(), &rootcoord.Catalog{Txn: txnkv, Snapshot: skv})
	assert.Nil(t, err)
	mockSnapshotKV := &mockTestKV{
		SnapShotKV: skv,
		loadWithPrefix: func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return skv.LoadWithPrefix(key, ts)
		},
	}
	mockTxnKV := &mockTestTxnKV{
		TxnKV:          txnkv,
		loadWithPrefix: func(key string) ([]string, []string, error) { return txnkv.LoadWithPrefix(key) },
		save:           func(key, value string) error { return txnkv.Save(key, value) },
		multiSave:      func(kvs map[string]string) error { return txnkv.MultiSave(kvs) },
		multiSaveAndRemoveWithPrefix: func(kvs map[string]string, removal []string) error {
			return txnkv.MultiSaveAndRemoveWithPrefix(kvs, removal)
		},
		remove: func(key string) error { return txnkv.Remove(key) },
	}

	mockMt, err := NewMetaTable(context.TODO(), &rootcoord.Catalog{Txn: mockTxnKV, Snapshot: mockSnapshotKV})
	assert.Nil(t, err)
	return mockMt, mockSnapshotKV, mockTxnKV, func() {
		etcdCli.Close()
	}
}

func TestMetaTable(t *testing.T) {
	const (
		collName        = "testColl"
		collNameInvalid = "testColl_invalid"
		aliasName1      = "alias1"
		aliasName2      = "alias2"
		collID          = typeutil.UniqueID(1)
		collIDInvalid   = typeutil.UniqueID(2)
		partIDDefault   = typeutil.UniqueID(10)
		partID          = typeutil.UniqueID(20)
		partName        = "testPart"
		partIDInvalid   = typeutil.UniqueID(21)
		segID           = typeutil.UniqueID(100)
		segID2          = typeutil.UniqueID(101)
		fieldID         = typeutil.UniqueID(110)
		fieldID2        = typeutil.UniqueID(111)
		indexID         = typeutil.UniqueID(10000)
		indexID2        = typeutil.UniqueID(10001)
		buildID         = typeutil.UniqueID(201)
		indexName       = "testColl_index_110"
	)

	rand.Seed(time.Now().UnixNano())
	randVal := rand.Int()
	Params.Init()
	rootPath := fmt.Sprintf("/test/meta/%d", randVal)

	var vtso typeutil.Timestamp
	ftso := func() typeutil.Timestamp {
		vtso++
		return vtso
	}

	etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
	require.Nil(t, err)
	defer etcdCli.Close()

	skv, err := rootcoord.NewMetaSnapshot(etcdCli, rootPath, TimestampPrefix, 7)
	assert.Nil(t, err)
	assert.NotNil(t, skv)
	txnKV := etcdkv.NewEtcdKV(etcdCli, rootPath)
	mt, err := NewMetaTable(context.TODO(), &rootcoord.Catalog{Txn: txnKV, Snapshot: skv})
	assert.Nil(t, err)

	collInfo := &model.Collection{
		CollectionID: collID,
		Name:         collName,
		AutoID:       false,
		Fields: []*model.Field{
			{
				FieldID:      fieldID,
				Name:         "field110",
				IsPrimaryKey: false,
				Description:  "",
				DataType:     schemapb.DataType_FloatVector,
				TypeParams: []*commonpb.KeyValuePair{
					{
						Key:   "field110-k1",
						Value: "field110-v1",
					},
					{
						Key:   "field110-k2",
						Value: "field110-v2",
					},
				},
				IndexParams: []*commonpb.KeyValuePair{
					{
						Key:   "field110-i1",
						Value: "field110-v1",
					},
					{
						Key:   "field110-i2",
						Value: "field110-v2",
					},
				},
			},
		},
		FieldIDToIndexID: []common.Int64Tuple{
			{
				Key:   fieldID,
				Value: indexID,
			},
		},
		CreateTime: 0,
		Partitions: []*model.Partition{
			{
				PartitionID:               partIDDefault,
				PartitionName:             Params.CommonCfg.DefaultPartitionName,
				PartitionCreatedTimestamp: 0,
			},
		},
		VirtualChannelNames: []string{
			fmt.Sprintf("dmChannel_%dv%d", collID, 0),
			fmt.Sprintf("dmChannel_%dv%d", collID, 1),
		},
		PhysicalChannelNames: []string{
			funcutil.ToPhysicalChannel(fmt.Sprintf("dmChannel_%dv%d", collID, 0)),
			funcutil.ToPhysicalChannel(fmt.Sprintf("dmChannel_%dv%d", collID, 1)),
		},
	}

	idxInfo := []*model.Index{
		{
			IndexName: indexName,
			IndexID:   indexID,
			FieldID:   fieldID,
			IndexParams: []*commonpb.KeyValuePair{
				{
					Key:   "field110-i1",
					Value: "field110-v1",
				},
				{
					Key:   "field110-i2",
					Value: "field110-v2",
				},
			},
		},
	}

	var wg sync.WaitGroup
	wg.Add(1)
	t.Run("add collection", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()

		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)
		assert.Equal(t, uint64(1), ts)

		collMeta, err := mt.GetCollectionByName(collName, ts)
		assert.Nil(t, err)
		assert.Equal(t, collMeta.CreateTime, ts)
		assert.Equal(t, collMeta.Partitions[0].PartitionCreatedTimestamp, ts)

		assert.Equal(t, partIDDefault, collMeta.Partitions[0].PartitionID)
		assert.Equal(t, 1, len(collMeta.Partitions))
		assert.True(t, mt.HasCollection(collInfo.CollectionID, 0))

		field, err := mt.GetFieldSchema(collName, "field110")
		assert.Nil(t, err)
		assert.Equal(t, collInfo.Fields[0].FieldID, field.FieldID)
	})

	wg.Add(1)
	t.Run("add alias", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		exists := mt.IsAlias(aliasName1)
		assert.False(t, exists)
		err = mt.AddAlias(aliasName1, collName, ts)
		assert.Nil(t, err)
		aliases := mt.ListAliases(collID)
		assert.Equal(t, aliases, []string{aliasName1})
		exists = mt.IsAlias(aliasName1)
		assert.True(t, exists)
	})
	wg.Add(1)
	t.Run("alter alias", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		err = mt.AlterAlias(aliasName1, collName, ts)
		assert.Nil(t, err)
		err = mt.AlterAlias(aliasName1, collNameInvalid, ts)
		assert.NotNil(t, err)
	})

	wg.Add(1)
	t.Run("delete alias", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		err = mt.DropAlias(aliasName1, ts)
		assert.Nil(t, err)
	})

	wg.Add(1)
	t.Run("not load alias when load collection meta", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		err = mt.AddAlias(aliasName1, collName, ts)
		assert.Nil(t, err)
		err = mt.reloadFromCatalog()
		assert.Nil(t, err)
		_, ok := mt.collName2ID[aliasName1]
		assert.False(t, ok)
	})

	wg.Add(1)
	t.Run("add partition", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		err = mt.AddPartition(collID, partName, partID, ts, "")
		assert.Nil(t, err)
		//assert.Equal(t, ts, uint64(2))

		collMeta, ok := mt.collID2Meta[collID]
		assert.True(t, ok)
		assert.Equal(t, 2, len(collMeta.Partitions))
		assert.Equal(t, collMeta.Partitions[1].PartitionName, partName)
		assert.Equal(t, ts, collMeta.Partitions[1].PartitionCreatedTimestamp)
	})

	wg.Add(1)
	t.Run("add segment index", func(t *testing.T) {
		defer wg.Done()
		alreadyExists, err := mt.AddIndex(collName, "field110", idxInfo[0], []typeutil.UniqueID{segID})
		assert.Nil(t, err)
		assert.False(t, alreadyExists)

		index := model.Index{
			CollectionID: collID,
			FieldID:      fieldID,
			IndexID:      indexID,
			SegmentIndexes: map[int64]model.SegmentIndex{
				segID: {
					Segment: model.Segment{
						SegmentID:   segID,
						PartitionID: partID,
					},
					BuildID:     buildID,
					EnableIndex: true,
				},
			},
		}
		err = mt.AlterIndex(&index)
		assert.Nil(t, err)

		// it's legal to add index twice
		err = mt.AlterIndex(&index)
		assert.Nil(t, err)

		idxID, ok := mt.segID2IndexID[segID]
		assert.True(t, ok)
		indexMeta, ok := mt.indexID2Meta[idxID]
		assert.True(t, ok)
		segIdx, ok := indexMeta.SegmentIndexes[segID]
		assert.True(t, ok)
		assert.True(t, segIdx.EnableIndex)
	})

	wg.Add(1)
	t.Run("add diff index with same index name", func(t *testing.T) {
		defer wg.Done()
		params := []*commonpb.KeyValuePair{
			{
				Key:   "field110-k1",
				Value: "field110-v1",
			},
			{
				Key:   "field110-k2",
				Value: "field110-v2",
			},
		}
		idxInfo := &model.Index{
			IndexName:   "testColl_index_110",
			IndexID:     indexID,
			IndexParams: params,
		}

		alreadyExists, err := mt.AddIndex("collTest", "field110", idxInfo, []typeutil.UniqueID{segID})
		assert.NotNil(t, err)
		assert.False(t, alreadyExists)
	})

	wg.Add(1)
	t.Run("get not indexed segments", func(t *testing.T) {
		defer wg.Done()
		params := []*commonpb.KeyValuePair{
			{
				Key:   "field110-i1",
				Value: "field110-v1",
			},
			{
				Key:   "field110-i2",
				Value: "field110-v2",
			},
		}

		tparams := []*commonpb.KeyValuePair{
			{
				Key:   "field110-k1",
				Value: "field110-v1",
			},
			{
				Key:   "field110-k2",
				Value: "field110-v2",
			},
		}
		idxInfo := &model.Index{
			IndexName:   "field110",
			IndexID:     2000,
			IndexParams: params,
		}

		_, _, err := mt.GetNotIndexedSegments("collTest", "field110", idxInfo, nil)
		assert.NotNil(t, err)
		seg, field, err := mt.GetNotIndexedSegments(collName, "field110", idxInfo, []typeutil.UniqueID{segID, segID2})
		assert.Nil(t, err)
		assert.Equal(t, 1, len(seg))
		assert.Equal(t, segID2, seg[0])
		assert.True(t, EqualKeyPairArray(field.TypeParams, tparams))

		params = []*commonpb.KeyValuePair{
			{
				Key:   "field110-i1",
				Value: "field110-v1",
			},
		}
		idxInfo.IndexParams = params
		idxInfo.IndexID = 2001
		idxInfo.IndexName = "field110-1"

		seg, field, err = mt.GetNotIndexedSegments(collName, "field110", idxInfo, []typeutil.UniqueID{segID, segID2})
		assert.Nil(t, err)
		assert.Equal(t, 2, len(seg))
		assert.Equal(t, segID, seg[0])
		assert.Equal(t, segID2, seg[1])
		assert.True(t, EqualKeyPairArray(field.TypeParams, tparams))
	})

	wg.Add(1)
	t.Run("get index by name", func(t *testing.T) {
		defer wg.Done()
		_, idx, err := mt.GetIndexByName(collName, indexName)
		assert.Nil(t, err)
		assert.Equal(t, 2, len(idx))
		assert.Equal(t, idxInfo[0].IndexID, idx[0].IndexID)
		params := []*commonpb.KeyValuePair{
			{
				Key:   "field110-i1",
				Value: "field110-v1",
			},
			{
				Key:   "field110-i2",
				Value: "field110-v2",
			},
		}
		assert.True(t, EqualKeyPairArray(idx[0].IndexParams, params))

		_, idx, err = mt.GetIndexByName(collName, "idx201")
		assert.Nil(t, err)
		assert.Zero(t, len(idx))
	})

	wg.Add(1)
	t.Run("drop partition", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		id, err := mt.DeletePartition(collID, partName, ts, "")
		assert.Nil(t, err)
		assert.Equal(t, partID, id)
	})

	wg.Add(1)
	t.Run("drop collection", func(t *testing.T) {
		defer wg.Done()
		ts := ftso()
		err = mt.DeleteCollection(collIDInvalid, ts, "")
		assert.NotNil(t, err)
		ts2 := ftso()
		err = mt.AddAlias(aliasName2, collName, ts2)
		assert.Nil(t, err)
		err = mt.DeleteCollection(collID, ts, "")
		assert.Nil(t, err)
		ts3 := ftso()
		err = mt.DropAlias(aliasName2, ts3)
		assert.NotNil(t, err)
	})

	wg.Add(1)
	t.Run("delete credential", func(t *testing.T) {
		defer wg.Done()

		err = mt.DeleteCredential("")
		assert.Nil(t, err)

		err = mt.DeleteCredential("abcxyz")
		assert.Nil(t, err)
	})

	/////////////////////////// these tests should run at last, it only used to hit the error lines ////////////////////////
	txnkv := etcdkv.NewEtcdKV(etcdCli, rootPath)
	mockKV := &mockTestKV{
		loadWithPrefix: func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		},
	}
	mockTxnKV := &mockTestTxnKV{
		TxnKV:          txnkv,
		loadWithPrefix: func(key string) ([]string, []string, error) { return txnkv.LoadWithPrefix(key) },
		save:           func(key, value string) error { return txnkv.Save(key, value) },
		multiSave:      func(kvs map[string]string) error { return txnkv.MultiSave(kvs) },
		multiSaveAndRemoveWithPrefix: func(kvs map[string]string, removal []string) error {
			return txnkv.MultiSaveAndRemoveWithPrefix(kvs, removal)
		},
		remove: func(key string) error { return txnkv.Remove(key) },
	}

	mt, err = NewMetaTable(context.TODO(), &rootcoord.Catalog{Txn: mockTxnKV, Snapshot: mockKV})
	assert.Nil(t, err)

	wg.Add(1)
	t.Run("add collection failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return fmt.Errorf("multi save error")
		}
		collInfo.Partitions = []*model.Partition{}
		assert.Error(t, mt.AddCollection(collInfo, 0, ""))
	})

	wg.Add(1)
	t.Run("delete collection failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		mockKV.multiSaveAndRemoveWithPrefix = func(save map[string]string, keys []string, ts typeutil.Timestamp) error {
			return fmt.Errorf("multi save and remove with prefix error")
		}
		ts := ftso()
		assert.Error(t, mt.DeleteCollection(collInfo.CollectionID, ts, ""))
	})

	wg.Add(1)
	t.Run("get collection failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.save = func(key string, value string, ts typeutil.Timestamp) error {
			return nil
		}

		ts := ftso()
		collInfo.Partitions = []*model.Partition{}
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		mt.collID2Meta = make(map[int64]model.Collection)
		_, err = mt.GetCollectionByName(collInfo.Name, 0)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("can't find collection %s with id %d", collInfo.Name, collInfo.CollectionID))

	})

	wg.Add(1)
	t.Run("add partition failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.save = func(key string, value string, ts typeutil.Timestamp) error {
			return nil
		}
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		ts := ftso()
		collInfo.Partitions = []*model.Partition{}
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		ts = ftso()
		err = mt.AddPartition(2, "no-part", 22, ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, "can't find collection. id = 2")

		coll := mt.collID2Meta[collInfo.CollectionID]
		coll.Partitions = make([]*model.Partition, Params.RootCoordCfg.MaxPartitionNum)
		mt.collID2Meta[coll.CollectionID] = coll
		err = mt.AddPartition(coll.CollectionID, "no-part", 22, ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("maximum partition's number should be limit to %d", Params.RootCoordCfg.MaxPartitionNum))

		coll.Partitions = []*model.Partition{{PartitionID: partID, PartitionName: partName, PartitionCreatedTimestamp: ftso()}}
		mt.collID2Meta[coll.CollectionID] = coll

		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return fmt.Errorf("multi save error")
		}
		tmpSaveFunc := mockKV.save
		mockKV.save = func(key, value string, ts typeutil.Timestamp) error {
			return errors.New("mock")
		}
		assert.Error(t, mt.AddPartition(coll.CollectionID, "no-part", 22, ts, ""))
		mockKV.save = tmpSaveFunc
		//err = mt.AddPartition(coll.CollectionID, "no-part", 22, ts, nil)
		//assert.NotNil(t, err)
		//assert.EqualError(t, err, "multi save error")

		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}

		collInfo.Partitions = []*model.Partition{}
		ts = ftso()
		err = mt.AddPartition(coll.CollectionID, partName, 22, ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("partition name = %s already exists", partName))
		err = mt.AddPartition(coll.CollectionID, "no-part", partID, ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("partition id = %d already exists", partID))
	})

	wg.Add(1)
	t.Run("has partition failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		assert.False(t, mt.HasPartition(collInfo.CollectionID, "no-partName", 0))

		mt.collID2Meta = make(map[int64]model.Collection)
		assert.False(t, mt.HasPartition(collInfo.CollectionID, partName, 0))
	})

	wg.Add(1)
	t.Run("delete partition failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{{PartitionID: partID, PartitionName: partName, PartitionCreatedTimestamp: ftso()}}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		ts = ftso()
		_, err = mt.DeletePartition(collInfo.CollectionID, Params.CommonCfg.DefaultPartitionName, ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, "default partition cannot be deleted")

		_, err = mt.DeletePartition(collInfo.CollectionID, "abc", ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, "partition abc does not exist")

		mockKV.save = func(key, value string, ts typeutil.Timestamp) error { return errors.New("mocked error") }
		_, err = mt.DeletePartition(collInfo.CollectionID, partName, ts, "")
		assert.Error(t, err)

		mt.collID2Meta = make(map[int64]model.Collection)
		_, err = mt.DeletePartition(collInfo.CollectionID, "abc", ts, "")
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("can't find collection id = %d", collInfo.CollectionID))
	})

	wg.Add(1)
	t.Run("add index failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		mockKV.save = func(key, value string, ts typeutil.Timestamp) error {
			return nil
		}
		err = mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		index := &model.Index{
			CollectionID: collID,
			FieldID:      fieldID,
			IndexID:      indexID,
			SegmentIndexes: map[int64]model.SegmentIndex{
				segID: {
					Segment: model.Segment{
						SegmentID:   segID,
						PartitionID: partID,
					},
					BuildID:     buildID,
					CreateTime:  10,
					EnableIndex: false,
				},
			},
		}
		err = mt.AlterIndex(index)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("index id = %d not found", index.IndexID))

		mt.collID2Meta = make(map[int64]model.Collection)
		err = mt.AlterIndex(index)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("collection id = %d not found", collInfo.CollectionID))

		err = mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts = ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		newIdx := *index
		newIdx.SegmentIndexes = map[int64]model.SegmentIndex{
			segID: {
				Segment: model.Segment{
					SegmentID:   segID,
					PartitionID: partID,
				},
				BuildID:     buildID,
				CreateTime:  10,
				EnableIndex: true,
			},
		}

		mt.indexID2Meta[indexID] = index
		mockTxnKV.multiSave = func(kvs map[string]string) error {
			return fmt.Errorf("save error")
		}
		assert.Error(t, mt.AlterIndex(&newIdx))
	})

	wg.Add(1)
	t.Run("drop index failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		mockKV.save = func(key string, value string, ts typeutil.Timestamp) error {
			return nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)
		mt.indexID2Meta[indexID] = idxInfo[0]

		_, _, err = mt.DropIndex("abc", "abc", "abc")
		assert.NotNil(t, err)
		assert.EqualError(t, err, "collection name = abc not exist")

		mt.collName2ID["abc"] = 2
		_, _, err = mt.DropIndex("abc", "abc", "abc")
		assert.NotNil(t, err)
		assert.EqualError(t, err, "collection name  = abc not has meta")

		_, _, err = mt.DropIndex(collInfo.Name, "abc", "abc")
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("collection %s doesn't have filed abc", collInfo.Name))

		coll := mt.collID2Meta[collInfo.CollectionID]
		coll.FieldIDToIndexID = []common.Int64Tuple{
			{
				Key:   fieldID2,
				Value: indexID2,
			},
			{
				Key:   fieldID,
				Value: indexID,
			},
		}
		mt.collID2Meta[coll.CollectionID] = coll
		mt.indexID2Meta = make(map[int64]*model.Index)
		idxID, isDroped, err := mt.DropIndex(collInfo.Name, collInfo.Fields[0].Name, idxInfo[0].IndexName)
		assert.Zero(t, idxID)
		assert.False(t, isDroped)
		assert.Nil(t, err)

		err = mt.reloadFromCatalog()
		assert.Nil(t, err)
		collInfo.Partitions = []*model.Partition{}
		ts = ftso()
		err = mt.AddCollection(collInfo, ts, "")
		mt.indexID2Meta[indexID] = idxInfo[0]

		assert.Nil(t, err)
		mockTxnKV.multiSaveAndRemoveWithPrefix = func(saves map[string]string, removals []string) error {
			return fmt.Errorf("multi save and remove with prefix error")
		}

		_, _, err = mt.DropIndex(collInfo.Name, collInfo.Fields[0].Name, idxInfo[0].IndexName)
		assert.Error(t, err)
	})

	wg.Add(1)
	t.Run("get segment index info by id", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockTxnKV.multiSave = func(kvs map[string]string) error {
			return nil
		}
		mockTxnKV.save = func(key, value string) error {
			return nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		alreadyExists, err := mt.AddIndex(collName, "field110", idxInfo[0], []typeutil.UniqueID{segID})
		assert.Nil(t, err)
		assert.False(t, alreadyExists)

		segIdxInfo := model.Index{
			CollectionID: collID,
			SegmentIndexes: map[int64]model.SegmentIndex{
				segID: {
					Segment: model.Segment{
						SegmentID:   segID,
						PartitionID: partID,
					},
					BuildID: buildID,
				},
			},
			FieldID: fieldID,
			IndexID: indexID,
		}
		err = mt.AlterIndex(&segIdxInfo)
		assert.Nil(t, err)
		idx, err := mt.GetSegmentIndexInfoByID(segID, segIdxInfo.FieldID, idxInfo[0].IndexName)
		assert.Nil(t, err)
		assert.Equal(t, segIdxInfo.IndexID, idx.IndexID)

		_, err = mt.GetSegmentIndexInfoByID(segID, segIdxInfo.FieldID, "abc")
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("can't find index name = abc on segment = %d, with filed id = %d", segID, segIdxInfo.FieldID))

		_, err = mt.GetSegmentIndexInfoByID(segID, 11, idxInfo[0].IndexName)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("can't find index name = %s on segment = %d, with filed id = 11", idxInfo[0].IndexName, segID))
	})

	wg.Add(1)
	t.Run("get field schema failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		mockKV.save = func(key string, value string, ts typeutil.Timestamp) error {
			return nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		mt.collID2Meta = make(map[int64]model.Collection)
		_, err = mt.getFieldSchemaInternal(collInfo.Name, collInfo.Fields[0].Name)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("collection %s not found", collInfo.Name))

		mt.collName2ID = make(map[string]int64)
		_, err = mt.getFieldSchemaInternal(collInfo.Name, collInfo.Fields[0].Name)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("collection %s not found", collInfo.Name))
	})

	wg.Add(1)
	t.Run("is segment indexed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)
		idx := &pb.SegmentIndexInfo{
			IndexID:   30,
			FieldID:   31,
			SegmentID: 32,
		}
		idxMeta := make(map[int64]pb.SegmentIndexInfo)
		idxMeta[idx.IndexID] = *idx

		field := model.Field{
			FieldID: 31,
		}
		assert.False(t, mt.IsSegmentIndexed(idx.SegmentID, &field, nil))
		field.FieldID = 34
		assert.False(t, mt.IsSegmentIndexed(idx.SegmentID, &field, nil))
	})

	wg.Add(1)
	t.Run("get not indexed segments", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		idx := &model.Index{
			IndexName: "no-idx",
			IndexID:   456,
			IndexParams: []*commonpb.KeyValuePair{
				{
					Key:   "no-idx-k1",
					Value: "no-idx-v1",
				},
			},
		}

		mt.collName2ID["abc"] = 123
		_, _, err = mt.GetNotIndexedSegments("abc", "no-field", idx, nil)
		assert.NotNil(t, err)
		assert.EqualError(t, err, "collection abc not found")

		mockKV.multiSave = func(kvs map[string]string, ts typeutil.Timestamp) error {
			return nil
		}
		mockKV.save = func(key string, value string, ts typeutil.Timestamp) error {
			return nil
		}
		err = mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)

		_, _, err = mt.GetNotIndexedSegments(collInfo.Name, "no-field", idx, nil)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("collection %s doesn't have filed no-field", collInfo.Name))

		mt.indexID2Meta = make(map[int64]*model.Index)
		_, _, err = mt.GetNotIndexedSegments(collInfo.Name, collInfo.Fields[0].Name, idx, nil)
		assert.Nil(t, err)
	})

	wg.Add(1)
	t.Run("get index by name failed", func(t *testing.T) {
		defer wg.Done()
		mockKV.loadWithPrefix = func(key string, ts typeutil.Timestamp) ([]string, []string, error) {
			return nil, nil, nil
		}
		err := mt.reloadFromCatalog()
		assert.Nil(t, err)

		mt.collName2ID["abc"] = 123
		_, _, err = mt.GetIndexByName("abc", "hij")
		assert.NotNil(t, err)
		assert.EqualError(t, err, "collection abc not found")

		mockTxnKV.multiSave = func(kvs map[string]string) error {
			return nil
		}
		mockTxnKV.save = func(key string, value string) error {
			return nil
		}
		err = mt.reloadFromCatalog()
		assert.Nil(t, err)

		collInfo.Partitions = []*model.Partition{}
		ts := ftso()
		err = mt.AddCollection(collInfo, ts, "")
		assert.Nil(t, err)
		mt.indexID2Meta = make(map[int64]*model.Index)
		_, _, err = mt.GetIndexByName(collInfo.Name, idxInfo[0].IndexName)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("index id = %d not found", idxInfo[0].IndexID))

		_, err = mt.GetIndexByID(idxInfo[0].IndexID)
		assert.NotNil(t, err)
		assert.EqualError(t, err, fmt.Sprintf("cannot find index, id = %d", idxInfo[0].IndexID))
	})

	wg.Add(1)
	t.Run("add credential failed", func(t *testing.T) {
		defer wg.Done()
		mockTxnKV.save = func(key, value string) error {
			return fmt.Errorf("save error")
		}
		err = mt.AddCredential(&internalpb.CredentialInfo{Username: "x", EncryptedPassword: "a\xc5z"})
		assert.NotNil(t, err)
	})

	wg.Add(1)
	t.Run("delete credential failed", func(t *testing.T) {
		defer wg.Done()
		mockTxnKV.remove = func(key string) error {
			return fmt.Errorf("delete error")
		}
		err := mt.DeleteCredential("")
		assert.Error(t, err)
	})

	wg.Wait()
}

func TestRbacCreateRole(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error
	err = mt.CreateRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: ""})
	assert.NotNil(t, err)

	mockTxnKV.save = func(key, value string) error {
		return nil
	}
	err = mt.CreateRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: "role1"})
	assert.Nil(t, err)

	mockTxnKV.save = func(key, value string) error {
		return fmt.Errorf("save error")
	}
	err = mt.CreateRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: "role2"})
	assert.NotNil(t, err)
}

func TestRbacDropRole(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error

	mockTxnKV.remove = func(key string) error {
		return nil
	}

	err = mt.DropRole(util.DefaultTenant, "role1")
	assert.Nil(t, err)

	mockTxnKV.remove = func(key string) error {
		return fmt.Errorf("delete error")
	}

	err = mt.DropRole(util.DefaultTenant, "role2")
	assert.NotNil(t, err)
}

func TestRbacOperateRole(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error

	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: " "}, &milvuspb.RoleEntity{Name: "role"}, milvuspb.OperateUserRoleType_AddUserToRole)
	assert.NotNil(t, err)

	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, &milvuspb.RoleEntity{Name: "  "}, milvuspb.OperateUserRoleType_AddUserToRole)
	assert.NotNil(t, err)

	mockTxnKV.save = func(key, value string) error {
		return nil
	}
	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, &milvuspb.RoleEntity{Name: "role"}, milvuspb.OperateUserRoleType(100))
	assert.NotNil(t, err)

	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, &milvuspb.RoleEntity{Name: "role"}, milvuspb.OperateUserRoleType_AddUserToRole)
	assert.Nil(t, err)

	mockTxnKV.save = func(key, value string) error {
		return fmt.Errorf("save error")
	}
	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, &milvuspb.RoleEntity{Name: "role"}, milvuspb.OperateUserRoleType_AddUserToRole)
	assert.NotNil(t, err)

	mockTxnKV.remove = func(key string) error {
		return nil
	}
	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, &milvuspb.RoleEntity{Name: "role"}, milvuspb.OperateUserRoleType_RemoveUserFromRole)
	assert.Nil(t, err)

	mockTxnKV.remove = func(key string) error {
		return fmt.Errorf("remove error")
	}
	err = mt.OperateUserRole(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, &milvuspb.RoleEntity{Name: "role"}, milvuspb.OperateUserRoleType_RemoveUserFromRole)
	assert.NotNil(t, err)
}

func TestRbacSelectRole(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error

	_, err = mt.SelectRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: ""}, false)
	assert.NotNil(t, err)

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{}, []string{}, fmt.Errorf("load with prefix error")
	}
	_, err = mt.SelectRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: "role"}, true)
	assert.NotNil(t, err)

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/key1", key + "/key2", key + "/a/err"}, []string{"value1", "value2", "values3"}, nil
	}
	mockTxnKV.load = func(key string) (string, error) {
		return "", nil
	}
	results, _ := mt.SelectRole(util.DefaultTenant, nil, false)
	assert.Equal(t, 2, len(results))
	results, _ = mt.SelectRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: "role"}, false)
	assert.Equal(t, 1, len(results))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{}, []string{}, fmt.Errorf("load with prefix error")
	}
	_, err = mt.SelectRole(util.DefaultTenant, nil, false)
	assert.NotNil(t, err)

	mockTxnKV.load = func(key string) (string, error) {
		return "", fmt.Errorf("load error")
	}
	_, err = mt.SelectRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: "role"}, false)
	assert.NotNil(t, err)

	roleName := "role1"
	mockTxnKV.load = func(key string) (string, error) {
		return "", nil
	}
	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/user1/" + roleName, key + "/user2/role2", key + "/user3/" + roleName, key + "/err"}, []string{"value1", "value2", "values3", "value4"}, nil
	}
	results, err = mt.SelectRole(util.DefaultTenant, &milvuspb.RoleEntity{Name: roleName}, true)
	assert.Nil(t, err)
	assert.Equal(t, 1, len(results))
	assert.Equal(t, 2, len(results[0].Users))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		if key == rootcoord.RoleMappingPrefix {
			return []string{key + "/user1/role2", key + "/user2/role2", key + "/user1/role1", key + "/user2/role1"}, []string{"value1", "value2", "values3", "value4"}, nil
		} else if key == rootcoord.RolePrefix {
			return []string{key + "/role1", key + "/role2", key + "/role3"}, []string{"value1", "value2", "values3"}, nil
		} else {
			return []string{}, []string{}, fmt.Errorf("load with prefix error")
		}
	}
	results, err = mt.SelectRole(util.DefaultTenant, nil, true)
	assert.Nil(t, err)
	assert.Equal(t, 3, len(results))
	for _, result := range results {
		if result.Role.Name == "role1" {
			assert.Equal(t, 2, len(result.Users))
		} else if result.Role.Name == "role2" {
			assert.Equal(t, 2, len(result.Users))
		}
	}
}

func TestRbacSelectUser(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error

	_, err = mt.SelectUser(util.DefaultTenant, &milvuspb.UserEntity{Name: ""}, false)
	assert.NotNil(t, err)

	credentialInfo := internalpb.CredentialInfo{
		EncryptedPassword: "password",
	}
	credentialInfoByte, _ := json.Marshal(credentialInfo)

	mockTxnKV.load = func(key string) (string, error) {
		return string(credentialInfoByte), nil
	}
	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/key1", key + "/key2"}, []string{string(credentialInfoByte), string(credentialInfoByte)}, nil
	}
	results, err := mt.SelectUser(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, false)
	assert.Nil(t, err)
	assert.Equal(t, 1, len(results))
	results, err = mt.SelectUser(util.DefaultTenant, nil, false)
	assert.Nil(t, err)
	assert.Equal(t, 2, len(results))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/key1", key + "/key2", key + "/a/err"}, []string{"value1", "value2", "values3"}, nil
	}

	mockTxnKV.load = func(key string) (string, error) {
		return string(credentialInfoByte), nil
	}
	results, err = mt.SelectUser(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, true)
	assert.Nil(t, err)
	assert.Equal(t, 1, len(results))
	assert.Equal(t, 2, len(results[0].Roles))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		logger.Debug("simfg", zap.String("key", key))
		if strings.Contains(key, rootcoord.RoleMappingPrefix) {
			if strings.Contains(key, "user1") {
				return []string{key + "/role2", key + "/role1", key + "/role3"}, []string{"value1", "value4", "value2"}, nil
			} else if strings.Contains(key, "user2") {
				return []string{key + "/role2"}, []string{"value1"}, nil
			}
			return []string{}, []string{}, nil
		} else if key == rootcoord.CredentialPrefix {
			return []string{key + "/user1", key + "/user2", key + "/user3"}, []string{string(credentialInfoByte), string(credentialInfoByte), string(credentialInfoByte)}, nil
		} else {
			return []string{}, []string{}, fmt.Errorf("load with prefix error")
		}
	}
	results, err = mt.SelectUser(util.DefaultTenant, nil, true)
	assert.Nil(t, err)
	assert.Equal(t, 3, len(results))
	for _, result := range results {
		if result.User.Name == "user1" {
			assert.Equal(t, 3, len(result.Roles))
		} else if result.User.Name == "user2" {
			assert.Equal(t, 1, len(result.Roles))
		}
	}

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{}, []string{}, nil
	}
	_, err = mt.SelectUser(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, true)
	assert.Nil(t, err)

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{}, []string{}, fmt.Errorf("load with prefix error")
	}
	_, err = mt.SelectUser(util.DefaultTenant, &milvuspb.UserEntity{Name: "user"}, true)
	assert.NotNil(t, err)

	_, err = mt.SelectUser(util.DefaultTenant, nil, true)
	assert.NotNil(t, err)
}

func TestRbacOperatePrivilege(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error

	entity := &milvuspb.GrantEntity{}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	entity.ObjectName = "col1"
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	entity.Object = &milvuspb.ObjectEntity{Name: commonpb.ObjectType_Collection.String()}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	entity.Role = &milvuspb.RoleEntity{Name: "admin"}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	entity.Grantor = &milvuspb.GrantorEntity{}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	entity.Grantor.Privilege = &milvuspb.PrivilegeEntity{Name: commonpb.ObjectPrivilege_PrivilegeLoad.String()}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	err = mt.OperatePrivilege(util.DefaultTenant, entity, 100)
	assert.NotNil(t, err)

	mockTxnKV.save = func(key, value string) error {
		return nil
	}
	mockTxnKV.load = func(key string) (string, error) {
		return "fail", fmt.Errorf("load error")
	}
	entity.Grantor.User = &milvuspb.UserEntity{Name: "user2"}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Revoke)
	assert.NotNil(t, err)
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)
	mockTxnKV.load = func(key string) (string, error) {
		return "unmarshal", nil
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)

	mockTxnKV.load = func(key string) (string, error) {
		return "fail", common.NewKeyNotExistError(key)
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.Nil(t, err)

	grantPrivilegeEntity := &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "aaa"}, Privilege: &milvuspb.PrivilegeEntity{Name: commonpb.ObjectPrivilege_PrivilegeLoad.String()}},
	}}
	mockTxnKV.load = func(key string) (string, error) {
		grantPrivilegeEntityByte, _ := proto.Marshal(grantPrivilegeEntity)
		return string(grantPrivilegeEntityByte), nil
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.Nil(t, err)
	entity.Grantor.Privilege = &milvuspb.PrivilegeEntity{Name: commonpb.ObjectPrivilege_PrivilegeRelease.String()}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Revoke)
	assert.NotNil(t, err)
	entity.Grantor.Privilege = &milvuspb.PrivilegeEntity{Name: commonpb.ObjectPrivilege_PrivilegeLoad.String()}
	grantPrivilegeEntity = &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "user2"}, Privilege: &milvuspb.PrivilegeEntity{Name: commonpb.ObjectPrivilege_PrivilegeLoad.String()}},
	}}
	mockTxnKV.load = func(key string) (string, error) {
		grantPrivilegeEntityByte, _ := proto.Marshal(grantPrivilegeEntity)
		return string(grantPrivilegeEntityByte), nil
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.Nil(t, err)
	mockTxnKV.remove = func(key string) error {
		return fmt.Errorf("remove error")
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Revoke)
	assert.NotNil(t, err)
	mockTxnKV.remove = func(key string) error {
		return nil
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Revoke)
	assert.Nil(t, err)

	grantPrivilegeEntity = &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "u3"}, Privilege: &milvuspb.PrivilegeEntity{Name: commonpb.ObjectPrivilege_PrivilegeInsert.String()}},
	}}
	mockTxnKV.load = func(key string) (string, error) {
		grantPrivilegeEntityByte, _ := proto.Marshal(grantPrivilegeEntity)
		return string(grantPrivilegeEntityByte), nil
	}
	mockTxnKV.save = func(key, value string) error {
		return fmt.Errorf("save error")
	}
	err = mt.OperatePrivilege(util.DefaultTenant, entity, milvuspb.OperatePrivilegeType_Grant)
	assert.NotNil(t, err)
}

func TestRbacSelectGrant(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	var err error

	entity := &milvuspb.GrantEntity{}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)

	//entity.Role = &milvuspb.RoleEntity{Name: "admin"}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)

	mockTxnKV.load = func(key string) (string, error) {
		return "unmarshal error", nil
	}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)

	entity.Role = &milvuspb.RoleEntity{Name: "role1"}
	entity.ObjectName = "col1"
	entity.Object = &milvuspb.ObjectEntity{Name: "Collection"}

	grantPrivilegeEntity := &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "aaa"}, Privilege: &milvuspb.PrivilegeEntity{Name: "111"}},
		{User: &milvuspb.UserEntity{Name: "bbb"}, Privilege: &milvuspb.PrivilegeEntity{Name: "222"}},
		{User: &milvuspb.UserEntity{Name: "ccc"}, Privilege: &milvuspb.PrivilegeEntity{Name: "*"}},
	}}
	grantPrivilegeEntityByte, _ := proto.Marshal(grantPrivilegeEntity)

	mockTxnKV.load = func(key string) (string, error) {
		return "unmarshal error", nil
	}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)

	mockTxnKV.load = func(key string) (string, error) {
		return string(grantPrivilegeEntityByte), nil
	}
	results, err := mt.SelectGrant(util.DefaultTenant, entity)
	assert.Nil(t, err)
	assert.Equal(t, 3, len(results))

	mockTxnKV.load = func(key string) (string, error) {
		return "", fmt.Errorf("load error")
	}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)

	grantPrivilegeEntity2 := &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "ccc"}, Privilege: &milvuspb.PrivilegeEntity{Name: "333"}},
	}}
	grantPrivilegeEntityByte2, _ := proto.Marshal(grantPrivilegeEntity2)

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/collection/col1", key + "/collection/col2", key + "/collection/a/col3"}, []string{string(grantPrivilegeEntityByte), string(grantPrivilegeEntityByte2), string(grantPrivilegeEntityByte2)}, nil
	}
	entity.ObjectName = ""
	entity.Object = nil
	results, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.Nil(t, err)
	assert.Equal(t, 4, len(results))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return nil, nil, fmt.Errorf("load with prefix error")
	}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/collection/col1"}, []string{"unmarshal error"}, nil
	}
	_, err = mt.SelectGrant(util.DefaultTenant, entity)
	assert.NotNil(t, err)
}

func TestRbacListPolicy(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{}, []string{}, fmt.Errorf("load with prefix err")
	}
	policies, err := mt.ListPolicy(util.DefaultTenant)
	assert.NotNil(t, err)
	assert.Equal(t, 0, len(policies))

	grantPrivilegeEntity := &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "aaa"}, Privilege: &milvuspb.PrivilegeEntity{Name: "111"}},
		{User: &milvuspb.UserEntity{Name: "bbb"}, Privilege: &milvuspb.PrivilegeEntity{Name: "222"}},
	}}
	grantPrivilegeEntityByte, _ := proto.Marshal(grantPrivilegeEntity)
	grantPrivilegeEntity2 := &milvuspb.GrantPrivilegeEntity{Entities: []*milvuspb.GrantorEntity{
		{User: &milvuspb.UserEntity{Name: "ccc"}, Privilege: &milvuspb.PrivilegeEntity{Name: "333"}},
	}}
	grantPrivilegeEntityByte2, _ := proto.Marshal(grantPrivilegeEntity2)
	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/alice/collection/col1", key + "/tom/collection/col2", key + "/tom/collection/a/col2"}, []string{string(grantPrivilegeEntityByte), string(grantPrivilegeEntityByte2), string(grantPrivilegeEntityByte2)}, nil
	}
	policies, err = mt.ListPolicy(util.DefaultTenant)
	assert.Nil(t, err)
	assert.Equal(t, 3, len(policies))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/alice/collection/col1"}, []string{"unmarshal error"}, nil
	}
	_, err = mt.ListPolicy(util.DefaultTenant)
	assert.Nil(t, err)
}

func TestRbacListUserRole(t *testing.T) {
	mt, _, mockTxnKV, closeCli := generateMetaTable(t)
	defer closeCli()
	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{}, []string{}, fmt.Errorf("load with prefix err")
	}
	userRoles, err := mt.ListUserRole(util.DefaultTenant)
	assert.NotNil(t, err)
	assert.Equal(t, 0, len(userRoles))

	mockTxnKV.loadWithPrefix = func(key string) ([]string, []string, error) {
		return []string{key + "/user1/role2", key + "/user2/role2", key + "/user1/role1", key + "/user2/role1", key + "/user2/role1/a"}, []string{"value1", "value2", "values3", "value4", "value5"}, nil
	}
	userRoles, err = mt.ListUserRole(util.DefaultTenant)
	assert.Nil(t, err)
	assert.Equal(t, 4, len(userRoles))
}

func TestMetaWithTimestamp(t *testing.T) {
	const (
		collID1   = typeutil.UniqueID(1)
		collID2   = typeutil.UniqueID(2)
		collName1 = "t1"
		collName2 = "t2"
		partID1   = 11
		partID2   = 12
		partName1 = "p1"
		partName2 = "p2"
	)
	rand.Seed(time.Now().UnixNano())
	randVal := rand.Int()
	Params.Init()
	rootPath := fmt.Sprintf("/test/meta/%d", randVal)

	var tsoStart typeutil.Timestamp = 100
	vtso := tsoStart
	ftso := func() typeutil.Timestamp {
		vtso++
		return vtso
	}
	etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
	assert.Nil(t, err)
	defer etcdCli.Close()

	skv, err := rootcoord.NewMetaSnapshot(etcdCli, rootPath, TimestampPrefix, 7)
	assert.Nil(t, err)
	assert.NotNil(t, skv)
	txnKV := etcdkv.NewEtcdKV(etcdCli, rootPath)
	mt, err := NewMetaTable(context.TODO(), &rootcoord.Catalog{Txn: txnKV, Snapshot: skv})
	assert.Nil(t, err)

	collInfo := &model.Collection{
		CollectionID: collID1,
		Name:         collName1,
	}

	collInfo.Partitions = []*model.Partition{{PartitionID: partID1, PartitionName: partName1, PartitionCreatedTimestamp: ftso()}}
	t1 := ftso()
	err = mt.AddCollection(collInfo, t1, "")
	assert.Nil(t, err)

	collInfo.CollectionID = collID2
	collInfo.Partitions = []*model.Partition{{PartitionID: partID2, PartitionName: partName2, PartitionCreatedTimestamp: ftso()}}
	collInfo.Name = collName2
	t2 := ftso()
	err = mt.AddCollection(collInfo, t2, "")
	assert.Nil(t, err)

	assert.True(t, mt.HasCollection(collID1, 0))
	assert.True(t, mt.HasCollection(collID2, 0))

	assert.True(t, mt.HasCollection(collID1, t2))
	assert.True(t, mt.HasCollection(collID2, t2))

	assert.True(t, mt.HasCollection(collID1, t1))
	assert.False(t, mt.HasCollection(collID2, t1))

	assert.False(t, mt.HasCollection(collID1, tsoStart))
	assert.False(t, mt.HasCollection(collID2, tsoStart))

	c1, err := mt.GetCollectionByID(collID1, 0)
	assert.Nil(t, err)
	c2, err := mt.GetCollectionByID(collID2, 0)
	assert.Nil(t, err)
	assert.Equal(t, collID1, c1.CollectionID)
	assert.Equal(t, collID2, c2.CollectionID)

	c1, err = mt.GetCollectionByID(collID1, t2)
	assert.Nil(t, err)
	c2, err = mt.GetCollectionByID(collID2, t2)
	assert.Nil(t, err)
	assert.Equal(t, collID1, c1.CollectionID)
	assert.Equal(t, collID2, c2.CollectionID)

	c1, err = mt.GetCollectionByID(collID1, t1)
	assert.Nil(t, err)
	c2, err = mt.GetCollectionByID(collID2, t1)
	assert.NotNil(t, err)
	assert.Equal(t, int64(1), c1.CollectionID)

	c1, err = mt.GetCollectionByID(collID1, tsoStart)
	assert.NotNil(t, err)
	c2, err = mt.GetCollectionByID(collID2, tsoStart)
	assert.NotNil(t, err)

	c1, err = mt.GetCollectionByName(collName1, 0)
	assert.Nil(t, err)
	c2, err = mt.GetCollectionByName(collName2, 0)
	assert.Nil(t, err)
	assert.Equal(t, int64(1), c1.CollectionID)
	assert.Equal(t, int64(2), c2.CollectionID)

	c1, err = mt.GetCollectionByName(collName1, t2)
	assert.Nil(t, err)
	c2, err = mt.GetCollectionByName(collName2, t2)
	assert.Nil(t, err)
	assert.Equal(t, int64(1), c1.CollectionID)
	assert.Equal(t, int64(2), c2.CollectionID)

	c1, err = mt.GetCollectionByName(collName1, t1)
	assert.Nil(t, err)
	c2, err = mt.GetCollectionByName(collName2, t1)
	assert.NotNil(t, err)
	assert.Equal(t, int64(1), c1.CollectionID)

	c1, err = mt.GetCollectionByName(collName1, tsoStart)
	assert.NotNil(t, err)
	c2, err = mt.GetCollectionByName(collName2, tsoStart)
	assert.NotNil(t, err)

	getKeys := func(m map[string]*model.Collection) []string {
		keys := make([]string, 0, len(m))
		for key := range m {
			keys = append(keys, key)
		}
		return keys
	}

	s1, err := mt.ListCollections(0)
	assert.Nil(t, err)
	assert.Equal(t, 2, len(s1))
	assert.ElementsMatch(t, getKeys(s1), []string{collName1, collName2})

	s1, err = mt.ListCollections(t2)
	assert.Nil(t, err)
	assert.Equal(t, 2, len(s1))
	assert.ElementsMatch(t, getKeys(s1), []string{collName1, collName2})

	s1, err = mt.ListCollections(t1)
	assert.Nil(t, err)
	assert.Equal(t, 1, len(s1))
	assert.ElementsMatch(t, getKeys(s1), []string{collName1})

	s1, err = mt.ListCollections(tsoStart)
	assert.Nil(t, err)
	assert.Equal(t, 0, len(s1))

	p1, err := mt.GetPartitionByName(collID1, partName1, 0)
	assert.Nil(t, err)
	p2, err := mt.GetPartitionByName(collID2, partName2, 0)
	assert.Nil(t, err)
	assert.Equal(t, int64(partID1), p1)
	assert.Equal(t, int64(partID2), p2)

	p1, err = mt.GetPartitionByName(collID1, partName1, t2)
	assert.Nil(t, err)
	p2, err = mt.GetPartitionByName(collID2, partName2, t2)
	assert.Nil(t, err)
	assert.Equal(t, int64(11), p1)
	assert.Equal(t, int64(12), p2)

	p1, err = mt.GetPartitionByName(1, partName1, t1)
	assert.Nil(t, err)
	_, err = mt.GetPartitionByName(2, partName2, t1)
	assert.NotNil(t, err)
	assert.Equal(t, int64(11), p1)

	_, err = mt.GetPartitionByName(1, partName1, tsoStart)
	assert.NotNil(t, err)
	_, err = mt.GetPartitionByName(2, partName2, tsoStart)
	assert.NotNil(t, err)

	var cID UniqueID
	cID, err = mt.GetCollectionIDByName(collName1)
	assert.NoError(t, err)
	assert.Equal(t, collID1, cID)

	_, err = mt.GetCollectionIDByName("badname")
	assert.Error(t, err)

	name, err := mt.GetCollectionNameByID(collID2)
	assert.Nil(t, err)
	assert.Equal(t, collName2, name)

	_, err = mt.GetCollectionNameByID(int64(999))
	assert.Error(t, err)
}

func TestFixIssue10540(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	randVal := rand.Int()
	Params.Init()
	rootPath := fmt.Sprintf("/test/meta/%d", randVal)

	etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
	assert.Nil(t, err)
	defer etcdCli.Close()

	skv, err := rootcoord.NewMetaSnapshot(etcdCli, rootPath, TimestampPrefix, 7)
	assert.Nil(t, err)
	assert.NotNil(t, skv)
	//txnKV := etcdkv.NewEtcdKVWithClient(etcdCli, rootPath)
	txnKV := memkv.NewMemoryKV()
	// compose rc7 legace tombstone cases
	txnKV.Save(path.Join(rootcoord.SegmentIndexMetaPrefix, "2"), string(rootcoord.SuffixSnapshotTombstone))
	txnKV.Save(path.Join(rootcoord.IndexMetaPrefix, "3"), string(rootcoord.SuffixSnapshotTombstone))

	_, err = NewMetaTable(context.TODO(), &rootcoord.Catalog{Txn: txnKV, Snapshot: skv})
	assert.Nil(t, err)
}

func TestMetaTable_GetSegmentIndexInfos(t *testing.T) {
	meta := &MetaTable{
		segID2IndexID: make(map[typeutil.UniqueID]typeutil.UniqueID, 1),
		indexID2Meta:  make(map[typeutil.UniqueID]*model.Index, 1),
	}

	segID := typeutil.UniqueID(100)
	_, err := meta.GetSegmentIndexInfos(segID)
	assert.Error(t, err)

	meta.segID2IndexID[segID] = 5
	meta.indexID2Meta[5] = &model.Index{
		CollectionID: 1,
		FieldID:      4,
		IndexID:      5,
		SegmentIndexes: map[int64]model.SegmentIndex{
			segID: {
				Segment: model.Segment{
					SegmentID:   segID,
					PartitionID: 2,
				},
				BuildID:     6,
				EnableIndex: true,
			},
		},
	}

	indexMeta, err := meta.GetSegmentIndexInfos(segID)
	assert.NoError(t, err)
	segmentIndex, ok := indexMeta.SegmentIndexes[segID]
	assert.True(t, ok)
	assert.Equal(t, typeutil.UniqueID(1), indexMeta.CollectionID)
	segment := segmentIndex.Segment
	assert.Equal(t, typeutil.UniqueID(2), segment.PartitionID)
	assert.Equal(t, segID, segment.SegmentID)
	assert.Equal(t, typeutil.UniqueID(4), indexMeta.FieldID)
	assert.Equal(t, typeutil.UniqueID(5), indexMeta.IndexID)
	assert.Equal(t, typeutil.UniqueID(6), segmentIndex.BuildID)
	assert.Equal(t, true, segmentIndex.EnableIndex)
}

func TestMetaTable_unlockGetCollectionInfo(t *testing.T) {
	t.Run("normal case", func(t *testing.T) {
		mt := &MetaTable{
			collName2ID: map[string]typeutil.UniqueID{"test": 100},
			collID2Meta: map[typeutil.UniqueID]model.Collection{
				100: {CollectionID: 100, Name: "test"},
			},
		}
		info, err := mt.getCollectionInfoInternal("test")
		assert.NoError(t, err)
		assert.Equal(t, UniqueID(100), info.CollectionID)
		assert.Equal(t, "test", info.Name)
	})

	t.Run("collection name not found", func(t *testing.T) {
		mt := &MetaTable{collName2ID: nil, collAlias2ID: nil}
		_, err := mt.getCollectionInfoInternal("test")
		assert.Error(t, err)
	})

	t.Run("name found, meta not found", func(t *testing.T) {
		mt := &MetaTable{
			collName2ID:  map[string]typeutil.UniqueID{"test": 100},
			collAlias2ID: nil,
			collID2Meta:  nil,
		}
		_, err := mt.getCollectionInfoInternal("test")
		assert.Error(t, err)
	})

	t.Run("alias found, meta not found", func(t *testing.T) {
		mt := &MetaTable{
			collName2ID:  nil,
			collAlias2ID: map[string]typeutil.UniqueID{"test": 100},
			collID2Meta:  nil,
		}
		_, err := mt.getCollectionInfoInternal("test")
		assert.Error(t, err)
	})
}

func TestMetaTable_checkFieldCanBeIndexed(t *testing.T) {
	t.Run("field not indexed", func(t *testing.T) {
		mt := &MetaTable{}
		collMeta := model.Collection{
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{
			FieldID: 101,
		}
		idxInfo := &model.Index{}
		err := mt.checkFieldCanBeIndexed(collMeta, fieldSchema, idxInfo)
		assert.NoError(t, err)
	})

	t.Run("field already indexed", func(t *testing.T) {
		mt := &MetaTable{
			indexID2Meta: map[typeutil.UniqueID]*model.Index{
				200: {IndexID: 200, IndexName: "test"},
			},
		}
		collMeta := model.Collection{
			Name:             "test",
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{Name: "test", FieldID: 100}
		idxInfo := &model.Index{IndexName: "not_test"}
		err := mt.checkFieldCanBeIndexed(collMeta, fieldSchema, idxInfo)
		assert.Error(t, err)
	})

	t.Run("unexpected", func(t *testing.T) {
		mt := &MetaTable{
			// index meta incomplete.
			indexID2Meta: map[typeutil.UniqueID]*model.Index{},
		}
		collMeta := model.Collection{
			Name:             "test",
			CollectionID:     1000,
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{Name: "test", FieldID: 100}
		idxInfo := &model.Index{IndexName: "not_test"}
		err := mt.checkFieldCanBeIndexed(collMeta, fieldSchema, idxInfo)
		assert.NoError(t, err)
	})
}

func TestMetaTable_checkFieldIndexDuplicate(t *testing.T) {
	t.Run("index already exists", func(t *testing.T) {
		mt := &MetaTable{
			indexID2Meta: map[typeutil.UniqueID]*model.Index{
				200: {IndexID: 200, IndexName: "test"},
			},
		}
		collMeta := model.Collection{
			Name:             "test",
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{Name: "test", FieldID: 101}
		idxInfo := &model.Index{IndexName: "test"}
		_, _, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
		assert.Error(t, err)
	})

	t.Run("index parameters mismatch", func(t *testing.T) {
		mt := &MetaTable{
			indexID2Meta: map[typeutil.UniqueID]*model.Index{
				200: {IndexID: 200, IndexName: "test",
					IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "Value"}}},
			},
		}
		collMeta := model.Collection{
			Name:             "test",
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{Name: "test", FieldID: 100}
		idxInfo := &model.Index{IndexName: "test", IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "not_Value"}}}
		_, _, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
		assert.Error(t, err)
	})

	t.Run("index parameters match", func(t *testing.T) {
		mt := &MetaTable{
			indexID2Meta: map[typeutil.UniqueID]*model.Index{
				200: {IndexID: 200, IndexName: "test",
					IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "Value"}}},
			},
		}
		collMeta := model.Collection{
			Name:             "test",
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{Name: "test", FieldID: 100}
		idxInfo := &model.Index{IndexName: "test", IndexParams: []*commonpb.KeyValuePair{{Key: "Key", Value: "Value"}}}
		duplicate, dupIdxInfo, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
		assert.NoError(t, err)
		assert.True(t, duplicate)
		assert.Equal(t, idxInfo.IndexName, dupIdxInfo.IndexName)
	})

	t.Run("field not found", func(t *testing.T) {
		mt := &MetaTable{}
		collMeta := model.Collection{
			FieldIDToIndexID: []common.Int64Tuple{{Key: 100, Value: 200}},
		}
		fieldSchema := model.Field{
			FieldID: 101,
		}
		idxInfo := &model.Index{}
		duplicate, dupIdxInfo, err := mt.checkFieldIndexDuplicate(collMeta, fieldSchema, idxInfo)
		assert.NoError(t, err)
		assert.False(t, duplicate)
		assert.Nil(t, dupIdxInfo)
	})
}

func TestMetaTable_GetInitBuildIDs(t *testing.T) {
	var (
		collName  = "GetInitBuildID-Coll"
		indexName = "GetInitBuildID-Index"
	)
	mt := &MetaTable{
		collID2Meta: map[typeutil.UniqueID]model.Collection{
			1: {
				FieldIDToIndexID: []common.Int64Tuple{
					{
						Key:   1,
						Value: 1,
					},
					{
						Key:   2,
						Value: 2,
					},
					{
						Key:   3,
						Value: 3,
					},
				},
			},
		},
		collName2ID: map[string]typeutil.UniqueID{
			"GetInitBuildID-Coll-1": 2,
		},
		indexID2Meta: map[typeutil.UniqueID]*model.Index{
			1: {
				IndexName: "GetInitBuildID-Index-1",
				IndexID:   1,
				SegmentIndexes: map[int64]model.SegmentIndex{
					4: {
						Segment: model.Segment{
							SegmentID: 4,
						},
						EnableIndex: true,
						CreateTime:  5,
					},
				},
				CreateTime: 10,
			},
			2: {
				IndexName:  "GetInitBuildID-Index-2",
				IndexID:    2,
				CreateTime: 10,
			},
		},
	}
	t.Run("coll not exist", func(t *testing.T) {
		buildIDs, err := mt.GetInitBuildIDs("collName", indexName)
		assert.Error(t, err)
		assert.Nil(t, buildIDs)
	})

	mt.collName2ID[collName] = 1

	t.Run("index not exist", func(t *testing.T) {
		buildIDs, err := mt.GetInitBuildIDs(collName, indexName)
		assert.Error(t, err)
		assert.Nil(t, buildIDs)
	})

	mt.indexID2Meta[3] = &model.Index{
		IndexName:  indexName,
		IndexID:    3,
		CreateTime: 10,
		SegmentIndexes: map[int64]model.SegmentIndex{
			5: {
				Segment: model.Segment{
					SegmentID: 5,
				},
				EnableIndex: true,
				CreateTime:  5,
			},
		},
	}

	mt.segID2IndexID = map[typeutil.UniqueID]typeutil.UniqueID{
		4: 1,
		5: 3,
	}

	t.Run("success", func(t *testing.T) {
		buildIDs, err := mt.GetInitBuildIDs(collName, indexName)
		assert.NoError(t, err)
		assert.Equal(t, 1, len(buildIDs))
	})
}

func TestMetaTable_GetDroppedIndex(t *testing.T) {
	mt := &MetaTable{
		collID2Meta: map[typeutil.UniqueID]model.Collection{
			1: {
				FieldIDToIndexID: []common.Int64Tuple{
					{
						Key:   1,
						Value: 1,
					},
					{
						Key:   2,
						Value: 2,
					},
					{
						Key:   3,
						Value: 3,
					},
				},
			},
		},
		indexID2Meta: map[typeutil.UniqueID]*model.Index{
			1: {
				IndexName: "GetInitBuildID-Index-1",
				IndexID:   1,
				IsDeleted: true,
			},
			2: {
				IndexName: "GetInitBuildID-Index-2",
				IndexID:   2,
				IsDeleted: false,
			},
		},
	}

	droppedIndexID := mt.GetDroppedIndex()
	indexInfo, ok := droppedIndexID[1]
	assert.True(t, ok)
	assert.Equal(t, 1, len(indexInfo))
}

func TestMetaTable_AlignSegmentsMeta(t *testing.T) {
	var (
		indexName = "GetDroppedIndex-Index"
		collID    = UniqueID(1)
		partID    = UniqueID(10)
		indexID   = UniqueID(1000)
	)
	mt := &MetaTable{
		catalog: &rootcoord.Catalog{
			Txn: &mockTestTxnKV{
				multiRemove: func(keys []string) error {
					return nil
				},
			},
		},
		collID2Meta: map[typeutil.UniqueID]model.Collection{
			collID: {
				FieldIDToIndexID: []common.Int64Tuple{
					{
						Key:   1,
						Value: 1,
					},
				},
			},
		},
		partID2IndexedSegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{
			partID: {
				100: true,
				101: true,
				102: true,
			},
		},
		indexID2Meta: map[typeutil.UniqueID]*model.Index{
			indexID: {
				IndexName: indexName,
				IndexID:   1,
				IsDeleted: true,
			},
		},
	}
	t.Run("success", func(t *testing.T) {
		mt.AlignSegmentsMeta(collID, partID, map[UniqueID]struct{}{101: {}, 102: {}, 103: {}})
	})

	t.Run("txn error", func(t *testing.T) {
		txn := &mockTestTxnKV{
			multiRemove: func(keys []string) error {
				return fmt.Errorf("error occurred")
			},
		}
		mt.catalog = &rootcoord.Catalog{Txn: txn}
		mt.AlignSegmentsMeta(collID, partID, map[UniqueID]struct{}{103: {}, 104: {}, 105: {}})
	})
}

func TestMetaTable_MarkIndexDeleted(t *testing.T) {
	var (
		collName  = "MarkIndexDeleted-Coll"
		fieldName = "MarkIndexDeleted-Field"
		indexName = "MarkIndexDeleted-Index"
		collID    = UniqueID(1)
		partID    = UniqueID(10)
		fieldID   = UniqueID(100)
		indexID   = UniqueID(1000)
	)
	mt := &MetaTable{
		ctx: context.TODO(),
		collID2Meta: map[typeutil.UniqueID]model.Collection{
			collID: {
				FieldIDToIndexID: []common.Int64Tuple{
					{
						Key:   101,
						Value: 1001,
					},
				},
				Fields: []*model.Field{
					{
						FieldID: fieldID,
						Name:    fieldName,
					},
				},
			},
		},
		collName2ID: map[string]typeutil.UniqueID{
			collName: collID,
		},
		partID2IndexedSegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{
			partID: {
				100: true,
				101: true,
				102: true,
			},
		},
		indexID2Meta: map[typeutil.UniqueID]*model.Index{
			1001: {
				IndexName: indexName,
				IndexID:   indexID,
				IsDeleted: true,
			},
		},
	}

	t.Run("get collection meta failed", func(t *testing.T) {
		err := mt.MarkIndexDeleted("collName", fieldName, indexName)
		assert.Error(t, err)
	})

	t.Run("get field meta failed", func(t *testing.T) {
		err := mt.MarkIndexDeleted(collName, "fieldName", indexName)
		assert.Error(t, err)

		err = mt.MarkIndexDeleted(collName, fieldName, indexName)
		assert.NoError(t, err)
	})

	t.Run("indexMeta error", func(t *testing.T) {
		collMeta := model.Collection{
			FieldIDToIndexID: []common.Int64Tuple{
				{
					Key:   fieldID,
					Value: indexID,
				},
			},
			Fields: []*model.Field{
				{
					FieldID: fieldID,
					Name:    fieldName,
				},
			},
		}
		mt.collID2Meta[collID] = collMeta

		err := mt.MarkIndexDeleted(collName, fieldName, indexName)
		assert.Error(t, err)

		mt.indexID2Meta[indexID] = &model.Index{
			IndexName: "indexName",
			IndexID:   indexID,
		}

		err = mt.MarkIndexDeleted(collName, fieldName, indexName)
		assert.NoError(t, err)
	})

	t.Run("txn save failed", func(t *testing.T) {
		mt.indexID2Meta[indexID] = &model.Index{
			IndexName: indexName,
			IndexID:   indexID,
		}
		mc := &MockedCatalog{}
		targetErr := errors.New("alter add index fail")

		mc.On("AlterIndex").Return(targetErr)
		mt.catalog = mc
		err := mt.MarkIndexDeleted(collName, fieldName, indexName)
		assert.Error(t, err)
		assert.True(t, errors.Is(err, targetErr))

		mc = &MockedCatalog{}
		mc.On("AlterIndex").Return(nil)
		mt.catalog = mc
		err = mt.MarkIndexDeleted(collName, fieldName, indexName)
		assert.NoError(t, err)
		err = mt.MarkIndexDeleted(collName, fieldName, indexName)
		assert.NoError(t, err)
	})
}

type MockedCatalog struct {
	mock.Mock
	metastore.RootCoordCatalog
	alterIndexParamsVerification  func(ctx context.Context, oldIndex *model.Index, newIndex *model.Index, alterType metastore.AlterType)
	createIndexParamsVerification func(ctx context.Context, col *model.Collection, index *model.Index)
	dropIndexParamsVerification   func(ctx context.Context, collectionInfo *model.Collection, dropIdxID typeutil.UniqueID)
}

func (mc *MockedCatalog) ListCollections(ctx context.Context, ts typeutil.Timestamp) (map[string]*model.Collection, error) {
	args := mc.Called()
	return args.Get(0).(map[string]*model.Collection), nil
}

func (mc *MockedCatalog) ListIndexes(ctx context.Context) ([]*model.Index, error) {
	args := mc.Called()
	return args.Get(0).([]*model.Index), nil
}

func (mc *MockedCatalog) ListAliases(ctx context.Context, ts typeutil.Timestamp) ([]*model.Alias, error) {
	args := mc.Called()
	return args.Get(0).([]*model.Alias), nil
}

func (mc *MockedCatalog) AlterIndex(ctx context.Context, oldIndex *model.Index, newIndex *model.Index, alterType metastore.AlterType) error {
	if mc.alterIndexParamsVerification != nil {
		mc.alterIndexParamsVerification(ctx, oldIndex, newIndex, alterType)
	}
	args := mc.Called()
	err := args.Get(0)
	if err == nil {
		return nil
	}
	return err.(error)
}

func (mc *MockedCatalog) CreateIndex(ctx context.Context, col *model.Collection, index *model.Index) error {
	if mc.createIndexParamsVerification != nil {
		mc.createIndexParamsVerification(ctx, col, index)
	}
	args := mc.Called()
	err := args.Get(0)
	if err == nil {
		return nil
	}
	return err.(error)
}

func (mc *MockedCatalog) DropIndex(ctx context.Context, collectionInfo *model.Collection,
	dropIdxID typeutil.UniqueID) error {
	if mc.dropIndexParamsVerification != nil {
		mc.dropIndexParamsVerification(ctx, collectionInfo, dropIdxID)
	}

	args := mc.Called()
	err := args.Get(0)
	if err == nil {
		return nil
	}
	return err.(error)
}

func TestMetaTable_ReloadFromKV(t *testing.T) {
	mc := &MockedCatalog{}

	collectionName := "cn"
	collInfo := &model.Collection{
		CollectionID: 1,
		Name:         collectionName,
		AutoID:       false,
		Fields: []*model.Field{
			{
				FieldID:      1,
				Name:         "field110",
				IsPrimaryKey: false,
				Description:  "",
				DataType:     schemapb.DataType_FloatVector,
				TypeParams: []*commonpb.KeyValuePair{
					{
						Key:   "field110-k1",
						Value: "field110-v1",
					},
				},
				IndexParams: []*commonpb.KeyValuePair{
					{
						Key:   "field110-i1",
						Value: "field110-v1",
					},
				},
			},
		},
		FieldIDToIndexID: []common.Int64Tuple{{Key: 1, Value: 1}},
		Partitions: []*model.Partition{
			{
				PartitionID:               1,
				PartitionName:             Params.CommonCfg.DefaultPartitionName,
				PartitionCreatedTimestamp: 0,
			},
		},
		Aliases: []string{"a", "b"},
	}
	collections := map[string]*model.Collection{collectionName: collInfo}
	mc.On("ListCollections").Return(collections, nil)

	indexes := []*model.Index{
		{
			CollectionID: 1,
			IndexName:    "idx",
			IndexID:      1,
			FieldID:      1,
			IndexParams: []*commonpb.KeyValuePair{
				{
					Key:   "field110-i1",
					Value: "field110-v1",
				},
			},
			SegmentIndexes: map[int64]model.SegmentIndex{
				1: {
					Segment: model.Segment{
						SegmentID:   1,
						PartitionID: 1,
					},
					BuildID:     1000,
					EnableIndex: true,
				},
			},
		},
	}
	mc.On("ListIndexes").Return(indexes, nil)

	alias1 := *collInfo
	alias1.Name = collInfo.Aliases[0]

	alias2 := *collInfo
	alias2.Name = collInfo.Aliases[1]
	mc.On("ListAliases").Return([]*model.Alias{
		{
			CollectionID: collInfo.CollectionID,
			Name:         collInfo.Aliases[0],
		},
		{
			CollectionID: collInfo.CollectionID,
			Name:         collInfo.Aliases[1],
		},
	}, nil)

	mt := &MetaTable{}
	mt.catalog = mc
	mt.reloadFromCatalog()

	assert.True(t, len(mt.collID2Meta) == 1)
	assert.Equal(t, mt.collID2Meta[1], *collInfo)

	assert.True(t, len(mt.collName2ID) == 1)

	assert.True(t, len(mt.collAlias2ID) == 2)
	ret, ok := mt.collAlias2ID[collInfo.Aliases[0]]
	assert.True(t, ok)
	assert.Equal(t, int64(1), ret)

	assert.True(t, len(mt.partID2IndexedSegID) == 1)
	ret2, ok := mt.partID2IndexedSegID[1]
	assert.True(t, ok)
	assert.True(t, len(ret2) == 1)
	ret3, ok := ret2[typeutil.UniqueID(1)]
	assert.True(t, ok)
	assert.True(t, ret3)

	assert.True(t, len(mt.segID2IndexID) == 1)
	segID, ok := mt.segID2IndexID[1]
	assert.True(t, ok)
	assert.Equal(t, int64(1), segID)

	assert.True(t, len(mt.indexID2Meta) == 1)
	meta, ok := mt.indexID2Meta[1]
	assert.True(t, ok)
	assert.Equal(t, indexes[0], meta)
}

func TestMetaTable_AddIndex(t *testing.T) {
	var (
		collName   = "MarkIndexDeleted-Coll"
		fieldName  = "MarkIndexDeleted-Field"
		fieldName2 = "MarkIndexDeleted-Field2"
		indexName  = "MarkIndexDeleted-Index"
		collID     = UniqueID(1)
		fieldID    = UniqueID(100)
		fieldID2   = UniqueID(101)
		indexID    = UniqueID(1000)
		indexID2   = UniqueID(1001)
		segID      = UniqueID(10000)
	)

	colMeta := model.Collection{
		CollectionID: collID,
		Name:         collName,
		FieldIDToIndexID: []common.Int64Tuple{
			{
				Key:   fieldID,
				Value: indexID,
			},
		},
		Fields: []*model.Field{
			{
				FieldID:     fieldID,
				Name:        fieldName,
				DataType:    schemapb.DataType_FloatVector,
				IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			},
			{
				FieldID:     fieldID2,
				Name:        fieldName2,
				DataType:    schemapb.DataType_FloatVector,
				IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			},
		},
	}

	t.Run("test index already exists", func(t *testing.T) {
		mt := &MetaTable{
			collID2Meta: map[typeutil.UniqueID]model.Collection{collID: colMeta},
			collName2ID: map[string]typeutil.UniqueID{collName: collID},
		}
		mt.indexID2Meta = map[typeutil.UniqueID]*model.Index{
			indexID: {
				IndexName:   indexName,
				IndexID:     indexID,
				IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
				IsDeleted:   false,
				CreateTime:  0,
			},
		}

		mc := &MockedCatalog{}
		mc.On("AlterIndex").Return(nil)
		mc.alterIndexParamsVerification = func(ctx context.Context, oldIndex *model.Index, newIndex *model.Index, alterType metastore.AlterType) {
			assert.NotNil(t, oldIndex)
			assert.NotNil(t, newIndex)
			assert.Equal(t, metastore.ADD, alterType)
			assert.Equal(t, oldIndex.IndexID, newIndex.IndexID)
			assert.Equal(t, oldIndex.IsDeleted, newIndex.IsDeleted)
			assert.Equal(t, oldIndex.IndexParams, newIndex.IndexParams)
			assert.Equal(t, oldIndex.CreateTime, uint64(0))
			assert.Equal(t, newIndex.CreateTime, uint64(100))
		}
		mt.catalog = mc

		idxInfo := &model.Index{
			IndexName:   indexName,
			IndexID:     indexID,
			IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			CreateTime:  100,
		}

		ret, err := mt.AddIndex(collName, fieldName, idxInfo, []UniqueID{segID})
		assert.True(t, ret)

		idxMeta, ok := mt.indexID2Meta[indexID]
		assert.True(t, ok)
		assert.Equal(t, uint64(100), idxMeta.CreateTime)
		assert.NoError(t, err)
	})

	t.Run("test add index firstly(create index)", func(t *testing.T) {
		mt := &MetaTable{
			collID2Meta:  map[typeutil.UniqueID]model.Collection{collID: colMeta},
			collName2ID:  map[string]typeutil.UniqueID{collName: collID},
			indexID2Meta: make(map[typeutil.UniqueID]*model.Index),
		}

		idxInfo := &model.Index{
			FieldID:     fieldID2,
			IndexName:   indexName,
			IndexID:     indexID2,
			IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			CreateTime:  100,
		}

		mc := &MockedCatalog{}
		mc.On("CreateIndex").Return(errors.New("create index fail"))
		mt.catalog = mc
		ret, err := mt.AddIndex(collName, fieldName2, idxInfo, []UniqueID{segID})
		assert.False(t, ret)
		assert.Error(t, err)
		_, ok := mt.indexID2Meta[indexID2]
		assert.False(t, ok)

		mc = &MockedCatalog{}
		mc.On("CreateIndex").Return(nil)
		mc.createIndexParamsVerification = func(ctx context.Context, col *model.Collection, index *model.Index) {
			assert.NotNil(t, col)
			assert.NotNil(t, index)
			assert.NotEqual(t, colMeta, col)
			assert.Equal(t, 2, len(col.FieldIDToIndexID))
			assert.Equal(t, fieldID, col.FieldIDToIndexID[0].Key)
			assert.Equal(t, fieldID2, col.FieldIDToIndexID[1].Key)
			assert.Equal(t, indexID2, index.IndexID)
			assert.Equal(t, uint64(100), index.CreateTime)
			assert.Equal(t, 1, len(index.SegmentIndexes))
			assert.Equal(t, segID, index.SegmentIndexes[segID].SegmentID)
			assert.Equal(t, false, index.SegmentIndexes[segID].EnableIndex)
		}

		mt.catalog = mc
		ret, err = mt.AddIndex(collName, fieldName2, idxInfo, []UniqueID{segID})
		assert.False(t, ret)
		assert.NoError(t, err)

		idxMeta, ok := mt.indexID2Meta[indexID2]
		assert.True(t, ok)
		assert.Equal(t, uint64(100), idxMeta.CreateTime)
		assert.NoError(t, err)

		newColMeta, ok := mt.collID2Meta[collID]
		assert.True(t, ok)
		assert.Equal(t, collID, newColMeta.CollectionID)
		assert.Equal(t, 2, len(newColMeta.FieldIDToIndexID))
		assert.Equal(t, fieldID, newColMeta.FieldIDToIndexID[0].Key)
		assert.Equal(t, indexID, newColMeta.FieldIDToIndexID[0].Value)
		assert.Equal(t, fieldID2, newColMeta.FieldIDToIndexID[1].Key)
		assert.Equal(t, indexID2, newColMeta.FieldIDToIndexID[1].Value)
	})
}

func TestMetaTable_RecycleDroppedIndex(t *testing.T) {
	colName := "c"
	fieldName := "f1"
	indexName1 := "idx1"
	indexName2 := "idx2"

	colMeta := model.Collection{
		CollectionID: 1,
		Name:         colName,
		FieldIDToIndexID: []common.Int64Tuple{
			{
				Key:   1,
				Value: 1,
			},
			{
				Key:   1,
				Value: 2,
			},
		},
		Fields: []*model.Field{
			{
				FieldID:     1,
				Name:        fieldName,
				DataType:    schemapb.DataType_FloatVector,
				IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			},
		},
		Partitions: []*model.Partition{
			{
				PartitionID:   1,
				PartitionName: "p",
			},
		},
	}

	mt := &MetaTable{
		collID2Meta: map[typeutil.UniqueID]model.Collection{1: colMeta},
		collName2ID: map[string]typeutil.UniqueID{colName: 1},
		segID2IndexID: map[typeutil.UniqueID]typeutil.UniqueID{
			1: 1,
		},
		partID2IndexedSegID: map[typeutil.UniqueID]map[typeutil.UniqueID]bool{
			1: {
				1: true,
			},
		},
	}
	mt.indexID2Meta = map[typeutil.UniqueID]*model.Index{
		1: {
			IndexName:   indexName1,
			IndexID:     1,
			IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			IsDeleted:   true,
			SegmentIndexes: map[int64]model.SegmentIndex{
				1: {
					Segment: model.Segment{
						SegmentID:   1,
						PartitionID: 1,
					},
					BuildID:     1000,
					EnableIndex: true,
				},
			},
		},
		2: {
			IndexName:   indexName2,
			IndexID:     2,
			IndexParams: []*commonpb.KeyValuePair{{Key: "index_type", Value: DefaultIndexType}},
			IsDeleted:   false,
			SegmentIndexes: map[int64]model.SegmentIndex{
				1: {
					Segment: model.Segment{
						SegmentID:   1,
						PartitionID: 1,
					},
					BuildID:     1000,
					EnableIndex: true,
				},
			},
		},
	}

	mc := &MockedCatalog{}
	mc.On("DropIndex").Return(nil)
	mc.dropIndexParamsVerification = func(ctx context.Context, collectionInfo *model.Collection, dropIdxID typeutil.UniqueID) {
		assert.NotNil(t, collectionInfo)
		assert.Equal(t, int64(1), dropIdxID)
		assert.Equal(t, int64(1), collectionInfo.CollectionID)
		assert.Equal(t, 1, len(collectionInfo.FieldIDToIndexID))
		assert.Equal(t, int64(1), collectionInfo.FieldIDToIndexID[0].Key)
		assert.Equal(t, int64(2), collectionInfo.FieldIDToIndexID[0].Value)
	}
	mt.catalog = mc

	mt.RecycleDroppedIndex()
	newColMeta, ok := mt.collID2Meta[1]
	assert.True(t, ok)
	assert.Equal(t, int64(1), newColMeta.CollectionID)
	assert.Equal(t, 1, len(newColMeta.FieldIDToIndexID))
	assert.Equal(t, int64(1), newColMeta.FieldIDToIndexID[0].Key)
	assert.Equal(t, int64(2), newColMeta.FieldIDToIndexID[0].Value)

	assert.Equal(t, 1, len(mt.indexID2Meta))
	idxMeta, ok := mt.indexID2Meta[2]
	assert.True(t, ok)
	assert.Equal(t, false, idxMeta.IsDeleted)

	assert.Equal(t, 0, len(mt.segID2IndexID))
	assert.Equal(t, 1, len(mt.partID2IndexedSegID))
}