test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 19:39:21 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

fix: [2.4] Make querycoord panick when rg metastore sync fail (#34106) (#34127)

Browse Source 
Cherry-pick from master
pr: #34106
See also #34047

When `unassignNode` sync resource group with node removed failed

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2024-06-26 10:04:03 +08:00 committed by GitHub
parent 3e51256fb4
commit 26b2e1d43c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 29 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/querycoordv2/meta/resource_manager.go
View File

@ -820,12 +820,11 @@ func (rm *ResourceManager) unassignNode(node int64) (string, error) {
mrg.UnassignNode(node)
rg := mrg.ToResourceGroup()
if err := rm.catalog.SaveResourceGroup(rg.GetMeta()); err != nil {
log.Warn("unassign node from resource group",
log.Fatal("unassign node from resource group",
zap.String("rgName", rg.GetName()),
zap.Int64("node", node),
zap.Error(err),
)
return "", merr.WrapErrResourceGroupServiceAvailable()
}
// Commit updates to memory.

28
internal/querycoordv2/meta/resource_manager_test.go
View File

@ -18,11 +18,13 @@ package meta
import (
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v2/rgpb"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/kv/mocks"
"github.com/milvus-io/milvus/internal/metastore/kv/querycoord"
"github.com/milvus-io/milvus/internal/querycoordv2/params"
"github.com/milvus-io/milvus/internal/querycoordv2/session"
@ -591,3 +593,29 @@ func (suite *ResourceManagerSuite) TestIncomingNode() {
suite.NoError(err)
suite.Len(nodes, 1)
}
func (suite *ResourceManagerSuite) TestUnassignFail() {
// suite.man
mockKV := mocks.NewMetaKv(suite.T())
mockKV.EXPECT().MultiSave(mock.Anything).Return(nil).Once()
store := querycoord.NewCatalog(mockKV)
suite.manager = NewResourceManager(store, session.NewNodeManager())
suite.manager.UpdateResourceGroups(map[string]*rgpb.ResourceGroupConfig{
"rg1": newResourceGroupConfig(20, 30),
})
suite.manager.nodeMgr.Add(session.NewNodeInfo(session.ImmutableNodeInfo{
NodeID: 1,
Address: "localhost",
Hostname: "localhost",
}))
suite.manager.HandleNodeUp(1)
mockKV.EXPECT().MultiSave(mock.Anything).Return(merr.WrapErrServiceInternal("mocked")).Once()
suite.Panics(func() {
suite.manager.HandleNodeDown(1)
})
}