Fix proxy panicking when QueryCoord offline (#23254)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2023-04-07 17:30:28 +08:00 committed by GitHub
parent dc091166d3
commit 132c010271
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 53 additions and 8 deletions

View File

@ -3531,18 +3531,15 @@ func (node *Proxy) GetReplicas(ctx context.Context, req *milvuspb.GetReplicasReq
req.CollectionID, _ = globalMetaCache.GetCollectionID(ctx, req.GetCollectionName())
}
resp, err := node.queryCoord.GetReplicas(ctx, req)
r, err := node.queryCoord.GetReplicas(ctx, req)
if err != nil {
log.Error("Failed to get replicas from Query Coordinator",
log.Warn("Failed to get replicas from Query Coordinator",
zap.Error(err))
resp.Status.ErrorCode = commonpb.ErrorCode_UnexpectedError
resp.Status.Reason = err.Error()
resp.Status = merr.Status(err)
return resp, nil
}
log.Debug("received get replicas response",
zap.Any("resp", resp),
zap.Error(err))
return resp, nil
log.Debug("received get replicas response", zap.String("resp", r.String()))
return r, nil
}
// GetCompactionState gets the compaction state of multiple segments

View File

@ -487,3 +487,51 @@ func TestProxy_GetFlushAllState(t *testing.T) {
assert.Equal(t, resp.GetStatus().GetErrorCode(), commonpb.ErrorCode_UnexpectedError)
})
}
func TestProxy_GetReplicas(t *testing.T) {
factory := dependency.NewDefaultFactory(true)
ctx := context.Background()
node, err := NewProxy(ctx, factory)
assert.NoError(t, err)
node.stateCode.Store(commonpb.StateCode_Healthy)
node.tsoAllocator = &timestampAllocator{
tso: newMockTimestampAllocatorInterface(),
}
mockQC := types.NewMockQueryCoord(t)
mockRC := mocks.NewRootCoord(t)
node.queryCoord = mockQC
node.rootCoord = mockRC
// set expectations
successStatus := &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}
t.Run("success", func(t *testing.T) {
mockQC.EXPECT().GetReplicas(mock.Anything, mock.AnythingOfType("*milvuspb.GetReplicasRequest")).Return(&milvuspb.GetReplicasResponse{Status: successStatus}, nil)
resp, err := node.GetReplicas(ctx, &milvuspb.GetReplicasRequest{
CollectionID: 1000,
})
assert.NoError(t, err)
assert.Equal(t, resp.GetStatus().GetErrorCode(), commonpb.ErrorCode_Success)
})
t.Run("proxy_not_healthy", func(t *testing.T) {
node.stateCode.Store(commonpb.StateCode_Abnormal)
resp, err := node.GetReplicas(ctx, &milvuspb.GetReplicasRequest{
CollectionID: 1000,
})
assert.NoError(t, err)
assert.Equal(t, resp.GetStatus().GetErrorCode(), commonpb.ErrorCode_UnexpectedError)
node.stateCode.Store(commonpb.StateCode_Healthy)
})
t.Run("QueryCoordClient_returnsError", func(t *testing.T) {
mockQC.ExpectedCalls = nil
mockQC.EXPECT().GetReplicas(mock.Anything, mock.AnythingOfType("*milvuspb.GetReplicasRequest")).Return(nil, errors.New("mocked"))
resp, err := node.GetReplicas(ctx, &milvuspb.GetReplicasRequest{
CollectionID: 1000,
})
assert.NoError(t, err)
assert.Equal(t, resp.GetStatus().GetErrorCode(), commonpb.ErrorCode_UnexpectedError)
})
}