2021-07-13 14:16:00 +08:00
|
|
|
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
|
|
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
|
|
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
|
|
|
|
|
|
package querycoord
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2021-09-22 17:50:00 +08:00
|
|
|
"fmt"
|
2021-07-13 14:16:00 +08:00
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
|
2021-09-15 20:40:07 +08:00
|
|
|
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
|
2021-07-16 10:21:55 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
2021-07-13 14:16:00 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/commonpb"
|
|
|
|
"github.com/milvus-io/milvus/internal/proto/querypb"
|
2021-09-22 17:50:00 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/typeutil"
|
2021-07-13 14:16:00 +08:00
|
|
|
)
|
|
|
|
|
2021-08-26 14:17:54 +08:00
|
|
|
//func waitQueryNodeOnline(cluster *queryNodeCluster, nodeID int64)
|
|
|
|
|
2021-09-22 17:50:00 +08:00
|
|
|
func removeNodeSession(id int64) error {
|
|
|
|
kv, err := etcdkv.NewEtcdKV(Params.EtcdEndpoints, Params.MetaRootPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
err = kv.Remove(fmt.Sprintf("session/"+typeutil.QueryNodeRole+"-%d", id))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func removeAllSession() error {
|
|
|
|
kv, err := etcdkv.NewEtcdKV(Params.EtcdEndpoints, Params.MetaRootPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
err = kv.RemoveWithPrefix("session")
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-09-15 20:40:07 +08:00
|
|
|
func waitAllQueryNodeOffline(cluster Cluster, nodes map[int64]Node) bool {
|
2021-09-22 17:50:00 +08:00
|
|
|
reDoCount := 40
|
2021-08-26 14:17:54 +08:00
|
|
|
for {
|
|
|
|
if reDoCount <= 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
allOffline := true
|
|
|
|
for nodeID := range nodes {
|
|
|
|
_, err := cluster.getNodeByID(nodeID)
|
|
|
|
if err == nil {
|
|
|
|
allOffline = false
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if allOffline {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
log.Debug("wait all queryNode offline")
|
2021-09-24 18:29:55 +08:00
|
|
|
time.Sleep(100 * time.Millisecond)
|
2021-08-26 14:17:54 +08:00
|
|
|
reDoCount--
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-26 22:01:57 +08:00
|
|
|
func waitQueryNodeOnline(cluster Cluster, nodeID int64) {
|
|
|
|
for {
|
|
|
|
online, err := cluster.isOnline(nodeID)
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if online {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-13 14:16:00 +08:00
|
|
|
func TestQueryNode_MultiNode_stop(t *testing.T) {
|
2021-09-15 20:40:07 +08:00
|
|
|
refreshParams()
|
2021-07-13 14:16:00 +08:00
|
|
|
baseCtx := context.Background()
|
|
|
|
|
|
|
|
queryCoord, err := startQueryCoord(baseCtx)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
queryNode1, err := startQueryNodeServer(baseCtx)
|
|
|
|
assert.Nil(t, err)
|
2021-09-26 22:01:57 +08:00
|
|
|
waitQueryNodeOnline(queryCoord.cluster, queryNode1.queryNodeID)
|
2021-07-13 14:16:00 +08:00
|
|
|
|
2021-09-26 22:01:57 +08:00
|
|
|
queryNode2, err := startQueryNodeServer(baseCtx)
|
2021-07-13 14:16:00 +08:00
|
|
|
assert.Nil(t, err)
|
2021-09-26 22:01:57 +08:00
|
|
|
waitQueryNodeOnline(queryCoord.cluster, queryNode2.queryNodeID)
|
2021-07-13 14:16:00 +08:00
|
|
|
|
|
|
|
queryNode1.stop()
|
2021-09-22 17:50:00 +08:00
|
|
|
err = removeNodeSession(queryNode1.queryNodeID)
|
|
|
|
assert.Nil(t, err)
|
2021-07-13 14:16:00 +08:00
|
|
|
|
|
|
|
queryCoord.LoadCollection(baseCtx, &querypb.LoadCollectionRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_LoadCollection,
|
|
|
|
},
|
|
|
|
CollectionID: defaultCollectionID,
|
|
|
|
Schema: genCollectionSchema(defaultCollectionID, false),
|
|
|
|
})
|
|
|
|
_, err = queryCoord.ReleaseCollection(baseCtx, &querypb.ReleaseCollectionRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_ReleaseCollection,
|
|
|
|
},
|
|
|
|
CollectionID: defaultCollectionID,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2021-09-24 18:29:55 +08:00
|
|
|
time.Sleep(100 * time.Millisecond)
|
2021-09-15 20:40:07 +08:00
|
|
|
nodes, err := queryCoord.cluster.onlineNodes()
|
2021-07-16 10:21:55 +08:00
|
|
|
assert.Nil(t, err)
|
2021-09-26 22:01:57 +08:00
|
|
|
queryNode2.stop()
|
|
|
|
err = removeNodeSession(queryNode2.queryNodeID)
|
2021-09-22 17:50:00 +08:00
|
|
|
assert.Nil(t, err)
|
2021-07-16 10:21:55 +08:00
|
|
|
|
2021-08-26 14:17:54 +08:00
|
|
|
allNodeOffline := waitAllQueryNodeOffline(queryCoord.cluster, nodes)
|
|
|
|
assert.Equal(t, allNodeOffline, true)
|
2021-07-13 14:16:00 +08:00
|
|
|
queryCoord.Stop()
|
2021-09-22 17:50:00 +08:00
|
|
|
err = removeAllSession()
|
|
|
|
assert.Nil(t, err)
|
2021-07-13 14:16:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestQueryNode_MultiNode_reStart(t *testing.T) {
|
2021-09-15 20:40:07 +08:00
|
|
|
refreshParams()
|
2021-07-13 14:16:00 +08:00
|
|
|
baseCtx := context.Background()
|
|
|
|
|
|
|
|
queryCoord, err := startQueryCoord(baseCtx)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
queryNode1, err := startQueryNodeServer(baseCtx)
|
|
|
|
assert.Nil(t, err)
|
2021-09-26 22:01:57 +08:00
|
|
|
waitQueryNodeOnline(queryCoord.cluster, queryNode1.queryNodeID)
|
2021-07-13 14:16:00 +08:00
|
|
|
|
2021-09-24 18:29:55 +08:00
|
|
|
time.Sleep(100 * time.Millisecond)
|
2021-07-13 14:16:00 +08:00
|
|
|
queryCoord.LoadCollection(baseCtx, &querypb.LoadCollectionRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_LoadCollection,
|
|
|
|
},
|
|
|
|
CollectionID: defaultCollectionID,
|
|
|
|
Schema: genCollectionSchema(defaultCollectionID, false),
|
|
|
|
})
|
|
|
|
queryNode1.stop()
|
2021-09-22 17:50:00 +08:00
|
|
|
err = removeNodeSession(queryNode1.queryNodeID)
|
|
|
|
assert.Nil(t, err)
|
2021-07-13 14:16:00 +08:00
|
|
|
queryNode3, err := startQueryNodeServer(baseCtx)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-09-24 18:29:55 +08:00
|
|
|
time.Sleep(100 * time.Millisecond)
|
2021-07-13 14:16:00 +08:00
|
|
|
_, err = queryCoord.ReleaseCollection(baseCtx, &querypb.ReleaseCollectionRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_ReleaseCollection,
|
|
|
|
},
|
|
|
|
CollectionID: defaultCollectionID,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2021-09-15 20:40:07 +08:00
|
|
|
nodes, err := queryCoord.cluster.onlineNodes()
|
2021-07-16 10:21:55 +08:00
|
|
|
assert.Nil(t, err)
|
2021-07-13 14:16:00 +08:00
|
|
|
queryNode3.stop()
|
2021-09-22 17:50:00 +08:00
|
|
|
err = removeNodeSession(queryNode3.queryNodeID)
|
|
|
|
assert.Nil(t, err)
|
2021-07-16 10:21:55 +08:00
|
|
|
|
2021-08-26 14:17:54 +08:00
|
|
|
allNodeOffline := waitAllQueryNodeOffline(queryCoord.cluster, nodes)
|
|
|
|
assert.Equal(t, allNodeOffline, true)
|
2021-07-13 14:16:00 +08:00
|
|
|
queryCoord.Stop()
|
2021-09-22 17:50:00 +08:00
|
|
|
err = removeAllSession()
|
|
|
|
assert.Nil(t, err)
|
2021-07-13 14:16:00 +08:00
|
|
|
}
|
2021-08-17 10:06:11 +08:00
|
|
|
|
|
|
|
func TestQueryNode_getMetrics(t *testing.T) {
|
|
|
|
log.Info("TestQueryNode_getMetrics, todo")
|
|
|
|
}
|
2021-09-15 20:40:07 +08:00
|
|
|
|
|
|
|
func TestNewQueryNode(t *testing.T) {
|
|
|
|
refreshParams()
|
|
|
|
baseCtx, cancel := context.WithCancel(context.Background())
|
|
|
|
kv, err := etcdkv.NewEtcdKV(Params.EtcdEndpoints, Params.MetaRootPath)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
queryNode1, err := startQueryNodeServer(baseCtx)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
addr := queryNode1.session.Address
|
|
|
|
nodeID := queryNode1.queryNodeID
|
|
|
|
node, err := newQueryNode(baseCtx, addr, nodeID, kv)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
err = node.start()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
cancel()
|
|
|
|
node.stop()
|
|
|
|
queryNode1.stop()
|
2021-09-22 17:50:00 +08:00
|
|
|
err = removeAllSession()
|
|
|
|
assert.Nil(t, err)
|
2021-09-15 20:40:07 +08:00
|
|
|
}
|
2021-09-18 15:51:51 +08:00
|
|
|
|
|
|
|
func TestReleaseCollectionOnOfflineNode(t *testing.T) {
|
|
|
|
refreshParams()
|
|
|
|
baseCtx, cancel := context.WithCancel(context.Background())
|
|
|
|
kv, err := etcdkv.NewEtcdKV(Params.EtcdEndpoints, Params.MetaRootPath)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
node, err := newQueryNode(baseCtx, "test", 100, kv)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
node.setState(offline)
|
|
|
|
req := &querypb.ReleaseCollectionRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_ReleaseCollection,
|
|
|
|
},
|
|
|
|
CollectionID: defaultCollectionID,
|
|
|
|
}
|
|
|
|
|
|
|
|
err = node.releaseCollection(baseCtx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
cancel()
|
|
|
|
}
|