2021-10-25 19:44:37 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-04-19 11:35:38 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-10-25 19:44:37 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-04-19 11:35:38 +08:00
|
|
|
//
|
2021-10-25 19:44:37 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2021-06-22 10:42:07 +08:00
|
|
|
package datacoord
|
2021-04-13 09:47:02 +08:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2021-12-19 20:00:42 +08:00
|
|
|
"fmt"
|
2021-08-16 11:00:09 +08:00
|
|
|
"math/rand"
|
2021-08-20 17:50:12 +08:00
|
|
|
"os"
|
2022-02-15 15:07:48 +08:00
|
|
|
"os/signal"
|
2021-05-26 20:14:30 +08:00
|
|
|
"path"
|
2021-06-11 16:09:05 +08:00
|
|
|
"strconv"
|
2023-01-31 14:47:49 +08:00
|
|
|
"strings"
|
2022-10-18 13:39:26 +08:00
|
|
|
"sync"
|
2022-02-15 15:07:48 +08:00
|
|
|
"syscall"
|
2021-04-13 09:47:02 +08:00
|
|
|
"testing"
|
2021-04-24 11:29:15 +08:00
|
|
|
"time"
|
2021-04-13 09:47:02 +08:00
|
|
|
|
2023-02-26 11:31:49 +08:00
|
|
|
"github.com/cockroachdb/errors"
|
2022-10-10 20:31:22 +08:00
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/stretchr/testify/mock"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
clientv3 "go.etcd.io/etcd/client/v3"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2022-10-16 20:49:27 +08:00
|
|
|
"github.com/milvus-io/milvus-proto/go-api/commonpb"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
|
2023-03-04 23:21:50 +08:00
|
|
|
"github.com/milvus-io/milvus-proto/go-api/msgpb"
|
2022-10-16 20:49:27 +08:00
|
|
|
"github.com/milvus-io/milvus-proto/go-api/schemapb"
|
2021-11-19 13:57:12 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/common"
|
2023-01-12 19:49:40 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/indexnode"
|
2021-09-01 10:13:15 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
2023-01-04 19:37:36 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/metastore/model"
|
2022-03-03 21:57:56 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/mq/msgstream"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/datapb"
|
2022-09-16 11:32:48 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/indexpb"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/internalpb"
|
2022-09-26 18:06:54 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
|
|
|
|
"github.com/milvus-io/milvus/internal/storage"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/types"
|
2022-04-07 22:05:32 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/dependency"
|
2021-12-29 14:35:21 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/etcd"
|
2023-01-04 19:37:36 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/funcutil"
|
|
|
|
"github.com/milvus-io/milvus/internal/util/metautil"
|
2021-12-20 15:47:38 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/metricsinfo"
|
2023-01-04 19:37:36 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/paramtable"
|
2021-05-24 09:44:49 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/sessionutil"
|
2023-01-04 19:37:36 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/typeutil"
|
2021-04-13 09:47:02 +08:00
|
|
|
)
|
|
|
|
|
2021-08-20 17:50:12 +08:00
|
|
|
func TestMain(m *testing.M) {
|
2023-01-31 14:47:49 +08:00
|
|
|
// init embed etcd
|
|
|
|
embedetcdServer, tempDir, err := etcd.StartTestEmbedEtcdServer()
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err.Error())
|
|
|
|
}
|
|
|
|
defer os.RemoveAll(tempDir)
|
|
|
|
defer embedetcdServer.Close()
|
|
|
|
|
|
|
|
addrs := etcd.GetEmbedEtcdEndpoints(embedetcdServer)
|
|
|
|
|
2022-11-04 14:25:38 +08:00
|
|
|
paramtable.Init()
|
2023-02-20 21:14:25 +08:00
|
|
|
paramtable.Get().Save(Params.EtcdCfg.Endpoints.Key, strings.Join(addrs, ","))
|
|
|
|
|
2021-08-16 11:00:09 +08:00
|
|
|
rand.Seed(time.Now().UnixNano())
|
2021-08-20 17:50:12 +08:00
|
|
|
os.Exit(m.Run())
|
2021-08-16 11:00:09 +08:00
|
|
|
}
|
|
|
|
|
2021-04-16 16:30:55 +08:00
|
|
|
func TestGetSegmentInfoChannel(t *testing.T) {
|
2021-05-26 19:06:56 +08:00
|
|
|
svr := newTestServer(t, nil)
|
2021-04-16 16:30:55 +08:00
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
t.Run("get segment info channel", func(t *testing.T) {
|
|
|
|
resp, err := svr.GetSegmentInfoChannel(context.TODO())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
2022-12-07 18:01:19 +08:00
|
|
|
assert.EqualValues(t, Params.CommonCfg.DataCoordSegmentInfo.GetValue(), resp.Value)
|
2021-04-16 16:30:55 +08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAssignSegmentID(t *testing.T) {
|
2021-04-24 11:29:15 +08:00
|
|
|
const collID = 100
|
|
|
|
const collIDInvalid = 101
|
|
|
|
const partID = 0
|
|
|
|
const channel0 = "channel0"
|
|
|
|
|
2021-07-23 21:58:33 +08:00
|
|
|
t.Run("assign segment normally", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-09-06 17:02:41 +08:00
|
|
|
ID: collID,
|
|
|
|
Schema: schema,
|
|
|
|
Partitions: []int64{},
|
|
|
|
})
|
2021-07-23 21:58:33 +08:00
|
|
|
req := &datapb.SegmentIDRequest{
|
|
|
|
Count: 1000,
|
|
|
|
ChannelName: channel0,
|
|
|
|
CollectionID: collID,
|
|
|
|
PartitionID: partID,
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.AssignSegmentID(context.TODO(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{req},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 1, len(resp.SegIDAssignments))
|
|
|
|
assign := resp.SegIDAssignments[0]
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, assign.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, collID, assign.CollectionID)
|
|
|
|
assert.EqualValues(t, partID, assign.PartitionID)
|
|
|
|
assert.EqualValues(t, channel0, assign.ChannelName)
|
|
|
|
assert.EqualValues(t, 1000, assign.Count)
|
|
|
|
})
|
2021-04-16 16:30:55 +08:00
|
|
|
|
2022-06-07 16:56:07 +08:00
|
|
|
t.Run("assign segment for bulkload", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-06-07 16:56:07 +08:00
|
|
|
ID: collID,
|
|
|
|
Schema: schema,
|
|
|
|
Partitions: []int64{},
|
|
|
|
})
|
|
|
|
req := &datapb.SegmentIDRequest{
|
|
|
|
Count: 1000,
|
|
|
|
ChannelName: channel0,
|
|
|
|
CollectionID: collID,
|
|
|
|
PartitionID: partID,
|
|
|
|
IsImport: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.AssignSegmentID(context.TODO(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{req},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 1, len(resp.SegIDAssignments))
|
|
|
|
assign := resp.SegIDAssignments[0]
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, assign.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, collID, assign.CollectionID)
|
|
|
|
assert.EqualValues(t, partID, assign.PartitionID)
|
|
|
|
assert.EqualValues(t, channel0, assign.ChannelName)
|
|
|
|
assert.EqualValues(t, 1000, assign.Count)
|
|
|
|
})
|
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
req := &datapb.SegmentIDRequest{
|
|
|
|
Count: 100,
|
|
|
|
ChannelName: channel0,
|
|
|
|
CollectionID: collID,
|
|
|
|
PartitionID: partID,
|
|
|
|
}
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.AssignSegmentID(context.Background(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{req},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
|
|
|
|
2021-07-23 21:58:33 +08:00
|
|
|
t.Run("assign segment with invalid collection", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2022-09-26 18:06:54 +08:00
|
|
|
svr.rootCoordClient = &mockRootCoord{
|
2021-11-18 22:31:34 +08:00
|
|
|
RootCoord: svr.rootCoordClient,
|
|
|
|
collID: collID,
|
|
|
|
}
|
2021-09-06 17:02:41 +08:00
|
|
|
schema := newTestSchema()
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-09-06 17:02:41 +08:00
|
|
|
ID: collID,
|
|
|
|
Schema: schema,
|
|
|
|
Partitions: []int64{},
|
|
|
|
})
|
2021-07-23 21:58:33 +08:00
|
|
|
req := &datapb.SegmentIDRequest{
|
|
|
|
Count: 1000,
|
|
|
|
ChannelName: channel0,
|
|
|
|
CollectionID: collIDInvalid,
|
|
|
|
PartitionID: partID,
|
|
|
|
}
|
2021-04-16 16:30:55 +08:00
|
|
|
|
2021-07-23 21:58:33 +08:00
|
|
|
resp, err := svr.AssignSegmentID(context.TODO(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{req},
|
2021-04-16 16:30:55 +08:00
|
|
|
})
|
2021-07-23 21:58:33 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 0, len(resp.SegIDAssignments))
|
|
|
|
})
|
2021-04-16 16:30:55 +08:00
|
|
|
}
|
|
|
|
|
2022-09-26 18:06:54 +08:00
|
|
|
type mockRootCoord struct {
|
2021-11-18 22:31:34 +08:00
|
|
|
types.RootCoord
|
|
|
|
collID UniqueID
|
|
|
|
}
|
|
|
|
|
2023-01-04 16:37:35 +08:00
|
|
|
func (r *mockRootCoord) DescribeCollectionInternal(ctx context.Context, req *milvuspb.DescribeCollectionRequest) (*milvuspb.DescribeCollectionResponse, error) {
|
2021-11-18 22:31:34 +08:00
|
|
|
if req.CollectionID != r.collID {
|
|
|
|
return &milvuspb.DescribeCollectionResponse{
|
|
|
|
Status: &commonpb.Status{
|
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
|
|
|
Reason: "Collection not found",
|
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
return r.RootCoord.DescribeCollection(ctx, req)
|
|
|
|
}
|
|
|
|
|
2022-09-26 18:06:54 +08:00
|
|
|
func (r *mockRootCoord) ReportImport(context.Context, *rootcoordpb.ImportResult) (*commonpb.Status, error) {
|
|
|
|
return &commonpb.Status{
|
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
|
|
|
Reason: "something bad",
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2021-04-16 16:30:55 +08:00
|
|
|
func TestFlush(t *testing.T) {
|
2021-07-12 17:24:25 +08:00
|
|
|
req := &datapb.FlushRequest{
|
2021-04-16 16:30:55 +08:00
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_Flush,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
DbID: 0,
|
|
|
|
CollectionID: 0,
|
2021-07-12 17:24:25 +08:00
|
|
|
}
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{ID: 0, Schema: schema, Partitions: []int64{}})
|
2021-09-06 11:12:42 +08:00
|
|
|
allocations, err := svr.segmentManager.AllocSegment(context.TODO(), 0, 1, "channel-1", 1)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 1, len(allocations))
|
|
|
|
expireTs := allocations[0].ExpireTime
|
|
|
|
segID := allocations[0].SegmentID
|
|
|
|
|
|
|
|
resp, err := svr.Flush(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
2021-12-15 10:53:16 +08:00
|
|
|
|
|
|
|
svr.meta.SetCurrentRows(segID, 1)
|
2021-09-06 11:12:42 +08:00
|
|
|
ids, err := svr.segmentManager.GetFlushableSegments(context.TODO(), "channel-1", expireTs)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 1, len(ids))
|
|
|
|
assert.EqualValues(t, segID, ids[0])
|
|
|
|
})
|
|
|
|
|
2023-01-31 12:41:53 +08:00
|
|
|
t.Run("bulkload segment", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
|
|
|
svr.meta.AddCollection(&collectionInfo{ID: 0, Schema: schema, Partitions: []int64{}})
|
|
|
|
|
|
|
|
allocations, err := svr.segmentManager.allocSegmentForImport(context.TODO(), 0, 1, "channel-1", 1, 100)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
expireTs := allocations.ExpireTime
|
|
|
|
segID := allocations.SegmentID
|
|
|
|
|
|
|
|
resp, err := svr.Flush(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 0, len(resp.SegmentIDs))
|
|
|
|
// should not flush anything since this is a normal flush
|
|
|
|
svr.meta.SetCurrentRows(segID, 1)
|
|
|
|
ids, err := svr.segmentManager.GetFlushableSegments(context.TODO(), "channel-1", expireTs)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 0, len(ids))
|
|
|
|
|
|
|
|
req := &datapb.FlushRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_Flush,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
DbID: 0,
|
|
|
|
CollectionID: 0,
|
|
|
|
IsImport: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err = svr.Flush(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.SegmentIDs))
|
|
|
|
|
|
|
|
ids, err = svr.segmentManager.GetFlushableSegments(context.TODO(), "channel-1", expireTs)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, 1, len(ids))
|
|
|
|
assert.EqualValues(t, segID, ids[0])
|
|
|
|
})
|
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.Flush(context.Background(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
2021-04-16 16:30:55 +08:00
|
|
|
}
|
|
|
|
|
2021-05-26 19:06:56 +08:00
|
|
|
//func TestGetComponentStates(t *testing.T) {
|
|
|
|
//svr := newTestServer(t)
|
|
|
|
//defer closeTestServer(t, svr)
|
|
|
|
//cli := newMockDataNodeClient(1)
|
|
|
|
//err := cli.Init()
|
|
|
|
//assert.Nil(t, err)
|
|
|
|
//err = cli.Start()
|
|
|
|
//assert.Nil(t, err)
|
|
|
|
|
|
|
|
//err = svr.cluster.Register(&dataNode{
|
|
|
|
//id: 1,
|
|
|
|
//address: struct {
|
|
|
|
//ip string
|
|
|
|
//port int64
|
|
|
|
//}{
|
|
|
|
//ip: "",
|
|
|
|
//port: 0,
|
|
|
|
//},
|
|
|
|
//client: cli,
|
|
|
|
//channelNum: 0,
|
|
|
|
//})
|
|
|
|
//assert.Nil(t, err)
|
|
|
|
|
|
|
|
//resp, err := svr.GetComponentStates(context.TODO())
|
|
|
|
//assert.Nil(t, err)
|
|
|
|
//assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
2022-10-10 15:55:22 +08:00
|
|
|
//assert.EqualValues(t, commonpb.StateCode_Healthy, resp.State.StateCode)
|
2021-05-26 19:06:56 +08:00
|
|
|
//assert.EqualValues(t, 1, len(resp.SubcomponentStates))
|
2022-10-10 15:55:22 +08:00
|
|
|
//assert.EqualValues(t, commonpb.StateCode_Healthy, resp.SubcomponentStates[0].StateCode)
|
2021-05-26 19:06:56 +08:00
|
|
|
//}
|
2021-04-16 16:30:55 +08:00
|
|
|
|
|
|
|
func TestGetTimeTickChannel(t *testing.T) {
|
2021-05-26 19:06:56 +08:00
|
|
|
svr := newTestServer(t, nil)
|
2021-04-16 16:30:55 +08:00
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetTimeTickChannel(context.TODO())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
2022-12-07 18:01:19 +08:00
|
|
|
assert.EqualValues(t, Params.CommonCfg.DataCoordTimeTick.GetValue(), resp.Value)
|
2021-04-16 16:30:55 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestGetSegmentStates(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
t.Run("normal cases", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
segment := &datapb.SegmentInfo{
|
|
|
|
ID: 1000,
|
|
|
|
CollectionID: 100,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "c1",
|
|
|
|
NumOfRows: 0,
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-09-06 17:02:41 +08:00
|
|
|
ChannelName: "c1",
|
|
|
|
MsgID: []byte{},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segment))
|
|
|
|
assert.Nil(t, err)
|
2021-04-16 16:30:55 +08:00
|
|
|
|
2021-09-06 17:02:41 +08:00
|
|
|
cases := []struct {
|
|
|
|
description string
|
|
|
|
id UniqueID
|
|
|
|
expected bool
|
|
|
|
expectedState commonpb.SegmentState
|
|
|
|
}{
|
|
|
|
{"get existed segment", 1000, true, commonpb.SegmentState_Growing},
|
|
|
|
{"get non-existed segment", 10, false, commonpb.SegmentState_Growing},
|
|
|
|
}
|
2021-04-16 16:30:55 +08:00
|
|
|
|
2021-09-06 17:02:41 +08:00
|
|
|
for _, test := range cases {
|
|
|
|
t.Run(test.description, func(t *testing.T) {
|
|
|
|
resp, err := svr.GetSegmentStates(context.TODO(), &datapb.GetSegmentStatesRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: 0,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
SegmentIDs: []int64{test.id},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.States))
|
|
|
|
if test.expected {
|
|
|
|
assert.EqualValues(t, test.expectedState, resp.States[0].State)
|
|
|
|
}
|
2021-04-16 16:30:55 +08:00
|
|
|
})
|
2021-09-06 17:02:41 +08:00
|
|
|
}
|
|
|
|
})
|
2021-09-06 11:12:42 +08:00
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetSegmentStates(context.TODO(), &datapb.GetSegmentStatesRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: 0,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
SegmentIDs: []int64{0},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
2021-04-16 16:30:55 +08:00
|
|
|
}
|
|
|
|
|
2021-04-26 09:45:54 +08:00
|
|
|
func TestGetInsertBinlogPaths(t *testing.T) {
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2021-04-26 09:45:54 +08:00
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
info := &datapb.SegmentInfo{
|
|
|
|
ID: 0,
|
|
|
|
Binlogs: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "dev/datacoord/testsegment/1/part1",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "dev/datacoord/testsegment/1/part2",
|
|
|
|
},
|
2021-09-06 11:12:42 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2021-11-29 22:35:41 +08:00
|
|
|
State: commonpb.SegmentState_Growing,
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
2021-09-28 22:04:06 +08:00
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(info))
|
|
|
|
assert.Nil(t, err)
|
2021-09-06 11:12:42 +08:00
|
|
|
req := &datapb.GetInsertBinlogPathsRequest{
|
|
|
|
SegmentID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetInsertBinlogPaths(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with invalid segment id", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
info := &datapb.SegmentInfo{
|
|
|
|
ID: 0,
|
|
|
|
Binlogs: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "dev/datacoord/testsegment/1/part1",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "dev/datacoord/testsegment/1/part2",
|
|
|
|
},
|
2021-09-06 11:12:42 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2021-11-29 22:35:41 +08:00
|
|
|
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
2021-09-28 21:54:14 +08:00
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(info))
|
|
|
|
assert.Nil(t, err)
|
2021-09-06 11:12:42 +08:00
|
|
|
req := &datapb.GetInsertBinlogPathsRequest{
|
|
|
|
SegmentID: 1,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetInsertBinlogPaths(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetInsertBinlogPaths(context.TODO(), &datapb.GetInsertBinlogPathsRequest{
|
|
|
|
SegmentID: 0,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
2021-04-26 09:45:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestGetCollectionStatistics(t *testing.T) {
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2021-04-26 09:45:54 +08:00
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
req := &datapb.GetCollectionStatisticsRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetCollectionStatistics(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
|
|
|
|
})
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetCollectionStatistics(context.Background(), &datapb.GetCollectionStatisticsRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestGetPartitionStatistics(t *testing.T) {
|
|
|
|
t.Run("normal cases", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
req := &datapb.GetPartitionStatisticsRequest{
|
|
|
|
CollectionID: 0,
|
2022-07-18 09:58:28 +08:00
|
|
|
PartitionIDs: []int64{0},
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
|
|
|
resp, err := svr.GetPartitionStatistics(context.Background(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
})
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetPartitionStatistics(context.Background(), &datapb.GetPartitionStatisticsRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
2021-04-26 09:45:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestGetSegmentInfo(t *testing.T) {
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2021-04-26 09:45:54 +08:00
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
segInfo := &datapb.SegmentInfo{
|
2022-12-01 16:31:16 +08:00
|
|
|
ID: 0,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
NumOfRows: 100,
|
|
|
|
Binlogs: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 0, 1, 801),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 0, 1, 802),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 0, 1, 803),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
2021-10-01 20:58:10 +08:00
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segInfo))
|
|
|
|
assert.Nil(t, err)
|
2021-09-06 11:12:42 +08:00
|
|
|
|
|
|
|
req := &datapb.GetSegmentInfoRequest{
|
|
|
|
SegmentIDs: []int64{0},
|
|
|
|
}
|
|
|
|
resp, err := svr.GetSegmentInfo(svr.ctx, req)
|
2022-12-01 16:31:16 +08:00
|
|
|
assert.Equal(t, 1, len(resp.GetInfos()))
|
|
|
|
// Check that # of rows is corrected from 100 to 60.
|
|
|
|
assert.EqualValues(t, 60, resp.GetInfos()[0].GetNumOfRows())
|
2021-09-06 11:12:42 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
})
|
|
|
|
t.Run("with wrong segment id", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
2021-11-29 22:35:41 +08:00
|
|
|
ID: 0,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
2021-10-01 21:00:46 +08:00
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segInfo))
|
|
|
|
assert.Nil(t, err)
|
2021-09-06 11:12:42 +08:00
|
|
|
|
|
|
|
req := &datapb.GetSegmentInfoRequest{
|
|
|
|
SegmentIDs: []int64{0, 1},
|
|
|
|
}
|
|
|
|
resp, err := svr.GetSegmentInfo(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode)
|
|
|
|
})
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetSegmentInfo(context.Background(), &datapb.GetSegmentInfoRequest{
|
|
|
|
SegmentIDs: []int64{},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
2022-06-17 18:24:12 +08:00
|
|
|
t.Run("with dropped segment", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: 0,
|
|
|
|
State: commonpb.SegmentState_Dropped,
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segInfo))
|
|
|
|
assert.Nil(t, err)
|
2021-09-06 11:12:42 +08:00
|
|
|
|
2022-06-17 18:24:12 +08:00
|
|
|
req := &datapb.GetSegmentInfoRequest{
|
|
|
|
SegmentIDs: []int64{0},
|
|
|
|
IncludeUnHealthy: false,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetSegmentInfo(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, 0, len(resp.Infos))
|
|
|
|
|
|
|
|
req = &datapb.GetSegmentInfoRequest{
|
|
|
|
SegmentIDs: []int64{0},
|
|
|
|
IncludeUnHealthy: true,
|
|
|
|
}
|
|
|
|
resp2, err := svr.GetSegmentInfo(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, 1, len(resp2.Infos))
|
|
|
|
})
|
2022-12-13 16:17:22 +08:00
|
|
|
|
|
|
|
t.Run("with channel checkpoint", func(t *testing.T) {
|
|
|
|
mockVChannel := "fake-by-dev-rootcoord-dml-1-testgetsegmentinfo-v0"
|
|
|
|
mockPChannel := "fake-by-dev-rootcoord-dml-1"
|
|
|
|
|
2023-03-04 23:21:50 +08:00
|
|
|
pos := &msgpb.MsgPosition{
|
2022-12-13 16:17:22 +08:00
|
|
|
ChannelName: mockPChannel,
|
|
|
|
MsgID: []byte{},
|
|
|
|
Timestamp: 1000,
|
|
|
|
}
|
|
|
|
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: 0,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segInfo))
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
req := &datapb.GetSegmentInfoRequest{
|
|
|
|
SegmentIDs: []int64{0},
|
|
|
|
}
|
|
|
|
// no channel checkpoint
|
|
|
|
resp, err := svr.GetSegmentInfo(svr.ctx, req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.Equal(t, 0, len(resp.GetChannelCheckpoint()))
|
|
|
|
|
|
|
|
// with nil insert channel of segment
|
|
|
|
err = svr.meta.UpdateChannelCheckpoint(mockVChannel, pos)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
resp, err = svr.GetSegmentInfo(svr.ctx, req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.Equal(t, 0, len(resp.GetChannelCheckpoint()))
|
|
|
|
|
|
|
|
// normal test
|
|
|
|
segInfo.InsertChannel = mockVChannel
|
|
|
|
segInfo.ID = 2
|
|
|
|
req.SegmentIDs = []int64{2}
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(segInfo))
|
|
|
|
assert.NoError(t, err)
|
|
|
|
resp, err = svr.GetSegmentInfo(svr.ctx, req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.Equal(t, 1, len(resp.GetChannelCheckpoint()))
|
|
|
|
assert.Equal(t, mockPChannel, resp.ChannelCheckpoint[mockVChannel].ChannelName)
|
|
|
|
assert.Equal(t, Timestamp(1000), resp.ChannelCheckpoint[mockVChannel].Timestamp)
|
|
|
|
})
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
2021-04-26 09:45:54 +08:00
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
func TestGetComponentStates(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2021-11-19 13:57:12 +08:00
|
|
|
resp, err := svr.GetComponentStates(context.Background())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, common.NotRegisteredID, resp.State.NodeID)
|
|
|
|
svr.session = &sessionutil.Session{}
|
|
|
|
svr.session.UpdateRegistered(true)
|
2021-09-06 11:12:42 +08:00
|
|
|
type testCase struct {
|
2022-10-10 15:55:22 +08:00
|
|
|
state commonpb.StateCode
|
|
|
|
code commonpb.StateCode
|
2021-04-26 09:45:54 +08:00
|
|
|
}
|
2021-09-06 11:12:42 +08:00
|
|
|
cases := []testCase{
|
2022-10-10 15:55:22 +08:00
|
|
|
{state: commonpb.StateCode_Abnormal, code: commonpb.StateCode_Abnormal},
|
|
|
|
{state: commonpb.StateCode_Initializing, code: commonpb.StateCode_Initializing},
|
|
|
|
{state: commonpb.StateCode_Healthy, code: commonpb.StateCode_Healthy},
|
2021-09-06 11:12:42 +08:00
|
|
|
}
|
|
|
|
for _, tc := range cases {
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(tc.state)
|
2021-09-06 11:12:42 +08:00
|
|
|
resp, err := svr.GetComponentStates(context.Background())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, tc.code, resp.GetState().GetStateCode())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestGetFlushedSegments(t *testing.T) {
|
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
type testCase struct {
|
|
|
|
collID int64
|
|
|
|
partID int64
|
|
|
|
searchPartID int64
|
|
|
|
flushedSegments []int64
|
|
|
|
unflushedSegments []int64
|
|
|
|
expected []int64
|
|
|
|
}
|
|
|
|
cases := []testCase{
|
|
|
|
{
|
|
|
|
collID: 1,
|
|
|
|
partID: 1,
|
|
|
|
searchPartID: 1,
|
|
|
|
flushedSegments: []int64{1, 2, 3},
|
|
|
|
unflushedSegments: []int64{4},
|
|
|
|
expected: []int64{1, 2, 3},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 1,
|
|
|
|
partID: 2,
|
|
|
|
searchPartID: 2,
|
|
|
|
flushedSegments: []int64{5, 6},
|
|
|
|
unflushedSegments: []int64{},
|
|
|
|
expected: []int64{5, 6},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 2,
|
|
|
|
partID: 3,
|
|
|
|
searchPartID: 3,
|
|
|
|
flushedSegments: []int64{11, 12},
|
|
|
|
unflushedSegments: []int64{},
|
|
|
|
expected: []int64{11, 12},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 1,
|
|
|
|
searchPartID: -1,
|
|
|
|
expected: []int64{1, 2, 3, 5, 6},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 2,
|
|
|
|
searchPartID: -1,
|
|
|
|
expected: []int64{11, 12},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
|
|
for _, fs := range tc.flushedSegments {
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: fs,
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.partID,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
}
|
|
|
|
assert.Nil(t, svr.meta.AddSegment(NewSegmentInfo(segInfo)))
|
|
|
|
}
|
|
|
|
for _, us := range tc.unflushedSegments {
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: us,
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.partID,
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
}
|
|
|
|
assert.Nil(t, svr.meta.AddSegment(NewSegmentInfo(segInfo)))
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.GetFlushedSegments(context.Background(), &datapb.GetFlushedSegmentsRequest{
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.searchPartID,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
|
|
|
|
|
|
assert.ElementsMatch(t, tc.expected, resp.GetSegments())
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetFlushedSegments(context.Background(), &datapb.GetFlushedSegmentsRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
|
|
|
})
|
2021-04-26 09:45:54 +08:00
|
|
|
}
|
|
|
|
|
2022-10-08 11:51:02 +08:00
|
|
|
func TestGetSegmentsByStates(t *testing.T) {
|
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
type testCase struct {
|
|
|
|
collID int64
|
|
|
|
partID int64
|
|
|
|
searchPartID int64
|
|
|
|
flushedSegments []int64
|
|
|
|
sealedSegments []int64
|
|
|
|
growingSegments []int64
|
|
|
|
expected []int64
|
|
|
|
}
|
|
|
|
cases := []testCase{
|
|
|
|
{
|
|
|
|
collID: 1,
|
|
|
|
partID: 1,
|
|
|
|
searchPartID: 1,
|
|
|
|
flushedSegments: []int64{1, 2, 3},
|
|
|
|
sealedSegments: []int64{4},
|
|
|
|
growingSegments: []int64{5},
|
|
|
|
expected: []int64{1, 2, 3, 4},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 1,
|
|
|
|
partID: 2,
|
|
|
|
searchPartID: 2,
|
|
|
|
flushedSegments: []int64{6, 7},
|
|
|
|
sealedSegments: []int64{},
|
|
|
|
growingSegments: []int64{8},
|
|
|
|
expected: []int64{6, 7},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 2,
|
|
|
|
partID: 3,
|
|
|
|
searchPartID: 3,
|
|
|
|
flushedSegments: []int64{9, 10},
|
|
|
|
sealedSegments: []int64{},
|
|
|
|
growingSegments: []int64{11},
|
|
|
|
expected: []int64{9, 10},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 1,
|
|
|
|
searchPartID: -1,
|
|
|
|
expected: []int64{1, 2, 3, 4, 6, 7},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
collID: 2,
|
|
|
|
searchPartID: -1,
|
|
|
|
expected: []int64{9, 10},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
|
|
for _, fs := range tc.flushedSegments {
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: fs,
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.partID,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
}
|
|
|
|
assert.Nil(t, svr.meta.AddSegment(NewSegmentInfo(segInfo)))
|
|
|
|
}
|
|
|
|
for _, us := range tc.sealedSegments {
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: us,
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.partID,
|
|
|
|
State: commonpb.SegmentState_Sealed,
|
|
|
|
}
|
|
|
|
assert.Nil(t, svr.meta.AddSegment(NewSegmentInfo(segInfo)))
|
|
|
|
}
|
|
|
|
for _, us := range tc.growingSegments {
|
|
|
|
segInfo := &datapb.SegmentInfo{
|
|
|
|
ID: us,
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.partID,
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
}
|
|
|
|
assert.Nil(t, svr.meta.AddSegment(NewSegmentInfo(segInfo)))
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.GetSegmentsByStates(context.Background(), &datapb.GetSegmentsByStatesRequest{
|
|
|
|
CollectionID: tc.collID,
|
|
|
|
PartitionID: tc.searchPartID,
|
|
|
|
States: []commonpb.SegmentState{commonpb.SegmentState_Sealed, commonpb.SegmentState_Flushed},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
|
|
|
|
|
|
assert.ElementsMatch(t, tc.expected, resp.GetSegments())
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetSegmentsByStates(context.Background(), &datapb.GetSegmentsByStatesRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-10-14 19:20:35 +08:00
|
|
|
func TestService_WatchServices(t *testing.T) {
|
2022-02-15 15:07:48 +08:00
|
|
|
sc := make(chan os.Signal, 1)
|
|
|
|
signal.Notify(sc, syscall.SIGINT)
|
|
|
|
defer signal.Reset(syscall.SIGINT)
|
2022-04-07 22:05:32 +08:00
|
|
|
factory := dependency.NewDefaultFactory(true)
|
2021-12-29 14:35:21 +08:00
|
|
|
svr := CreateServer(context.TODO(), factory)
|
2022-02-15 15:07:48 +08:00
|
|
|
svr.session = &sessionutil.Session{
|
|
|
|
TriggerKill: true,
|
|
|
|
}
|
2021-10-15 15:56:35 +08:00
|
|
|
svr.serverLoopWg.Add(1)
|
2021-10-14 19:20:35 +08:00
|
|
|
|
|
|
|
ech := make(chan *sessionutil.SessionEvent)
|
2022-05-31 16:36:03 +08:00
|
|
|
svr.dnEventCh = ech
|
2021-10-14 19:20:35 +08:00
|
|
|
|
|
|
|
flag := false
|
2022-02-15 15:07:48 +08:00
|
|
|
closed := false
|
|
|
|
sigDone := make(chan struct{}, 1)
|
|
|
|
sigQuit := make(chan struct{}, 1)
|
2021-10-14 19:20:35 +08:00
|
|
|
|
|
|
|
go func() {
|
2021-10-29 21:30:49 +08:00
|
|
|
svr.watchService(context.Background())
|
2021-10-14 19:20:35 +08:00
|
|
|
flag = true
|
2022-02-15 15:07:48 +08:00
|
|
|
sigDone <- struct{}{}
|
|
|
|
}()
|
|
|
|
go func() {
|
|
|
|
<-sc
|
|
|
|
closed = true
|
|
|
|
sigQuit <- struct{}{}
|
2021-10-14 19:20:35 +08:00
|
|
|
}()
|
|
|
|
|
|
|
|
close(ech)
|
2022-02-15 15:07:48 +08:00
|
|
|
<-sigDone
|
|
|
|
<-sigQuit
|
2021-10-14 19:20:35 +08:00
|
|
|
assert.True(t, flag)
|
2022-02-15 15:07:48 +08:00
|
|
|
assert.True(t, closed)
|
2021-10-14 19:20:35 +08:00
|
|
|
|
|
|
|
ech = make(chan *sessionutil.SessionEvent)
|
|
|
|
|
|
|
|
flag = false
|
2022-05-31 16:36:03 +08:00
|
|
|
svr.dnEventCh = ech
|
2021-10-14 19:20:35 +08:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2021-10-15 15:56:35 +08:00
|
|
|
svr.serverLoopWg.Add(1)
|
|
|
|
|
2021-10-14 19:20:35 +08:00
|
|
|
go func() {
|
2021-10-29 21:30:49 +08:00
|
|
|
svr.watchService(ctx)
|
2021-10-14 19:20:35 +08:00
|
|
|
flag = true
|
2022-02-15 15:07:48 +08:00
|
|
|
sigDone <- struct{}{}
|
2021-10-14 19:20:35 +08:00
|
|
|
}()
|
|
|
|
|
|
|
|
ech <- nil
|
|
|
|
cancel()
|
2022-02-15 15:07:48 +08:00
|
|
|
<-sigDone
|
2021-10-14 19:20:35 +08:00
|
|
|
assert.True(t, flag)
|
|
|
|
}
|
|
|
|
|
2022-07-07 14:44:21 +08:00
|
|
|
//func TestServer_watchCoord(t *testing.T) {
|
|
|
|
// Params.Init()
|
|
|
|
// etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
|
|
|
|
// assert.Nil(t, err)
|
|
|
|
// etcdKV := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath)
|
|
|
|
// assert.NotNil(t, etcdKV)
|
|
|
|
// factory := dependency.NewDefaultFactory(true)
|
|
|
|
// svr := CreateServer(context.TODO(), factory)
|
|
|
|
// svr.session = &sessionutil.Session{
|
|
|
|
// TriggerKill: true,
|
|
|
|
// }
|
|
|
|
// svr.kvClient = etcdKV
|
|
|
|
//
|
|
|
|
// dnCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
// //icCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
// qcCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
// rcCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
//
|
|
|
|
// svr.dnEventCh = dnCh
|
|
|
|
// //svr.icEventCh = icCh
|
|
|
|
// svr.qcEventCh = qcCh
|
|
|
|
// svr.rcEventCh = rcCh
|
|
|
|
//
|
|
|
|
// segRefer, err := NewSegmentReferenceManager(etcdKV, nil)
|
|
|
|
// assert.NoError(t, err)
|
|
|
|
// assert.NotNil(t, segRefer)
|
|
|
|
// svr.segReferManager = segRefer
|
|
|
|
//
|
|
|
|
// sc := make(chan os.Signal, 1)
|
|
|
|
// signal.Notify(sc, syscall.SIGINT)
|
|
|
|
// defer signal.Reset(syscall.SIGINT)
|
|
|
|
// closed := false
|
|
|
|
// sigQuit := make(chan struct{}, 1)
|
|
|
|
//
|
|
|
|
// svr.serverLoopWg.Add(1)
|
|
|
|
// go func() {
|
|
|
|
// svr.watchService(context.Background())
|
|
|
|
// }()
|
|
|
|
//
|
|
|
|
// go func() {
|
|
|
|
// <-sc
|
|
|
|
// closed = true
|
|
|
|
// sigQuit <- struct{}{}
|
|
|
|
// }()
|
|
|
|
//
|
|
|
|
// icCh <- &sessionutil.SessionEvent{
|
|
|
|
// EventType: sessionutil.SessionAddEvent,
|
|
|
|
// Session: &sessionutil.Session{
|
|
|
|
// ServerID: 1,
|
|
|
|
// },
|
|
|
|
// }
|
|
|
|
// icCh <- &sessionutil.SessionEvent{
|
|
|
|
// EventType: sessionutil.SessionDelEvent,
|
|
|
|
// Session: &sessionutil.Session{
|
|
|
|
// ServerID: 1,
|
|
|
|
// },
|
|
|
|
// }
|
|
|
|
// close(icCh)
|
|
|
|
// <-sigQuit
|
|
|
|
// svr.serverLoopWg.Wait()
|
|
|
|
// assert.True(t, closed)
|
|
|
|
//}
|
2022-05-31 16:36:03 +08:00
|
|
|
|
2023-01-04 19:37:36 +08:00
|
|
|
//func TestServer_watchQueryCoord(t *testing.T) {
|
|
|
|
// Params.Init()
|
|
|
|
// etcdCli, err := etcd.GetEtcdClient(
|
|
|
|
// Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
|
|
|
|
// Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
|
|
|
|
// Params.EtcdCfg.Endpoints.GetAsStrings(),
|
|
|
|
// Params.EtcdCfg.EtcdTLSCert.GetValue(),
|
|
|
|
// Params.EtcdCfg.EtcdTLSKey.GetValue(),
|
|
|
|
// Params.EtcdCfg.EtcdTLSCACert.GetValue(),
|
|
|
|
// Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
|
|
|
|
// assert.Nil(t, err)
|
|
|
|
// etcdKV := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath.GetValue())
|
|
|
|
// assert.NotNil(t, etcdKV)
|
|
|
|
// factory := dependency.NewDefaultFactory(true)
|
|
|
|
// svr := CreateServer(context.TODO(), factory)
|
|
|
|
// svr.session = &sessionutil.Session{
|
|
|
|
// TriggerKill: true,
|
|
|
|
// }
|
|
|
|
// svr.kvClient = etcdKV
|
|
|
|
//
|
|
|
|
// dnCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
// //icCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
// qcCh := make(chan *sessionutil.SessionEvent)
|
|
|
|
//
|
|
|
|
// svr.dnEventCh = dnCh
|
|
|
|
//
|
|
|
|
// segRefer, err := NewSegmentReferenceManager(etcdKV, nil)
|
|
|
|
// assert.NoError(t, err)
|
|
|
|
// assert.NotNil(t, segRefer)
|
|
|
|
//
|
|
|
|
// sc := make(chan os.Signal, 1)
|
|
|
|
// signal.Notify(sc, syscall.SIGINT)
|
|
|
|
// defer signal.Reset(syscall.SIGINT)
|
|
|
|
// closed := false
|
|
|
|
// sigQuit := make(chan struct{}, 1)
|
|
|
|
//
|
|
|
|
// svr.serverLoopWg.Add(1)
|
|
|
|
// go func() {
|
|
|
|
// svr.watchService(context.Background())
|
|
|
|
// }()
|
|
|
|
//
|
|
|
|
// go func() {
|
|
|
|
// <-sc
|
|
|
|
// closed = true
|
|
|
|
// sigQuit <- struct{}{}
|
|
|
|
// }()
|
|
|
|
//
|
|
|
|
// qcCh <- &sessionutil.SessionEvent{
|
|
|
|
// EventType: sessionutil.SessionAddEvent,
|
|
|
|
// Session: &sessionutil.Session{
|
|
|
|
// ServerID: 2,
|
|
|
|
// },
|
|
|
|
// }
|
|
|
|
// qcCh <- &sessionutil.SessionEvent{
|
|
|
|
// EventType: sessionutil.SessionDelEvent,
|
|
|
|
// Session: &sessionutil.Session{
|
|
|
|
// ServerID: 2,
|
|
|
|
// },
|
|
|
|
// }
|
|
|
|
// close(qcCh)
|
|
|
|
// <-sigQuit
|
|
|
|
// svr.serverLoopWg.Wait()
|
|
|
|
// assert.True(t, closed)
|
|
|
|
//}
|
2022-06-15 12:20:10 +08:00
|
|
|
|
2022-08-12 13:20:39 +08:00
|
|
|
func TestServer_ShowConfigurations(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2022-12-09 14:31:21 +08:00
|
|
|
pattern := "datacoord.Port"
|
2022-08-12 13:20:39 +08:00
|
|
|
req := &internalpb.ShowConfigurationsRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_WatchQueryChannels,
|
|
|
|
MsgID: rand.Int63(),
|
|
|
|
},
|
|
|
|
Pattern: pattern,
|
|
|
|
}
|
|
|
|
|
|
|
|
// server is closed
|
2022-10-10 15:55:22 +08:00
|
|
|
stateSave := svr.stateCode.Load()
|
|
|
|
svr.stateCode.Store(commonpb.StateCode_Initializing)
|
2022-08-12 13:20:39 +08:00
|
|
|
resp, err := svr.ShowConfigurations(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode)
|
|
|
|
|
|
|
|
// normal case
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(stateSave)
|
2022-08-12 13:20:39 +08:00
|
|
|
|
|
|
|
resp, err = svr.ShowConfigurations(svr.ctx, req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.Equal(t, 1, len(resp.Configuations))
|
|
|
|
assert.Equal(t, "datacoord.port", resp.Configuations[0].Key)
|
|
|
|
}
|
|
|
|
|
2021-09-01 10:13:15 +08:00
|
|
|
func TestServer_GetMetrics(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
var err error
|
|
|
|
|
|
|
|
// server is closed
|
2022-10-10 15:55:22 +08:00
|
|
|
stateSave := svr.stateCode.Load()
|
|
|
|
svr.stateCode.Store(commonpb.StateCode_Initializing)
|
2021-09-01 10:13:15 +08:00
|
|
|
resp, err := svr.GetMetrics(svr.ctx, &milvuspb.GetMetricsRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(stateSave)
|
2021-09-01 10:13:15 +08:00
|
|
|
|
|
|
|
// failed to parse metric type
|
|
|
|
invalidRequest := "invalid request"
|
|
|
|
resp, err = svr.GetMetrics(svr.ctx, &milvuspb.GetMetricsRequest{
|
|
|
|
Request: invalidRequest,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
|
|
|
|
// unsupported metric type
|
|
|
|
unsupportedMetricType := "unsupported"
|
|
|
|
req, err := metricsinfo.ConstructRequestByMetricType(unsupportedMetricType)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
resp, err = svr.GetMetrics(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
|
|
|
|
// normal case
|
|
|
|
req, err = metricsinfo.ConstructRequestByMetricType(metricsinfo.SystemInfoMetrics)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
resp, err = svr.GetMetrics(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
log.Info("TestServer_GetMetrics",
|
|
|
|
zap.String("name", resp.ComponentName),
|
|
|
|
zap.String("response", resp.Response))
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestServer_getSystemInfoMetrics(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
req, err := metricsinfo.ConstructRequestByMetricType(metricsinfo.SystemInfoMetrics)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
resp, err := svr.getSystemInfoMetrics(svr.ctx, req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
log.Info("TestServer_getSystemInfoMetrics",
|
|
|
|
zap.String("name", resp.ComponentName),
|
|
|
|
zap.String("response", resp.Response))
|
|
|
|
|
|
|
|
var coordTopology metricsinfo.DataCoordTopology
|
|
|
|
err = metricsinfo.UnmarshalTopology(resp.Response, &coordTopology)
|
|
|
|
assert.Nil(t, err)
|
2023-01-06 14:21:37 +08:00
|
|
|
assert.Equal(t, len(svr.cluster.GetSessions()), len(coordTopology.Cluster.ConnectedDataNodes))
|
|
|
|
for _, nodeMetrics := range coordTopology.Cluster.ConnectedDataNodes {
|
2021-09-01 10:13:15 +08:00
|
|
|
assert.Equal(t, false, nodeMetrics.HasError)
|
|
|
|
assert.Equal(t, 0, len(nodeMetrics.ErrorReason))
|
|
|
|
_, err = metricsinfo.MarshalComponentInfos(nodeMetrics)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-12 00:22:42 +08:00
|
|
|
type spySegmentManager struct {
|
|
|
|
spyCh chan struct{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// AllocSegment allocates rows and record the allocation.
|
|
|
|
func (s *spySegmentManager) AllocSegment(ctx context.Context, collectionID UniqueID, partitionID UniqueID, channelName string, requestRows int64) ([]*Allocation, error) {
|
|
|
|
panic("not implemented") // TODO: Implement
|
|
|
|
}
|
|
|
|
|
2022-06-27 13:56:17 +08:00
|
|
|
func (s *spySegmentManager) allocSegmentForImport(ctx context.Context, collectionID UniqueID, partitionID UniqueID, channelName string, requestRows int64, taskID int64) (*Allocation, error) {
|
2022-06-07 16:56:07 +08:00
|
|
|
panic("not implemented") // TODO: Implement
|
|
|
|
}
|
|
|
|
|
2021-11-12 00:22:42 +08:00
|
|
|
// DropSegment drops the segment from manager.
|
|
|
|
func (s *spySegmentManager) DropSegment(ctx context.Context, segmentID UniqueID) {
|
|
|
|
}
|
|
|
|
|
|
|
|
// SealAllSegments seals all segments of collection with collectionID and return sealed segments
|
2023-01-31 12:41:53 +08:00
|
|
|
func (s *spySegmentManager) SealAllSegments(ctx context.Context, collectionID UniqueID, segIDs []UniqueID, isImport bool) ([]UniqueID, error) {
|
2021-11-12 00:22:42 +08:00
|
|
|
panic("not implemented") // TODO: Implement
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetFlushableSegments returns flushable segment ids
|
|
|
|
func (s *spySegmentManager) GetFlushableSegments(ctx context.Context, channel string, ts Timestamp) ([]UniqueID, error) {
|
|
|
|
panic("not implemented") // TODO: Implement
|
|
|
|
}
|
|
|
|
|
|
|
|
// ExpireAllocations notifies segment status to expire old allocations
|
|
|
|
func (s *spySegmentManager) ExpireAllocations(channel string, ts Timestamp) error {
|
|
|
|
panic("not implemented") // TODO: Implement
|
|
|
|
}
|
|
|
|
|
|
|
|
// DropSegmentsOfChannel drops all segments in a channel
|
|
|
|
func (s *spySegmentManager) DropSegmentsOfChannel(ctx context.Context, channel string) {
|
|
|
|
s.spyCh <- struct{}{}
|
|
|
|
}
|
|
|
|
|
2021-05-20 11:34:45 +08:00
|
|
|
func TestSaveBinlogPaths(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
t.Run("Normal SaveRequest", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2021-05-20 11:34:45 +08:00
|
|
|
|
2022-09-16 11:32:48 +08:00
|
|
|
// vecFieldID := int64(201)
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-09-16 11:32:48 +08:00
|
|
|
ID: 0,
|
|
|
|
})
|
2021-05-20 11:34:45 +08:00
|
|
|
|
2021-09-06 17:02:41 +08:00
|
|
|
segments := []struct {
|
|
|
|
id UniqueID
|
|
|
|
collectionID UniqueID
|
|
|
|
}{
|
2021-11-12 00:22:42 +08:00
|
|
|
{0, 0},
|
|
|
|
{1, 0},
|
2021-07-12 17:24:25 +08:00
|
|
|
}
|
2021-09-06 17:02:41 +08:00
|
|
|
for _, segment := range segments {
|
|
|
|
s := &datapb.SegmentInfo{
|
2021-11-12 00:22:42 +08:00
|
|
|
ID: segment.id,
|
|
|
|
CollectionID: segment.collectionID,
|
|
|
|
InsertChannel: "ch1",
|
2021-11-29 22:35:41 +08:00
|
|
|
State: commonpb.SegmentState_Growing,
|
2021-09-06 17:02:41 +08:00
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
}
|
|
|
|
|
2021-11-12 00:22:42 +08:00
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2021-11-12 00:22:42 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-05-20 11:34:45 +08:00
|
|
|
ctx := context.Background()
|
|
|
|
resp, err := svr.SaveBinlogPaths(ctx, &datapb.SaveBinlogPathsRequest{
|
2021-05-21 16:54:29 +08:00
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
Timestamp: uint64(time.Now().Unix()),
|
|
|
|
},
|
2021-11-12 00:22:42 +08:00
|
|
|
SegmentID: 1,
|
2021-05-20 11:34:45 +08:00
|
|
|
CollectionID: 0,
|
2021-08-19 13:00:12 +08:00
|
|
|
Field2BinlogPaths: []*datapb.FieldBinlog{
|
2021-05-26 12:21:55 +08:00
|
|
|
{
|
2021-08-19 13:00:12 +08:00
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
2023-02-27 10:41:46 +08:00
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo1",
|
|
|
|
EntriesNum: 5,
|
2021-12-19 20:00:42 +08:00
|
|
|
},
|
|
|
|
{
|
2023-02-27 10:41:46 +08:00
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo2",
|
|
|
|
EntriesNum: 5,
|
2021-12-19 20:00:42 +08:00
|
|
|
},
|
2021-05-28 15:13:51 +08:00
|
|
|
},
|
2021-05-26 12:21:55 +08:00
|
|
|
},
|
2021-05-20 11:34:45 +08:00
|
|
|
},
|
2021-06-04 11:45:45 +08:00
|
|
|
CheckPoints: []*datapb.CheckPoint{
|
|
|
|
{
|
2023-02-27 10:41:46 +08:00
|
|
|
SegmentID: 1,
|
2023-03-04 23:21:50 +08:00
|
|
|
Position: &msgpb.MsgPosition{
|
2021-06-04 11:45:45 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
2023-02-27 10:41:46 +08:00
|
|
|
NumOfRows: 12,
|
2021-06-04 11:45:45 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Flushed: false,
|
2021-05-20 11:34:45 +08:00
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, resp.ErrorCode, commonpb.ErrorCode_Success)
|
|
|
|
|
2023-03-03 14:13:49 +08:00
|
|
|
segment := svr.meta.GetHealthySegment(1)
|
2021-08-19 13:00:12 +08:00
|
|
|
assert.NotNil(t, segment)
|
|
|
|
binlogs := segment.GetBinlogs()
|
|
|
|
assert.EqualValues(t, 1, len(binlogs))
|
|
|
|
fieldBinlogs := binlogs[0]
|
|
|
|
assert.NotNil(t, fieldBinlogs)
|
|
|
|
assert.EqualValues(t, 2, len(fieldBinlogs.GetBinlogs()))
|
|
|
|
assert.EqualValues(t, 1, fieldBinlogs.GetFieldID())
|
2022-11-18 15:35:09 +08:00
|
|
|
assert.EqualValues(t, "/by-dev/test/0/1/1/1/Allo1", fieldBinlogs.GetBinlogs()[0].GetLogPath())
|
|
|
|
assert.EqualValues(t, "/by-dev/test/0/1/1/1/Allo2", fieldBinlogs.GetBinlogs()[1].GetLogPath())
|
2021-05-20 11:34:45 +08:00
|
|
|
|
2023-02-27 10:41:46 +08:00
|
|
|
assert.EqualValues(t, segment.DmlPosition.ChannelName, "ch1")
|
|
|
|
assert.EqualValues(t, segment.DmlPosition.MsgID, []byte{1, 2, 3})
|
|
|
|
assert.EqualValues(t, segment.NumOfRows, 10)
|
2021-05-20 11:34:45 +08:00
|
|
|
})
|
2021-09-06 11:12:42 +08:00
|
|
|
|
2023-03-03 14:13:49 +08:00
|
|
|
t.Run("SaveDroppedSegment", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
// vecFieldID := int64(201)
|
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
})
|
|
|
|
|
|
|
|
segments := []struct {
|
|
|
|
id UniqueID
|
|
|
|
collectionID UniqueID
|
|
|
|
}{
|
|
|
|
{0, 0},
|
|
|
|
{1, 0},
|
|
|
|
}
|
|
|
|
for _, segment := range segments {
|
|
|
|
s := &datapb.SegmentInfo{
|
|
|
|
ID: segment.id,
|
|
|
|
CollectionID: segment.collectionID,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Dropped,
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
resp, err := svr.SaveBinlogPaths(ctx, &datapb.SaveBinlogPathsRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
Timestamp: uint64(time.Now().Unix()),
|
|
|
|
},
|
|
|
|
SegmentID: 1,
|
|
|
|
CollectionID: 0,
|
|
|
|
Field2BinlogPaths: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo1",
|
|
|
|
EntriesNum: 5,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo2",
|
|
|
|
EntriesNum: 5,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
CheckPoints: []*datapb.CheckPoint{
|
|
|
|
{
|
|
|
|
SegmentID: 1,
|
2023-03-04 23:21:50 +08:00
|
|
|
Position: &msgpb.MsgPosition{
|
2023-03-03 14:13:49 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
NumOfRows: 12,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Flushed: false,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, resp.ErrorCode, commonpb.ErrorCode_Success)
|
|
|
|
|
|
|
|
segment := svr.meta.GetSegment(1)
|
|
|
|
assert.NotNil(t, segment)
|
|
|
|
binlogs := segment.GetBinlogs()
|
|
|
|
assert.EqualValues(t, 0, len(binlogs))
|
|
|
|
assert.EqualValues(t, segment.NumOfRows, 0)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("SaveUnhealthySegment", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
// vecFieldID := int64(201)
|
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
})
|
|
|
|
|
|
|
|
segments := []struct {
|
|
|
|
id UniqueID
|
|
|
|
collectionID UniqueID
|
|
|
|
}{
|
|
|
|
{0, 0},
|
|
|
|
{1, 0},
|
|
|
|
}
|
|
|
|
for _, segment := range segments {
|
|
|
|
s := &datapb.SegmentInfo{
|
|
|
|
ID: segment.id,
|
|
|
|
CollectionID: segment.collectionID,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_NotExist,
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
resp, err := svr.SaveBinlogPaths(ctx, &datapb.SaveBinlogPathsRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
Timestamp: uint64(time.Now().Unix()),
|
|
|
|
},
|
|
|
|
SegmentID: 1,
|
|
|
|
CollectionID: 0,
|
|
|
|
Field2BinlogPaths: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo1",
|
|
|
|
EntriesNum: 5,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo2",
|
|
|
|
EntriesNum: 5,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
CheckPoints: []*datapb.CheckPoint{
|
|
|
|
{
|
|
|
|
SegmentID: 1,
|
2023-03-04 23:21:50 +08:00
|
|
|
Position: &msgpb.MsgPosition{
|
2023-03-03 14:13:49 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
NumOfRows: 12,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Flushed: false,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, resp.ErrorCode, commonpb.ErrorCode_SegmentNotFound)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("SaveNotExistSegment", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
// vecFieldID := int64(201)
|
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
})
|
|
|
|
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
resp, err := svr.SaveBinlogPaths(ctx, &datapb.SaveBinlogPathsRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
Timestamp: uint64(time.Now().Unix()),
|
|
|
|
},
|
|
|
|
SegmentID: 1,
|
|
|
|
CollectionID: 0,
|
|
|
|
Field2BinlogPaths: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo1",
|
|
|
|
EntriesNum: 5,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/1/1/Allo2",
|
|
|
|
EntriesNum: 5,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
CheckPoints: []*datapb.CheckPoint{
|
|
|
|
{
|
|
|
|
SegmentID: 1,
|
2023-03-04 23:21:50 +08:00
|
|
|
Position: &msgpb.MsgPosition{
|
2023-03-03 14:13:49 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
NumOfRows: 12,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Flushed: false,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, resp.ErrorCode, commonpb.ErrorCode_SegmentNotFound)
|
|
|
|
})
|
|
|
|
|
2022-05-27 16:20:00 +08:00
|
|
|
t.Run("with channel not matched", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
require.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2022-05-27 16:20:00 +08:00
|
|
|
require.Nil(t, err)
|
|
|
|
s := &datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
InsertChannel: "ch2",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
}
|
|
|
|
svr.meta.AddSegment(NewSegmentInfo(s))
|
|
|
|
|
|
|
|
resp, err := svr.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{
|
|
|
|
SegmentID: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_MetaFailed, resp.GetErrorCode())
|
|
|
|
})
|
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.SaveBinlogPaths(context.Background(), &datapb.SaveBinlogPathsRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetReason())
|
|
|
|
})
|
2021-12-02 16:39:33 +08:00
|
|
|
/*
|
|
|
|
t.Run("test save dropped segment and remove channel", func(t *testing.T) {
|
|
|
|
spyCh := make(chan struct{}, 1)
|
2023-01-12 19:49:40 +08:00
|
|
|
svr := newTestServer(t, nil, WithSegmentManager(&spySegmentManager{spyCh: spyCh}))
|
2021-12-02 16:39:33 +08:00
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{ID: 1})
|
2021-12-02 16:39:33 +08:00
|
|
|
err := svr.meta.AddSegment(&SegmentInfo{
|
2022-08-20 10:24:51 +08:00
|
|
|
Segment: &datapb.SegmentInfo{
|
2021-12-02 16:39:33 +08:00
|
|
|
ID: 1,
|
|
|
|
CollectionID: 1,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2021-11-12 00:22:42 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
err = svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 1})
|
2021-12-02 16:39:33 +08:00
|
|
|
assert.Nil(t, err)
|
2021-11-12 00:22:42 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
_, err = svr.SaveBinlogPaths(context.TODO(), &datapb.SaveBinlogPathsRequest{
|
|
|
|
SegmentID: 1,
|
|
|
|
Dropped: true,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
<-spyCh
|
|
|
|
})*/
|
2021-05-20 11:34:45 +08:00
|
|
|
}
|
|
|
|
|
2021-11-29 22:35:41 +08:00
|
|
|
func TestDropVirtualChannel(t *testing.T) {
|
|
|
|
t.Run("normal DropVirtualChannel", func(t *testing.T) {
|
|
|
|
spyCh := make(chan struct{}, 1)
|
2023-01-12 19:49:40 +08:00
|
|
|
svr := newTestServer(t, nil, WithSegmentManager(&spySegmentManager{spyCh: spyCh}))
|
2021-11-29 22:35:41 +08:00
|
|
|
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2022-09-16 11:32:48 +08:00
|
|
|
vecFieldID := int64(201)
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-09-16 11:32:48 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: &schemapb.CollectionSchema{
|
|
|
|
Fields: []*schemapb.FieldSchema{
|
|
|
|
{
|
|
|
|
FieldID: vecFieldID,
|
|
|
|
DataType: schemapb.DataType_FloatVector,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
2021-11-29 22:35:41 +08:00
|
|
|
type testSegment struct {
|
|
|
|
id UniqueID
|
|
|
|
collectionID UniqueID
|
|
|
|
}
|
|
|
|
segments := make([]testSegment, 0, maxOperationsPerTxn) // test batch overflow
|
|
|
|
for i := 0; i < maxOperationsPerTxn; i++ {
|
|
|
|
segments = append(segments, testSegment{
|
|
|
|
id: int64(i),
|
|
|
|
collectionID: 0,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
for idx, segment := range segments {
|
|
|
|
s := &datapb.SegmentInfo{
|
|
|
|
ID: segment.id,
|
|
|
|
CollectionID: segment.collectionID,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
}
|
|
|
|
if idx%2 == 0 {
|
|
|
|
s.Binlogs = []*datapb.FieldBinlog{
|
|
|
|
{FieldID: 1},
|
|
|
|
}
|
|
|
|
s.Statslogs = []*datapb.FieldBinlog{
|
|
|
|
{FieldID: 1},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
}
|
2021-12-03 18:55:34 +08:00
|
|
|
// add non matched segments
|
|
|
|
os := &datapb.SegmentInfo{
|
|
|
|
ID: maxOperationsPerTxn + 100,
|
|
|
|
CollectionID: 0,
|
|
|
|
InsertChannel: "ch2",
|
|
|
|
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
}
|
|
|
|
|
|
|
|
svr.meta.AddSegment(NewSegmentInfo(os))
|
2021-11-29 22:35:41 +08:00
|
|
|
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
require.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2021-11-29 22:35:41 +08:00
|
|
|
require.Nil(t, err)
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
req := &datapb.DropVirtualChannelRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
Timestamp: uint64(time.Now().Unix()),
|
|
|
|
},
|
|
|
|
ChannelName: "ch1",
|
|
|
|
Segments: make([]*datapb.DropVirtualChannelSegment, 0, maxOperationsPerTxn),
|
|
|
|
}
|
|
|
|
for _, segment := range segments {
|
|
|
|
seg2Drop := &datapb.DropVirtualChannelSegment{
|
|
|
|
SegmentID: segment.id,
|
|
|
|
CollectionID: segment.collectionID,
|
|
|
|
Field2BinlogPaths: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/2/1/Allo1",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/2/1/Allo2",
|
|
|
|
},
|
2021-11-29 22:35:41 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Field2StatslogPaths: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/2/1/stats1",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/by-dev/test/0/1/2/1/stats2",
|
|
|
|
},
|
2021-11-29 22:35:41 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2021-12-19 20:00:42 +08:00
|
|
|
Deltalogs: []*datapb.FieldBinlog{
|
2021-11-29 22:35:41 +08:00
|
|
|
{
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
EntriesNum: 1,
|
|
|
|
LogPath: "/by-dev/test/0/1/2/1/delta1",
|
|
|
|
},
|
|
|
|
},
|
2021-11-29 22:35:41 +08:00
|
|
|
},
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
CheckPoint: &msgpb.MsgPosition{
|
2021-11-29 22:35:41 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-11-29 22:35:41 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
NumOfRows: 10,
|
|
|
|
}
|
|
|
|
req.Segments = append(req.Segments, seg2Drop)
|
|
|
|
}
|
|
|
|
resp, err := svr.DropVirtualChannel(ctx, req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
|
|
|
|
|
|
<-spyCh
|
|
|
|
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2021-11-29 22:35:41 +08:00
|
|
|
require.Nil(t, err)
|
|
|
|
|
|
|
|
//resend
|
|
|
|
resp, err = svr.DropVirtualChannel(ctx, req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with channel not matched", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
require.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2021-11-29 22:35:41 +08:00
|
|
|
require.Nil(t, err)
|
|
|
|
|
|
|
|
resp, err := svr.DropVirtualChannel(context.Background(), &datapb.DropVirtualChannelRequest{
|
|
|
|
ChannelName: "ch2",
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2022-05-27 16:20:00 +08:00
|
|
|
assert.Equal(t, commonpb.ErrorCode_MetaFailed, resp.GetStatus().GetErrorCode())
|
2021-11-29 22:35:41 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.DropVirtualChannel(context.Background(), &datapb.DropVirtualChannelRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-05-25 15:35:37 +08:00
|
|
|
func TestDataNodeTtChannel(t *testing.T) {
|
|
|
|
genMsg := func(msgType commonpb.MsgType, ch string, t Timestamp) *msgstream.DataNodeTtMsg {
|
|
|
|
return &msgstream.DataNodeTtMsg{
|
|
|
|
BaseMsg: msgstream.BaseMsg{
|
|
|
|
HashValues: []uint32{0},
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DataNodeTtMsg: msgpb.DataNodeTtMsg{
|
2021-05-25 15:35:37 +08:00
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: msgType,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: t,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
ChannelName: ch,
|
|
|
|
Timestamp: t,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
2021-08-04 16:59:25 +08:00
|
|
|
t.Run("Test segment flush after tt", func(t *testing.T) {
|
2022-09-27 18:04:53 +08:00
|
|
|
ch := make(chan any, 1)
|
2021-08-04 16:59:25 +08:00
|
|
|
svr := newTestServer(t, ch)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-08-04 16:59:25 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
Partitions: []int64{0},
|
|
|
|
})
|
|
|
|
|
2022-04-07 22:05:32 +08:00
|
|
|
ttMsgStream, err := svr.factory.NewMsgStream(context.TODO())
|
2021-08-04 16:59:25 +08:00
|
|
|
assert.Nil(t, err)
|
2022-12-07 18:01:19 +08:00
|
|
|
ttMsgStream.AsProducer([]string{Params.CommonCfg.DataCoordTimeTick.GetValue()})
|
2021-08-04 16:59:25 +08:00
|
|
|
defer ttMsgStream.Close()
|
2021-10-14 15:44:34 +08:00
|
|
|
info := &NodeInfo{
|
2021-08-04 16:59:25 +08:00
|
|
|
Address: "localhost:7777",
|
2021-10-14 15:44:34 +08:00
|
|
|
NodeID: 0,
|
2021-08-04 16:59:25 +08:00
|
|
|
}
|
2021-10-15 17:12:35 +08:00
|
|
|
err = svr.cluster.Register(info)
|
|
|
|
assert.Nil(t, err)
|
2021-05-25 15:35:37 +08:00
|
|
|
|
|
|
|
resp, err := svr.AssignSegmentID(context.TODO(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{
|
|
|
|
{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
ChannelName: "ch-1",
|
|
|
|
Count: 100,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.SegIDAssignments))
|
|
|
|
assign := resp.SegIDAssignments[0]
|
|
|
|
|
|
|
|
resp2, err := svr.Flush(context.TODO(), &datapb.FlushRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_Flush,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
DbID: 0,
|
|
|
|
CollectionID: 0,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2021-06-23 16:56:11 +08:00
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp2.Status.ErrorCode)
|
2021-05-25 15:35:37 +08:00
|
|
|
|
|
|
|
msgPack := msgstream.MsgPack{}
|
|
|
|
msg := genMsg(commonpb.MsgType_DataNodeTt, "ch-1", assign.ExpireTime)
|
2023-03-04 23:21:50 +08:00
|
|
|
msg.SegmentsStats = append(msg.SegmentsStats, &commonpb.SegmentStats{
|
2021-12-15 10:53:16 +08:00
|
|
|
SegmentID: assign.GetSegID(),
|
|
|
|
NumRows: 1,
|
|
|
|
})
|
2021-05-25 15:35:37 +08:00
|
|
|
msgPack.Msgs = append(msgPack.Msgs, msg)
|
2021-10-03 22:17:56 +08:00
|
|
|
err = ttMsgStream.Produce(&msgPack)
|
|
|
|
assert.Nil(t, err)
|
2021-05-25 15:35:37 +08:00
|
|
|
|
|
|
|
flushMsg := <-ch
|
|
|
|
flushReq := flushMsg.(*datapb.FlushSegmentsRequest)
|
|
|
|
assert.EqualValues(t, 1, len(flushReq.SegmentIDs))
|
|
|
|
assert.EqualValues(t, assign.SegID, flushReq.SegmentIDs[0])
|
|
|
|
})
|
|
|
|
|
2021-08-04 16:59:25 +08:00
|
|
|
t.Run("flush segment with different channels", func(t *testing.T) {
|
2022-09-27 18:04:53 +08:00
|
|
|
ch := make(chan any, 1)
|
2021-08-04 16:59:25 +08:00
|
|
|
svr := newTestServer(t, ch)
|
|
|
|
defer closeTestServer(t, svr)
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-08-04 16:59:25 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
Partitions: []int64{0},
|
|
|
|
})
|
2022-04-07 22:05:32 +08:00
|
|
|
ttMsgStream, err := svr.factory.NewMsgStream(context.TODO())
|
2021-08-04 16:59:25 +08:00
|
|
|
assert.Nil(t, err)
|
2022-12-07 18:01:19 +08:00
|
|
|
ttMsgStream.AsProducer([]string{Params.CommonCfg.DataCoordTimeTick.GetValue()})
|
2021-08-04 16:59:25 +08:00
|
|
|
defer ttMsgStream.Close()
|
2021-10-14 15:44:34 +08:00
|
|
|
info := &NodeInfo{
|
2021-08-04 16:59:25 +08:00
|
|
|
Address: "localhost:7777",
|
2021-10-14 15:44:34 +08:00
|
|
|
NodeID: 0,
|
2021-08-04 16:59:25 +08:00
|
|
|
}
|
2021-10-15 17:12:35 +08:00
|
|
|
err = svr.cluster.Register(info)
|
|
|
|
assert.Nil(t, err)
|
2021-08-04 16:59:25 +08:00
|
|
|
resp, err := svr.AssignSegmentID(context.TODO(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{
|
|
|
|
{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
ChannelName: "ch-1",
|
|
|
|
Count: 100,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
ChannelName: "ch-2",
|
|
|
|
Count: 100,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 2, len(resp.SegIDAssignments))
|
|
|
|
var assign *datapb.SegmentIDAssignment
|
|
|
|
for _, segment := range resp.SegIDAssignments {
|
|
|
|
if segment.GetChannelName() == "ch-1" {
|
|
|
|
assign = segment
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert.NotNil(t, assign)
|
|
|
|
resp2, err := svr.Flush(context.TODO(), &datapb.FlushRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_Flush,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
DbID: 0,
|
|
|
|
CollectionID: 0,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp2.Status.ErrorCode)
|
|
|
|
|
|
|
|
msgPack := msgstream.MsgPack{}
|
|
|
|
msg := genMsg(commonpb.MsgType_DataNodeTt, "ch-1", assign.ExpireTime)
|
2023-03-04 23:21:50 +08:00
|
|
|
msg.SegmentsStats = append(msg.SegmentsStats, &commonpb.SegmentStats{
|
2021-12-15 10:53:16 +08:00
|
|
|
SegmentID: assign.GetSegID(),
|
|
|
|
NumRows: 1,
|
|
|
|
})
|
2021-08-04 16:59:25 +08:00
|
|
|
msgPack.Msgs = append(msgPack.Msgs, msg)
|
2021-10-03 22:17:56 +08:00
|
|
|
err = ttMsgStream.Produce(&msgPack)
|
|
|
|
assert.Nil(t, err)
|
2021-08-04 16:59:25 +08:00
|
|
|
flushMsg := <-ch
|
|
|
|
flushReq := flushMsg.(*datapb.FlushSegmentsRequest)
|
|
|
|
assert.EqualValues(t, 1, len(flushReq.SegmentIDs))
|
|
|
|
assert.EqualValues(t, assign.SegID, flushReq.SegmentIDs[0])
|
|
|
|
})
|
2021-08-16 11:00:09 +08:00
|
|
|
|
|
|
|
t.Run("test expire allocation after receiving tt msg", func(t *testing.T) {
|
2022-09-27 18:04:53 +08:00
|
|
|
ch := make(chan any, 1)
|
2021-08-16 11:00:09 +08:00
|
|
|
helper := ServerHelper{
|
|
|
|
eventAfterHandleDataNodeTt: func() { ch <- struct{}{} },
|
|
|
|
}
|
2023-01-12 19:49:40 +08:00
|
|
|
svr := newTestServer(t, nil, WithServerHelper(helper))
|
2021-08-16 11:00:09 +08:00
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-08-16 11:00:09 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
Partitions: []int64{0},
|
|
|
|
})
|
|
|
|
|
2022-04-07 22:05:32 +08:00
|
|
|
ttMsgStream, err := svr.factory.NewMsgStream(context.TODO())
|
2021-08-16 11:00:09 +08:00
|
|
|
assert.Nil(t, err)
|
2022-12-07 18:01:19 +08:00
|
|
|
ttMsgStream.AsProducer([]string{Params.CommonCfg.DataCoordTimeTick.GetValue()})
|
2021-08-16 11:00:09 +08:00
|
|
|
defer ttMsgStream.Close()
|
2021-10-14 15:44:34 +08:00
|
|
|
node := &NodeInfo{
|
|
|
|
NodeID: 0,
|
2021-08-16 11:00:09 +08:00
|
|
|
Address: "localhost:7777",
|
|
|
|
}
|
2021-10-15 17:12:35 +08:00
|
|
|
err = svr.cluster.Register(node)
|
|
|
|
assert.Nil(t, err)
|
2021-08-16 11:00:09 +08:00
|
|
|
|
|
|
|
resp, err := svr.AssignSegmentID(context.TODO(), &datapb.AssignSegmentIDRequest{
|
|
|
|
NodeID: 0,
|
|
|
|
PeerRole: "",
|
|
|
|
SegmentIDRequests: []*datapb.SegmentIDRequest{
|
|
|
|
{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
ChannelName: "ch-1",
|
|
|
|
Count: 100,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.SegIDAssignments))
|
|
|
|
|
|
|
|
assignedSegmentID := resp.SegIDAssignments[0].SegID
|
2023-03-03 14:13:49 +08:00
|
|
|
segment := svr.meta.GetHealthySegment(assignedSegmentID)
|
2021-08-16 11:00:09 +08:00
|
|
|
assert.EqualValues(t, 1, len(segment.allocations))
|
|
|
|
|
|
|
|
msgPack := msgstream.MsgPack{}
|
|
|
|
msg := genMsg(commonpb.MsgType_DataNodeTt, "ch-1", resp.SegIDAssignments[0].ExpireTime)
|
|
|
|
msgPack.Msgs = append(msgPack.Msgs, msg)
|
2021-10-03 22:17:56 +08:00
|
|
|
err = ttMsgStream.Produce(&msgPack)
|
|
|
|
assert.Nil(t, err)
|
2021-08-16 11:00:09 +08:00
|
|
|
|
|
|
|
<-ch
|
2023-03-03 14:13:49 +08:00
|
|
|
segment = svr.meta.GetHealthySegment(assignedSegmentID)
|
2021-08-16 11:00:09 +08:00
|
|
|
assert.EqualValues(t, 0, len(segment.allocations))
|
|
|
|
})
|
2021-05-25 15:35:37 +08:00
|
|
|
}
|
|
|
|
|
2022-11-12 21:09:04 +08:00
|
|
|
func TestGetChannelSeekPosition(t *testing.T) {
|
|
|
|
startPos1 := []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch1",
|
|
|
|
Data: []byte{1, 2, 3},
|
2022-11-10 22:13:04 +08:00
|
|
|
},
|
2022-11-12 21:09:04 +08:00
|
|
|
}
|
|
|
|
startPosNonExist := []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch2",
|
|
|
|
Data: []byte{4, 5, 6},
|
2022-11-10 22:13:04 +08:00
|
|
|
},
|
2022-11-12 21:09:04 +08:00
|
|
|
}
|
2022-11-10 22:13:04 +08:00
|
|
|
|
2022-11-12 21:09:04 +08:00
|
|
|
tests := []struct {
|
|
|
|
testName string
|
2023-03-04 23:21:50 +08:00
|
|
|
channelCP *msgpb.MsgPosition
|
|
|
|
segDMLPos []*msgpb.MsgPosition
|
2022-11-12 21:09:04 +08:00
|
|
|
collStartPos []*commonpb.KeyDataPair
|
|
|
|
channelName string
|
2023-03-04 23:21:50 +08:00
|
|
|
expectedPos *msgpb.MsgPosition
|
2022-11-12 21:09:04 +08:00
|
|
|
}{
|
|
|
|
{"test-with-channelCP",
|
2023-03-04 23:21:50 +08:00
|
|
|
&msgpb.MsgPosition{ChannelName: "ch1", Timestamp: 100},
|
|
|
|
[]*msgpb.MsgPosition{{ChannelName: "ch1", Timestamp: 50}, {ChannelName: "ch1", Timestamp: 200}},
|
2022-11-12 21:09:04 +08:00
|
|
|
startPos1,
|
2023-03-04 23:21:50 +08:00
|
|
|
"ch1", &msgpb.MsgPosition{ChannelName: "ch1", Timestamp: 100}},
|
2022-11-12 21:09:04 +08:00
|
|
|
|
|
|
|
{"test-with-segmentDMLPos",
|
|
|
|
nil,
|
2023-03-04 23:21:50 +08:00
|
|
|
[]*msgpb.MsgPosition{{ChannelName: "ch1", Timestamp: 50}, {ChannelName: "ch1", Timestamp: 200}},
|
2022-11-12 21:09:04 +08:00
|
|
|
startPos1,
|
2023-03-04 23:21:50 +08:00
|
|
|
"ch1", &msgpb.MsgPosition{ChannelName: "ch1", Timestamp: 50}},
|
2022-11-12 21:09:04 +08:00
|
|
|
|
|
|
|
{"test-with-collStartPos",
|
|
|
|
nil,
|
|
|
|
nil,
|
|
|
|
startPos1,
|
2023-03-04 23:21:50 +08:00
|
|
|
"ch1", &msgpb.MsgPosition{ChannelName: "ch1", MsgID: startPos1[0].Data}},
|
2022-11-12 21:09:04 +08:00
|
|
|
|
|
|
|
{"test-non-exist-channel-1",
|
|
|
|
nil,
|
|
|
|
nil,
|
|
|
|
startPosNonExist,
|
|
|
|
"ch1", nil},
|
|
|
|
|
|
|
|
{"test-non-exist-channel-2",
|
|
|
|
nil,
|
|
|
|
nil,
|
|
|
|
nil,
|
|
|
|
"ch1", nil},
|
|
|
|
}
|
|
|
|
for _, test := range tests {
|
|
|
|
t.Run(test.testName, func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
|
|
|
if test.collStartPos != nil {
|
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
Schema: schema,
|
|
|
|
StartPositions: test.collStartPos,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
for i, segPos := range test.segDMLPos {
|
|
|
|
seg := &datapb.SegmentInfo{
|
|
|
|
ID: UniqueID(i),
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
DmlPosition: segPos,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(seg))
|
|
|
|
assert.NoError(t, err)
|
|
|
|
}
|
|
|
|
if test.channelCP != nil {
|
|
|
|
err := svr.meta.UpdateChannelCheckpoint(test.channelCP.ChannelName, test.channelCP)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
}
|
2022-11-10 22:13:04 +08:00
|
|
|
|
2022-11-12 21:09:04 +08:00
|
|
|
seekPos := svr.handler.(*ServerHandler).GetChannelSeekPosition(&channel{
|
|
|
|
Name: test.channelName,
|
|
|
|
CollectionID: 0}, allPartitionID)
|
|
|
|
if test.expectedPos == nil {
|
|
|
|
assert.True(t, seekPos == nil)
|
|
|
|
} else {
|
|
|
|
assert.Equal(t, test.expectedPos.ChannelName, seekPos.ChannelName)
|
|
|
|
assert.Equal(t, test.expectedPos.Timestamp, seekPos.Timestamp)
|
|
|
|
assert.ElementsMatch(t, test.expectedPos.MsgID, seekPos.MsgID)
|
|
|
|
}
|
2022-11-10 22:13:04 +08:00
|
|
|
})
|
2022-11-12 21:09:04 +08:00
|
|
|
}
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
2022-10-11 16:51:22 +08:00
|
|
|
func TestGetDataVChanPositions(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
2022-10-11 17:59:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-10-11 16:51:22 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: schema,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch1",
|
|
|
|
Data: []byte{8, 9, 10},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
2022-10-11 17:59:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-10-11 16:51:22 +08:00
|
|
|
ID: 1,
|
|
|
|
Schema: schema,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch0",
|
|
|
|
Data: []byte{8, 9, 10},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
s1 := &datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2022-10-11 16:51:22 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s1))
|
|
|
|
require.Nil(t, err)
|
|
|
|
s2 := &datapb.SegmentInfo{
|
|
|
|
ID: 2,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2022-10-11 16:51:22 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2022-10-11 16:51:22 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
Timestamp: 1,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s2))
|
|
|
|
require.Nil(t, err)
|
|
|
|
s3 := &datapb.SegmentInfo{
|
|
|
|
ID: 3,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 1,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2022-10-11 16:51:22 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2022-10-11 16:51:22 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{11, 12, 13},
|
|
|
|
Timestamp: 2,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s3))
|
|
|
|
require.Nil(t, err)
|
|
|
|
|
|
|
|
t.Run("get unexisted channel", func(t *testing.T) {
|
|
|
|
vchan := svr.handler.GetDataVChanPositions(&channel{Name: "chx1", CollectionID: 0}, allPartitionID)
|
|
|
|
assert.Empty(t, vchan.UnflushedSegmentIds)
|
|
|
|
assert.Empty(t, vchan.FlushedSegmentIds)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("get existed channel", func(t *testing.T) {
|
|
|
|
vchan := svr.handler.GetDataVChanPositions(&channel{Name: "ch1", CollectionID: 0}, allPartitionID)
|
|
|
|
assert.EqualValues(t, 1, len(vchan.FlushedSegmentIds))
|
|
|
|
assert.EqualValues(t, 1, vchan.FlushedSegmentIds[0])
|
|
|
|
assert.EqualValues(t, 2, len(vchan.UnflushedSegmentIds))
|
|
|
|
assert.ElementsMatch(t, []int64{s2.ID, s3.ID}, vchan.UnflushedSegmentIds)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("empty collection", func(t *testing.T) {
|
|
|
|
infos := svr.handler.GetDataVChanPositions(&channel{Name: "ch0_suffix", CollectionID: 1}, allPartitionID)
|
|
|
|
assert.EqualValues(t, 1, infos.CollectionID)
|
|
|
|
assert.EqualValues(t, 0, len(infos.FlushedSegmentIds))
|
|
|
|
assert.EqualValues(t, 0, len(infos.UnflushedSegmentIds))
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("filter partition", func(t *testing.T) {
|
|
|
|
infos := svr.handler.GetDataVChanPositions(&channel{Name: "ch1", CollectionID: 0}, 1)
|
|
|
|
assert.EqualValues(t, 0, infos.CollectionID)
|
|
|
|
assert.EqualValues(t, 0, len(infos.FlushedSegmentIds))
|
|
|
|
assert.EqualValues(t, 1, len(infos.UnflushedSegmentIds))
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("empty collection with passed positions", func(t *testing.T) {
|
|
|
|
vchannel := "ch_no_segment_1"
|
|
|
|
pchannel := funcutil.ToPhysicalChannel(vchannel)
|
|
|
|
infos := svr.handler.GetDataVChanPositions(&channel{
|
|
|
|
Name: vchannel,
|
|
|
|
CollectionID: 0,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{{Key: pchannel, Data: []byte{14, 15, 16}}},
|
|
|
|
}, allPartitionID)
|
|
|
|
assert.EqualValues(t, 0, infos.CollectionID)
|
|
|
|
assert.EqualValues(t, vchannel, infos.ChannelName)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestGetQueryVChanPositions(t *testing.T) {
|
2021-05-28 15:13:51 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-05-28 15:13:51 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: schema,
|
2021-10-09 20:57:33 +08:00
|
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch1",
|
|
|
|
Data: []byte{8, 9, 10},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-10-09 20:57:33 +08:00
|
|
|
ID: 1,
|
|
|
|
Schema: schema,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch0",
|
|
|
|
Data: []byte{8, 9, 10},
|
|
|
|
},
|
|
|
|
},
|
2021-05-28 15:13:51 +08:00
|
|
|
})
|
2021-10-09 20:57:33 +08:00
|
|
|
|
2023-01-04 19:37:36 +08:00
|
|
|
err := svr.meta.CreateIndex(&model.Index{
|
|
|
|
TenantID: "",
|
|
|
|
CollectionID: 0,
|
|
|
|
FieldID: 2,
|
|
|
|
IndexID: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-07-12 17:24:25 +08:00
|
|
|
s1 := &datapb.SegmentInfo{
|
2021-05-28 15:13:51 +08:00
|
|
|
ID: 1,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "ch1",
|
2021-06-04 11:45:45 +08:00
|
|
|
State: commonpb.SegmentState_Flushed,
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-11-13 08:45:09 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
2021-07-12 17:24:25 +08:00
|
|
|
}
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s1))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegmentIndex(&model.SegmentIndex{
|
|
|
|
SegmentID: 1,
|
|
|
|
BuildID: 1,
|
|
|
|
IndexID: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.FinishTask(&indexpb.IndexTaskInfo{
|
|
|
|
BuildID: 1,
|
|
|
|
State: commonpb.IndexState_Finished,
|
|
|
|
})
|
2021-05-28 15:13:51 +08:00
|
|
|
assert.Nil(t, err)
|
2021-07-12 17:24:25 +08:00
|
|
|
s2 := &datapb.SegmentInfo{
|
2021-05-31 18:47:32 +08:00
|
|
|
ID: 2,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "ch1",
|
2021-06-04 11:45:45 +08:00
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-11-13 08:45:09 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
MsgGroup: "",
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-06-04 11:45:45 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
2021-11-16 21:41:50 +08:00
|
|
|
Timestamp: 1,
|
2021-06-04 11:45:45 +08:00
|
|
|
},
|
2021-07-12 17:24:25 +08:00
|
|
|
}
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s2))
|
2021-05-31 18:47:32 +08:00
|
|
|
assert.Nil(t, err)
|
2021-07-12 17:24:25 +08:00
|
|
|
s3 := &datapb.SegmentInfo{
|
2021-06-04 11:45:45 +08:00
|
|
|
ID: 3,
|
|
|
|
CollectionID: 0,
|
2021-11-13 08:45:09 +08:00
|
|
|
PartitionID: 1,
|
2021-06-04 11:45:45 +08:00
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-11-13 08:45:09 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
MsgGroup: "",
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-11-13 08:45:09 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{11, 12, 13},
|
|
|
|
MsgGroup: "",
|
2021-11-16 21:41:50 +08:00
|
|
|
Timestamp: 2,
|
2021-11-13 08:45:09 +08:00
|
|
|
},
|
2021-07-12 17:24:25 +08:00
|
|
|
}
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s3))
|
2021-05-28 15:13:51 +08:00
|
|
|
assert.Nil(t, err)
|
2023-01-04 19:37:36 +08:00
|
|
|
//mockResp := &indexpb.GetIndexInfoResponse{
|
|
|
|
// Status: &commonpb.Status{},
|
|
|
|
// SegmentInfo: map[int64]*indexpb.SegmentInfo{
|
|
|
|
// s1.ID: {
|
|
|
|
// CollectionID: s1.CollectionID,
|
|
|
|
// SegmentID: s1.ID,
|
|
|
|
// EnableIndex: true,
|
|
|
|
// IndexInfos: []*indexpb.IndexFilePathInfo{
|
|
|
|
// {
|
|
|
|
// SegmentID: s1.ID,
|
|
|
|
// FieldID: 2,
|
|
|
|
// },
|
|
|
|
// },
|
|
|
|
// },
|
|
|
|
// },
|
|
|
|
//}
|
2021-05-28 15:13:51 +08:00
|
|
|
|
|
|
|
t.Run("get unexisted channel", func(t *testing.T) {
|
2022-10-11 16:51:22 +08:00
|
|
|
vchan := svr.handler.GetQueryVChanPositions(&channel{Name: "chx1", CollectionID: 0}, allPartitionID)
|
2022-06-16 12:00:10 +08:00
|
|
|
assert.Empty(t, vchan.UnflushedSegmentIds)
|
|
|
|
assert.Empty(t, vchan.FlushedSegmentIds)
|
2021-05-28 15:13:51 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("get existed channel", func(t *testing.T) {
|
2022-10-11 16:51:22 +08:00
|
|
|
vchan := svr.handler.GetQueryVChanPositions(&channel{Name: "ch1", CollectionID: 0}, allPartitionID)
|
2022-06-16 12:00:10 +08:00
|
|
|
assert.EqualValues(t, 1, len(vchan.FlushedSegmentIds))
|
|
|
|
assert.EqualValues(t, 1, vchan.FlushedSegmentIds[0])
|
|
|
|
assert.EqualValues(t, 2, len(vchan.UnflushedSegmentIds))
|
2022-10-11 16:51:22 +08:00
|
|
|
assert.ElementsMatch(t, []int64{s2.ID, s3.ID}, vchan.UnflushedSegmentIds)
|
2021-05-28 15:13:51 +08:00
|
|
|
})
|
2021-10-09 20:57:33 +08:00
|
|
|
|
|
|
|
t.Run("empty collection", func(t *testing.T) {
|
2022-10-11 16:51:22 +08:00
|
|
|
infos := svr.handler.GetQueryVChanPositions(&channel{Name: "ch0_suffix", CollectionID: 1}, allPartitionID)
|
2021-10-14 15:44:34 +08:00
|
|
|
assert.EqualValues(t, 1, infos.CollectionID)
|
2022-06-16 12:00:10 +08:00
|
|
|
assert.EqualValues(t, 0, len(infos.FlushedSegmentIds))
|
|
|
|
assert.EqualValues(t, 0, len(infos.UnflushedSegmentIds))
|
2021-10-09 20:57:33 +08:00
|
|
|
})
|
2021-11-13 08:45:09 +08:00
|
|
|
|
|
|
|
t.Run("filter partition", func(t *testing.T) {
|
2022-10-11 16:51:22 +08:00
|
|
|
infos := svr.handler.GetQueryVChanPositions(&channel{Name: "ch1", CollectionID: 0}, 1)
|
2021-11-13 08:45:09 +08:00
|
|
|
assert.EqualValues(t, 0, infos.CollectionID)
|
2022-06-16 12:00:10 +08:00
|
|
|
assert.EqualValues(t, 0, len(infos.FlushedSegmentIds))
|
|
|
|
assert.EqualValues(t, 1, len(infos.UnflushedSegmentIds))
|
2021-11-13 08:45:09 +08:00
|
|
|
})
|
2022-08-22 14:42:52 +08:00
|
|
|
|
|
|
|
t.Run("empty collection with passed positions", func(t *testing.T) {
|
|
|
|
vchannel := "ch_no_segment_1"
|
|
|
|
pchannel := funcutil.ToPhysicalChannel(vchannel)
|
2022-10-11 16:51:22 +08:00
|
|
|
infos := svr.handler.GetQueryVChanPositions(&channel{
|
2022-08-22 14:42:52 +08:00
|
|
|
Name: vchannel,
|
|
|
|
CollectionID: 0,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{{Key: pchannel, Data: []byte{14, 15, 16}}},
|
|
|
|
}, allPartitionID)
|
|
|
|
assert.EqualValues(t, 0, infos.CollectionID)
|
|
|
|
assert.EqualValues(t, vchannel, infos.ChannelName)
|
|
|
|
})
|
2021-05-28 15:13:51 +08:00
|
|
|
}
|
|
|
|
|
2021-11-25 15:13:15 +08:00
|
|
|
func TestShouldDropChannel(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
schema := newTestSchema()
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-11-25 15:13:15 +08:00
|
|
|
ID: 0,
|
|
|
|
Schema: schema,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch1",
|
|
|
|
Data: []byte{8, 9, 10},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2021-11-25 15:13:15 +08:00
|
|
|
ID: 1,
|
|
|
|
Schema: schema,
|
|
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
|
|
{
|
|
|
|
Key: "ch0",
|
|
|
|
Data: []byte{8, 9, 10},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
2021-12-02 16:39:33 +08:00
|
|
|
/*
|
|
|
|
s1 := &datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Dropped,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-12-02 16:39:33 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-12-02 16:39:33 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
s2 := &datapb.SegmentInfo{
|
|
|
|
ID: 2,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
CompactionFrom: []int64{4, 5},
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-12-02 16:39:33 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
MsgGroup: "",
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-12-02 16:39:33 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{1, 2, 3},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 1,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
s3 := &datapb.SegmentInfo{
|
|
|
|
ID: 3,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 1,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-12-02 16:39:33 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{8, 9, 10},
|
|
|
|
MsgGroup: "",
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-12-02 16:39:33 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
MsgID: []byte{11, 12, 13},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 2,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
s4 := &datapb.SegmentInfo{
|
|
|
|
ID: 4,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 1,
|
|
|
|
InsertChannel: "ch1",
|
|
|
|
State: commonpb.SegmentState_Growing,
|
|
|
|
}*/
|
|
|
|
/*
|
|
|
|
t.Run("channel without segments", func(t *testing.T) {
|
|
|
|
r := svr.handler.CheckShouldDropChannel("ch1")
|
|
|
|
assert.True(t, r)
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
})
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
t.Run("channel with all dropped segments", func(t *testing.T) {
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s1))
|
|
|
|
require.NoError(t, err)
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
r := svr.handler.CheckShouldDropChannel("ch1")
|
|
|
|
assert.True(t, r)
|
|
|
|
})
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
t.Run("channel with all dropped segments and flushed compacted segments", func(t *testing.T) {
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(s2))
|
|
|
|
require.Nil(t, err)
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
r := svr.handler.CheckShouldDropChannel("ch1")
|
|
|
|
assert.False(t, r)
|
|
|
|
})
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
t.Run("channel with other state segments", func(t *testing.T) {
|
|
|
|
err := svr.meta.DropSegment(2)
|
|
|
|
require.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s3))
|
|
|
|
require.Nil(t, err)
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
r := svr.handler.CheckShouldDropChannel("ch1")
|
|
|
|
assert.False(t, r)
|
|
|
|
})
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
t.Run("channel with dropped segment and with segment without start position", func(t *testing.T) {
|
|
|
|
err := svr.meta.DropSegment(3)
|
|
|
|
require.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(s4))
|
|
|
|
require.Nil(t, err)
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
r := svr.handler.CheckShouldDropChannel("ch1")
|
|
|
|
assert.True(t, r)
|
|
|
|
})
|
|
|
|
*/
|
|
|
|
t.Run("channel name not in kv", func(t *testing.T) {
|
|
|
|
assert.False(t, svr.handler.CheckShouldDropChannel("ch99"))
|
2021-11-25 15:13:15 +08:00
|
|
|
})
|
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
t.Run("channel in remove flag", func(t *testing.T) {
|
2022-08-20 10:24:51 +08:00
|
|
|
err := svr.meta.catalog.MarkChannelDeleted(context.TODO(), "ch1")
|
2021-12-02 16:39:33 +08:00
|
|
|
require.NoError(t, err)
|
2021-11-25 15:13:15 +08:00
|
|
|
|
2021-12-02 16:39:33 +08:00
|
|
|
assert.True(t, svr.handler.CheckShouldDropChannel("ch1"))
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("channel name not matched", func(t *testing.T) {
|
|
|
|
assert.False(t, svr.handler.CheckShouldDropChannel("ch2"))
|
2021-11-25 15:13:15 +08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-06-07 09:47:36 +08:00
|
|
|
func TestGetRecoveryInfo(t *testing.T) {
|
|
|
|
|
|
|
|
t.Run("test get recovery info with no segments", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
2021-09-06 17:02:41 +08:00
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
2021-06-07 09:47:36 +08:00
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 0, len(resp.GetBinlogs()))
|
|
|
|
assert.EqualValues(t, 1, len(resp.GetChannels()))
|
|
|
|
assert.Nil(t, resp.GetChannels()[0].SeekPosition)
|
|
|
|
})
|
|
|
|
|
|
|
|
createSegment := func(id, collectionID, partitionID, numOfRows int64, posTs uint64,
|
|
|
|
channel string, state commonpb.SegmentState) *datapb.SegmentInfo {
|
|
|
|
return &datapb.SegmentInfo{
|
|
|
|
ID: id,
|
|
|
|
CollectionID: collectionID,
|
|
|
|
PartitionID: partitionID,
|
|
|
|
InsertChannel: channel,
|
|
|
|
NumOfRows: numOfRows,
|
|
|
|
State: state,
|
2023-03-04 23:21:50 +08:00
|
|
|
DmlPosition: &msgpb.MsgPosition{
|
2021-06-07 09:47:36 +08:00
|
|
|
ChannelName: channel,
|
|
|
|
MsgID: []byte{},
|
|
|
|
Timestamp: posTs,
|
|
|
|
},
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2021-06-15 19:23:55 +08:00
|
|
|
ChannelName: "",
|
|
|
|
MsgID: []byte{},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
2021-06-07 09:47:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-28 14:30:26 +08:00
|
|
|
t.Run("test get earliest position of flushed segments as seek position", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
2021-09-06 17:02:41 +08:00
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-09-16 11:32:48 +08:00
|
|
|
Schema: newTestSchema(),
|
|
|
|
})
|
2022-11-10 22:13:04 +08:00
|
|
|
|
2023-03-04 23:21:50 +08:00
|
|
|
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
|
2022-11-10 22:13:04 +08:00
|
|
|
ChannelName: "vchan1",
|
|
|
|
Timestamp: 10,
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.meta.CreateIndex(&model.Index{
|
|
|
|
TenantID: "",
|
|
|
|
CollectionID: 0,
|
|
|
|
FieldID: 2,
|
|
|
|
IndexID: 0,
|
|
|
|
IndexName: "",
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-06-07 09:47:36 +08:00
|
|
|
seg1 := createSegment(0, 0, 0, 100, 10, "vchan1", commonpb.SegmentState_Flushed)
|
2022-12-01 16:31:16 +08:00
|
|
|
seg1.Binlogs = []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 0, 1, 901),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 0, 1, 902),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 0, 1, 903),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2021-06-07 09:47:36 +08:00
|
|
|
seg2 := createSegment(1, 0, 0, 100, 20, "vchan1", commonpb.SegmentState_Flushed)
|
2022-12-01 16:31:16 +08:00
|
|
|
seg2.Binlogs = []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
EntriesNum: 30,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 1, 1, 801),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 70,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 1, 1, 802),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2022-11-10 22:13:04 +08:00
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg1))
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.Nil(t, err)
|
2021-07-12 17:24:25 +08:00
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg2))
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.Nil(t, err)
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.meta.AddSegmentIndex(&model.SegmentIndex{
|
|
|
|
SegmentID: seg1.ID,
|
|
|
|
BuildID: seg1.ID,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.FinishTask(&indexpb.IndexTaskInfo{
|
|
|
|
BuildID: seg1.ID,
|
|
|
|
State: commonpb.IndexState_Finished,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegmentIndex(&model.SegmentIndex{
|
|
|
|
SegmentID: seg2.ID,
|
|
|
|
BuildID: seg2.ID,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.FinishTask(&indexpb.IndexTaskInfo{
|
|
|
|
BuildID: seg2.ID,
|
|
|
|
State: commonpb.IndexState_Finished,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2021-06-07 09:47:36 +08:00
|
|
|
|
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.GetChannels()))
|
2022-06-16 12:00:10 +08:00
|
|
|
assert.EqualValues(t, 0, len(resp.GetChannels()[0].GetUnflushedSegmentIds()))
|
2022-12-01 16:31:16 +08:00
|
|
|
assert.ElementsMatch(t, []int64{0, 1}, resp.GetChannels()[0].GetFlushedSegmentIds())
|
2021-10-28 14:30:26 +08:00
|
|
|
assert.EqualValues(t, 10, resp.GetChannels()[0].GetSeekPosition().GetTimestamp())
|
2022-12-01 16:31:16 +08:00
|
|
|
assert.EqualValues(t, 2, len(resp.GetBinlogs()))
|
|
|
|
// Row count corrected from 100 + 100 -> 100 + 60.
|
|
|
|
assert.EqualValues(t, 160, resp.GetBinlogs()[0].GetNumOfRows()+resp.GetBinlogs()[1].GetNumOfRows())
|
2021-06-07 09:47:36 +08:00
|
|
|
})
|
|
|
|
|
2021-06-15 19:23:55 +08:00
|
|
|
t.Run("test get recovery of unflushed segments ", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
2021-09-06 17:02:41 +08:00
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
})
|
|
|
|
|
2023-03-04 23:21:50 +08:00
|
|
|
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
|
2022-11-10 22:13:04 +08:00
|
|
|
ChannelName: "vchan1",
|
|
|
|
Timestamp: 0,
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
2021-06-07 09:47:36 +08:00
|
|
|
seg1 := createSegment(3, 0, 0, 100, 30, "vchan1", commonpb.SegmentState_Growing)
|
2022-12-01 16:31:16 +08:00
|
|
|
seg1.Binlogs = []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 3, 1, 901),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 3, 1, 902),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 20,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 3, 1, 903),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2021-06-07 09:47:36 +08:00
|
|
|
seg2 := createSegment(4, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Growing)
|
2022-12-01 16:31:16 +08:00
|
|
|
seg2.Binlogs = []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
EntriesNum: 30,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 4, 1, 801),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
EntriesNum: 70,
|
|
|
|
LogPath: metautil.BuildInsertLogPath("a", 0, 0, 4, 1, 802),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2022-11-10 22:13:04 +08:00
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg1))
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.Nil(t, err)
|
2021-07-12 17:24:25 +08:00
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg2))
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.Nil(t, err)
|
2023-01-04 19:37:36 +08:00
|
|
|
//svr.indexCoord.(*mocks.MockIndexCoord).EXPECT().GetIndexInfos(mock.Anything, mock.Anything).Return(nil, nil)
|
2021-06-07 09:47:36 +08:00
|
|
|
|
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
2021-06-15 19:23:55 +08:00
|
|
|
assert.EqualValues(t, 0, len(resp.GetBinlogs()))
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.EqualValues(t, 1, len(resp.GetChannels()))
|
2021-06-15 19:23:55 +08:00
|
|
|
assert.NotNil(t, resp.GetChannels()[0].SeekPosition)
|
2021-11-12 18:27:10 +08:00
|
|
|
assert.NotEqual(t, 0, resp.GetChannels()[0].GetSeekPosition().GetTimestamp())
|
2021-06-07 09:47:36 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test get binlogs", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
2022-09-16 11:32:48 +08:00
|
|
|
Schema: newTestSchema(),
|
|
|
|
})
|
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
2021-09-06 17:02:41 +08:00
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
2021-06-07 09:47:36 +08:00
|
|
|
binlogReq := &datapb.SaveBinlogPathsRequest{
|
|
|
|
SegmentID: 0,
|
|
|
|
CollectionID: 0,
|
2021-08-19 13:00:12 +08:00
|
|
|
Field2BinlogPaths: []*datapb.FieldBinlog{
|
2021-06-07 09:47:36 +08:00
|
|
|
{
|
2021-08-19 13:00:12 +08:00
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/binlog/file1",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/binlog/file2",
|
|
|
|
},
|
2021-06-07 09:47:36 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2021-10-25 22:52:27 +08:00
|
|
|
Field2StatslogPaths: []*datapb.FieldBinlog{
|
|
|
|
{
|
|
|
|
FieldID: 1,
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
LogPath: "/stats_log/file1",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
LogPath: "/stats_log/file2",
|
|
|
|
},
|
2021-10-25 22:52:27 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2021-12-19 20:00:42 +08:00
|
|
|
Deltalogs: []*datapb.FieldBinlog{
|
2021-10-25 22:52:27 +08:00
|
|
|
{
|
2021-12-19 20:00:42 +08:00
|
|
|
Binlogs: []*datapb.Binlog{
|
|
|
|
{
|
|
|
|
TimestampFrom: 0,
|
|
|
|
TimestampTo: 1,
|
|
|
|
LogPath: "/stats_log/file1",
|
|
|
|
LogSize: 1,
|
|
|
|
},
|
|
|
|
},
|
2021-10-25 22:52:27 +08:00
|
|
|
},
|
|
|
|
},
|
2021-06-07 09:47:36 +08:00
|
|
|
}
|
2022-09-16 11:32:48 +08:00
|
|
|
segment := createSegment(0, 0, 1, 100, 10, "vchan1", commonpb.SegmentState_Flushed)
|
2021-08-19 13:00:12 +08:00
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segment))
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.Nil(t, err)
|
2021-11-12 00:22:42 +08:00
|
|
|
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.meta.CreateIndex(&model.Index{
|
|
|
|
TenantID: "",
|
|
|
|
CollectionID: 0,
|
|
|
|
FieldID: 2,
|
|
|
|
IndexID: 0,
|
|
|
|
IndexName: "",
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegmentIndex(&model.SegmentIndex{
|
|
|
|
SegmentID: segment.ID,
|
|
|
|
BuildID: segment.ID,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.FinishTask(&indexpb.IndexTaskInfo{
|
|
|
|
BuildID: segment.ID,
|
|
|
|
State: commonpb.IndexState_Finished,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-11-12 00:22:42 +08:00
|
|
|
err = svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
2022-09-16 11:32:48 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "vchan1", CollectionID: 0})
|
2021-11-12 00:22:42 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-08-19 13:00:12 +08:00
|
|
|
sResp, err := svr.SaveBinlogPaths(context.TODO(), binlogReq)
|
2021-06-07 09:47:36 +08:00
|
|
|
assert.Nil(t, err)
|
2021-08-19 13:00:12 +08:00
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, sResp.ErrorCode)
|
2021-06-07 09:47:36 +08:00
|
|
|
|
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
2022-09-16 11:32:48 +08:00
|
|
|
PartitionID: 1,
|
2021-06-07 09:47:36 +08:00
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.GetBinlogs()))
|
|
|
|
assert.EqualValues(t, 0, resp.GetBinlogs()[0].GetSegmentID())
|
|
|
|
assert.EqualValues(t, 1, len(resp.GetBinlogs()[0].GetFieldBinlogs()))
|
|
|
|
assert.EqualValues(t, 1, resp.GetBinlogs()[0].GetFieldBinlogs()[0].GetFieldID())
|
2021-12-19 20:00:42 +08:00
|
|
|
for i, binlog := range resp.GetBinlogs()[0].GetFieldBinlogs()[0].GetBinlogs() {
|
|
|
|
assert.Equal(t, fmt.Sprintf("/binlog/file%d", i+1), binlog.GetLogPath())
|
|
|
|
}
|
2021-06-07 09:47:36 +08:00
|
|
|
})
|
2021-11-23 18:59:15 +08:00
|
|
|
t.Run("with dropped segments", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
2021-11-23 18:59:15 +08:00
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
})
|
|
|
|
|
2023-03-04 23:21:50 +08:00
|
|
|
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
|
2022-11-10 22:13:04 +08:00
|
|
|
ChannelName: "vchan1",
|
|
|
|
Timestamp: 0,
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
2021-11-23 18:59:15 +08:00
|
|
|
seg1 := createSegment(7, 0, 0, 100, 30, "vchan1", commonpb.SegmentState_Growing)
|
|
|
|
seg2 := createSegment(8, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Dropped)
|
2022-11-10 22:13:04 +08:00
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg1))
|
2021-11-23 18:59:15 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg2))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 0, len(resp.GetBinlogs()))
|
|
|
|
assert.EqualValues(t, 1, len(resp.GetChannels()))
|
|
|
|
assert.NotNil(t, resp.GetChannels()[0].SeekPosition)
|
|
|
|
assert.NotEqual(t, 0, resp.GetChannels()[0].GetSeekPosition().GetTimestamp())
|
2022-06-16 12:00:10 +08:00
|
|
|
assert.Len(t, resp.GetChannels()[0].GetDroppedSegmentIds(), 1)
|
|
|
|
assert.Equal(t, UniqueID(8), resp.GetChannels()[0].GetDroppedSegmentIds()[0])
|
2021-11-23 18:59:15 +08:00
|
|
|
})
|
|
|
|
|
2022-11-11 18:03:05 +08:00
|
|
|
t.Run("with fake segments", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
})
|
|
|
|
|
2023-03-04 23:21:50 +08:00
|
|
|
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
|
2022-11-11 18:03:05 +08:00
|
|
|
ChannelName: "vchan1",
|
|
|
|
Timestamp: 0,
|
|
|
|
})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
seg1 := createSegment(7, 0, 0, 100, 30, "vchan1", commonpb.SegmentState_Growing)
|
|
|
|
seg2 := createSegment(8, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Flushed)
|
|
|
|
seg2.IsFake = true
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg1))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg2))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 0, len(resp.GetBinlogs()))
|
|
|
|
assert.EqualValues(t, 1, len(resp.GetChannels()))
|
|
|
|
assert.NotNil(t, resp.GetChannels()[0].SeekPosition)
|
|
|
|
assert.NotEqual(t, 0, resp.GetChannels()[0].GetSeekPosition().GetTimestamp())
|
|
|
|
})
|
|
|
|
|
2023-02-12 18:56:33 +08:00
|
|
|
t.Run("with continuous compaction", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 0,
|
|
|
|
Schema: newTestSchema(),
|
|
|
|
})
|
|
|
|
|
2023-03-04 23:21:50 +08:00
|
|
|
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
|
2023-02-12 18:56:33 +08:00
|
|
|
ChannelName: "vchan1",
|
|
|
|
Timestamp: 0,
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
seg1 := createSegment(9, 0, 0, 100, 30, "vchan1", commonpb.SegmentState_Dropped)
|
|
|
|
seg2 := createSegment(10, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Dropped)
|
|
|
|
seg3 := createSegment(11, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Dropped)
|
|
|
|
seg3.CompactionFrom = []int64{9, 10}
|
|
|
|
seg4 := createSegment(12, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Dropped)
|
|
|
|
seg5 := createSegment(13, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Flushed)
|
|
|
|
seg5.CompactionFrom = []int64{11, 12}
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg1))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg2))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg3))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg4))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.AddSegment(NewSegmentInfo(seg5))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.meta.CreateIndex(&model.Index{
|
|
|
|
TenantID: "",
|
|
|
|
CollectionID: 0,
|
|
|
|
FieldID: 2,
|
|
|
|
IndexID: 0,
|
|
|
|
IndexName: "_default_idx_2",
|
|
|
|
IsDeleted: false,
|
|
|
|
CreateTime: 0,
|
|
|
|
TypeParams: nil,
|
|
|
|
IndexParams: nil,
|
|
|
|
IsAutoIndex: false,
|
|
|
|
UserIndexParams: nil,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
svr.meta.segments.SetSegmentIndex(seg4.ID, &model.SegmentIndex{
|
|
|
|
SegmentID: seg4.ID,
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
NumRows: 100,
|
|
|
|
IndexID: 0,
|
|
|
|
BuildID: 0,
|
|
|
|
NodeID: 0,
|
|
|
|
IndexVersion: 1,
|
|
|
|
IndexState: commonpb.IndexState_Finished,
|
|
|
|
FailReason: "",
|
|
|
|
IsDeleted: false,
|
|
|
|
CreateTime: 0,
|
|
|
|
IndexFileKeys: nil,
|
|
|
|
IndexSize: 0,
|
|
|
|
})
|
|
|
|
|
|
|
|
req := &datapb.GetRecoveryInfoRequest{
|
|
|
|
CollectionID: 0,
|
|
|
|
PartitionID: 0,
|
|
|
|
}
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), req)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.NotNil(t, resp.GetChannels()[0].SeekPosition)
|
|
|
|
assert.NotEqual(t, 0, resp.GetChannels()[0].GetSeekPosition().GetTimestamp())
|
|
|
|
assert.Len(t, resp.GetChannels()[0].GetDroppedSegmentIds(), 0)
|
|
|
|
assert.ElementsMatch(t, []UniqueID{9, 10}, resp.GetChannels()[0].GetUnflushedSegmentIds())
|
|
|
|
assert.ElementsMatch(t, []UniqueID{12}, resp.GetChannels()[0].GetFlushedSegmentIds())
|
|
|
|
})
|
|
|
|
|
2021-09-06 11:12:42 +08:00
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.GetRecoveryInfo(context.TODO(), &datapb.GetRecoveryInfoRequest{})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
2021-06-07 09:47:36 +08:00
|
|
|
}
|
|
|
|
|
2021-11-05 22:25:00 +08:00
|
|
|
func TestGetCompactionState(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.DataCoordCfg.EnableCompaction.Key, "true")
|
|
|
|
defer paramtable.Get().Reset(Params.DataCoordCfg.EnableCompaction.Key)
|
2021-11-05 22:25:00 +08:00
|
|
|
t.Run("test get compaction state with new compactionhandler", func(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-05 22:25:00 +08:00
|
|
|
|
|
|
|
svr.compactionHandler = &mockCompactionHandler{
|
|
|
|
methods: map[string]interface{}{
|
2021-11-09 14:47:02 +08:00
|
|
|
"getCompactionTasksBySignalID": func(signalID int64) []*compactionTask {
|
|
|
|
return []*compactionTask{
|
|
|
|
{state: completed},
|
|
|
|
}
|
2021-11-05 22:25:00 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
resp, err := svr.GetCompactionState(context.Background(), &milvuspb.GetCompactionStateRequest{})
|
2021-11-05 22:25:00 +08:00
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
2021-11-09 14:47:02 +08:00
|
|
|
assert.Equal(t, commonpb.CompactionState_Completed, resp.GetState())
|
2021-11-05 22:25:00 +08:00
|
|
|
})
|
|
|
|
t.Run("test get compaction state in running", func(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-05 22:25:00 +08:00
|
|
|
|
|
|
|
svr.compactionHandler = &mockCompactionHandler{
|
|
|
|
methods: map[string]interface{}{
|
2021-11-09 14:47:02 +08:00
|
|
|
"getCompactionTasksBySignalID": func(signalID int64) []*compactionTask {
|
|
|
|
return []*compactionTask{
|
|
|
|
{state: executing},
|
|
|
|
{state: executing},
|
|
|
|
{state: executing},
|
|
|
|
{state: completed},
|
|
|
|
{state: completed},
|
2022-08-23 15:50:52 +08:00
|
|
|
{state: failed, plan: &datapb.CompactionPlan{PlanID: 1}},
|
|
|
|
{state: timeout, plan: &datapb.CompactionPlan{PlanID: 2}},
|
|
|
|
{state: timeout},
|
|
|
|
{state: timeout},
|
2021-11-09 14:47:02 +08:00
|
|
|
{state: timeout},
|
|
|
|
}
|
2021-11-05 22:25:00 +08:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
resp, err := svr.GetCompactionState(context.Background(), &milvuspb.GetCompactionStateRequest{CompactionID: 1})
|
2021-11-05 22:25:00 +08:00
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
2021-11-09 14:47:02 +08:00
|
|
|
assert.Equal(t, commonpb.CompactionState_Executing, resp.GetState())
|
2021-11-05 22:25:00 +08:00
|
|
|
assert.EqualValues(t, 3, resp.GetExecutingPlanNo())
|
|
|
|
assert.EqualValues(t, 2, resp.GetCompletedPlanNo())
|
2022-08-23 15:50:52 +08:00
|
|
|
assert.EqualValues(t, 1, resp.GetFailedPlanNo())
|
|
|
|
assert.EqualValues(t, 4, resp.GetTimeoutPlanNo())
|
2021-11-05 22:25:00 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Abnormal)
|
2021-11-05 22:25:00 +08:00
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
resp, err := svr.GetCompactionState(context.Background(), &milvuspb.GetCompactionStateRequest{})
|
2021-11-05 22:25:00 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
2022-11-04 14:25:38 +08:00
|
|
|
assert.Equal(t, msgDataCoordIsUnhealthy(paramtable.GetNodeID()), resp.GetStatus().GetReason())
|
2021-11-05 22:25:00 +08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestManualCompaction(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.DataCoordCfg.EnableCompaction.Key, "true")
|
|
|
|
defer paramtable.Get().Reset(Params.DataCoordCfg.EnableCompaction.Key)
|
2021-11-05 22:25:00 +08:00
|
|
|
t.Run("test manual compaction successfully", func(t *testing.T) {
|
2021-12-09 11:03:08 +08:00
|
|
|
svr := &Server{allocator: &MockAllocator{}}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-05 22:25:00 +08:00
|
|
|
svr.compactionTrigger = &mockCompactionTrigger{
|
|
|
|
methods: map[string]interface{}{
|
2022-10-10 20:31:22 +08:00
|
|
|
"forceTriggerCompaction": func(collectionID int64) (UniqueID, error) {
|
2021-11-05 22:25:00 +08:00
|
|
|
return 1, nil
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
resp, err := svr.ManualCompaction(context.TODO(), &milvuspb.ManualCompactionRequest{
|
2021-11-05 22:25:00 +08:00
|
|
|
CollectionID: 1,
|
|
|
|
Timetravel: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test manual compaction failure", func(t *testing.T) {
|
2021-12-09 11:03:08 +08:00
|
|
|
svr := &Server{allocator: &MockAllocator{}}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-05 22:25:00 +08:00
|
|
|
svr.compactionTrigger = &mockCompactionTrigger{
|
|
|
|
methods: map[string]interface{}{
|
2022-10-10 20:31:22 +08:00
|
|
|
"forceTriggerCompaction": func(collectionID int64) (UniqueID, error) {
|
2021-11-05 22:25:00 +08:00
|
|
|
return 0, errors.New("mock error")
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
resp, err := svr.ManualCompaction(context.TODO(), &milvuspb.ManualCompactionRequest{
|
2021-11-05 22:25:00 +08:00
|
|
|
CollectionID: 1,
|
|
|
|
Timetravel: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test manual compaction with closed server", func(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Abnormal)
|
2021-11-05 22:25:00 +08:00
|
|
|
svr.compactionTrigger = &mockCompactionTrigger{
|
|
|
|
methods: map[string]interface{}{
|
2022-10-10 20:31:22 +08:00
|
|
|
"forceTriggerCompaction": func(collectionID int64) (UniqueID, error) {
|
2021-11-05 22:25:00 +08:00
|
|
|
return 1, nil
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
resp, err := svr.ManualCompaction(context.TODO(), &milvuspb.ManualCompactionRequest{
|
2021-11-05 22:25:00 +08:00
|
|
|
CollectionID: 1,
|
|
|
|
Timetravel: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode)
|
2022-11-04 14:25:38 +08:00
|
|
|
assert.Equal(t, msgDataCoordIsUnhealthy(paramtable.GetNodeID()), resp.Status.Reason)
|
2021-11-05 22:25:00 +08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
func TestGetCompactionStateWithPlans(t *testing.T) {
|
|
|
|
t.Run("test get compaction state successfully", func(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
|
|
|
|
2021-11-09 14:47:02 +08:00
|
|
|
svr.compactionHandler = &mockCompactionHandler{
|
|
|
|
methods: map[string]interface{}{
|
|
|
|
"getCompactionTasksBySignalID": func(signalID int64) []*compactionTask {
|
|
|
|
return []*compactionTask{
|
|
|
|
{
|
|
|
|
triggerInfo: &compactionSignal{id: 1},
|
|
|
|
state: executing,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.GetCompactionStateWithPlans(context.TODO(), &milvuspb.GetCompactionPlansRequest{
|
|
|
|
CompactionID: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.Equal(t, commonpb.CompactionState_Executing, resp.State)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test get compaction state with closed server", func(t *testing.T) {
|
|
|
|
svr := &Server{}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Abnormal)
|
2021-11-09 14:47:02 +08:00
|
|
|
svr.compactionHandler = &mockCompactionHandler{
|
|
|
|
methods: map[string]interface{}{
|
|
|
|
"getCompactionTasksBySignalID": func(signalID int64) []*compactionTask {
|
|
|
|
return []*compactionTask{
|
|
|
|
{
|
|
|
|
triggerInfo: &compactionSignal{id: 1},
|
|
|
|
state: executing,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.GetCompactionStateWithPlans(context.TODO(), &milvuspb.GetCompactionPlansRequest{
|
|
|
|
CompactionID: 1,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode)
|
2022-11-04 14:25:38 +08:00
|
|
|
assert.Equal(t, msgDataCoordIsUnhealthy(paramtable.GetNodeID()), resp.Status.Reason)
|
2021-11-09 14:47:02 +08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-09-06 17:02:41 +08:00
|
|
|
func TestOptions(t *testing.T) {
|
2022-03-28 22:33:27 +08:00
|
|
|
kv := getMetaKv(t)
|
|
|
|
defer func() {
|
|
|
|
kv.RemoveWithPrefix("")
|
|
|
|
kv.Close()
|
|
|
|
}()
|
|
|
|
|
2023-01-12 19:49:40 +08:00
|
|
|
t.Run("WithRootCoordCreator", func(t *testing.T) {
|
2021-09-06 17:02:41 +08:00
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2021-12-29 14:35:21 +08:00
|
|
|
var crt rootCoordCreatorFunc = func(ctx context.Context, metaRoot string, etcdClient *clientv3.Client) (types.RootCoord, error) {
|
2021-09-06 17:02:41 +08:00
|
|
|
return nil, errors.New("dummy")
|
|
|
|
}
|
2023-01-12 19:49:40 +08:00
|
|
|
opt := WithRootCoordCreator(crt)
|
2021-09-06 17:02:41 +08:00
|
|
|
assert.NotNil(t, opt)
|
|
|
|
svr.rootCoordClientCreator = nil
|
|
|
|
opt(svr)
|
|
|
|
// testify cannot compare function directly
|
|
|
|
// the behavior is actually undefined
|
|
|
|
assert.NotNil(t, crt)
|
|
|
|
assert.NotNil(t, svr.rootCoordClientCreator)
|
|
|
|
})
|
2023-01-12 19:49:40 +08:00
|
|
|
t.Run("WithCluster", func(t *testing.T) {
|
2022-03-28 22:33:27 +08:00
|
|
|
defer kv.RemoveWithPrefix("")
|
|
|
|
|
2021-10-14 15:44:34 +08:00
|
|
|
sessionManager := NewSessionManager()
|
2021-11-17 23:25:12 +08:00
|
|
|
channelManager, err := NewChannelManager(kv, newMockHandler())
|
2021-10-14 15:44:34 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
cluster := NewCluster(sessionManager, channelManager)
|
2021-09-06 17:02:41 +08:00
|
|
|
assert.Nil(t, err)
|
2023-01-12 19:49:40 +08:00
|
|
|
opt := WithCluster(cluster)
|
2021-09-06 17:02:41 +08:00
|
|
|
assert.NotNil(t, opt)
|
|
|
|
svr := newTestServer(t, nil, opt)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
2021-10-14 15:44:34 +08:00
|
|
|
assert.Same(t, cluster, svr.cluster)
|
2021-09-06 17:02:41 +08:00
|
|
|
})
|
2023-01-12 19:49:40 +08:00
|
|
|
t.Run("WithDataNodeCreator", func(t *testing.T) {
|
2021-09-18 19:10:07 +08:00
|
|
|
var target int64
|
2021-12-14 15:31:07 +08:00
|
|
|
var val = rand.Int63()
|
2023-01-12 19:49:40 +08:00
|
|
|
opt := WithDataNodeCreator(func(context.Context, string) (types.DataNode, error) {
|
2021-09-18 19:10:07 +08:00
|
|
|
target = val
|
|
|
|
return nil, nil
|
|
|
|
})
|
|
|
|
assert.NotNil(t, opt)
|
|
|
|
|
2022-04-07 22:05:32 +08:00
|
|
|
factory := dependency.NewDefaultFactory(true)
|
2021-09-18 19:10:07 +08:00
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr := CreateServer(context.TODO(), factory, opt)
|
2021-09-18 19:10:07 +08:00
|
|
|
dn, err := svr.dataNodeCreator(context.Background(), "")
|
|
|
|
assert.Nil(t, dn)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, target, val)
|
|
|
|
})
|
2021-09-06 17:02:41 +08:00
|
|
|
}
|
|
|
|
|
2021-10-14 15:44:34 +08:00
|
|
|
type mockPolicyFactory struct {
|
|
|
|
ChannelPolicyFactoryV1
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewRegisterPolicy create a new register policy
|
|
|
|
func (p *mockPolicyFactory) NewRegisterPolicy() RegisterPolicy {
|
|
|
|
return EmptyRegister
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewDeregisterPolicy create a new dereigster policy
|
|
|
|
func (p *mockPolicyFactory) NewDeregisterPolicy() DeregisterPolicy {
|
|
|
|
return EmptyDeregisterPolicy
|
|
|
|
}
|
|
|
|
|
2021-09-06 17:02:41 +08:00
|
|
|
func TestHandleSessionEvent(t *testing.T) {
|
2022-03-28 22:33:27 +08:00
|
|
|
kv := getMetaKv(t)
|
|
|
|
defer func() {
|
|
|
|
kv.RemoveWithPrefix("")
|
|
|
|
kv.Close()
|
|
|
|
}()
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
|
|
defer cancel()
|
|
|
|
|
2021-11-17 23:25:12 +08:00
|
|
|
channelManager, err := NewChannelManager(kv, newMockHandler(), withFactory(&mockPolicyFactory{}))
|
2021-10-14 15:44:34 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
sessionManager := NewSessionManager()
|
|
|
|
cluster := NewCluster(sessionManager, channelManager)
|
2021-09-06 17:02:41 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2022-03-28 22:33:27 +08:00
|
|
|
err = cluster.Startup(ctx, nil)
|
2021-10-15 17:12:35 +08:00
|
|
|
assert.Nil(t, err)
|
2021-10-14 15:44:34 +08:00
|
|
|
defer cluster.Close()
|
2021-09-06 17:02:41 +08:00
|
|
|
|
2023-01-12 19:49:40 +08:00
|
|
|
svr := newTestServer(t, nil, WithCluster(cluster))
|
2021-09-06 17:02:41 +08:00
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
t.Run("handle events", func(t *testing.T) {
|
|
|
|
// None event
|
|
|
|
evt := &sessionutil.SessionEvent{
|
|
|
|
EventType: sessionutil.SessionNoneEvent,
|
|
|
|
Session: &sessionutil.Session{
|
|
|
|
ServerID: 0,
|
|
|
|
ServerName: "",
|
|
|
|
Address: "",
|
|
|
|
Exclusive: false,
|
|
|
|
},
|
|
|
|
}
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.handleSessionEvent(context.Background(), typeutil.DataNodeRole, evt)
|
2021-10-15 17:12:35 +08:00
|
|
|
assert.Nil(t, err)
|
2021-09-06 17:02:41 +08:00
|
|
|
|
|
|
|
evt = &sessionutil.SessionEvent{
|
|
|
|
EventType: sessionutil.SessionAddEvent,
|
|
|
|
Session: &sessionutil.Session{
|
|
|
|
ServerID: 101,
|
|
|
|
ServerName: "DN101",
|
|
|
|
Address: "DN127.0.0.101",
|
|
|
|
Exclusive: false,
|
|
|
|
},
|
|
|
|
}
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.handleSessionEvent(context.Background(), typeutil.DataNodeRole, evt)
|
2021-10-15 17:12:35 +08:00
|
|
|
assert.Nil(t, err)
|
2021-10-14 15:44:34 +08:00
|
|
|
dataNodes := svr.cluster.GetSessions()
|
2021-09-06 17:02:41 +08:00
|
|
|
assert.EqualValues(t, 1, len(dataNodes))
|
2021-10-14 15:44:34 +08:00
|
|
|
assert.EqualValues(t, "DN127.0.0.101", dataNodes[0].info.Address)
|
2021-09-06 17:02:41 +08:00
|
|
|
|
|
|
|
evt = &sessionutil.SessionEvent{
|
|
|
|
EventType: sessionutil.SessionDelEvent,
|
|
|
|
Session: &sessionutil.Session{
|
|
|
|
ServerID: 101,
|
|
|
|
ServerName: "DN101",
|
|
|
|
Address: "DN127.0.0.101",
|
|
|
|
Exclusive: false,
|
|
|
|
},
|
|
|
|
}
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.handleSessionEvent(context.Background(), typeutil.DataNodeRole, evt)
|
2021-10-15 17:12:35 +08:00
|
|
|
assert.Nil(t, err)
|
2021-10-14 15:44:34 +08:00
|
|
|
dataNodes = svr.cluster.GetSessions()
|
2021-09-06 17:02:41 +08:00
|
|
|
assert.EqualValues(t, 0, len(dataNodes))
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("nil evt", func(t *testing.T) {
|
|
|
|
assert.NotPanics(t, func() {
|
2023-01-04 19:37:36 +08:00
|
|
|
err = svr.handleSessionEvent(context.Background(), typeutil.DataNodeRole, nil)
|
2021-10-15 17:12:35 +08:00
|
|
|
assert.Nil(t, err)
|
2021-09-06 17:02:41 +08:00
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
type rootCoordSegFlushComplete struct {
|
|
|
|
mockRootCoordService
|
|
|
|
flag bool
|
|
|
|
}
|
|
|
|
|
2022-12-01 16:31:16 +08:00
|
|
|
// SegmentFlushCompleted, override default behavior
|
2021-09-06 17:02:41 +08:00
|
|
|
func (rc *rootCoordSegFlushComplete) SegmentFlushCompleted(ctx context.Context, req *datapb.SegmentFlushCompletedMsg) (*commonpb.Status, error) {
|
|
|
|
if rc.flag {
|
|
|
|
return &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, nil
|
|
|
|
}
|
|
|
|
return &commonpb.Status{ErrorCode: commonpb.ErrorCode_UnexpectedError}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestPostFlush(t *testing.T) {
|
|
|
|
t.Run("segment not found", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
err := svr.postFlush(context.Background(), 1)
|
2023-02-26 11:31:49 +08:00
|
|
|
assert.EqualValues(t, "segment not found, might be a faked segemnt, ignore post flush", err.Error())
|
2021-09-06 17:02:41 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("success post flush", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
svr.rootCoordClient = &rootCoordSegFlushComplete{flag: true}
|
|
|
|
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(&datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
CollectionID: 1,
|
|
|
|
PartitionID: 1,
|
|
|
|
State: commonpb.SegmentState_Flushing,
|
|
|
|
}))
|
|
|
|
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
err = svr.postFlush(context.Background(), 1)
|
|
|
|
assert.Nil(t, err)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-11-23 10:55:14 +08:00
|
|
|
func TestGetFlushState(t *testing.T) {
|
|
|
|
t.Run("get flush state with all flushed segments", func(t *testing.T) {
|
|
|
|
svr := &Server{
|
|
|
|
meta: &meta{
|
|
|
|
segments: &SegmentsInfo{
|
|
|
|
segments: map[int64]*SegmentInfo{
|
|
|
|
1: {
|
|
|
|
SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
2: {
|
|
|
|
SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
ID: 2,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-23 10:55:14 +08:00
|
|
|
resp, err := svr.GetFlushState(context.TODO(), &milvuspb.GetFlushStateRequest{SegmentIDs: []int64{1, 2}})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, &milvuspb.GetFlushStateResponse{
|
|
|
|
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
|
|
|
Flushed: true,
|
|
|
|
}, resp)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("get flush state with unflushed segments", func(t *testing.T) {
|
|
|
|
svr := &Server{
|
|
|
|
meta: &meta{
|
|
|
|
segments: &SegmentsInfo{
|
|
|
|
segments: map[int64]*SegmentInfo{
|
|
|
|
1: {
|
|
|
|
SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
2: {
|
|
|
|
SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
ID: 2,
|
|
|
|
State: commonpb.SegmentState_Sealed,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-23 10:55:14 +08:00
|
|
|
|
|
|
|
resp, err := svr.GetFlushState(context.TODO(), &milvuspb.GetFlushStateRequest{SegmentIDs: []int64{1, 2}})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, &milvuspb.GetFlushStateResponse{
|
|
|
|
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
|
|
|
Flushed: false,
|
|
|
|
}, resp)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("get flush state with compacted segments", func(t *testing.T) {
|
|
|
|
svr := &Server{
|
|
|
|
meta: &meta{
|
|
|
|
segments: &SegmentsInfo{
|
|
|
|
segments: map[int64]*SegmentInfo{
|
|
|
|
1: {
|
|
|
|
SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
ID: 1,
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
2: {
|
|
|
|
SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
ID: 2,
|
|
|
|
State: commonpb.SegmentState_Dropped,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
2022-10-10 15:55:22 +08:00
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
2021-11-23 10:55:14 +08:00
|
|
|
|
|
|
|
resp, err := svr.GetFlushState(context.TODO(), &milvuspb.GetFlushStateRequest{SegmentIDs: []int64{1, 2}})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, &milvuspb.GetFlushStateResponse{
|
|
|
|
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
|
|
|
Flushed: true,
|
|
|
|
}, resp)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-04-06 15:33:32 +08:00
|
|
|
func TestDataCoordServer_SetSegmentState(t *testing.T) {
|
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
segment := &datapb.SegmentInfo{
|
|
|
|
ID: 1000,
|
|
|
|
CollectionID: 100,
|
|
|
|
PartitionID: 0,
|
|
|
|
InsertChannel: "c1",
|
|
|
|
NumOfRows: 0,
|
|
|
|
State: commonpb.SegmentState_Growing,
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2022-04-06 15:33:32 +08:00
|
|
|
ChannelName: "c1",
|
|
|
|
MsgID: []byte{},
|
|
|
|
MsgGroup: "",
|
|
|
|
Timestamp: 0,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
err := svr.meta.AddSegment(NewSegmentInfo(segment))
|
|
|
|
assert.Nil(t, err)
|
|
|
|
// Set segment state.
|
|
|
|
svr.SetSegmentState(context.TODO(), &datapb.SetSegmentStateRequest{
|
|
|
|
SegmentId: 1000,
|
|
|
|
NewState: commonpb.SegmentState_Flushed,
|
|
|
|
})
|
|
|
|
// Verify that the state has been updated.
|
|
|
|
resp, err := svr.GetSegmentStates(context.TODO(), &datapb.GetSegmentStatesRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: 0,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
SegmentIDs: []int64{1000},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.States))
|
|
|
|
assert.EqualValues(t, commonpb.SegmentState_Flushed, resp.States[0].State)
|
|
|
|
})
|
|
|
|
|
2022-11-03 14:41:35 +08:00
|
|
|
t.Run("dataCoord meta set state not exists", func(t *testing.T) {
|
|
|
|
meta, err := newMemoryMeta()
|
2022-09-27 18:04:53 +08:00
|
|
|
assert.NoError(t, err)
|
|
|
|
svr := newTestServerWithMeta(t, nil, meta)
|
2022-04-06 15:33:32 +08:00
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
// Set segment state.
|
|
|
|
svr.SetSegmentState(context.TODO(), &datapb.SetSegmentStateRequest{
|
|
|
|
SegmentId: 1000,
|
|
|
|
NewState: commonpb.SegmentState_Flushed,
|
|
|
|
})
|
|
|
|
// Verify that the state has been updated.
|
|
|
|
resp, err := svr.GetSegmentStates(context.TODO(), &datapb.GetSegmentStatesRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: 0,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: 0,
|
|
|
|
},
|
|
|
|
SegmentIDs: []int64{1000},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
|
|
|
|
assert.EqualValues(t, 1, len(resp.States))
|
2022-07-08 13:48:21 +08:00
|
|
|
assert.EqualValues(t, commonpb.SegmentState_NotExist, resp.States[0].State)
|
2022-04-06 15:33:32 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
resp, err := svr.SetSegmentState(context.TODO(), &datapb.SetSegmentStateRequest{
|
|
|
|
SegmentId: 1000,
|
|
|
|
NewState: commonpb.SegmentState_Flushed,
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
|
|
assert.Equal(t, serverNotServingErrMsg, resp.GetStatus().GetReason())
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-06-02 18:54:04 +08:00
|
|
|
func TestDataCoord_Import(t *testing.T) {
|
2022-09-26 18:06:54 +08:00
|
|
|
storage.CheckBucketRetryAttempts = 2
|
|
|
|
|
2022-03-21 15:47:23 +08:00
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
2022-09-26 18:06:54 +08:00
|
|
|
svr.sessionManager.AddSession(&NodeInfo{
|
|
|
|
NodeID: 0,
|
|
|
|
Address: "localhost:8080",
|
|
|
|
})
|
2022-04-01 11:33:28 +08:00
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2022-04-01 11:33:28 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
resp, err := svr.Import(svr.ctx, &datapb.ImportTaskRequest{
|
|
|
|
ImportTask: &datapb.ImportTask{
|
|
|
|
CollectionId: 100,
|
|
|
|
PartitionId: 100,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.Status.GetErrorCode())
|
2022-09-26 18:06:54 +08:00
|
|
|
closeTestServer(t, svr)
|
2022-04-01 11:33:28 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("no free node", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
|
|
|
|
err := svr.channelManager.AddNode(0)
|
|
|
|
assert.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 0})
|
2022-04-01 11:33:28 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
resp, err := svr.Import(svr.ctx, &datapb.ImportTaskRequest{
|
|
|
|
ImportTask: &datapb.ImportTask{
|
|
|
|
CollectionId: 100,
|
|
|
|
PartitionId: 100,
|
|
|
|
},
|
|
|
|
WorkingNodes: []int64{0},
|
2022-03-21 15:47:23 +08:00
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2022-05-07 14:05:52 +08:00
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_UnexpectedError, resp.Status.GetErrorCode())
|
2022-09-26 18:06:54 +08:00
|
|
|
closeTestServer(t, svr)
|
2022-03-21 15:47:23 +08:00
|
|
|
})
|
2022-04-01 11:33:28 +08:00
|
|
|
|
|
|
|
t.Run("no datanode available", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
2023-01-06 14:33:36 +08:00
|
|
|
Params.BaseTable.Save("minio.address", "minio:9000")
|
|
|
|
defer Params.BaseTable.Reset("minio.address")
|
2022-04-01 11:33:28 +08:00
|
|
|
resp, err := svr.Import(svr.ctx, &datapb.ImportTaskRequest{
|
|
|
|
ImportTask: &datapb.ImportTask{
|
|
|
|
CollectionId: 100,
|
|
|
|
PartitionId: 100,
|
|
|
|
},
|
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_UnexpectedError, resp.Status.GetErrorCode())
|
2022-09-26 18:06:54 +08:00
|
|
|
closeTestServer(t, svr)
|
2022-04-01 11:33:28 +08:00
|
|
|
})
|
|
|
|
|
2022-03-21 15:47:23 +08:00
|
|
|
t.Run("with closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
|
2022-04-01 11:33:28 +08:00
|
|
|
resp, err := svr.Import(svr.ctx, &datapb.ImportTaskRequest{
|
|
|
|
ImportTask: &datapb.ImportTask{
|
|
|
|
CollectionId: 100,
|
|
|
|
PartitionId: 100,
|
|
|
|
},
|
2022-03-21 15:47:23 +08:00
|
|
|
})
|
|
|
|
assert.Nil(t, err)
|
2022-03-22 15:11:24 +08:00
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.GetErrorCode())
|
2022-11-04 14:25:38 +08:00
|
|
|
assert.Equal(t, msgDataCoordIsUnhealthy(paramtable.GetNodeID()), resp.Status.GetReason())
|
2022-03-21 15:47:23 +08:00
|
|
|
})
|
2022-04-20 14:03:40 +08:00
|
|
|
|
|
|
|
t.Run("test update segment stat", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
|
|
|
|
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
|
2023-03-04 23:21:50 +08:00
|
|
|
Stats: []*commonpb.SegmentStats{{
|
2022-04-20 14:03:40 +08:00
|
|
|
SegmentID: 100,
|
|
|
|
NumRows: int64(1),
|
|
|
|
}},
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
|
2022-09-26 18:06:54 +08:00
|
|
|
closeTestServer(t, svr)
|
2022-04-20 14:03:40 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test update segment stat w/ closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
|
|
|
|
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
|
2023-03-04 23:21:50 +08:00
|
|
|
Stats: []*commonpb.SegmentStats{{
|
2022-04-20 14:03:40 +08:00
|
|
|
SegmentID: 100,
|
|
|
|
NumRows: int64(1),
|
|
|
|
}},
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.GetErrorCode())
|
|
|
|
})
|
2023-02-27 10:41:46 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestDataCoord_SegmentStatistics(t *testing.T) {
|
|
|
|
t.Run("test update imported segment stat", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
|
|
|
|
seg1 := &datapb.SegmentInfo{
|
|
|
|
ID: 100,
|
|
|
|
Binlogs: []*datapb.FieldBinlog{getFieldBinlogPathsWithEntry(101, 1, getInsertLogPath("log1", 100))},
|
|
|
|
Statslogs: []*datapb.FieldBinlog{getFieldBinlogPaths(101, getStatsLogPath("log2", 100))},
|
|
|
|
Deltalogs: []*datapb.FieldBinlog{getFieldBinlogPaths(101, getDeltaLogPath("log3", 100))},
|
|
|
|
State: commonpb.SegmentState_Importing,
|
|
|
|
}
|
|
|
|
|
|
|
|
info := NewSegmentInfo(seg1)
|
|
|
|
svr.meta.AddSegment(info)
|
|
|
|
|
|
|
|
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
|
2023-03-04 23:21:50 +08:00
|
|
|
Stats: []*commonpb.SegmentStats{{
|
2023-02-27 10:41:46 +08:00
|
|
|
SegmentID: 100,
|
|
|
|
NumRows: int64(1),
|
|
|
|
}},
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
2023-03-03 14:13:49 +08:00
|
|
|
assert.Equal(t, svr.meta.GetHealthySegment(100).currRows, int64(1))
|
2023-02-27 10:41:46 +08:00
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
})
|
2022-05-31 16:36:03 +08:00
|
|
|
|
2023-02-27 10:41:46 +08:00
|
|
|
t.Run("test update flushed segment stat", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
|
|
|
|
seg1 := &datapb.SegmentInfo{
|
|
|
|
ID: 100,
|
|
|
|
Binlogs: []*datapb.FieldBinlog{getFieldBinlogPathsWithEntry(101, 1, getInsertLogPath("log1", 100))},
|
|
|
|
Statslogs: []*datapb.FieldBinlog{getFieldBinlogPaths(101, getStatsLogPath("log2", 100))},
|
|
|
|
Deltalogs: []*datapb.FieldBinlog{getFieldBinlogPaths(101, getDeltaLogPath("log3", 100))},
|
|
|
|
State: commonpb.SegmentState_Flushed,
|
|
|
|
}
|
|
|
|
|
|
|
|
info := NewSegmentInfo(seg1)
|
|
|
|
svr.meta.AddSegment(info)
|
|
|
|
|
|
|
|
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
|
2023-03-04 23:21:50 +08:00
|
|
|
Stats: []*commonpb.SegmentStats{{
|
2023-02-27 10:41:46 +08:00
|
|
|
SegmentID: 100,
|
|
|
|
NumRows: int64(1),
|
|
|
|
}},
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
2023-03-03 14:13:49 +08:00
|
|
|
assert.Equal(t, svr.meta.GetHealthySegment(100).currRows, int64(0))
|
2023-02-27 10:41:46 +08:00
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
})
|
2022-06-02 18:54:04 +08:00
|
|
|
}
|
|
|
|
|
2022-09-26 18:06:54 +08:00
|
|
|
func TestDataCoord_SaveImportSegment(t *testing.T) {
|
2022-06-02 18:54:04 +08:00
|
|
|
t.Run("test add segment", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
2022-10-10 20:31:22 +08:00
|
|
|
svr.meta.AddCollection(&collectionInfo{
|
|
|
|
ID: 100,
|
|
|
|
})
|
2022-09-26 18:06:54 +08:00
|
|
|
seg := buildSegment(100, 100, 100, "ch1", false)
|
|
|
|
svr.meta.AddSegment(seg)
|
|
|
|
svr.sessionManager.AddSession(&NodeInfo{
|
|
|
|
NodeID: 110,
|
|
|
|
Address: "localhost:8080",
|
|
|
|
})
|
2022-06-02 18:54:04 +08:00
|
|
|
err := svr.channelManager.AddNode(110)
|
|
|
|
assert.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 100})
|
2022-06-02 18:54:04 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2022-09-26 18:06:54 +08:00
|
|
|
status, err := svr.SaveImportSegment(context.TODO(), &datapb.SaveImportSegmentRequest{
|
2022-06-02 18:54:04 +08:00
|
|
|
SegmentId: 100,
|
|
|
|
ChannelName: "ch1",
|
|
|
|
CollectionId: 100,
|
|
|
|
PartitionId: 100,
|
|
|
|
RowNum: int64(1),
|
2022-09-26 18:06:54 +08:00
|
|
|
SaveBinlogPathReq: &datapb.SaveBinlogPathsRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
2022-11-04 14:25:38 +08:00
|
|
|
SourceID: paramtable.GetNodeID(),
|
2022-09-26 18:06:54 +08:00
|
|
|
},
|
|
|
|
SegmentID: 100,
|
|
|
|
CollectionID: 100,
|
|
|
|
Importing: true,
|
|
|
|
StartPositions: []*datapb.SegmentStartPosition{
|
|
|
|
{
|
2023-03-04 23:21:50 +08:00
|
|
|
StartPosition: &msgpb.MsgPosition{
|
2022-09-26 18:06:54 +08:00
|
|
|
ChannelName: "ch1",
|
|
|
|
Timestamp: 1,
|
|
|
|
},
|
|
|
|
SegmentID: 100,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2022-06-02 18:54:04 +08:00
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test add segment w/ bad channel name", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
err := svr.channelManager.AddNode(110)
|
|
|
|
assert.Nil(t, err)
|
2022-08-22 14:42:52 +08:00
|
|
|
err = svr.channelManager.Watch(&channel{Name: "ch1", CollectionID: 100})
|
2022-06-02 18:54:04 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2022-09-26 18:06:54 +08:00
|
|
|
status, err := svr.SaveImportSegment(context.TODO(), &datapb.SaveImportSegmentRequest{
|
2022-06-02 18:54:04 +08:00
|
|
|
SegmentId: 100,
|
|
|
|
ChannelName: "non-channel",
|
|
|
|
CollectionId: 100,
|
|
|
|
PartitionId: 100,
|
|
|
|
RowNum: int64(1),
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.GetErrorCode())
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test add segment w/ closed server", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
closeTestServer(t, svr)
|
|
|
|
|
2022-09-26 18:06:54 +08:00
|
|
|
status, err := svr.SaveImportSegment(context.TODO(), &datapb.SaveImportSegmentRequest{})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_DataCoordNA, status.GetErrorCode())
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestDataCoord_UnsetIsImportingState(t *testing.T) {
|
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
seg := buildSegment(100, 100, 100, "ch1", false)
|
|
|
|
svr.meta.AddSegment(seg)
|
|
|
|
|
|
|
|
status, err := svr.UnsetIsImportingState(context.Background(), &datapb.UnsetIsImportingStateRequest{
|
|
|
|
SegmentIds: []int64{100},
|
|
|
|
})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode())
|
|
|
|
|
|
|
|
// Trying to unset state of a segment that does not exist.
|
|
|
|
status, err = svr.UnsetIsImportingState(context.Background(), &datapb.UnsetIsImportingStateRequest{
|
|
|
|
SegmentIds: []int64{999},
|
|
|
|
})
|
2022-06-02 18:54:04 +08:00
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.GetErrorCode())
|
2022-05-31 16:36:03 +08:00
|
|
|
})
|
2022-03-21 15:47:23 +08:00
|
|
|
}
|
|
|
|
|
2022-11-10 22:13:04 +08:00
|
|
|
func TestDataCoordServer_UpdateChannelCheckpoint(t *testing.T) {
|
|
|
|
mockVChannel := "fake-by-dev-rootcoord-dml-1-testchannelcp-v0"
|
|
|
|
mockPChannel := "fake-by-dev-rootcoord-dml-1"
|
|
|
|
|
|
|
|
t.Run("UpdateChannelCheckpoint", func(t *testing.T) {
|
|
|
|
svr := newTestServer(t, nil)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
|
|
|
|
req := &datapb.UpdateChannelCheckpointRequest{
|
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
SourceID: paramtable.GetNodeID(),
|
|
|
|
},
|
|
|
|
VChannel: mockVChannel,
|
2023-03-04 23:21:50 +08:00
|
|
|
Position: &msgpb.MsgPosition{
|
2022-11-10 22:13:04 +08:00
|
|
|
ChannelName: mockPChannel,
|
|
|
|
Timestamp: 1000,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := svr.UpdateChannelCheckpoint(context.TODO(), req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.ErrorCode)
|
|
|
|
|
|
|
|
req.Position = nil
|
|
|
|
resp, err = svr.UpdateChannelCheckpoint(context.TODO(), req)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.EqualValues(t, commonpb.ErrorCode_UnexpectedError, resp.ErrorCode)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-02-23 11:09:52 +08:00
|
|
|
// https://github.com/milvus-io/milvus/issues/15659
|
|
|
|
func TestIssue15659(t *testing.T) {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
s := &Server{
|
|
|
|
helper: ServerHelper{
|
|
|
|
eventAfterHandleDataNodeTt: func() {},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ms := &MockClosePanicMsgstream{}
|
2022-03-11 20:09:59 +08:00
|
|
|
|
|
|
|
msgChan := make(chan *msgstream.MsgPack)
|
|
|
|
go func() {
|
|
|
|
msgChan <- &msgstream.MsgPack{}
|
|
|
|
}()
|
|
|
|
ms.On("Chan").Return(msgChan)
|
|
|
|
|
2022-02-23 11:09:52 +08:00
|
|
|
ch := make(chan struct{})
|
|
|
|
go func() {
|
|
|
|
assert.NotPanics(t, func() {
|
|
|
|
s.serverLoopWg.Add(1)
|
|
|
|
s.handleDataNodeTimetickMsgstream(ctx, ms)
|
|
|
|
close(ch)
|
|
|
|
})
|
|
|
|
}()
|
|
|
|
cancel()
|
|
|
|
<-ch
|
|
|
|
}
|
|
|
|
|
|
|
|
type MockClosePanicMsgstream struct {
|
|
|
|
mock.Mock
|
|
|
|
msgstream.MsgStream
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ms *MockClosePanicMsgstream) Close() {
|
|
|
|
panic("mocked close panic")
|
|
|
|
}
|
|
|
|
|
2022-03-11 20:09:59 +08:00
|
|
|
func (ms *MockClosePanicMsgstream) Chan() <-chan *msgstream.MsgPack {
|
2022-02-23 11:09:52 +08:00
|
|
|
args := ms.Called()
|
2022-03-11 20:09:59 +08:00
|
|
|
return args.Get(0).(chan *msgstream.MsgPack)
|
2022-02-23 11:09:52 +08:00
|
|
|
}
|
|
|
|
|
2022-09-27 18:04:53 +08:00
|
|
|
func newTestServer(t *testing.T, receiveCh chan any, opts ...Option) *Server {
|
2022-03-28 22:33:27 +08:00
|
|
|
var err error
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.CommonCfg.DataCoordTimeTick.Key, Params.CommonCfg.DataCoordTimeTick.GetValue()+strconv.Itoa(rand.Int()))
|
2022-04-07 22:05:32 +08:00
|
|
|
factory := dependency.NewDefaultFactory(true)
|
2021-05-21 19:28:52 +08:00
|
|
|
|
2022-11-30 18:23:15 +08:00
|
|
|
etcdCli, err := etcd.GetEtcdClient(
|
|
|
|
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
|
|
|
|
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
|
|
|
|
Params.EtcdCfg.Endpoints.GetAsStrings(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSKey.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
|
2021-05-21 19:28:52 +08:00
|
|
|
assert.Nil(t, err)
|
2022-11-17 18:59:09 +08:00
|
|
|
sessKey := path.Join(Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
|
2021-05-26 20:14:30 +08:00
|
|
|
_, err = etcdCli.Delete(context.Background(), sessKey, clientv3.WithPrefix())
|
2021-05-21 19:28:52 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2021-12-29 14:35:21 +08:00
|
|
|
svr := CreateServer(context.TODO(), factory, opts...)
|
|
|
|
svr.SetEtcdClient(etcdCli)
|
2021-09-18 19:10:07 +08:00
|
|
|
svr.dataNodeCreator = func(ctx context.Context, addr string) (types.DataNode, error) {
|
2021-05-26 19:06:56 +08:00
|
|
|
return newMockDataNodeClient(0, receiveCh)
|
2021-04-13 09:47:02 +08:00
|
|
|
}
|
2021-12-29 14:35:21 +08:00
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
2021-06-21 17:28:03 +08:00
|
|
|
return newMockRootCoordService(), nil
|
2021-05-28 09:55:21 +08:00
|
|
|
}
|
2022-03-28 22:33:27 +08:00
|
|
|
|
2021-04-13 09:47:02 +08:00
|
|
|
err = svr.Init()
|
|
|
|
assert.Nil(t, err)
|
2023-02-10 17:40:32 +08:00
|
|
|
if Params.DataCoordCfg.EnableActiveStandby.GetAsBool() {
|
|
|
|
assert.Equal(t, commonpb.StateCode_StandBy, svr.stateCode.Load().(commonpb.StateCode))
|
|
|
|
} else {
|
|
|
|
assert.Equal(t, commonpb.StateCode_Initializing, svr.stateCode.Load().(commonpb.StateCode))
|
|
|
|
}
|
2021-12-15 11:47:10 +08:00
|
|
|
err = svr.Register()
|
|
|
|
assert.Nil(t, err)
|
2023-02-10 17:40:32 +08:00
|
|
|
err = svr.Start()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.StateCode_Healthy, svr.stateCode.Load().(commonpb.StateCode))
|
2022-03-28 22:33:27 +08:00
|
|
|
|
|
|
|
// Stop channal watch state watcher in tests
|
|
|
|
if svr.channelManager != nil && svr.channelManager.stopChecker != nil {
|
|
|
|
svr.channelManager.stopChecker()
|
|
|
|
}
|
|
|
|
|
2021-04-16 16:30:55 +08:00
|
|
|
return svr
|
|
|
|
}
|
|
|
|
|
2022-09-27 18:04:53 +08:00
|
|
|
func newTestServerWithMeta(t *testing.T, receiveCh chan any, meta *meta, opts ...Option) *Server {
|
|
|
|
var err error
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.CommonCfg.DataCoordTimeTick.Key, Params.CommonCfg.DataCoordTimeTick.GetValue()+strconv.Itoa(rand.Int()))
|
2022-09-27 18:04:53 +08:00
|
|
|
factory := dependency.NewDefaultFactory(true)
|
|
|
|
|
2022-11-30 18:23:15 +08:00
|
|
|
etcdCli, err := etcd.GetEtcdClient(
|
|
|
|
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
|
|
|
|
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
|
|
|
|
Params.EtcdCfg.Endpoints.GetAsStrings(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSKey.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
|
2022-09-27 18:04:53 +08:00
|
|
|
assert.Nil(t, err)
|
2022-11-17 18:59:09 +08:00
|
|
|
sessKey := path.Join(Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
|
2022-09-27 18:04:53 +08:00
|
|
|
_, err = etcdCli.Delete(context.Background(), sessKey, clientv3.WithPrefix())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
svr := CreateServer(context.TODO(), factory, opts...)
|
|
|
|
svr.SetEtcdClient(etcdCli)
|
|
|
|
svr.dataNodeCreator = func(ctx context.Context, addr string) (types.DataNode, error) {
|
|
|
|
return newMockDataNodeClient(0, receiveCh)
|
|
|
|
}
|
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
2023-01-04 19:37:36 +08:00
|
|
|
//indexCoord := mocks.NewMockIndexCoord(t)
|
|
|
|
//indexCoord.EXPECT().GetIndexInfos(mock.Anything, mock.Anything).Return(nil, nil).Maybe()
|
|
|
|
//svr.indexCoord = indexCoord
|
2022-09-27 18:04:53 +08:00
|
|
|
|
|
|
|
err = svr.Init()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
svr.meta = meta
|
|
|
|
|
|
|
|
err = svr.Start()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.Register()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
// Stop channal watch state watcher in tests
|
|
|
|
if svr.channelManager != nil && svr.channelManager.stopChecker != nil {
|
|
|
|
svr.channelManager.stopChecker()
|
|
|
|
}
|
|
|
|
|
|
|
|
return svr
|
|
|
|
}
|
|
|
|
|
2021-04-16 16:30:55 +08:00
|
|
|
func closeTestServer(t *testing.T, svr *Server) {
|
|
|
|
err := svr.Stop()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.CleanMeta()
|
|
|
|
assert.Nil(t, err)
|
2021-04-13 09:47:02 +08:00
|
|
|
}
|
2022-05-31 16:36:03 +08:00
|
|
|
|
2022-09-27 18:04:53 +08:00
|
|
|
func newTestServer2(t *testing.T, receiveCh chan any, opts ...Option) *Server {
|
2022-05-31 16:36:03 +08:00
|
|
|
var err error
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Init()
|
|
|
|
paramtable.Get().Save(Params.CommonCfg.DataCoordTimeTick.Key, Params.CommonCfg.DataCoordTimeTick.GetValue()+strconv.Itoa(rand.Int()))
|
2022-05-31 16:36:03 +08:00
|
|
|
factory := dependency.NewDefaultFactory(true)
|
|
|
|
|
2022-11-30 18:23:15 +08:00
|
|
|
etcdCli, err := etcd.GetEtcdClient(
|
|
|
|
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
|
|
|
|
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
|
|
|
|
Params.EtcdCfg.Endpoints.GetAsStrings(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSKey.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
|
2022-05-31 16:36:03 +08:00
|
|
|
assert.Nil(t, err)
|
2022-11-17 18:59:09 +08:00
|
|
|
sessKey := path.Join(Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
|
2022-05-31 16:36:03 +08:00
|
|
|
_, err = etcdCli.Delete(context.Background(), sessKey, clientv3.WithPrefix())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
svr := CreateServer(context.TODO(), factory, opts...)
|
|
|
|
svr.SetEtcdClient(etcdCli)
|
|
|
|
svr.dataNodeCreator = func(ctx context.Context, addr string) (types.DataNode, error) {
|
|
|
|
return newMockDataNodeClient(0, receiveCh)
|
|
|
|
}
|
|
|
|
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
|
|
|
|
return newMockRootCoordService(), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
err = svr.Init()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.Start()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
_, err = etcdCli.Delete(context.Background(), sessKey, clientv3.WithPrefix())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
err = svr.Register()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
// Stop channal watch state watcher in tests
|
|
|
|
if svr.channelManager != nil && svr.channelManager.stopChecker != nil {
|
|
|
|
svr.channelManager.stopChecker()
|
|
|
|
}
|
|
|
|
|
|
|
|
return svr
|
|
|
|
}
|
|
|
|
|
2022-10-18 13:39:26 +08:00
|
|
|
func Test_CheckHealth(t *testing.T) {
|
|
|
|
t.Run("not healthy", func(t *testing.T) {
|
|
|
|
ctx := context.Background()
|
|
|
|
s := &Server{session: &sessionutil.Session{ServerID: 1}}
|
|
|
|
s.stateCode.Store(commonpb.StateCode_Abnormal)
|
|
|
|
resp, err := s.CheckHealth(ctx, &milvuspb.CheckHealthRequest{})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, false, resp.IsHealthy)
|
|
|
|
assert.NotEmpty(t, resp.Reasons)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("data node health check is ok", func(t *testing.T) {
|
|
|
|
svr := &Server{session: &sessionutil.Session{ServerID: 1}}
|
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
|
|
|
healthClient := &mockDataNodeClient{
|
|
|
|
id: 1,
|
|
|
|
state: commonpb.StateCode_Healthy}
|
|
|
|
sm := NewSessionManager()
|
|
|
|
sm.sessions = struct {
|
|
|
|
sync.RWMutex
|
|
|
|
data map[int64]*Session
|
|
|
|
}{data: map[int64]*Session{1: {
|
|
|
|
client: healthClient,
|
|
|
|
clientCreator: func(ctx context.Context, addr string) (types.DataNode, error) {
|
|
|
|
return healthClient, nil
|
|
|
|
},
|
|
|
|
}}}
|
|
|
|
|
|
|
|
svr.sessionManager = sm
|
|
|
|
ctx := context.Background()
|
|
|
|
resp, err := svr.CheckHealth(ctx, &milvuspb.CheckHealthRequest{})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, true, resp.IsHealthy)
|
|
|
|
assert.Empty(t, resp.Reasons)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("data node health check is fail", func(t *testing.T) {
|
|
|
|
svr := &Server{session: &sessionutil.Session{ServerID: 1}}
|
|
|
|
svr.stateCode.Store(commonpb.StateCode_Healthy)
|
|
|
|
unhealthClient := &mockDataNodeClient{
|
|
|
|
id: 1,
|
|
|
|
state: commonpb.StateCode_Abnormal}
|
|
|
|
sm := NewSessionManager()
|
|
|
|
sm.sessions = struct {
|
|
|
|
sync.RWMutex
|
|
|
|
data map[int64]*Session
|
|
|
|
}{data: map[int64]*Session{1: {
|
|
|
|
client: unhealthClient,
|
|
|
|
clientCreator: func(ctx context.Context, addr string) (types.DataNode, error) {
|
|
|
|
return unhealthClient, nil
|
|
|
|
},
|
|
|
|
}}}
|
|
|
|
svr.sessionManager = sm
|
|
|
|
ctx := context.Background()
|
|
|
|
resp, err := svr.CheckHealth(ctx, &milvuspb.CheckHealthRequest{})
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, false, resp.IsHealthy)
|
|
|
|
assert.NotEmpty(t, resp.Reasons)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-01-04 19:37:36 +08:00
|
|
|
//func Test_initServiceDiscovery(t *testing.T) {
|
|
|
|
// server := newTestServer2(t, nil)
|
|
|
|
// assert.NotNil(t, server)
|
|
|
|
//
|
|
|
|
// segmentID := rand.Int63()
|
|
|
|
// err := server.meta.AddSegment(&SegmentInfo{
|
|
|
|
// SegmentInfo: &datapb.SegmentInfo{
|
|
|
|
// ID: segmentID,
|
|
|
|
// CollectionID: rand.Int63(),
|
|
|
|
// PartitionID: rand.Int63(),
|
|
|
|
// NumOfRows: 100,
|
|
|
|
// },
|
|
|
|
// currRows: 100,
|
|
|
|
// })
|
|
|
|
// assert.Nil(t, err)
|
|
|
|
//
|
|
|
|
// qcSession := sessionutil.NewSession(context.Background(), Params.EtcdCfg.MetaRootPath.GetValue(), server.etcdCli)
|
|
|
|
// qcSession.Init(typeutil.QueryCoordRole, "localhost:19532", true, true)
|
|
|
|
// qcSession.Register()
|
|
|
|
// //req := &datapb.AcquireSegmentLockRequest{
|
|
|
|
// // NodeID: qcSession.ServerID,
|
|
|
|
// // SegmentIDs: []UniqueID{segmentID},
|
|
|
|
// //}
|
|
|
|
// //resp, err := server.AcquireSegmentLock(context.TODO(), req)
|
|
|
|
// //assert.Nil(t, err)
|
|
|
|
// //assert.Equal(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
|
|
|
|
//
|
|
|
|
// sessKey := path.Join(Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot, typeutil.QueryCoordRole)
|
|
|
|
// _, err = server.etcdCli.Delete(context.Background(), sessKey, clientv3.WithPrefix())
|
|
|
|
// assert.Nil(t, err)
|
|
|
|
//
|
|
|
|
// //for {
|
|
|
|
// // if !server.segReferManager.HasSegmentLock(segmentID) {
|
|
|
|
// // break
|
|
|
|
// // }
|
|
|
|
// //}
|
|
|
|
//
|
|
|
|
// closeTestServer(t, server)
|
|
|
|
//}
|
2022-06-02 19:42:03 +08:00
|
|
|
|
2022-09-25 15:56:51 +08:00
|
|
|
func Test_newChunkManagerFactory(t *testing.T) {
|
2022-06-02 19:42:03 +08:00
|
|
|
server := newTestServer2(t, nil)
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.DataCoordCfg.EnableGarbageCollection.Key, "true")
|
2022-06-02 19:42:03 +08:00
|
|
|
|
|
|
|
t.Run("err_minio_bad_address", func(t *testing.T) {
|
2023-01-19 14:53:44 +08:00
|
|
|
paramtable.Get().Save(Params.CommonCfg.StorageType.Key, "minio")
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.MinioCfg.Address.Key, "host:9000:bad")
|
|
|
|
defer paramtable.Get().Reset(Params.MinioCfg.Address.Key)
|
2022-09-25 15:56:51 +08:00
|
|
|
storageCli, err := server.newChunkManagerFactory()
|
|
|
|
assert.Nil(t, storageCli)
|
2022-06-02 19:42:03 +08:00
|
|
|
assert.Error(t, err)
|
2022-07-22 22:10:28 +08:00
|
|
|
assert.Contains(t, err.Error(), "too many colons in address")
|
2022-06-02 19:42:03 +08:00
|
|
|
})
|
|
|
|
|
2022-07-22 22:10:28 +08:00
|
|
|
t.Run("local storage init", func(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.CommonCfg.StorageType.Key, "local")
|
|
|
|
defer paramtable.Get().Reset(Params.CommonCfg.StorageType.Key)
|
2022-09-25 15:56:51 +08:00
|
|
|
storageCli, err := server.newChunkManagerFactory()
|
|
|
|
assert.NotNil(t, storageCli)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func Test_initGarbageCollection(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.DataCoordCfg.EnableGarbageCollection.Key, "true")
|
|
|
|
defer paramtable.Get().Reset(Params.DataCoordCfg.EnableGarbageCollection.Key)
|
|
|
|
|
2022-09-25 15:56:51 +08:00
|
|
|
server := newTestServer2(t, nil)
|
|
|
|
|
|
|
|
t.Run("ok", func(t *testing.T) {
|
|
|
|
storageCli, err := server.newChunkManagerFactory()
|
|
|
|
assert.NotNil(t, storageCli)
|
2022-06-02 19:42:03 +08:00
|
|
|
assert.NoError(t, err)
|
2022-09-25 15:56:51 +08:00
|
|
|
server.initGarbageCollection(storageCli)
|
2022-06-02 19:42:03 +08:00
|
|
|
})
|
2022-09-26 18:06:54 +08:00
|
|
|
t.Run("err_minio_bad_address", func(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.CommonCfg.StorageType.Key, "minio")
|
|
|
|
paramtable.Get().Save(Params.MinioCfg.Address.Key, "host:9000:bad")
|
|
|
|
defer paramtable.Get().Reset(Params.MinioCfg.Address.Key)
|
2022-09-26 18:06:54 +08:00
|
|
|
storageCli, err := server.newChunkManagerFactory()
|
|
|
|
assert.Nil(t, storageCli)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Contains(t, err.Error(), "too many colons in address")
|
|
|
|
})
|
2022-06-02 19:42:03 +08:00
|
|
|
}
|
2022-09-29 18:35:02 +08:00
|
|
|
|
|
|
|
func testDataCoordBase(t *testing.T, opts ...Option) *Server {
|
|
|
|
var err error
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.CommonCfg.DataCoordTimeTick.Key, Params.CommonCfg.DataCoordTimeTick.GetValue()+strconv.Itoa(rand.Int()))
|
2022-09-29 18:35:02 +08:00
|
|
|
factory := dependency.NewDefaultFactory(true)
|
|
|
|
|
2023-01-12 19:49:40 +08:00
|
|
|
ctx := context.Background()
|
2022-11-30 18:23:15 +08:00
|
|
|
etcdCli, err := etcd.GetEtcdClient(
|
|
|
|
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
|
|
|
|
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
|
|
|
|
Params.EtcdCfg.Endpoints.GetAsStrings(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSKey.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
|
|
|
|
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
|
2022-09-29 18:35:02 +08:00
|
|
|
assert.Nil(t, err)
|
2022-11-17 18:59:09 +08:00
|
|
|
sessKey := path.Join(Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
|
2023-01-12 19:49:40 +08:00
|
|
|
_, err = etcdCli.Delete(ctx, sessKey, clientv3.WithPrefix())
|
2022-09-29 18:35:02 +08:00
|
|
|
assert.Nil(t, err)
|
|
|
|
|
2023-01-12 19:49:40 +08:00
|
|
|
svr := CreateServer(ctx, factory, opts...)
|
2022-09-29 18:35:02 +08:00
|
|
|
svr.SetEtcdClient(etcdCli)
|
2023-01-12 19:49:40 +08:00
|
|
|
svr.SetDataNodeCreator(func(ctx context.Context, addr string) (types.DataNode, error) {
|
2022-09-29 18:35:02 +08:00
|
|
|
return newMockDataNodeClient(0, nil)
|
2023-01-12 19:49:40 +08:00
|
|
|
})
|
|
|
|
svr.SetIndexNodeCreator(func(ctx context.Context, addr string) (types.IndexNode, error) {
|
|
|
|
return indexnode.NewMockIndexNodeComponent(ctx)
|
|
|
|
})
|
|
|
|
svr.SetRootCoord(newMockRootCoordService())
|
2022-09-29 18:35:02 +08:00
|
|
|
|
|
|
|
err = svr.Init()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.Start()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
err = svr.Register()
|
|
|
|
assert.Nil(t, err)
|
|
|
|
|
|
|
|
resp, err := svr.GetComponentStates(context.Background())
|
|
|
|
assert.Nil(t, err)
|
|
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
2022-10-10 15:55:22 +08:00
|
|
|
assert.Equal(t, commonpb.StateCode_Healthy, resp.GetState().GetStateCode())
|
2022-09-29 18:35:02 +08:00
|
|
|
|
|
|
|
// stop channal watch state watcher in tests
|
|
|
|
if svr.channelManager != nil && svr.channelManager.stopChecker != nil {
|
|
|
|
svr.channelManager.stopChecker()
|
|
|
|
}
|
|
|
|
|
|
|
|
return svr
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestDataCoord_DisableActiveStandby(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.DataCoordCfg.EnableActiveStandby.Key, "false")
|
2022-09-29 18:35:02 +08:00
|
|
|
svr := testDataCoordBase(t)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
}
|
|
|
|
|
|
|
|
// make sure the main functions work well when EnableActiveStandby=true
|
|
|
|
func TestDataCoord_EnableActiveStandby(t *testing.T) {
|
2022-12-07 18:01:19 +08:00
|
|
|
paramtable.Get().Save(Params.DataCoordCfg.EnableActiveStandby.Key, "true")
|
|
|
|
defer paramtable.Get().Reset(Params.DataCoordCfg.EnableActiveStandby.Key)
|
2022-09-29 18:35:02 +08:00
|
|
|
svr := testDataCoordBase(t)
|
|
|
|
defer closeTestServer(t, svr)
|
|
|
|
}
|